Compare commits
5 Commits
e9945d933f
...
b8f5104167
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b8f5104167 | ||
|
|
8997e3880e | ||
|
|
40f4b3ffab | ||
|
|
d648ba37e0 | ||
|
|
45ba6fb958 |
@ -52,6 +52,7 @@ Windows PowerShell 可运行:
|
|||||||
2. Proxy 容量使用 `Reserved -> Active` 原子转换,禁止超卖。
|
2. Proxy 容量使用 `Reserved -> Active` 原子转换,禁止超卖。
|
||||||
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`,不提供 Lease、
|
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`,不提供 Lease、
|
||||||
Release 或 Renewal。
|
Release 或 Renewal。
|
||||||
4. `pool.maxSize` 是当前未提取库存上限;`fetch.maxTotal` 是累计获取额度。
|
4. `pool.maxSize` 是当前未提取库存硬上限;`fetch.maxTotal` 是 Redis generation
|
||||||
|
内的累计获取停止阈值。
|
||||||
5. CONNECT 向客户端提交 200 后不透明重放。
|
5. CONNECT 向客户端提交 200 后不透明重放。
|
||||||
6. 公开监听必须有认证或 CIDR 访问保护。
|
6. 公开监听必须有认证或 CIDR 访问保护。
|
||||||
|
|||||||
@ -182,6 +182,7 @@ message ReportRuntimeRequest {
|
|||||||
uint64 ownership_epoch = 4;
|
uint64 ownership_epoch = 4;
|
||||||
repeated ProxyRuntime counters = 5;
|
repeated ProxyRuntime counters = 5;
|
||||||
google.protobuf.Timestamp observed_at = 6;
|
google.protobuf.Timestamp observed_at = 6;
|
||||||
|
uint64 report_sequence = 7;
|
||||||
}
|
}
|
||||||
|
|
||||||
message ProxyRuntime {
|
message ProxyRuntime {
|
||||||
|
|||||||
@ -15,7 +15,10 @@ import (
|
|||||||
"proxy-pool/internal/controller/bootstrap"
|
"proxy-pool/internal/controller/bootstrap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const configEnvironment = "PROXY_POOL_CONFIG"
|
const (
|
||||||
|
configEnvironment = "PROXY_POOL_CONFIG"
|
||||||
|
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
|
||||||
|
)
|
||||||
|
|
||||||
type environmentLookup func(string) string
|
type environmentLookup func(string) string
|
||||||
type controllerRun func(context.Context, bootstrap.Options) error
|
type controllerRun func(context.Context, bootstrap.Options) error
|
||||||
@ -54,7 +57,13 @@ func execute(
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
|
var fingerprintKey []byte
|
||||||
|
if getenv != nil {
|
||||||
|
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
|
||||||
|
}
|
||||||
|
err := run(ctx, bootstrap.Options{
|
||||||
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
|
||||||
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
|||||||
if name == configEnvironment {
|
if name == configEnvironment {
|
||||||
return "environment.yaml"
|
return "environment.yaml"
|
||||||
}
|
}
|
||||||
|
if name == fingerprintKeyEnvironment {
|
||||||
|
return "0123456789abcdef0123456789abcdef"
|
||||||
|
}
|
||||||
return ""
|
return ""
|
||||||
}, func(_ context.Context, options bootstrap.Options) error {
|
}, func(_ context.Context, options bootstrap.Options) error {
|
||||||
received = options
|
received = options
|
||||||
return nil
|
return nil
|
||||||
}, &bytes.Buffer{})
|
}, &bytes.Buffer{})
|
||||||
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
|
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
|
||||||
t.Fatalf("execute() = %d, options = %+v", code, received)
|
t.Fatalf("execute() = %d, options = %+v", code, received)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
|||||||
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
var received bootstrap.Options
|
var received bootstrap.Options
|
||||||
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
|
code := execute(context.Background(), nil, func(name string) string {
|
||||||
|
if name == configEnvironment {
|
||||||
|
return "environment.yaml"
|
||||||
|
}
|
||||||
|
return "0123456789abcdef0123456789abcdef"
|
||||||
|
}, func(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
options bootstrap.Options,
|
options bootstrap.Options,
|
||||||
) error {
|
) error {
|
||||||
|
|||||||
@ -1,5 +1,13 @@
|
|||||||
name: proxy-pool
|
name: proxy-pool
|
||||||
|
|
||||||
|
x-app-environment: &app-environment
|
||||||
|
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
||||||
|
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
||||||
|
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
||||||
|
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
||||||
|
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||||
|
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||||
|
|
||||||
x-app: &app
|
x-app: &app
|
||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
@ -9,13 +17,7 @@ x-app: &app
|
|||||||
networks: [frontend, backend]
|
networks: [frontend, backend]
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
environment:
|
environment: *app-environment
|
||||||
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
|
||||||
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
|
||||||
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
|
||||||
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
|
||||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
|
||||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
|
||||||
stop_grace_period: 45s
|
stop_grace_period: 45s
|
||||||
|
|
||||||
services:
|
services:
|
||||||
@ -44,6 +46,9 @@ services:
|
|||||||
controller:
|
controller:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-controller"]
|
command: ["proxy-controller"]
|
||||||
|
environment:
|
||||||
|
<<: *app-environment
|
||||||
|
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
@ -9,8 +9,8 @@ stringData:
|
|||||||
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
||||||
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
||||||
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
||||||
|
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
|
||||||
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
||||||
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
||||||
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
||||||
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
||||||
|
|
||||||
|
|||||||
@ -63,10 +63,14 @@ var _ activitypool.HealthStore = (*Adapter)(nil)
|
|||||||
var _ activitypool.InventoryReader = (*Adapter)(nil)
|
var _ activitypool.InventoryReader = (*Adapter)(nil)
|
||||||
var _ extraction.Store = (*Adapter)(nil)
|
var _ extraction.Store = (*Adapter)(nil)
|
||||||
var _ ownership.Repository = (*Adapter)(nil)
|
var _ ownership.Repository = (*Adapter)(nil)
|
||||||
|
var _ workerruntime.SessionWriter = (*Adapter)(nil)
|
||||||
|
var _ workerruntime.ReportWriter = (*Adapter)(nil)
|
||||||
|
var _ workerruntime.RuntimeReader = (*Adapter)(nil)
|
||||||
|
var _ pool.InventoryReader = (*Adapter)(nil)
|
||||||
```
|
```
|
||||||
|
|
||||||
Provider、Checker、Distribution 和 Ownership 只依赖各自需要的端口,不直接
|
Provider、Checker、Distribution、Ownership 和 Worker Runtime 只依赖各自需要
|
||||||
依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
|
的端口,不直接依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
|
||||||
|
|
||||||
`ownership.Repository` 改为适合远程存储的上下文感知接口:
|
`ownership.Repository` 改为适合远程存储的上下文感知接口:
|
||||||
|
|
||||||
@ -97,6 +101,11 @@ pp:{activity}:owners HASH proxyID -> ownership assignment
|
|||||||
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
|
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
|
||||||
pp:{activity}:epoch STRING ownership 全局递增代次
|
pp:{activity}:epoch STRING ownership 全局递增代次
|
||||||
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
|
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
|
||||||
|
pp:{activity}:worker-sessions HASH workerID -> 当前 Worker session
|
||||||
|
pp:{activity}:worker-session-expiry ZSET workerID -> session expiry milliseconds
|
||||||
|
pp:{activity}:worker-runtime HASH workerID -> 完整稀疏运行态报告
|
||||||
|
pp:{activity}:worker-runtime-expiry ZSET workerID -> report expiry milliseconds
|
||||||
|
pp:{activity}:owned:<digest> ZSET 单 Upstream 已分配 AVAILABLE Proxy
|
||||||
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
|
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
|
||||||
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
|
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
|
||||||
```
|
```
|
||||||
@ -167,6 +176,38 @@ Assign、Renew、BeginDrain 和 AcknowledgeDrain 分别使用有界小脚本,
|
|||||||
- Assign 与 Extract 并发竞争同一 Proxy 时,只允许一个操作成功。
|
- Assign 与 Extract 并发竞争同一 Proxy 时,只允许一个操作成功。
|
||||||
- Expire 使用 `limit` 分批回收过期 assignment,禁止无界返回。
|
- Expire 使用 `limit` 分批回收过期 assignment,禁止无界返回。
|
||||||
|
|
||||||
|
### Worker 运行态与容量汇总
|
||||||
|
|
||||||
|
Gateway 的 `Capacity` 使用一次打包原子读取取得同一时刻的 Active/Reserved,
|
||||||
|
`snapshot.Store` 周期生成完整稀疏报告。当前 Snapshot 已移除但仍有活动连接的
|
||||||
|
Proxy 继续以 `draining=true` 上报,直到 Active/Reserved 同时归零。
|
||||||
|
|
||||||
|
Redis Adapter 在单个 `{activity}` 原子边界内维护 Worker session 和运行态报告:
|
||||||
|
|
||||||
|
1. 新 Worker session 替换旧 session,并隔离旧实例后续写入。
|
||||||
|
2. session 保存 Controller 已 ACK 的 snapshot version 与 ownership epoch;报告
|
||||||
|
必须与 ACK 上界完全一致,不能通过自报超前 epoch 绕过所有权校验。
|
||||||
|
3. `report_sequence` 严格递增;同序号、同内容可幂等重放,冲突或倒序拒绝。
|
||||||
|
4. 非零计数必须匹配当前 Proxy owner、Worker ID 和 ownership epoch。
|
||||||
|
5. session/report TTL 使用 Redis 服务端时间;过期、缺失或损坏时容量 fail-closed。
|
||||||
|
6. 空报告清除该 Worker 的全部旧计数,稀疏报告中缺失的 Proxy 计数视为零。
|
||||||
|
|
||||||
|
`pool.InventoryReader` 低频返回单个 Upstream 的 `Managed` 和
|
||||||
|
`AvailableSlots`。Managed 统计 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING;
|
||||||
|
Available Slots 只统计超过 safety margin、状态为 AVAILABLE 且所有权与新鲜
|
||||||
|
Worker 运行态一致的 `max - active - reserved`。未分配 Proxy 可直接贡献 Max;
|
||||||
|
已分配但运行态未知的 Proxy 贡献零槽位。PostgreSQL 不保存这些短效报告或容量
|
||||||
|
明细,Gateway 每次请求也不访问 Redis。
|
||||||
|
|
||||||
|
Managed 直接读取现有 Upstream 权威计数;Available Slots 只扫描目标 Upstream
|
||||||
|
的未分配可用索引与已分配可用索引,不扫描全局 Proxy,也不受其他供应商活记录
|
||||||
|
数量影响。索引成员数超过单次扫描预算时直接返回不可用,不降级为近似容量。
|
||||||
|
生产规模验收仍需验证脚本 p95/p99、CPU、内存和过期风暴下的有界行为。
|
||||||
|
|
||||||
|
Gateway `snapshot.Store` 扫描有界的当前 Snapshot,并使用分片索引补充已移除但
|
||||||
|
仍非零的 runtime,不扫描全部历史 Proxy;历史 Capacity 注册表设硬上限,达到
|
||||||
|
上限时拒绝新 Snapshot 并保持旧视图,避免长期轮换造成无界内存增长。
|
||||||
|
|
||||||
### 短 TTL 清理
|
### 短 TTL 清理
|
||||||
|
|
||||||
Redis Hash 字段没有独立 TTL,因此使用三层有界清理:
|
Redis Hash 字段没有独立 TTL,因此使用三层有界清理:
|
||||||
@ -226,6 +267,8 @@ Redis `inventory` 是当前未提取 Proxy 数量的运行时真值:
|
|||||||
- Assign 与 Extract 并发互斥,以及 renew/drain/ACK/expire。
|
- Assign 与 Extract 并发互斥,以及 renew/drain/ACK/expire。
|
||||||
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
|
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
|
||||||
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
|
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
|
||||||
|
- Worker session 替换、运行态序号幂等/冲突、报告过期和 ownership epoch 隔离。
|
||||||
|
- 权威 Managed/Available Slots 聚合及扫描预算耗尽时的 fail-closed 行为。
|
||||||
|
|
||||||
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒,并执行
|
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒,并执行
|
||||||
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由
|
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由
|
||||||
|
|||||||
@ -70,9 +70,14 @@ Outbox。状态变化、审计和 Outbox 在同一事务提交。
|
|||||||
|
|
||||||
### 配置修订
|
### 配置修订
|
||||||
|
|
||||||
配置提交只保存管理面恢复所需的非敏感事实:配置版本、SHA-256 校验和、来源、
|
配置提交只保存管理面恢复所需的非敏感事实:配置版本、完整已解析配置的
|
||||||
Upstream 启用状态和 Routing 候选/当前选择。已解析 Secret、Provider Token、
|
HMAC-SHA-256 指纹、来源、Upstream 启用状态和 Routing 候选/当前选择。HMAC 使用
|
||||||
Proxy 凭据和完整运行时对象不进入 PostgreSQL。
|
独立外部高熵密钥,覆盖 Secret 轮换以驱动多副本收敛,同时避免普通摘要成为
|
||||||
|
低熵 Secret 的离线校验器。HMAC 密钥、已解析 Secret、Provider Token、Proxy
|
||||||
|
凭据、配置正文和完整运行时对象均不进入 PostgreSQL。
|
||||||
|
|
||||||
|
配置发布携带事务返回的全局 revision。本地 `config.Store` 只接受严格递增
|
||||||
|
revision,因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖较新运行配置。
|
||||||
|
|
||||||
配置重载以一个事务替换管理快照。新 Routing 的当前 Upstream 必须属于其候选集,
|
配置重载以一个事务替换管理快照。新 Routing 的当前 Upstream 必须属于其候选集,
|
||||||
所有引用的 Upstream 必须存在,名称与列表必须非空且唯一。校验失败发生在事务前,
|
所有引用的 Upstream 必须存在,名称与列表必须非空且唯一。校验失败发生在事务前,
|
||||||
|
|||||||
@ -39,10 +39,24 @@ Status 以一个权威管理快照决定 Upstream 集合和 Enabled 状态,只
|
|||||||
|
|
||||||
1. `FileConfigurationLoader` 通过 `config.LoadResolved` 严格解析、解析 Secret 引用
|
1. `FileConfigurationLoader` 通过 `config.LoadResolved` 严格解析、解析 Secret 引用
|
||||||
并完成全量校验。
|
并完成全量校验。
|
||||||
2. 从脱敏管理投影计算版本与校验和;Secret 值及其可验证摘要不进入管理状态。
|
2. 使用独立外部密钥对完整已解析配置计算 HMAC-SHA-256 指纹;PostgreSQL 只保存
|
||||||
|
不透明 HMAC,不保存密钥、配置正文或 Secret 明文,因此 URL、模板和 Secret
|
||||||
|
轮换都会产生新版本,也不能借数据库摘要离线猜测低熵 Secret。
|
||||||
3. 在同一 `adminstate` mutation 中提交配置修订、管理状态、审计和 Outbox。
|
3. 在同一 `adminstate` mutation 中提交配置修订、管理状态、审计和 Outbox。
|
||||||
4. 提交成功后由 `config.Store` 一次原子指针交换发布完整运行配置;提交失败时旧
|
4. 提交成功后由 `config.Store` 按 PostgreSQL revision 原子发布完整运行配置;
|
||||||
配置保持不变。幂等重放仍执行发布,以修复进程本地状态。
|
Store 只接受严格递增 revision,提交失败或迟到旧 revision 不覆盖当前配置。
|
||||||
|
幂等重放仅在本地 revision 落后时修复进程状态。
|
||||||
|
5. Provider Supervisor 在提交前构造预检所有启用 Upstream;发布后按新配置取消、
|
||||||
|
替换或新增 Runtime。enable 同样在管理状态 mutation 前预检目标 Runtime。
|
||||||
|
6. 其他 Controller 每秒比较本地指纹与 PostgreSQL 权威指纹;所有副本必须使用
|
||||||
|
相同 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。共享配置源已同步时严格重载、预检
|
||||||
|
并按 revision 发布,源尚未同步时停止旧 Provider Runtime,禁止旧 URL/Secret
|
||||||
|
在换主后继续调用。
|
||||||
|
|
||||||
|
disable 成功后会立即通知 Supervisor 取消目标 Runtime;每秒一次的权威状态对账
|
||||||
|
用于修复进程内通知丢失。已取得的分布式 Permit 仍按幂等、保守规则完成结算。
|
||||||
|
PostgreSQL 瞬时读取失败不会终止 Controller 或取消当前 Provider Runtime;
|
||||||
|
Supervisor 保留 last-known 状态并在下一周期重试。
|
||||||
|
|
||||||
主配置或 Secret 文件 I/O 故障归类为 503;语法、未知字段、引用和语义校验失败
|
主配置或 Secret 文件 I/O 故障归类为 503;语法、未知字段、引用和语义校验失败
|
||||||
归类为 422。请求取消和截止时间保持原始上下文错误,不误报为配置错误。
|
归类为 422。请求取消和截止时间保持原始上下文错误,不误报为配置错误。
|
||||||
|
|||||||
@ -34,6 +34,18 @@ sequenceDiagram
|
|||||||
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
||||||
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
||||||
|
|
||||||
|
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
|
||||||
|
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
|
||||||
|
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用
|
||||||
|
Controller 侧 Redis 服务端时间。session 同时保存 Controller 已接受的
|
||||||
|
`snapshot_version/ownership_epoch`;运行态报告必须与该 ACK 上界完全一致,
|
||||||
|
Worker 自报的超前版本或 epoch 也必须拒绝。
|
||||||
|
|
||||||
|
运行态上报是完整稀疏替换:只携带 Active/Reserved 非零的 Proxy,空列表表示
|
||||||
|
当前会话全部归零。Controller 只有在 Worker session、报告 TTL、Proxy ownership
|
||||||
|
和 ownership epoch 同时有效时才使用计数;报告缺失、过期或不一致时按零可用
|
||||||
|
容量 fail-closed,不能把未知计数解释成空闲容量。
|
||||||
|
|
||||||
## 3. Snapshot 与 Delta
|
## 3. Snapshot 与 Delta
|
||||||
|
|
||||||
完整 Snapshot 包含:
|
完整 Snapshot 包含:
|
||||||
|
|||||||
@ -16,10 +16,13 @@ Controller 入口已实现,源码运行方式为:
|
|||||||
go run ./cmd/proxy-controller -config CONFIG_FILE
|
go run ./cmd/proxy-controller -config CONFIG_FILE
|
||||||
```
|
```
|
||||||
|
|
||||||
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
|
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。启用 Admin 时还
|
||||||
|
必须设置 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`,值为至少 32 字节的独立高熵密钥;
|
||||||
|
所有 Controller 副本必须一致,且该密钥不得放入 YAML 或 PostgreSQL。该入口已装配
|
||||||
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
|
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
|
||||||
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
|
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
|
||||||
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
|
Provider 自动补池、分布式配额、动态重载和 Admin 低基数统计已装配。完整 Gateway、
|
||||||
|
Checker、Worker 控制面与代表性负载验证仍在后续实施范围。
|
||||||
|
|
||||||
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
||||||
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
||||||
@ -312,11 +315,30 @@ proxyAuth:
|
|||||||
|
|
||||||
- `pool.maxSize`:当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限,包括
|
- `pool.maxSize`:当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限,包括
|
||||||
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
|
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
|
||||||
- `fetch.maxTotal`:当前运行或计费周期内,从 Provider 成功获取的累计上限;
|
- `fetch.maxTotal`:当前 Redis generation 内的累计获取停止阈值;`0` 表示不设置。
|
||||||
`0` 表示不设置累计上限。
|
Redis 在调用前原子校验 `fetched total + pending expected + expected`,并在达到
|
||||||
|
阈值后停止发起新调用。
|
||||||
|
|
||||||
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置,
|
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置,
|
||||||
但不会恢复累计获取额度。
|
但不会恢复累计获取额度。Provider 调用结果不确定、响应无法解析或 Permit 过期时,
|
||||||
|
系统按 `estimatedIPsPerCall` 保守记账,避免故障或换主造成额度低估。Redis 全量
|
||||||
|
状态丢失会创建新 generation,因此需要由外部计费系统提供跨 generation 的长期额度。
|
||||||
|
|
||||||
|
`estimatedIPsPerCall` 是预留估值,不是通用的 Provider 响应硬限制。如果某次实际
|
||||||
|
合法返回量超过估值,系统只保留本地池容量允许的数量,但累计账本按实际合法数量
|
||||||
|
记账并停止后续调用;该次可能越过停止阈值。需要绝对硬上限时,必须同时在 Provider
|
||||||
|
请求参数中配置供应商支持的批量上限,并保证其不超过剩余额度。
|
||||||
|
|
||||||
|
当前实现限制单个 `pool.maxSize <= 1,000,000`、配置内 Upstream 总数不超过
|
||||||
|
`4,096`,并要求单代理并发、Refill 双水位、理论总槽位及其他传入 Redis Lua 的
|
||||||
|
累计/并发计数不超过 `2^53-1`。这些边界在配置加载和 Admin reload 提交前校验,
|
||||||
|
不会等到 Provider Runtime 启动后才失败。
|
||||||
|
|
||||||
|
`proxyAuth.type: response` 的用户名/密码是 Parser 到 Redis Activity Adapter 之间的
|
||||||
|
临时凭据。每次 Parser handoff 使用独立、幂等释放的 lease,避免并发 Fetch 互相
|
||||||
|
删除或轮换版本覆盖;成功复制到 TTL 活动记录、候选被截断或解析失败后都会释放。
|
||||||
|
内存 lease 上限按所有配置 Upstream 的 `pool.maxSize * fetch.maxInFlight` 汇总,
|
||||||
|
配置 reload 只提高上限,不预分配对应内存。
|
||||||
|
|
||||||
### 7.3 Fetch 限制
|
### 7.3 Fetch 限制
|
||||||
|
|
||||||
@ -376,6 +398,12 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
|
|||||||
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
|
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
|
||||||
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
||||||
|
|
||||||
|
启用 Admin 的多 Controller 部署必须让所有副本读取同一版本化配置源、Secret
|
||||||
|
版本和 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。Supervisor 每秒比较本地完整配置的
|
||||||
|
HMAC-SHA-256 与 PostgreSQL 权威值:管理状态暂时不可读时沿用 last-known
|
||||||
|
Runtime;指纹已更新但本地源尚未同步时停止旧 Provider,待源匹配并通过预检后
|
||||||
|
按全局 revision 恢复,避免旧凭据换主或迟到旧配置回写。
|
||||||
|
|
||||||
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`、
|
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`、
|
||||||
`/readyz` 和 `/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
|
`/readyz` 和 `/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
|
||||||
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标,Provider、提取和容量等
|
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标,Provider、提取和容量等
|
||||||
|
|||||||
@ -143,6 +143,15 @@ type Proxy struct {
|
|||||||
运行态 `active` 与 `reserved` 存在 Worker 本地、按 Proxy ID 分片,不写入
|
运行态 `active` 与 `reserved` 存在 Worker 本地、按 Proxy ID 分片,不写入
|
||||||
不可变 Snapshot。
|
不可变 Snapshot。
|
||||||
|
|
||||||
|
Worker 以有界周期批量上报运行态,而不是在每个 Gateway 请求上写 Redis。
|
||||||
|
报告通过公用 `workerruntime` seam 表达为完整稀疏替换:Gateway 从打包原子计数
|
||||||
|
读取同一时刻的 Active/Reserved;已从当前 Snapshot 移除但仍有连接的 Proxy
|
||||||
|
继续以 draining 状态上报。Controller 使用 session、单调 report sequence、
|
||||||
|
ownership epoch、Controller 已 ACK 的 snapshot/epoch 上界和 Redis 服务端 TTL
|
||||||
|
共同校验;缺失、过期或超前报告按零可用容量 fail-closed。运行态报告扫描有界
|
||||||
|
当前 Snapshot,并以分片索引补充已移除但仍非零的 runtime;历史 Capacity
|
||||||
|
注册表有硬上限,避免短 TTL Proxy 持续轮换导致心跳扫描与内存无界增长。
|
||||||
|
|
||||||
## 6. Proxy 状态机
|
## 6. Proxy 状态机
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@ -244,6 +253,31 @@ flowchart TD
|
|||||||
fail-closed,不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
|
fail-closed,不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
|
||||||
复核期间若仍有 pending Fetch,则等待下一轮再同步 Managed,避免重复计数。
|
复核期间若仍有 pending Fetch,则等待下一轮再同步 Managed,避免重复计数。
|
||||||
|
|
||||||
|
Controller Bootstrap 通过 `Provider Supervisor` 按权威配置和 PostgreSQL 管理状态
|
||||||
|
维护每个 Upstream 的独立 `UpstreamRuntime`,再与 HTTP Runtime 通过公用 lifecycle
|
||||||
|
Group 联动启动、取消和等待。Admin disable 会通知 Supervisor 取消对应 Runtime;
|
||||||
|
reload 在提交前预检新 Runtime,并在发布后逐个取消、替换或新增。低频对账用于修复
|
||||||
|
丢失通知。每个 Leader 任期重新创建本地补池预算与合并信号;Redis 是
|
||||||
|
requestInterval、maxInFlight 和 maxTotal 的唯一分布式裁决者。本地预算只负责
|
||||||
|
pool.maxSize。
|
||||||
|
|
||||||
|
maxTotal 使用 Redis generation 内的累计值与 pending expected 原子预留。调用结果
|
||||||
|
不确定、解析失败或结算丢失时按 expected 保守记账,换主后旧 Permit 仍可幂等结算。
|
||||||
|
如果 Provider 单次实际返回量超过 `estimatedIPsPerCall`,账本按实际合法数量记账并
|
||||||
|
停止后续调用,但该次可能越过停止阈值;严格硬封顶需要 Provider API 支持可控批量。
|
||||||
|
|
||||||
|
响应中携带的代理凭据先进入有界内存凭据表,Redis Activity Adapter 复制凭据材料
|
||||||
|
进入 TTL 活动记录后立即按版本 fence 释放临时引用;解析中途失败也会回收已写引用。
|
||||||
|
因此短 TTL、持续轮换的新代理地址不会耗尽启动时的凭据容量。Provider 结果只记录
|
||||||
|
每个 Upstream 的低基数 Empty/Error 计数,不保留响应体、代理地址或错误对象。
|
||||||
|
|
||||||
|
补池读取 `pool.InventoryReader` 返回的权威 `Managed` 与 `AvailableSlots`,不使用
|
||||||
|
AVAILABLE Proxy 数量乘固定并发的近似值。Available Slots 同时计入 Proxy 状态、
|
||||||
|
TTL safety margin、MaxConcurrency、Worker Active/Reserved 和当前 ownership;
|
||||||
|
未知 Worker 运行态贡献零槽位。Redis 只扫描目标 Upstream 的未分配/已分配
|
||||||
|
AVAILABLE 索引,Managed 读取已有权威计数;该读取属于 Controller 冷路径,
|
||||||
|
Gateway 热路径仍只访问本地 Snapshot 和本地原子计数。
|
||||||
|
|
||||||
### 8.1 Empty、Duplicate 与 Error
|
### 8.1 Empty、Duplicate 与 Error
|
||||||
|
|
||||||
- **Empty**:HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
|
- **Empty**:HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
|
||||||
@ -403,6 +437,12 @@ flowchart LR
|
|||||||
- Routing 立即对新请求生效;旧请求持有旧 Snapshot 完成。
|
- Routing 立即对新请求生效;旧请求持有旧 Snapshot 完成。
|
||||||
- 修改 API 地址或凭据版本会重建 Provider Adapter,但不会把错误计成 Empty。
|
- 修改 API 地址或凭据版本会重建 Provider Adapter,但不会把错误计成 Empty。
|
||||||
- 新配置任何校验失败时,保留旧版本并报告完整错误。
|
- 新配置任何校验失败时,保留旧版本并报告完整错误。
|
||||||
|
- 多 Controller 以 PostgreSQL 中的 HMAC-SHA-256 完整配置指纹检测修订;HMAC
|
||||||
|
密钥由外部 Secret 注入且所有副本一致。副本从共享配置源读取相同 revision,
|
||||||
|
预检后通过单调 revision 栅栏原子发布。权威指纹已变化但本地源仍旧时,旧
|
||||||
|
Provider fail-closed 停止,避免旧 URL/Secret 在 Leader 换主后继续使用。
|
||||||
|
- PostgreSQL 瞬时读取失败时沿用 last-known Provider Runtime 并重试,不连带终止
|
||||||
|
Distribution、Admin 或 Metrics。
|
||||||
|
|
||||||
## 15. 安全
|
## 15. 安全
|
||||||
|
|
||||||
@ -463,7 +503,7 @@ CPU、内存、网络、Go 版本、配置和上游响应模型下测得。
|
|||||||
| Provider 超时/500 | 计 Error、退避;不计 Empty,不影响已有 Proxy |
|
| Provider 超时/500 | 计 Error、退避;不计 Empty,不影响已有 Proxy |
|
||||||
| Provider 合法空响应 | Empty++;达到阈值触发相关 Routing 原子切换 |
|
| Provider 合法空响应 | Empty++;达到阈值触发相关 Routing 原子切换 |
|
||||||
| Redis 不可用 | Gateway 暂用未过期快照;停止 Fetch 入池、Extract 和所有权变更 |
|
| Redis 不可用 | Gateway 暂用未过期快照;停止 Fetch 入池、Extract 和所有权变更 |
|
||||||
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理状态变更和 Admin 审计 |
|
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理写入,Provider 沿用 last-known 状态并重试 |
|
||||||
| Controller 断线 | Worker 在 maxStaleAge 内继续;超限拒绝新流量并排空 |
|
| Controller 断线 | Worker 在 maxStaleAge 内继续;超限拒绝新流量并排空 |
|
||||||
| Worker 崩溃 | 所有权租约过期后重新分配;期间不双重所有 |
|
| Worker 崩溃 | 所有权租约过期后重新分配;期间不双重所有 |
|
||||||
| Checker 积压 | 降低普通复检频率,优先新 Proxy 与 SUSPECT,不无限排队 |
|
| Checker 积压 | 降低普通复检频率,优先新 Proxy 与 SUSPECT,不无限排队 |
|
||||||
|
|||||||
@ -195,11 +195,14 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
|
|||||||
已完成。pgx Adapter 已在真实 PostgreSQL 18 上运行同一公用契约,并验证
|
已完成。pgx Adapter 已在真实 PostgreSQL 18 上运行同一公用契约,并验证
|
||||||
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
|
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
|
||||||
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
|
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
|
||||||
聚合与配置重载接到同一公用 seam;严格文件加载、脱敏管理摘要及原子配置发布
|
聚合与配置重载接到同一公用 seam;严格文件加载、外部密钥 HMAC 管理指纹及
|
||||||
已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
|
revision 单调配置发布已通过失败路径和确定性并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
|
||||||
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
|
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
|
||||||
Distribution/Admin 服务构造、错误合并和资源关闭;Provider 调度及完整 HTTP
|
Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider Supervisor 已按
|
||||||
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`、
|
权威管理状态动态装配 Upstream,并与 HTTP Runtime 通过公用 lifecycle Group 联动
|
||||||
|
停机;Admin disable 会取消 Runtime,reload 在提交前预检并在发布后替换运行实例。
|
||||||
|
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
|
||||||
|
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、
|
||||||
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
||||||
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
||||||
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
||||||
@ -219,8 +222,23 @@ generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、
|
|||||||
回收及 Redis 状态丢失后的新 generation 自动重建;Redis 异常期间不发放请求。
|
回收及 Redis 状态丢失后的新 generation 自动重建;Redis 异常期间不发放请求。
|
||||||
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`,Pool Reconciler
|
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`,Pool Reconciler
|
||||||
已实现迟滞与 pending 槽位折算,FetchBudget 仅在无 pending 时同步 Redis 权威
|
已实现迟滞与 pending 槽位折算,FetchBudget 仅在无 pending 时同步 Redis 权威
|
||||||
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成,
|
Managed。Gateway 已增加打包原子 Active/Reserved 读取与完整稀疏运行态快照;
|
||||||
因此本轮不勾选 Task 10 的组合验收项。
|
公用 `workerruntime` session/report/read seam 同时提供并发安全内存参考实现和
|
||||||
|
生产 Redis Adapter。Redis 以服务端时间、Worker session、已 ACK snapshot/epoch、
|
||||||
|
单调 report sequence 和报告 TTL 原子隔离旧实例,并由 `pool.InventoryReader` 汇总
|
||||||
|
权威 Managed/Available Slots;真实 Redis 8.2 已覆盖空报告、幂等重放、倒序、
|
||||||
|
冲突、超前 epoch、过期、单 Upstream 扫描隔离和预算耗尽的 fail-closed 行为。
|
||||||
|
Redis Provider Permit 现已把 requestInterval、maxInFlight 与 maxTotal 放在同一
|
||||||
|
原子边界,按 expected 预留、实际合法数量结算,并支持失败保守计费、换主后结算和
|
||||||
|
过期回收。响应型代理凭据使用独立、有界 lease,在 Redis Upsert、候选截断或解析
|
||||||
|
失败后按版本释放;Provider Empty/Error 低基数计数已接入 Admin Status,配置删除
|
||||||
|
时回收历史统计容量。Redis inventory 扫描上限固定覆盖配置允许的最大池,支持小池
|
||||||
|
启动后动态扩容。Supervisor 以 PostgreSQL 权威 HMAC 指纹和 revision 栅栏协调
|
||||||
|
多副本 reload:管理库瞬断沿用 last-known 状态,本地共享源落后时停止旧 Provider,
|
||||||
|
源匹配并预检后自动替换,迟到旧 revision 不覆盖新配置。
|
||||||
|
WorkerControlPlane gRPC 接收端、session 签发/心跳、Snapshot ACK
|
||||||
|
账本、Client 分布式限流和健康执行链仍待完成,因此本轮不勾选 Task 10 的组合
|
||||||
|
验收项。
|
||||||
|
|
||||||
## Task 11: Checker and Health Reducer
|
## Task 11: Checker and Health Reducer
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,7 @@
|
|||||||
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||||
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||||
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
||||||
- [ ] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||||
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||||
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||||
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||||
@ -20,6 +20,7 @@
|
|||||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||||
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||||
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
||||||
|
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
|
||||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||||
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
||||||
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||||
|
|||||||
@ -19,9 +19,9 @@
|
|||||||
## 2. 本地拓扑模板
|
## 2. 本地拓扑模板
|
||||||
|
|
||||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
||||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
|
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
||||||
指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
|
配额、动态重载和 Admin 低基数统计已装配;完整 Gateway/Checker/Loadgen 与 Worker
|
||||||
后续任务。
|
控制面仍属于 `implementation-plan.md` 后续任务。
|
||||||
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
||||||
视为完整可运行拓扑。
|
视为完整可运行拓扑。
|
||||||
|
|
||||||
@ -40,6 +40,7 @@ Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动
|
|||||||
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
|
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
|
||||||
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
|
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
|
||||||
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
|
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
|
||||||
|
$env:PROXY_POOL_CONFIG_FINGERPRINT_KEY = "LOCAL_HIGH_ENTROPY_KEY_AT_LEAST_32_BYTES"
|
||||||
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
||||||
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
||||||
```
|
```
|
||||||
@ -90,6 +91,8 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
|
|||||||
1. 使用托管 PostgreSQL 和 Redis,分别配置 TLS、备份、监控和多可用区。
|
1. 使用托管 PostgreSQL 和 Redis,分别配置 TLS、备份、监控和多可用区。
|
||||||
2. 复制 `secret.example.yaml` 到环境私密配置系统,由 External Secrets、SOPS
|
2. 复制 `secret.example.yaml` 到环境私密配置系统,由 External Secrets、SOPS
|
||||||
或密钥管理平台生成 `proxy-pool-secrets`,不要提交真实 Secret。
|
或密钥管理平台生成 `proxy-pool-secrets`,不要提交真实 Secret。
|
||||||
|
`PROXY_POOL_CONFIG_FINGERPRINT_KEY` 必须使用至少 32 字节的高熵随机值,所有
|
||||||
|
Controller 副本保持一致,且与配置中的业务 Secret 分离管理。
|
||||||
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
|
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
|
||||||
和 LoadBalancer 注解。
|
和 LoadBalancer 注解。
|
||||||
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
||||||
@ -221,11 +224,15 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
|
|||||||
|
|
||||||
1. Gateway 继续使用最后有效 Snapshot。
|
1. Gateway 继续使用最后有效 Snapshot。
|
||||||
2. Redis 健康且运行配置有效时,Distribution 继续执行原子 Extract,Provider
|
2. Redis 健康且运行配置有效时,Distribution 继续执行原子 Extract,Provider
|
||||||
继续刷新 TTL 活动池。
|
按 last-known 管理状态继续刷新 TTL 活动池;状态读取失败不得触发全进程退出。
|
||||||
3. 拒绝配置版本、Upstream/Routing 管理状态和其他需要 Admin 审计/outbox 的写入;
|
3. 拒绝配置版本、Upstream/Routing 管理状态和其他需要 Admin 审计/outbox 的写入;
|
||||||
不得把 Proxy 明细临时落入 PostgreSQL。
|
不得把 Proxy 明细临时落入 PostgreSQL。
|
||||||
4. 恢复后核对迁移、管理事务回滚、Admin 审计与 outbox backlog;不存在 Proxy
|
4. 恢复后核对迁移、管理事务回滚、Admin 审计与 outbox backlog;不存在 Proxy
|
||||||
明细或逐次提取记录恢复步骤。
|
明细或逐次提取记录恢复步骤。
|
||||||
|
5. 若权威配置指纹已变化,确认每个 Controller 的共享配置源和 Secret 版本已同步;
|
||||||
|
同时确认 `PROXY_POOL_CONFIG_FINGERPRINT_KEY` 一致。指纹不匹配的副本会停止旧
|
||||||
|
Provider,匹配并预检成功后按 PostgreSQL revision 自动恢复;迟到旧 revision
|
||||||
|
不会覆盖较新本地配置。
|
||||||
|
|
||||||
### 7.3 Redis 不可用
|
### 7.3 Redis 不可用
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,7 @@
|
|||||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 命令已实现,Gateway/Checker/Loadgen 构建产物待实现 |
|
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 命令已实现,Gateway/Checker/Loadgen 构建产物待实现 |
|
||||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
|
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
|
||||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 生产入口待装配 |
|
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 生产入口待装配 |
|
||||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成;分布式 Leader 与运行装配待完成 |
|
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主;Routing 切换到 Drain 的编排待完成 |
|
||||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
||||||
|
|
||||||
## Routing 与 Upstream
|
## Routing 与 Upstream
|
||||||
@ -32,11 +32,11 @@
|
|||||||
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
|
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
|
||||||
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
|
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
|
||||||
| FETCH-003 | 错误使用指数退避和抖动,429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
|
| FETCH-003 | 错误使用指数退避和抖动,429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
|
||||||
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;Provider Fleet/bootstrap 接线待完成 |
|
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;生产 Supervisor/bootstrap 已通过 Admin disable 和 Provider HTTP 到隔离 Redis 库存的组合 fixture |
|
||||||
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
|
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
|
||||||
| FETCH-006 | 重复候选不当作 Empty,记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
|
| FETCH-006 | 重复候选不当作 Empty,记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
|
||||||
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
|
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
|
||||||
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | `FetchBudget` 并发预占/释放测试 |
|
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | 本地 `FetchBudget` 仅约束当前库存;Redis Permit 原子维护累计与 pending 额度,并通过换主、取消、幂等和过期保守结算测试 |
|
||||||
|
|
||||||
## Proxy 生命周期与容量
|
## Proxy 生命周期与容量
|
||||||
|
|
||||||
@ -46,7 +46,7 @@
|
|||||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
|
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
|
||||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
|
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Gateway 打包 Active/Reserved 报告、Worker session/ACK/sequence/TTL/ownership fence、单 Upstream 索引及 Redis 权威 Managed/Slots 汇总已通过内存与真实 Redis 测试;WorkerControlPlane 接线、目标健康和 Gateway reserve 策略仍待完成 |
|
||||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||||
|
|||||||
@ -22,11 +22,20 @@
|
|||||||
4. 连续 4 次 Empty 后 Valid 不切换;连续 5 次只从 A 切到 B。
|
4. 连续 4 次 Empty 后 Valid 不切换;连续 5 次只从 A 切到 B。
|
||||||
5. Error 和 DuplicateOnly 不累计 Empty。
|
5. Error 和 DuplicateOnly 不累计 Empty。
|
||||||
6. 100 个缺池信号只形成一个合并 Provider reconcile。
|
6. 100 个缺池信号只形成一个合并 Provider reconcile。
|
||||||
7. 并发 Fetch 不突破 `pool.maxSize` 与 `fetch.maxTotal`。
|
7. 并发 Fetch 不突破 `pool.maxSize`,且不会在 Redis `fetch.maxTotal` 停止阈值耗尽后发起新调用。
|
||||||
8. TTL safety margin 内不再分配。
|
8. TTL safety margin 内不再分配。
|
||||||
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
|
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
|
||||||
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
|
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
|
||||||
11. 所有 Upstream 不可用时严格执行显式策略。
|
11. Admin disable 取消目标 Provider Runtime;reload 构造失败不提交,成功时替换运行实例。
|
||||||
|
12. 短 TTL 响应凭据使用独立 lease;并发 Fetch、截断、解析失败和 Redis Upsert
|
||||||
|
后均准确释放且不互相撤销。
|
||||||
|
13. PostgreSQL 状态瞬断保留 last-known Provider;多副本本地指纹落后时停止旧
|
||||||
|
Runtime,共享源同步后自动预检并恢复;不同 HMAC 密钥不能误判为相同配置。
|
||||||
|
14. 并发配置提交与 Supervisor 同步按 revision 单调发布,迟到旧 revision 不覆盖
|
||||||
|
已发布新配置。
|
||||||
|
15. 配置删除的 Provider 统计项被回收,禁用但仍配置的统计项保留,容量可复用。
|
||||||
|
16. 小池启动后重载到大池时,Redis inventory 扫描上限仍覆盖配置允许的最大池。
|
||||||
|
17. 所有 Upstream 不可用时严格执行显式策略。
|
||||||
|
|
||||||
## 3. 基础质量门禁
|
## 3. 基础质量门禁
|
||||||
|
|
||||||
@ -74,16 +83,17 @@ soak 测试单独标记,不混入快速单测。
|
|||||||
|
|
||||||
## 5. 当前本地微基准
|
## 5. 当前本地微基准
|
||||||
|
|
||||||
2026-07-28,Windows/amd64、Intel Core Ultra 7 155H:
|
2026-07-30,Windows/amd64、Intel Core Ultra 7 155H:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
BenchmarkAcquire100kIndexed-22 3553592 640.0 ns/op 256 B/op 2 allocs/op
|
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
|
||||||
BenchmarkStoreApply100k-22 1 472.7 ms/op 654 MB/op 2700642 allocs/op
|
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
|
||||||
```
|
```
|
||||||
|
|
||||||
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
||||||
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
||||||
暂不自动回收曾出现过的 Proxy ID。后续需要基于 RCU/引用计数定义安全回收点。
|
暂不自动回收曾出现过的 Proxy ID,但注册表有 1,000,000 项硬上限,达到上限时
|
||||||
|
拒绝新 Snapshot 并保留旧视图。后续需要基于 RCU/引用计数定义安全回收点。
|
||||||
这些数据不包含网络、认证、Provider、存储或多 Worker 协调,不能作为
|
这些数据不包含网络、认证、Provider、存储或多 Worker 协调,不能作为
|
||||||
100k QPS 端到端验收结论。
|
100k QPS 端到端验收结论。
|
||||||
|
|
||||||
|
|||||||
@ -120,7 +120,7 @@ Redis Readiness 和 Admin Status;HTTP Runner 使用测试 Adapter,避免占
|
|||||||
|
|
||||||
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
|
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
|
||||||
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
|
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
|
||||||
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
|
持久化失败不发布、HMAC 管理指纹、revision 单调发布和原子配置 Store 并发读写。静态
|
||||||
导入边界测试禁止 Admin 引用 Redis Activity/Extract 与 Proxy 明细包。
|
导入边界测试禁止 Admin 引用 Redis Activity/Extract 与 Proxy 明细包。
|
||||||
|
|
||||||
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器,不复用开发数据。
|
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器,不复用开发数据。
|
||||||
|
|||||||
@ -20,6 +20,7 @@ const (
|
|||||||
defaultTemplateTimeout = 100 * time.Millisecond
|
defaultTemplateTimeout = 100 * time.Millisecond
|
||||||
defaultTemplateMaxBytes = int64(1 << 20)
|
defaultTemplateMaxBytes = int64(1 << 20)
|
||||||
defaultMaxCandidates = 10_000
|
defaultMaxCandidates = 10_000
|
||||||
|
credentialReleaseTimeout = time.Second
|
||||||
maxRegexPatterns = 64
|
maxRegexPatterns = 64
|
||||||
maxRegexPatternBytes = 1024
|
maxRegexPatternBytes = 1024
|
||||||
)
|
)
|
||||||
@ -137,7 +138,14 @@ func NewTemplateParser(
|
|||||||
return parser, nil
|
return parser, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
|
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
|
||||||
|
storedCredentials := make([]credentials.Reference, 0)
|
||||||
|
defer func() {
|
||||||
|
if resultErr == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.releaseCredentials(storedCredentials)
|
||||||
|
}()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
if len(tokens) > p.maxCandidates {
|
if len(tokens) > p.maxCandidates {
|
||||||
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
|
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
|
||||||
}
|
}
|
||||||
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
|
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
|
||||||
credentialIndexes := make(map[string]int)
|
credentialIndexes := make(map[string]int)
|
||||||
for _, token := range tokens {
|
for _, token := range tokens {
|
||||||
candidate, credential, ok := p.parseCandidate(token)
|
candidate, credential, ok := p.parseCandidate(token)
|
||||||
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
}
|
}
|
||||||
candidate.SecretRef = reference.SecretRef
|
candidate.SecretRef = reference.SecretRef
|
||||||
candidate.CredentialVersion = reference.CredentialVersion
|
candidate.CredentialVersion = reference.CredentialVersion
|
||||||
|
storedCredentials = append(storedCredentials, reference)
|
||||||
credentialKey = candidateCredentialKey(candidate)
|
credentialKey = candidateCredentialKey(candidate)
|
||||||
if index, exists := credentialIndexes[credentialKey]; exists {
|
if index, exists := credentialIndexes[credentialKey]; exists {
|
||||||
proxies[index] = candidate
|
proxies[index] = candidate
|
||||||
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
if len(tokens) > 0 && len(proxies) == 0 {
|
if len(tokens) > 0 && len(proxies) == 0 {
|
||||||
return nil, ErrInvalidProxyOutput
|
return nil, ErrInvalidProxyOutput
|
||||||
}
|
}
|
||||||
|
p.releaseUnusedCredentials(storedCredentials, proxies)
|
||||||
return proxies, nil
|
return proxies, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||||
|
references := make([]credentials.Reference, 0, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
references = append(references, credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
p.releaseCredentials(references)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) releaseUnusedCredentials(
|
||||||
|
stored []credentials.Reference,
|
||||||
|
candidates []proxyDomain.Proxy,
|
||||||
|
) {
|
||||||
|
retained := make(map[credentials.Reference]int, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
retained[credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||||
|
}]++
|
||||||
|
}
|
||||||
|
unused := make([]credentials.Reference, 0, len(stored))
|
||||||
|
for _, reference := range stored {
|
||||||
|
if retained[reference] > 0 {
|
||||||
|
retained[reference]--
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
unused = append(unused, reference)
|
||||||
|
}
|
||||||
|
p.releaseCredentials(unused)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaser, ok := p.credentialStore.(credentials.Releaser)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
|
||||||
|
defer cancel()
|
||||||
|
for _, reference := range references {
|
||||||
|
if releaseCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = releaser.Release(releaseCtx, reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
|
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
|
||||||
compiled, err := p.compileRegex(pattern)
|
compiled, err := p.compileRegex(pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
|
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(1)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
|
|||||||
if len(proxies) != 2 {
|
if len(proxies) != 2 {
|
||||||
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
|
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
|
||||||
}
|
}
|
||||||
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
|
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
|
||||||
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
|
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
|
||||||
|
}
|
||||||
|
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
|
||||||
|
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
|
||||||
|
store, err := credentials.NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||||
|
Provider: config.Provider{Protocols: []string{"http"}},
|
||||||
|
API: config.ProviderAPI{Template: strings.Join([]string{
|
||||||
|
"http://alice:first-password@192.0.2.10:8080",
|
||||||
|
"http://bob:second-password@192.0.2.11:8080",
|
||||||
|
}, "\n")},
|
||||||
|
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||||
|
Pool: config.Pool{MaxSize: 2},
|
||||||
|
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||||
|
}, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newTemplateParser(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after failed parse): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
|
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(2)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
|
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(1)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
|
|||||||
if value.Password != "new-secret" {
|
if value.Password != "new-secret" {
|
||||||
t.Fatalf("resolved latest password mismatch")
|
t.Fatalf("resolved latest password mismatch")
|
||||||
}
|
}
|
||||||
|
parser.ReleaseCandidates(proxies)
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after releasing latest candidates): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
|
||||||
|
store, err := credentials.NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||||
|
Provider: config.Provider{Protocols: []string{"http"}},
|
||||||
|
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
|
||||||
|
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||||
|
Pool: config.Pool{MaxSize: 1},
|
||||||
|
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||||
|
}, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewTemplateParser(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
proxies, err := parser.Parse(context.Background(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse(): %v", err)
|
||||||
|
}
|
||||||
|
parser.ReleaseCandidates(proxies)
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after ReleaseCandidates): %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {
|
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {
|
||||||
|
|||||||
@ -23,29 +23,41 @@ type Options struct {
|
|||||||
Credentials credentials.Store
|
Credentials credentials.Store
|
||||||
OperationTTL time.Duration
|
OperationTTL time.Duration
|
||||||
MaxCandidateScan int
|
MaxCandidateScan int
|
||||||
|
MaxRuntimeCounters int
|
||||||
|
MaxInventoryScan int
|
||||||
CleanupLimit int
|
CleanupLimit int
|
||||||
}
|
}
|
||||||
|
|
||||||
type Adapter struct {
|
type Adapter struct {
|
||||||
client redis.Scripter
|
client redis.Scripter
|
||||||
credentials credentials.Store
|
credentials credentials.Store
|
||||||
|
credentialReleaser credentials.Releaser
|
||||||
keys keyspace
|
keys keyspace
|
||||||
options Options
|
options Options
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(client redis.Scripter, options Options) (*Adapter, error) {
|
func New(client redis.Scripter, options Options) (*Adapter, error) {
|
||||||
options.Namespace = strings.TrimSpace(options.Namespace)
|
options.Namespace = strings.TrimSpace(options.Namespace)
|
||||||
|
if options.MaxRuntimeCounters == 0 {
|
||||||
|
options.MaxRuntimeCounters = options.MaxCandidateScan
|
||||||
|
}
|
||||||
|
if options.MaxInventoryScan == 0 {
|
||||||
|
options.MaxInventoryScan = options.MaxCandidateScan
|
||||||
|
}
|
||||||
if nilInterface(client) || nilInterface(options.Credentials) ||
|
if nilInterface(client) || nilInterface(options.Credentials) ||
|
||||||
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
|
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
|
||||||
options.MaxCandidateScan <= 0 || options.CleanupLimit <= 0 {
|
options.MaxCandidateScan <= 0 || options.MaxRuntimeCounters <= 0 ||
|
||||||
|
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
|
||||||
return nil, ErrInvalidOptions
|
return nil, ErrInvalidOptions
|
||||||
}
|
}
|
||||||
return &Adapter{
|
adapter := &Adapter{
|
||||||
client: client,
|
client: client,
|
||||||
credentials: options.Credentials,
|
credentials: options.Credentials,
|
||||||
keys: newKeyspace(options.Namespace),
|
keys: newKeyspace(options.Namespace),
|
||||||
options: options,
|
options: options,
|
||||||
}, nil
|
}
|
||||||
|
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
|
||||||
|
return adapter, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) Format(state fmt.State, _ rune) {
|
func (a *Adapter) Format(state fmt.State, _ rune) {
|
||||||
|
|||||||
@ -58,6 +58,8 @@ func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
|
|||||||
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
|
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
|
||||||
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
|
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
|
||||||
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
|
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
|
||||||
|
{name: "negative runtime counters", client: client, options: withMaxRuntimeCounters(valid, -1)},
|
||||||
|
{name: "negative inventory scan", client: client, options: withMaxInventoryScan(valid, -1)},
|
||||||
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
|
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@ -93,7 +95,9 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
|||||||
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
||||||
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
||||||
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
||||||
adapter.keys.stateInventory,
|
adapter.keys.stateInventory, adapter.keys.workerSessions,
|
||||||
|
adapter.keys.workerSessionExpiry, adapter.keys.workerRuntime,
|
||||||
|
adapter.keys.workerRuntimeExpiry,
|
||||||
}
|
}
|
||||||
for _, key := range staticKeys {
|
for _, key := range staticKeys {
|
||||||
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
||||||
@ -109,6 +113,7 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
|||||||
adapter.keys.region(raw),
|
adapter.keys.region(raw),
|
||||||
adapter.keys.carrier(raw),
|
adapter.keys.carrier(raw),
|
||||||
adapter.keys.upstream(raw),
|
adapter.keys.upstream(raw),
|
||||||
|
adapter.keys.owned(raw),
|
||||||
}
|
}
|
||||||
for _, key := range dynamicKeys {
|
for _, key := range dynamicKeys {
|
||||||
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
|
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
|
||||||
@ -161,6 +166,7 @@ func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
|
|||||||
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
|
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
|
||||||
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
|
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
|
||||||
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
||||||
|
OwnerIndexKey: "pp:{activity}:test:owned:index",
|
||||||
}
|
}
|
||||||
first, err := encodeProxyRecord(record)
|
first, err := encodeProxyRecord(record)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -260,6 +266,16 @@ func withMaxCandidateScan(options Options, limit int) Options {
|
|||||||
return options
|
return options
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func withMaxRuntimeCounters(options Options, limit int) Options {
|
||||||
|
options.MaxRuntimeCounters = limit
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
|
||||||
|
func withMaxInventoryScan(options Options, limit int) Options {
|
||||||
|
options.MaxInventoryScan = limit
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
|
||||||
func withCleanupLimit(options Options, limit int) Options {
|
func withCleanupLimit(options Options, limit int) Options {
|
||||||
options.CleanupLimit = limit
|
options.CleanupLimit = limit
|
||||||
return options
|
return options
|
||||||
|
|||||||
45
internal/adapters/redisactivity/capacity.go
Normal file
45
internal/adapters/redisactivity/capacity.go
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
package redisactivity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
controllerPool "proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ controllerPool.InventoryReader = (*Adapter)(nil)
|
||||||
|
|
||||||
|
func (a *Adapter) ReadInventory(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
safetyMargin time.Duration,
|
||||||
|
) (controllerPool.InventorySnapshot, error) {
|
||||||
|
if ctx == nil || a == nil || !runtimeClean(upstreamID) || safetyMargin < 0 {
|
||||||
|
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return controllerPool.InventorySnapshot{}, err
|
||||||
|
}
|
||||||
|
result, err := runScript(ctx, a.client, capacityScript, []string{
|
||||||
|
a.keys.records, a.keys.inventory, a.keys.upstream(upstreamID), a.keys.owned(upstreamID), a.keys.owners,
|
||||||
|
a.keys.workerSessions, a.keys.workerSessionExpiry,
|
||||||
|
a.keys.workerRuntime, a.keys.workerRuntimeExpiry,
|
||||||
|
}, upstreamID, durationMillis(safetyMargin), a.options.MaxInventoryScan, a.options.CleanupLimit)
|
||||||
|
if err != nil {
|
||||||
|
return controllerPool.InventorySnapshot{}, err
|
||||||
|
}
|
||||||
|
var reply capacityScriptReply
|
||||||
|
if err := decodeScriptResult(result, &reply); err != nil {
|
||||||
|
return controllerPool.InventorySnapshot{}, err
|
||||||
|
}
|
||||||
|
if reply.Status == scriptInvalid {
|
||||||
|
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
|
||||||
|
}
|
||||||
|
if reply.Status != scriptOK || reply.Managed < 0 || reply.AvailableSlots < 0 {
|
||||||
|
return controllerPool.InventorySnapshot{}, invalidScriptReply("capacity inventory is unavailable")
|
||||||
|
}
|
||||||
|
return controllerPool.InventorySnapshot{
|
||||||
|
Managed: reply.Managed, AvailableSlots: reply.AvailableSlots,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
105
internal/adapters/redisactivity/capacity_integration_test.go
Normal file
105
internal/adapters/redisactivity/capacity_integration_test.go
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package redisactivity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
controllerPool "proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRedisCapacityInventoryCombinesProxyAndWorkerRuntime(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
now := redisTestNow()
|
||||||
|
unowned := testProxy("proxy-unowned", "192.0.2.10")
|
||||||
|
unowned.MaxConcurrency = 10
|
||||||
|
owned := testProxy("proxy-owned", "192.0.2.11")
|
||||||
|
owned.MaxConcurrency = 10
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, unowned)
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
|
||||||
|
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||||
|
"proxy-owned", "worker-a", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Assign(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||||
|
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||||
|
Counters: []workerruntime.Counter{{ProxyID: "proxy-owned", Active: 3, Reserved: 2}},
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||||
|
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 15 {
|
||||||
|
t.Fatalf("ReadInventory() = %+v, %v; want managed=2 slots=15", inventory, err)
|
||||||
|
}
|
||||||
|
if _, ok := any(fixture.Adapter).(controllerPool.InventoryReader); !ok {
|
||||||
|
t.Fatal("Adapter does not implement pool.InventoryReader")
|
||||||
|
}
|
||||||
|
inventory, err = fixture.Adapter.ReadInventory(context.Background(), "provider-a", 2*time.Hour)
|
||||||
|
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 0 {
|
||||||
|
t.Fatalf("ReadInventory(safety margin) = %+v, %v", inventory, err)
|
||||||
|
}
|
||||||
|
fixture.Adapter.options.MaxInventoryScan = 1
|
||||||
|
if _, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0); err == nil {
|
||||||
|
t.Fatal("ReadInventory(over scan limit) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisCapacityInventoryFailsClosedForExpiredRuntime(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
now := redisTestNow()
|
||||||
|
owned := testProxy("proxy-owned", "192.0.2.11")
|
||||||
|
owned.MaxConcurrency = 10
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
|
||||||
|
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||||
|
"proxy-owned", "worker-a", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Assign(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||||
|
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||||
|
}, 100*time.Millisecond); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||||
|
}, 100*time.Millisecond); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(150 * time.Millisecond)
|
||||||
|
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||||
|
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 0 {
|
||||||
|
t.Fatalf("ReadInventory(expired runtime) = %+v, %v", inventory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisCapacityInventoryScanIsIsolatedPerUpstream(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
fixture.Adapter.options.MaxInventoryScan = 1
|
||||||
|
now := redisTestNow()
|
||||||
|
target := testProxy("proxy-target", "192.0.2.10")
|
||||||
|
target.MaxConcurrency = 4
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, target)
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(time.Second), 2*time.Minute,
|
||||||
|
testProxy("proxy-other-1", "192.0.2.11"))
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(2*time.Second), 2*time.Minute,
|
||||||
|
testProxy("proxy-other-2", "192.0.2.12"))
|
||||||
|
|
||||||
|
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||||
|
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 4 {
|
||||||
|
t.Fatalf("ReadInventory(provider-a) = %+v, %v; want isolated managed=1 slots=4", inventory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -37,6 +37,7 @@ type proxyRecord struct {
|
|||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
Tags map[string]string `json:"tags,omitempty"`
|
Tags map[string]string `json:"tags,omitempty"`
|
||||||
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
||||||
|
OwnerIndexKey string `json:"ownerIndexKey"`
|
||||||
IndexKeys []string `json:"indexKeys,omitempty"`
|
IndexKeys []string `json:"indexKeys,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -203,7 +204,8 @@ func validateProxyRecord(record proxyRecord) error {
|
|||||||
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
|
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
|
||||||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
|
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
|
||||||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
|
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
|
||||||
!validScheme(record.Scheme) || !validProxyState(record.State) {
|
!validScheme(record.Scheme) || !validProxyState(record.State) ||
|
||||||
|
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
|
||||||
return ErrInvalidRecord
|
return ErrInvalidRecord
|
||||||
}
|
}
|
||||||
for _, key := range record.IndexKeys {
|
for _, key := range record.IndexKeys {
|
||||||
|
|||||||
@ -20,6 +20,10 @@ type keyspace struct {
|
|||||||
epoch string
|
epoch string
|
||||||
inventory string
|
inventory string
|
||||||
stateInventory string
|
stateInventory string
|
||||||
|
workerSessions string
|
||||||
|
workerSessionExpiry string
|
||||||
|
workerRuntime string
|
||||||
|
workerRuntimeExpiry string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newKeyspace(namespace string) keyspace {
|
func newKeyspace(namespace string) keyspace {
|
||||||
@ -36,6 +40,10 @@ func newKeyspace(namespace string) keyspace {
|
|||||||
epoch: prefix + ":epoch",
|
epoch: prefix + ":epoch",
|
||||||
inventory: prefix + ":inventory",
|
inventory: prefix + ":inventory",
|
||||||
stateInventory: prefix + ":state-inventory",
|
stateInventory: prefix + ":state-inventory",
|
||||||
|
workerSessions: prefix + ":worker-sessions",
|
||||||
|
workerSessionExpiry: prefix + ":worker-session-expiry",
|
||||||
|
workerRuntime: prefix + ":worker-runtime",
|
||||||
|
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -67,6 +75,10 @@ func (keys keyspace) upstream(value string) string {
|
|||||||
return keys.facet("upstream", value)
|
return keys.facet("upstream", value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (keys keyspace) owned(value string) string {
|
||||||
|
return keys.facet("owned", value)
|
||||||
|
}
|
||||||
|
|
||||||
func (keys keyspace) facet(name, value string) string {
|
func (keys keyspace) facet(name, value string) string {
|
||||||
return keys.prefix + ":" + name + ":" + digestToken(value)
|
return keys.prefix + ":" + name + ":" + digestToken(value)
|
||||||
}
|
}
|
||||||
|
|||||||
248
internal/adapters/redisactivity/runtime.go
Normal file
248
internal/adapters/redisactivity/runtime.go
Normal file
@ -0,0 +1,248 @@
|
|||||||
|
package redisactivity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
|
)
|
||||||
|
|
||||||
|
const runtimeWireVersion = 1
|
||||||
|
|
||||||
|
const (
|
||||||
|
runtimeReplaceSession = "replace_session"
|
||||||
|
runtimeReplaceReport = "replace_report"
|
||||||
|
runtimeRead = "read"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runtimeSessionWire struct {
|
||||||
|
Version int `json:"version"`
|
||||||
|
WorkerID string `json:"workerId"`
|
||||||
|
InstanceID string `json:"instanceId"`
|
||||||
|
SessionID string `json:"sessionId"`
|
||||||
|
AckedSnapshotVersion string `json:"ackedSnapshotVersion"`
|
||||||
|
AckedOwnershipEpoch string `json:"ackedOwnershipEpoch"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeCounterWire struct {
|
||||||
|
ProxyID string `json:"proxyId"`
|
||||||
|
Active int64 `json:"active"`
|
||||||
|
Reserved int64 `json:"reserved"`
|
||||||
|
Draining bool `json:"draining"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeReportWire struct {
|
||||||
|
Version int `json:"version"`
|
||||||
|
WorkerID string `json:"workerId"`
|
||||||
|
SessionID string `json:"sessionId"`
|
||||||
|
Sequence string `json:"sequence"`
|
||||||
|
SnapshotVersion string `json:"snapshotVersion"`
|
||||||
|
OwnershipEpoch string `json:"ownershipEpoch"`
|
||||||
|
ObservedAtMS int64 `json:"observedAtMs"`
|
||||||
|
Counters []runtimeCounterWire `json:"counters"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeOwnedProxyWire struct {
|
||||||
|
ProxyID string `json:"proxyId"`
|
||||||
|
WorkerID string `json:"workerId"`
|
||||||
|
OwnershipEpoch string `json:"ownershipEpoch"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeSnapshotWire struct {
|
||||||
|
ProxyID string `json:"proxyId"`
|
||||||
|
Active int64 `json:"active"`
|
||||||
|
Reserved int64 `json:"reserved"`
|
||||||
|
Draining bool `json:"draining"`
|
||||||
|
Fresh bool `json:"fresh"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
_ workerruntime.SessionWriter = (*Adapter)(nil)
|
||||||
|
_ workerruntime.ReportWriter = (*Adapter)(nil)
|
||||||
|
_ workerruntime.RuntimeReader = (*Adapter)(nil)
|
||||||
|
)
|
||||||
|
|
||||||
|
func (a *Adapter) ReplaceSession(ctx context.Context, session workerruntime.Session, ttl time.Duration) error {
|
||||||
|
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !runtimeClean(session.WorkerID) || !runtimeClean(session.InstanceID) || !runtimeClean(session.SessionID) ||
|
||||||
|
session.AckedSnapshotVersion == 0 || session.AckedOwnershipEpoch == 0 || ttl <= 0 {
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(runtimeSessionWire{
|
||||||
|
Version: runtimeWireVersion, WorkerID: session.WorkerID,
|
||||||
|
InstanceID: session.InstanceID, SessionID: session.SessionID,
|
||||||
|
AckedSnapshotVersion: strconv.FormatUint(session.AckedSnapshotVersion, 10),
|
||||||
|
AckedOwnershipEpoch: strconv.FormatUint(session.AckedOwnershipEpoch, 10),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
}
|
||||||
|
reply, err := a.runRuntime(ctx, runtimeReplaceSession, durationMillis(ttl), payload, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch reply.Status {
|
||||||
|
case scriptOK:
|
||||||
|
return nil
|
||||||
|
case scriptInvalid:
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
case scriptStale:
|
||||||
|
return workerruntime.ErrStaleSession
|
||||||
|
default:
|
||||||
|
return invalidScriptReply("unexpected worker session reply")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) ReplaceRuntime(ctx context.Context, report workerruntime.Report, ttl time.Duration) error {
|
||||||
|
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
payload, digest, err := a.encodeRuntimeReport(report, ttl)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reply, err := a.runRuntime(ctx, runtimeReplaceReport, durationMillis(ttl), payload, digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch reply.Status {
|
||||||
|
case scriptOK:
|
||||||
|
return nil
|
||||||
|
case scriptInvalid:
|
||||||
|
return workerruntime.ErrInvalidReport
|
||||||
|
case scriptStale:
|
||||||
|
return workerruntime.ErrStaleReport
|
||||||
|
case scriptConflict:
|
||||||
|
return workerruntime.ErrConflictingReport
|
||||||
|
case scriptUnavailable:
|
||||||
|
return workerruntime.ErrStaleSession
|
||||||
|
default:
|
||||||
|
return invalidScriptReply("unexpected worker runtime reply")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) ReadRuntime(ctx context.Context, proxies []workerruntime.OwnedProxy) ([]workerruntime.Snapshot, error) {
|
||||||
|
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(proxies) > a.options.MaxRuntimeCounters {
|
||||||
|
return nil, workerruntime.ErrInvalidQuery
|
||||||
|
}
|
||||||
|
wires := make([]runtimeOwnedProxyWire, len(proxies))
|
||||||
|
seen := make(map[string]struct{}, len(proxies))
|
||||||
|
for index, proxy := range proxies {
|
||||||
|
if !runtimeClean(proxy.ProxyID) || !runtimeClean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
|
||||||
|
return nil, workerruntime.ErrInvalidQuery
|
||||||
|
}
|
||||||
|
key := proxy.WorkerID + "\x00" + proxy.ProxyID
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return nil, workerruntime.ErrInvalidQuery
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
wires[index] = runtimeOwnedProxyWire{
|
||||||
|
ProxyID: proxy.ProxyID, WorkerID: proxy.WorkerID,
|
||||||
|
OwnershipEpoch: strconv.FormatUint(proxy.OwnershipEpoch, 10),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(wires)
|
||||||
|
if err != nil {
|
||||||
|
return nil, workerruntime.ErrInvalidQuery
|
||||||
|
}
|
||||||
|
reply, err := a.runRuntime(ctx, runtimeRead, 0, payload, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if reply.Status == scriptInvalid {
|
||||||
|
return nil, workerruntime.ErrInvalidQuery
|
||||||
|
}
|
||||||
|
if reply.Status != scriptOK || len(reply.Snapshots) != len(proxies) {
|
||||||
|
return nil, invalidScriptReply("unexpected worker runtime read reply")
|
||||||
|
}
|
||||||
|
result := make([]workerruntime.Snapshot, len(reply.Snapshots))
|
||||||
|
for index, snapshot := range reply.Snapshots {
|
||||||
|
if snapshot.ProxyID != proxies[index].ProxyID || snapshot.Active < 0 || snapshot.Reserved < 0 {
|
||||||
|
return nil, invalidScriptReply("invalid worker runtime snapshot")
|
||||||
|
}
|
||||||
|
result[index] = workerruntime.Snapshot{
|
||||||
|
ProxyID: snapshot.ProxyID, Active: snapshot.Active, Reserved: snapshot.Reserved,
|
||||||
|
Draining: snapshot.Draining, Fresh: snapshot.Fresh,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) encodeRuntimeReport(report workerruntime.Report, ttl time.Duration) ([]byte, string, error) {
|
||||||
|
if ttl <= 0 || !runtimeClean(report.WorkerID) || !runtimeClean(report.SessionID) ||
|
||||||
|
report.Sequence == 0 || report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() ||
|
||||||
|
len(report.Counters) > a.options.MaxRuntimeCounters {
|
||||||
|
return nil, "", workerruntime.ErrInvalidReport
|
||||||
|
}
|
||||||
|
counters := append([]workerruntime.Counter(nil), report.Counters...)
|
||||||
|
sort.Slice(counters, func(left, right int) bool { return counters[left].ProxyID < counters[right].ProxyID })
|
||||||
|
wires := make([]runtimeCounterWire, len(counters))
|
||||||
|
for index, counter := range counters {
|
||||||
|
if !runtimeClean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 ||
|
||||||
|
(index > 0 && counters[index-1].ProxyID == counter.ProxyID) {
|
||||||
|
return nil, "", workerruntime.ErrInvalidReport
|
||||||
|
}
|
||||||
|
wires[index] = runtimeCounterWire{
|
||||||
|
ProxyID: counter.ProxyID, Active: counter.Active,
|
||||||
|
Reserved: counter.Reserved, Draining: counter.Draining,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(runtimeReportWire{
|
||||||
|
Version: runtimeWireVersion, WorkerID: report.WorkerID, SessionID: report.SessionID,
|
||||||
|
Sequence: strconv.FormatUint(report.Sequence, 10),
|
||||||
|
SnapshotVersion: strconv.FormatUint(report.SnapshotVersion, 10),
|
||||||
|
OwnershipEpoch: strconv.FormatUint(report.OwnershipEpoch, 10),
|
||||||
|
ObservedAtMS: report.ObservedAt.UTC().UnixMilli(), Counters: wires,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", workerruntime.ErrInvalidReport
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(payload)
|
||||||
|
return payload, hex.EncodeToString(digest[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) runRuntime(
|
||||||
|
ctx context.Context,
|
||||||
|
operation string,
|
||||||
|
ttlMS int64,
|
||||||
|
payload []byte,
|
||||||
|
digest string,
|
||||||
|
) (runtimeScriptReply, error) {
|
||||||
|
result, err := runScript(ctx, a.client, runtimeScript, []string{
|
||||||
|
a.keys.workerSessions, a.keys.workerSessionExpiry,
|
||||||
|
a.keys.workerRuntime, a.keys.workerRuntimeExpiry, a.keys.owners,
|
||||||
|
}, operation, ttlMS, a.options.CleanupLimit, string(payload), digest)
|
||||||
|
if err != nil {
|
||||||
|
return runtimeScriptReply{}, err
|
||||||
|
}
|
||||||
|
var reply runtimeScriptReply
|
||||||
|
if err := decodeScriptResult(result, &reply); err != nil {
|
||||||
|
return runtimeScriptReply{}, err
|
||||||
|
}
|
||||||
|
return reply, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRuntimeCall(ctx context.Context, adapter *Adapter) error {
|
||||||
|
if ctx == nil || adapter == nil {
|
||||||
|
return workerruntime.ErrInvalidStore
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runtimeClean(value string) bool {
|
||||||
|
return value != "" && strings.TrimSpace(value) == value
|
||||||
|
}
|
||||||
147
internal/adapters/redisactivity/runtime_integration_test.go
Normal file
147
internal/adapters/redisactivity/runtime_integration_test.go
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package redisactivity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRedisWorkerRuntimeReplacesSparseCountersAndFencesReports(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
now := redisTestNow()
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
||||||
|
testProxy("proxy-a", "192.0.2.10"))
|
||||||
|
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||||
|
"proxy-a", "worker-a", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Assign(): %v", err)
|
||||||
|
}
|
||||||
|
session := workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||||
|
AckedSnapshotVersion: 3, AckedOwnershipEpoch: assignment.Epoch,
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), session, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
report := workerruntime.Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||||
|
Counters: []workerruntime.Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(replay): %v", err)
|
||||||
|
}
|
||||||
|
conflict := report
|
||||||
|
conflict.Counters = []workerruntime.Counter{{ProxyID: "proxy-a", Active: 3}}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), conflict, time.Minute); !errors.Is(err, workerruntime.ErrConflictingReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
|
||||||
|
}
|
||||||
|
stale := report
|
||||||
|
stale.Sequence = 1
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), stale, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
query := []workerruntime.OwnedProxy{{
|
||||||
|
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
|
||||||
|
}}
|
||||||
|
got, err := fixture.Adapter.ReadRuntime(context.Background(), query)
|
||||||
|
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{
|
||||||
|
ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
report.Sequence = 3
|
||||||
|
report.Counters = nil
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(empty): %v", err)
|
||||||
|
}
|
||||||
|
got, err = fixture.Adapter.ReadRuntime(context.Background(), query)
|
||||||
|
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{ProxyID: "proxy-a", Fresh: true}) {
|
||||||
|
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b",
|
||||||
|
AckedSnapshotVersion: 4, AckedOwnershipEpoch: assignment.Epoch + 1,
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(new): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleSession) {
|
||||||
|
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisWorkerRuntimeExpiresFailClosed(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
now := redisTestNow()
|
||||||
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
||||||
|
testProxy("proxy-a", "192.0.2.10"))
|
||||||
|
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||||
|
"proxy-a", "worker-a", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Assign(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||||
|
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||||
|
}, 100*time.Millisecond); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||||
|
}, 100*time.Millisecond); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(150 * time.Millisecond)
|
||||||
|
got, err := fixture.Adapter.ReadRuntime(context.Background(), []workerruntime.OwnedProxy{{
|
||||||
|
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
|
||||||
|
}})
|
||||||
|
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||||
|
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisWorkerRuntimeRejectsEmptyReportBeyondAcknowledgedSnapshot(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
now := redisTestNow()
|
||||||
|
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||||
|
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
for name, report := range map[string]workerruntime.Report{
|
||||||
|
"version": {
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
},
|
||||||
|
"epoch": {
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: 10, ObservedAt: now,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime() error = %v, want ErrStaleReport", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisWorkerRuntimeAcceptsEmptyRead(t *testing.T) {
|
||||||
|
fixture := newRedisTestFixture(t)
|
||||||
|
got, err := fixture.Adapter.ReadRuntime(context.Background(), nil)
|
||||||
|
if err != nil || got == nil || len(got) != 0 {
|
||||||
|
t.Fatalf("ReadRuntime(empty) = %#v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -74,6 +74,17 @@ type statusScriptInventory struct {
|
|||||||
Extracted int64 `json:"extracted"`
|
Extracted int64 `json:"extracted"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type runtimeScriptReply struct {
|
||||||
|
Status scriptStatus `json:"status"`
|
||||||
|
Snapshots []runtimeSnapshotWire `json:"snapshots"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type capacityScriptReply struct {
|
||||||
|
Status scriptStatus `json:"status"`
|
||||||
|
Managed int `json:"managed"`
|
||||||
|
AvailableSlots int64 `json:"availableSlots,string"`
|
||||||
|
}
|
||||||
|
|
||||||
//go:embed scripts/upsert.lua
|
//go:embed scripts/upsert.lua
|
||||||
var upsertSource string
|
var upsertSource string
|
||||||
|
|
||||||
@ -92,6 +103,12 @@ var sweepSource string
|
|||||||
//go:embed scripts/status.lua
|
//go:embed scripts/status.lua
|
||||||
var statusSource string
|
var statusSource string
|
||||||
|
|
||||||
|
//go:embed scripts/runtime.lua
|
||||||
|
var runtimeSource string
|
||||||
|
|
||||||
|
//go:embed scripts/capacity.lua
|
||||||
|
var capacitySource string
|
||||||
|
|
||||||
var (
|
var (
|
||||||
upsertScript = redis.NewScript(upsertSource)
|
upsertScript = redis.NewScript(upsertSource)
|
||||||
healthScript = redis.NewScript(healthSource)
|
healthScript = redis.NewScript(healthSource)
|
||||||
@ -99,6 +116,8 @@ var (
|
|||||||
ownershipScript = redis.NewScript(ownershipSource)
|
ownershipScript = redis.NewScript(ownershipSource)
|
||||||
sweepScript = redis.NewScript(sweepSource)
|
sweepScript = redis.NewScript(sweepSource)
|
||||||
statusScript = redis.NewScript(statusSource)
|
statusScript = redis.NewScript(statusSource)
|
||||||
|
runtimeScript = redis.NewScript(runtimeSource)
|
||||||
|
capacityScript = redis.NewScript(capacitySource)
|
||||||
)
|
)
|
||||||
|
|
||||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
||||||
|
|||||||
177
internal/adapters/redisactivity/scripts/capacity.lua
Normal file
177
internal/adapters/redisactivity/scripts/capacity.lua
Normal file
@ -0,0 +1,177 @@
|
|||||||
|
local records_key = KEYS[1]
|
||||||
|
local inventory_key = KEYS[2]
|
||||||
|
local available_upstream_key = KEYS[3]
|
||||||
|
local owned_upstream_key = KEYS[4]
|
||||||
|
local owners_key = KEYS[5]
|
||||||
|
local sessions_key = KEYS[6]
|
||||||
|
local session_expiry_key = KEYS[7]
|
||||||
|
local runtime_key = KEYS[8]
|
||||||
|
local runtime_expiry_key = KEYS[9]
|
||||||
|
|
||||||
|
local upstream_id = ARGV[1]
|
||||||
|
local safety_margin_ms = tonumber(ARGV[2])
|
||||||
|
local scan_limit = tonumber(ARGV[3])
|
||||||
|
local cleanup_limit = tonumber(ARGV[4])
|
||||||
|
|
||||||
|
local function reply(status, managed, available_slots)
|
||||||
|
return cjson.encode({
|
||||||
|
status = status,
|
||||||
|
managed = managed or 0,
|
||||||
|
availableSlots = tostring(available_slots or 0)
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
|
local function now_ms()
|
||||||
|
local value = redis.call('TIME')
|
||||||
|
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
|
||||||
|
end
|
||||||
|
|
||||||
|
local function decode_table(value)
|
||||||
|
if not value then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
local ok, decoded = pcall(cjson.decode, value)
|
||||||
|
if not ok or type(decoded) ~= 'table' then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
return decoded
|
||||||
|
end
|
||||||
|
|
||||||
|
local function valid_uint(value)
|
||||||
|
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
||||||
|
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
||||||
|
end
|
||||||
|
|
||||||
|
local function compare_uint(left, right)
|
||||||
|
if string.len(left) ~= string.len(right) then
|
||||||
|
return string.len(left) < string.len(right) and -1 or 1
|
||||||
|
end
|
||||||
|
if left == right then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
return left < right and -1 or 1
|
||||||
|
end
|
||||||
|
|
||||||
|
local function cleanup(now)
|
||||||
|
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||||
|
for _, worker_id in ipairs(expired_sessions) do
|
||||||
|
redis.call('HDEL', sessions_key, worker_id)
|
||||||
|
redis.call('HDEL', runtime_key, worker_id)
|
||||||
|
redis.call('ZREM', session_expiry_key, worker_id)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||||
|
end
|
||||||
|
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||||
|
for _, worker_id in ipairs(expired_reports) do
|
||||||
|
redis.call('HDEL', runtime_key, worker_id)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if type(upstream_id) ~= 'string' or upstream_id == '' or not safety_margin_ms or safety_margin_ms < 0 or
|
||||||
|
not scan_limit or scan_limit <= 0 or not cleanup_limit or cleanup_limit <= 0 then
|
||||||
|
return reply('invalid', 0, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
local now = now_ms()
|
||||||
|
cleanup(now)
|
||||||
|
local threshold = now + safety_margin_ms
|
||||||
|
local available_ids = redis.call('ZRANGEBYSCORE', available_upstream_key, '(' .. threshold, '+inf',
|
||||||
|
'LIMIT', 0, scan_limit + 1)
|
||||||
|
if #available_ids > scan_limit then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
local remaining = scan_limit - #available_ids
|
||||||
|
local owned_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. threshold, '+inf',
|
||||||
|
'LIMIT', 0, remaining + 1)
|
||||||
|
if #owned_ids > remaining then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
local managed = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
||||||
|
if not managed or managed < 0 or managed ~= math.floor(managed) then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
local available_slots = 0
|
||||||
|
local worker_cache = {}
|
||||||
|
local seen = {}
|
||||||
|
for _, proxy_id in ipairs(available_ids) do
|
||||||
|
seen[proxy_id] = true
|
||||||
|
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||||
|
if not record or type(record.sourceUpstream) ~= 'string' or type(record.state) ~= 'string' or
|
||||||
|
record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
|
||||||
|
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
|
||||||
|
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
|
||||||
|
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
|
||||||
|
(record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HGET', owners_key, proxy_id) then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
available_slots = available_slots + record.maxConcurrency
|
||||||
|
end
|
||||||
|
|
||||||
|
for _, proxy_id in ipairs(owned_ids) do
|
||||||
|
if seen[proxy_id] then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||||
|
if not record or record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
|
||||||
|
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
|
||||||
|
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
|
||||||
|
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
|
||||||
|
type(record.ownerWorkerId) ~= 'string' or record.ownerWorkerId == '' then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
local owner_worker_id = record.ownerWorkerId
|
||||||
|
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||||
|
if not owner or owner.workerId ~= owner_worker_id or type(owner.epoch) ~= 'number' or
|
||||||
|
type(owner.expiresAtMs) ~= 'number' or owner.expiresAtMs <= now or
|
||||||
|
type(owner.draining) ~= 'boolean' then
|
||||||
|
return reply('unavailable', 0, 0)
|
||||||
|
end
|
||||||
|
local cached = worker_cache[owner_worker_id]
|
||||||
|
if not cached then
|
||||||
|
local session = decode_table(redis.call('HGET', sessions_key, owner_worker_id))
|
||||||
|
local report = decode_table(redis.call('HGET', runtime_key, owner_worker_id))
|
||||||
|
cached = {fresh = false, counters = {}}
|
||||||
|
if session and report and session.workerId == owner_worker_id and
|
||||||
|
report.workerId == owner_worker_id and session.sessionId == report.sessionId and
|
||||||
|
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
|
||||||
|
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
|
||||||
|
valid_uint(session.ackedSnapshotVersion) and valid_uint(session.ackedOwnershipEpoch) and
|
||||||
|
report.snapshotVersion == session.ackedSnapshotVersion and
|
||||||
|
report.ownershipEpoch == session.ackedOwnershipEpoch then
|
||||||
|
cached.fresh = true
|
||||||
|
cached.ownershipEpoch = report.ownershipEpoch
|
||||||
|
if type(report.counters) == 'table' then
|
||||||
|
for _, counter in pairs(report.counters) do
|
||||||
|
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
|
||||||
|
cached.counters[counter.proxyId] = counter
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
worker_cache[owner_worker_id] = cached
|
||||||
|
end
|
||||||
|
local owner_epoch = tostring(owner.epoch)
|
||||||
|
if cached.fresh and valid_uint(owner_epoch) and
|
||||||
|
compare_uint(cached.ownershipEpoch, owner_epoch) >= 0 then
|
||||||
|
local counter = cached.counters[proxy_id]
|
||||||
|
local active = 0
|
||||||
|
local reserved = 0
|
||||||
|
local draining = false
|
||||||
|
if counter then
|
||||||
|
active = counter.active
|
||||||
|
reserved = counter.reserved
|
||||||
|
draining = counter.draining
|
||||||
|
end
|
||||||
|
if type(active) == 'number' and type(reserved) == 'number' and active >= 0 and reserved >= 0 and
|
||||||
|
active == math.floor(active) and reserved == math.floor(reserved) and
|
||||||
|
not draining and not owner.draining then
|
||||||
|
local slots = record.maxConcurrency - active - reserved
|
||||||
|
if slots > 0 then
|
||||||
|
available_slots = available_slots + slots
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return reply('ok', managed, available_slots)
|
||||||
@ -113,6 +113,12 @@ local function remove_available(proxy_id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function remove_owned(proxy_id, record)
|
||||||
|
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
local function remove_proxy(proxy_id)
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
local raw = redis.call('HGET', records_key, proxy_id)
|
||||||
local record = nil
|
local record = nil
|
||||||
@ -120,6 +126,7 @@ local function remove_proxy(proxy_id)
|
|||||||
local decoded
|
local decoded
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
decoded, record = pcall(cjson.decode, raw)
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
remove_available(proxy_id, decoded and record or nil)
|
||||||
|
remove_owned(proxy_id, decoded and record or nil)
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -70,12 +70,19 @@ local function remove_available(id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function remove_owned(id, record)
|
||||||
|
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function remove_proxy(id)
|
local function remove_proxy(id)
|
||||||
local raw = redis.call('HGET', records_key, id)
|
local raw = redis.call('HGET', records_key, id)
|
||||||
local record = nil
|
local record = nil
|
||||||
if raw then
|
if raw then
|
||||||
record = cjson.decode(raw)
|
record = cjson.decode(raw)
|
||||||
remove_available(id, record)
|
remove_available(id, record)
|
||||||
|
remove_owned(id, record)
|
||||||
if is_managed(record.state) then
|
if is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
@ -142,6 +149,9 @@ if not raw then
|
|||||||
return finish({status = 'not_found'})
|
return finish({status = 'not_found'})
|
||||||
end
|
end
|
||||||
local record = cjson.decode(raw)
|
local record = cjson.decode(raw)
|
||||||
|
if type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' then
|
||||||
|
return finish({status = 'invalid'})
|
||||||
|
end
|
||||||
if tonumber(record.expiresAtMs) <= checked_at_ms then
|
if tonumber(record.expiresAtMs) <= checked_at_ms then
|
||||||
remove_proxy(proxy_id)
|
remove_proxy(proxy_id)
|
||||||
return finish({status = 'not_found'})
|
return finish({status = 'not_found'})
|
||||||
@ -184,12 +194,18 @@ end
|
|||||||
local encoded = cjson.encode(record)
|
local encoded = cjson.encode(record)
|
||||||
redis.call('HSET', records_key, proxy_id, encoded)
|
redis.call('HSET', records_key, proxy_id, encoded)
|
||||||
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
|
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
|
||||||
if next_state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > checked_at_ms then
|
remove_owned(proxy_id, record)
|
||||||
|
if next_state == 'AVAILABLE' and tonumber(record.usableUntilMs) > checked_at_ms then
|
||||||
|
if owned then
|
||||||
|
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||||
|
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||||
|
else
|
||||||
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
||||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
for _, index_key in ipairs(record.indexKeys or {}) do
|
||||||
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
||||||
touch(index_key, tonumber(record.expiresAtMs))
|
touch(index_key, tonumber(record.expiresAtMs))
|
||||||
end
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
touch(records_key, tonumber(record.expiresAtMs))
|
touch(records_key, tonumber(record.expiresAtMs))
|
||||||
touch(unique_key, tonumber(record.expiresAtMs))
|
touch(unique_key, tonumber(record.expiresAtMs))
|
||||||
|
|||||||
@ -94,6 +94,12 @@ local function remove_available(id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function remove_owned(id, record)
|
||||||
|
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function add_available(id, record, at_ms)
|
local function add_available(id, record, at_ms)
|
||||||
local usable_until_ms = record and tonumber(record.usableUntilMs)
|
local usable_until_ms = record and tonumber(record.usableUntilMs)
|
||||||
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
|
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
|
||||||
@ -118,6 +124,7 @@ local function remove_proxy(id)
|
|||||||
local decoded
|
local decoded
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
decoded, record = pcall(cjson.decode, raw)
|
||||||
remove_available(id, decoded and record or nil)
|
remove_available(id, decoded and record or nil)
|
||||||
|
remove_owned(id, decoded and record or nil)
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
@ -169,6 +176,7 @@ local function clear_owner(id, assignment, at_ms, restore)
|
|||||||
local raw_record = redis.call('HGET', records_key, id)
|
local raw_record = redis.call('HGET', records_key, id)
|
||||||
local record = decode_table(raw_record)
|
local record = decode_table(raw_record)
|
||||||
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
|
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
|
||||||
|
remove_owned(id, record)
|
||||||
record.ownerWorkerId = nil
|
record.ownerWorkerId = nil
|
||||||
redis.call('HSET', records_key, id, cjson.encode(record))
|
redis.call('HSET', records_key, id, cjson.encode(record))
|
||||||
if restore then
|
if restore then
|
||||||
@ -195,6 +203,7 @@ if operation == 'assign' then
|
|||||||
|
|
||||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||||
if not record or record.state ~= 'AVAILABLE' or
|
if not record or record.state ~= 'AVAILABLE' or
|
||||||
|
type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' or
|
||||||
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
|
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
|
||||||
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
|
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
|
||||||
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
|
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
|
||||||
@ -223,6 +232,8 @@ if operation == 'assign' then
|
|||||||
record.ownerWorkerId = worker_id
|
record.ownerWorkerId = worker_id
|
||||||
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
|
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
|
||||||
remove_available(proxy_id, record)
|
remove_available(proxy_id, record)
|
||||||
|
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||||
|
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||||
return finish({status = 'ok', record = encoded})
|
return finish({status = 'ok', record = encoded})
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -267,6 +278,8 @@ if operation == 'begin_drain' then
|
|||||||
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
|
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
|
||||||
local encoded = cjson.encode(current)
|
local encoded = cjson.encode(current)
|
||||||
redis.call('HSET', owners_key, proxy_id, encoded)
|
redis.call('HSET', owners_key, proxy_id, encoded)
|
||||||
|
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||||
|
remove_owned(proxy_id, record)
|
||||||
return finish({status = 'ok', record = encoded})
|
return finish({status = 'ok', record = encoded})
|
||||||
end
|
end
|
||||||
return finish({status = 'ok', record = cjson.encode(current)})
|
return finish({status = 'ok', record = cjson.encode(current)})
|
||||||
|
|||||||
229
internal/adapters/redisactivity/scripts/runtime.lua
Normal file
229
internal/adapters/redisactivity/scripts/runtime.lua
Normal file
@ -0,0 +1,229 @@
|
|||||||
|
local sessions_key = KEYS[1]
|
||||||
|
local session_expiry_key = KEYS[2]
|
||||||
|
local runtime_key = KEYS[3]
|
||||||
|
local runtime_expiry_key = KEYS[4]
|
||||||
|
local owners_key = KEYS[5]
|
||||||
|
|
||||||
|
local operation = ARGV[1]
|
||||||
|
local ttl_ms = tonumber(ARGV[2])
|
||||||
|
local cleanup_limit = tonumber(ARGV[3])
|
||||||
|
local payload = ARGV[4]
|
||||||
|
local digest = ARGV[5]
|
||||||
|
|
||||||
|
local function reply(status, snapshots)
|
||||||
|
if snapshots then
|
||||||
|
return cjson.encode({status = status, snapshots = snapshots})
|
||||||
|
end
|
||||||
|
return '{"status":' .. cjson.encode(status) .. ',"snapshots":[]}'
|
||||||
|
end
|
||||||
|
|
||||||
|
local function now_ms()
|
||||||
|
local value = redis.call('TIME')
|
||||||
|
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
|
||||||
|
end
|
||||||
|
|
||||||
|
local function decode_table(value)
|
||||||
|
if not value then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
local ok, decoded = pcall(cjson.decode, value)
|
||||||
|
if not ok or type(decoded) ~= 'table' then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
return decoded
|
||||||
|
end
|
||||||
|
|
||||||
|
local function valid_uint(value)
|
||||||
|
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
||||||
|
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
||||||
|
end
|
||||||
|
|
||||||
|
local function compare_uint(left, right)
|
||||||
|
if string.len(left) ~= string.len(right) then
|
||||||
|
return string.len(left) < string.len(right) and -1 or 1
|
||||||
|
end
|
||||||
|
if left == right then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
return left < right and -1 or 1
|
||||||
|
end
|
||||||
|
|
||||||
|
local function cleanup(now)
|
||||||
|
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||||
|
for _, worker_id in ipairs(expired_sessions) do
|
||||||
|
redis.call('HDEL', sessions_key, worker_id)
|
||||||
|
redis.call('HDEL', runtime_key, worker_id)
|
||||||
|
redis.call('ZREM', session_expiry_key, worker_id)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||||
|
end
|
||||||
|
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||||
|
for _, worker_id in ipairs(expired_reports) do
|
||||||
|
redis.call('HDEL', runtime_key, worker_id)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
local function valid_session(value)
|
||||||
|
return value and value.version == 1 and type(value.workerId) == 'string' and value.workerId ~= '' and
|
||||||
|
type(value.instanceId) == 'string' and value.instanceId ~= '' and
|
||||||
|
type(value.sessionId) == 'string' and value.sessionId ~= '' and
|
||||||
|
valid_uint(value.ackedSnapshotVersion) and valid_uint(value.ackedOwnershipEpoch)
|
||||||
|
end
|
||||||
|
|
||||||
|
local function valid_owner(value, worker_id, ownership_epoch, now)
|
||||||
|
return value and type(value.workerId) == 'string' and value.workerId == worker_id and
|
||||||
|
type(value.epoch) == 'number' and valid_uint(tostring(value.epoch)) and
|
||||||
|
compare_uint(ownership_epoch, tostring(value.epoch)) >= 0 and
|
||||||
|
type(value.expiresAtMs) == 'number' and value.expiresAtMs > now
|
||||||
|
end
|
||||||
|
|
||||||
|
local now = now_ms()
|
||||||
|
cleanup(now)
|
||||||
|
|
||||||
|
if operation == 'replace_session' then
|
||||||
|
if not ttl_ms or ttl_ms <= 0 then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local session = decode_table(payload)
|
||||||
|
if not valid_session(session) then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local current = decode_table(redis.call('HGET', sessions_key, session.workerId))
|
||||||
|
if current and valid_session(current) and current.sessionId == session.sessionId and
|
||||||
|
current.instanceId == session.instanceId and type(current.expiresAtMs) == 'number' and
|
||||||
|
current.expiresAtMs > now then
|
||||||
|
local epoch_order = compare_uint(session.ackedOwnershipEpoch, current.ackedOwnershipEpoch)
|
||||||
|
local version_order = compare_uint(session.ackedSnapshotVersion, current.ackedSnapshotVersion)
|
||||||
|
if epoch_order < 0 or (epoch_order == 0 and version_order < 0) then
|
||||||
|
return reply('stale')
|
||||||
|
end
|
||||||
|
if epoch_order > 0 or version_order > 0 then
|
||||||
|
redis.call('HDEL', runtime_key, session.workerId)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
redis.call('HDEL', runtime_key, session.workerId)
|
||||||
|
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
||||||
|
end
|
||||||
|
session.expiresAtMs = now + ttl_ms
|
||||||
|
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
|
||||||
|
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
|
||||||
|
return reply('ok')
|
||||||
|
end
|
||||||
|
|
||||||
|
if operation == 'replace_report' then
|
||||||
|
if not ttl_ms or ttl_ms <= 0 or type(digest) ~= 'string' or digest == '' then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local report = decode_table(payload)
|
||||||
|
if not report or report.version ~= 1 or type(report.workerId) ~= 'string' or report.workerId == '' or
|
||||||
|
type(report.sessionId) ~= 'string' or report.sessionId == '' or not valid_uint(report.sequence) or
|
||||||
|
not valid_uint(report.snapshotVersion) or not valid_uint(report.ownershipEpoch) or
|
||||||
|
type(report.observedAtMs) ~= 'number' or type(report.counters) ~= 'table' then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local session = decode_table(redis.call('HGET', sessions_key, report.workerId))
|
||||||
|
if not valid_session(session) or session.workerId ~= report.workerId or session.sessionId ~= report.sessionId or
|
||||||
|
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
|
||||||
|
return reply('unavailable')
|
||||||
|
end
|
||||||
|
if report.snapshotVersion ~= session.ackedSnapshotVersion or
|
||||||
|
report.ownershipEpoch ~= session.ackedOwnershipEpoch then
|
||||||
|
return reply('stale')
|
||||||
|
end
|
||||||
|
local current = decode_table(redis.call('HGET', runtime_key, report.workerId))
|
||||||
|
if current and current.sessionId == report.sessionId and valid_uint(current.sequence) then
|
||||||
|
local ordering = compare_uint(report.sequence, current.sequence)
|
||||||
|
if ordering < 0 then
|
||||||
|
return reply('stale')
|
||||||
|
end
|
||||||
|
if ordering == 0 then
|
||||||
|
if current.digest == digest then
|
||||||
|
return reply('ok')
|
||||||
|
end
|
||||||
|
return reply('conflict')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
local seen = {}
|
||||||
|
for _, counter in pairs(report.counters) do
|
||||||
|
if type(counter) ~= 'table' or type(counter.proxyId) ~= 'string' or counter.proxyId == '' or
|
||||||
|
type(counter.active) ~= 'number' or counter.active < 0 or counter.active ~= math.floor(counter.active) or
|
||||||
|
type(counter.reserved) ~= 'number' or counter.reserved < 0 or counter.reserved ~= math.floor(counter.reserved) or
|
||||||
|
type(counter.draining) ~= 'boolean' or seen[counter.proxyId] then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
seen[counter.proxyId] = true
|
||||||
|
local owner = decode_table(redis.call('HGET', owners_key, counter.proxyId))
|
||||||
|
if not valid_owner(owner, report.workerId, report.ownershipEpoch, now) then
|
||||||
|
return reply('stale')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
report.digest = digest
|
||||||
|
report.expiresAtMs = now + ttl_ms
|
||||||
|
redis.call('HSET', runtime_key, report.workerId, cjson.encode(report))
|
||||||
|
redis.call('ZADD', runtime_expiry_key, report.expiresAtMs, report.workerId)
|
||||||
|
session.expiresAtMs = now + ttl_ms
|
||||||
|
redis.call('HSET', sessions_key, report.workerId, cjson.encode(session))
|
||||||
|
redis.call('ZADD', session_expiry_key, session.expiresAtMs, report.workerId)
|
||||||
|
return reply('ok')
|
||||||
|
end
|
||||||
|
|
||||||
|
if operation == 'read' then
|
||||||
|
local queries = decode_table(payload)
|
||||||
|
if not queries then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
if next(queries) == nil then
|
||||||
|
return '{"status":"ok","snapshots":[]}'
|
||||||
|
end
|
||||||
|
local snapshots = cjson.decode('[]')
|
||||||
|
local cache = {}
|
||||||
|
for _, query in ipairs(queries) do
|
||||||
|
if type(query) ~= 'table' or type(query.proxyId) ~= 'string' or query.proxyId == '' or
|
||||||
|
type(query.workerId) ~= 'string' or query.workerId == '' or not valid_uint(query.ownershipEpoch) then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local snapshot = {proxyId = query.proxyId, active = 0, reserved = 0, draining = false, fresh = false}
|
||||||
|
local owner = decode_table(redis.call('HGET', owners_key, query.proxyId))
|
||||||
|
if valid_owner(owner, query.workerId, query.ownershipEpoch, now) and
|
||||||
|
compare_uint(query.ownershipEpoch, tostring(owner.epoch)) == 0 then
|
||||||
|
local cached = cache[query.workerId]
|
||||||
|
if not cached then
|
||||||
|
local session = decode_table(redis.call('HGET', sessions_key, query.workerId))
|
||||||
|
local report = decode_table(redis.call('HGET', runtime_key, query.workerId))
|
||||||
|
cached = {fresh = false, counters = {}}
|
||||||
|
if valid_session(session) and session.workerId == query.workerId and report and
|
||||||
|
report.workerId == query.workerId and report.sessionId == session.sessionId and
|
||||||
|
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
|
||||||
|
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
|
||||||
|
valid_uint(report.ownershipEpoch) and
|
||||||
|
report.snapshotVersion == session.ackedSnapshotVersion and
|
||||||
|
report.ownershipEpoch == session.ackedOwnershipEpoch then
|
||||||
|
cached.fresh = true
|
||||||
|
cached.ownershipEpoch = report.ownershipEpoch
|
||||||
|
if type(report.counters) == 'table' then
|
||||||
|
for _, counter in pairs(report.counters) do
|
||||||
|
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
|
||||||
|
cached.counters[counter.proxyId] = counter
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
cache[query.workerId] = cached
|
||||||
|
end
|
||||||
|
if cached.fresh and compare_uint(cached.ownershipEpoch, query.ownershipEpoch) >= 0 then
|
||||||
|
snapshot.fresh = true
|
||||||
|
local counter = cached.counters[query.proxyId]
|
||||||
|
if counter then
|
||||||
|
snapshot.active = counter.active
|
||||||
|
snapshot.reserved = counter.reserved
|
||||||
|
snapshot.draining = counter.draining
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
snapshots[#snapshots + 1] = snapshot
|
||||||
|
end
|
||||||
|
return reply('ok', snapshots)
|
||||||
|
end
|
||||||
|
|
||||||
|
return reply('invalid')
|
||||||
@ -73,6 +73,12 @@ local function remove_available(proxy_id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function remove_owned(proxy_id, record)
|
||||||
|
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
local function remove_proxy(proxy_id)
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
local raw = redis.call('HGET', records_key, proxy_id)
|
||||||
local record = nil
|
local record = nil
|
||||||
@ -80,6 +86,7 @@ local function remove_proxy(proxy_id)
|
|||||||
local decoded
|
local decoded
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
decoded, record = pcall(cjson.decode, raw)
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
remove_available(proxy_id, decoded and record or nil)
|
||||||
|
remove_owned(proxy_id, decoded and record or nil)
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -70,12 +70,19 @@ local function remove_available(proxy_id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function remove_owned(proxy_id, record)
|
||||||
|
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
local function remove_proxy(proxy_id)
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
local raw = redis.call('HGET', records_key, proxy_id)
|
||||||
local record = nil
|
local record = nil
|
||||||
if raw then
|
if raw then
|
||||||
record = cjson.decode(raw)
|
record = cjson.decode(raw)
|
||||||
remove_available(proxy_id, record)
|
remove_available(proxy_id, record)
|
||||||
|
remove_owned(proxy_id, record)
|
||||||
if is_managed(record.state) then
|
if is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
@ -123,6 +130,16 @@ local function add_available(proxy_id, record)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function sync_owned(proxy_id, record)
|
||||||
|
if record.state == 'AVAILABLE' and record.ownerWorkerId and record.ownerWorkerId ~= '' and
|
||||||
|
tonumber(record.usableUntilMs) > now_ms then
|
||||||
|
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||||
|
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||||
|
else
|
||||||
|
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
local function finish(reply)
|
local function finish(reply)
|
||||||
local encoded = cjson.encode(reply)
|
local encoded = cjson.encode(reply)
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
||||||
@ -132,6 +149,11 @@ end
|
|||||||
cleanup_expired()
|
cleanup_expired()
|
||||||
|
|
||||||
for _, candidate in ipairs(candidates) do
|
for _, candidate in ipairs(candidates) do
|
||||||
|
local decoded, incoming = pcall(cjson.decode, candidate.record)
|
||||||
|
if not decoded or type(incoming) ~= 'table' or
|
||||||
|
type(incoming.ownerIndexKey) ~= 'string' or incoming.ownerIndexKey == '' then
|
||||||
|
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
||||||
|
end
|
||||||
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
|
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
|
||||||
if mapped and mapped ~= candidate.uniqueDigest then
|
if mapped and mapped ~= candidate.uniqueDigest then
|
||||||
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
||||||
@ -174,6 +196,7 @@ for _, candidate in ipairs(candidates) do
|
|||||||
local encoded = cjson.encode(incoming)
|
local encoded = cjson.encode(incoming)
|
||||||
redis.call('HSET', records_key, incumbent_id, encoded)
|
redis.call('HSET', records_key, incumbent_id, encoded)
|
||||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
|
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
|
||||||
|
sync_owned(incumbent_id, incoming)
|
||||||
add_available(incumbent_id, incoming)
|
add_available(incumbent_id, incoming)
|
||||||
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
||||||
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
||||||
@ -192,6 +215,7 @@ for _, candidate in ipairs(candidates) do
|
|||||||
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
|
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
|
||||||
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
|
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
|
||||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
|
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
|
||||||
|
sync_owned(candidate.proxyId, incoming)
|
||||||
if is_managed(incoming.state) then
|
if is_managed(incoming.state) then
|
||||||
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
||||||
|
return newRedisFixtureWithCredentialCapacity(t, 10_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
||||||
if redisURL == "" {
|
if redisURL == "" {
|
||||||
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
|
|||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
t.Fatalf("ping test Redis: %v", err)
|
t.Fatalf("ping test Redis: %v", err)
|
||||||
}
|
}
|
||||||
credentialStore, err := credentials.NewMemoryStore(10_000)
|
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
|||||||
@ -15,7 +15,10 @@ import (
|
|||||||
"proxy-pool/internal/platform/credentials"
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxUpsertScriptBatch = 256
|
const (
|
||||||
|
maxUpsertScriptBatch = 256
|
||||||
|
transientCredentialReleaseTimeout = time.Second
|
||||||
|
)
|
||||||
|
|
||||||
type upsertCandidate struct {
|
type upsertCandidate struct {
|
||||||
ProxyID string `json:"proxyId"`
|
ProxyID string `json:"proxyId"`
|
||||||
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
|
|||||||
|
|
||||||
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
var result activitypool.UpsertResult
|
var result activitypool.UpsertResult
|
||||||
|
if a != nil {
|
||||||
|
defer a.releaseTransientCredentials(batch.Proxies)
|
||||||
|
}
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return result, activitypool.ErrInvalidBatch
|
return result, activitypool.ErrInvalidBatch
|
||||||
}
|
}
|
||||||
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
|
||||||
|
if a.credentialReleaser == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
|
||||||
|
defer cancel()
|
||||||
|
for _, candidate := range proxies {
|
||||||
|
if releaseCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef,
|
||||||
|
CredentialVersion: candidate.CredentialVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (a *Adapter) prepareUpsertCandidate(
|
func (a *Adapter) prepareUpsertCandidate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
upstreamID string,
|
upstreamID string,
|
||||||
@ -139,7 +165,8 @@ func (a *Adapter) prepareUpsertCandidate(
|
|||||||
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
|
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
|
||||||
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
|
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
|
||||||
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
|
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
|
||||||
Tags: cloneTags(candidate.Tags), IndexKeys: a.availableIndexKeys(candidate),
|
Tags: cloneTags(candidate.Tags), OwnerIndexKey: a.keys.owned(upstreamID),
|
||||||
|
IndexKeys: a.availableIndexKeys(candidate),
|
||||||
}
|
}
|
||||||
if candidate.LastCheckedAt != nil {
|
if candidate.LastCheckedAt != nil {
|
||||||
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
||||||
|
|||||||
@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
|
||||||
|
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
|
||||||
|
first := testProxy("proxy-a", "192.0.2.10")
|
||||||
|
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
|
||||||
|
Username: "first", Password: "first-password",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(first credential): %v", err)
|
||||||
|
}
|
||||||
|
first.SecretRef = firstReference.SecretRef
|
||||||
|
first.CredentialVersion = firstReference.CredentialVersion
|
||||||
|
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||||
|
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
|
||||||
|
Proxies: []proxyDomain.Proxy{first},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertFetched(first): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
|
||||||
|
Username: "second", Password: "second-password",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Put(second credential after upsert): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
||||||
fixture := newRedisTestFixture(t)
|
fixture := newRedisTestFixture(t)
|
||||||
now := redisTestNow()
|
now := redisTestNow()
|
||||||
|
|||||||
@ -55,6 +55,8 @@ func (adapter *Adapter) RunLeader(
|
|||||||
) error {
|
) error {
|
||||||
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
||||||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
||||||
|
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
@ -217,9 +219,10 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
|
|||||||
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
|
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
|
||||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
|
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
|
||||||
return nil, controllerProvider.ErrInvalidCoordination
|
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
|
||||||
|
return nil, false, controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
operationCtx, cancel := context.WithCancel(ctx)
|
operationCtx, cancel := context.WithCancel(ctx)
|
||||||
stop := context.AfterFunc(session.ctx, cancel)
|
stop := context.AfterFunc(session.ctx, cancel)
|
||||||
@ -229,32 +232,37 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
|||||||
}()
|
}()
|
||||||
permitToken, err := randomToken()
|
permitToken, err := randomToken()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
return nil, false, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
||||||
}
|
}
|
||||||
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
||||||
for operationCtx.Err() == nil {
|
for operationCtx.Err() == nil {
|
||||||
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
||||||
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
||||||
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
||||||
durationMillis(permitTTL),
|
durationMillis(permitTTL), expected, session.limits.MaxTotal,
|
||||||
)
|
)
|
||||||
if scriptErr != nil {
|
if scriptErr != nil {
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
if operationCtx.Err() != nil {
|
if operationCtx.Err() != nil {
|
||||||
return nil, operationCtx.Err()
|
return nil, false, operationCtx.Err()
|
||||||
}
|
}
|
||||||
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
switch reply.Status {
|
switch reply.Status {
|
||||||
case "ok":
|
case "ok":
|
||||||
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
|
return &requestPermit{
|
||||||
|
adapter: session.adapter, keys: session.keys, token: permitToken,
|
||||||
|
settlementTTL: permitTTL,
|
||||||
|
}, true, nil
|
||||||
|
case "quota_exhausted":
|
||||||
|
return nil, false, nil
|
||||||
case "stale":
|
case "stale":
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
case "rate_limited", "at_capacity":
|
case "rate_limited", "at_capacity":
|
||||||
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
||||||
if delay <= 0 {
|
if delay <= 0 {
|
||||||
@ -262,30 +270,42 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
|||||||
}
|
}
|
||||||
if err := wait(operationCtx, delay); err != nil {
|
if err := wait(operationCtx, delay); err != nil {
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return nil, controllerProvider.ErrCoordinationUnavailable
|
return nil, false, controllerProvider.ErrCoordinationUnavailable
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
return nil, operationCtx.Err()
|
return nil, false, operationCtx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
type requestPermit struct {
|
type requestPermit struct {
|
||||||
adapter *Adapter
|
adapter *Adapter
|
||||||
keys upstreamKeys
|
keys upstreamKeys
|
||||||
token string
|
token string
|
||||||
|
settlementTTL time.Duration
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
done bool
|
done bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (permit *requestPermit) Release(ctx context.Context) error {
|
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
|
||||||
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
|
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
|
||||||
|
return controllerProvider.ErrInvalidCoordination
|
||||||
|
}
|
||||||
|
return permit.finish(ctx, "complete_fetch", fetched)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (permit *requestPermit) Cancel(ctx context.Context) error {
|
||||||
|
return permit.finish(ctx, "cancel_fetch", 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (permit *requestPermit) finish(ctx context.Context, operation string, fetched int) error {
|
||||||
|
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" || permit.settlementTTL <= 0 {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
permit.mu.Lock()
|
permit.mu.Lock()
|
||||||
@ -293,15 +313,22 @@ func (permit *requestPermit) Release(ctx context.Context) error {
|
|||||||
if permit.done {
|
if permit.done {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
|
for ctx.Err() == nil {
|
||||||
|
reply, err := runScript(ctx, permit.adapter.client, permit.keys,
|
||||||
|
operation, permit.token, fetched, durationMillis(permit.settlementTTL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
if waitErr := wait(ctx, permit.adapter.options.RetryInterval); waitErr != nil {
|
||||||
|
return waitErr
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if reply.Status != "ok" {
|
if reply.Status != "ok" {
|
||||||
return controllerProvider.ErrCoordinationUnavailable
|
return controllerProvider.ErrCoordinationUnavailable
|
||||||
}
|
}
|
||||||
permit.done = true
|
permit.done = true
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func randomToken() (string, error) {
|
func randomToken() (string, error) {
|
||||||
|
|||||||
@ -89,6 +89,27 @@ func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunLeaderRejectsNegativeFetchQuota(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
||||||
|
t.Cleanup(func() { _ = client.Close() })
|
||||||
|
adapter, err := New(client, Options{
|
||||||
|
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
|
||||||
|
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(): %v", err)
|
||||||
|
}
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: -1,
|
||||||
|
}
|
||||||
|
err = adapter.RunLeader(context.Background(), "provider-a", limits,
|
||||||
|
func(context.Context, controllerProvider.LeaderSession) error { return nil })
|
||||||
|
if !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
|
||||||
|
t.Fatalf("RunLeader() error = %v, want ErrInvalidCoordination", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|||||||
@ -103,36 +103,201 @@ func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T)
|
|||||||
})
|
})
|
||||||
}()
|
}()
|
||||||
session := receiveSession(t, sessions)
|
session := receiveSession(t, sessions)
|
||||||
first, err := session.AcquireFetch(context.Background())
|
first, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
if err != nil {
|
if err != nil || !available {
|
||||||
t.Fatalf("first AcquireFetch(): %v", err)
|
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", first, available, err)
|
||||||
}
|
}
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
secondResult := make(chan permitResultFixture, 1)
|
secondResult := make(chan permitResultFixture, 1)
|
||||||
go func() {
|
go func() {
|
||||||
permit, acquireErr := session.AcquireFetch(context.Background())
|
permit, permitAvailable, acquireErr := session.AcquireFetch(context.Background(), 1)
|
||||||
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
|
secondResult <- permitResultFixture{permit: permit, available: permitAvailable, err: acquireErr}
|
||||||
}()
|
}()
|
||||||
select {
|
select {
|
||||||
case result := <-secondResult:
|
case result := <-secondResult:
|
||||||
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
||||||
case <-time.After(100 * time.Millisecond):
|
case <-time.After(100 * time.Millisecond):
|
||||||
}
|
}
|
||||||
if err := first.Release(context.Background()); err != nil {
|
if err := first.Complete(context.Background(), 1); err != nil {
|
||||||
t.Fatalf("first Release(): %v", err)
|
t.Fatalf("first Complete(): %v", err)
|
||||||
}
|
}
|
||||||
result := receivePermit(t, secondResult)
|
result := receivePermit(t, secondResult)
|
||||||
if result.err != nil || result.permit == nil {
|
if result.err != nil || !result.available || result.permit == nil {
|
||||||
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
|
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", result.permit, result.available, result.err)
|
||||||
}
|
}
|
||||||
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
||||||
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
||||||
}
|
}
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||||
t.Fatalf("second Release(): %v", err)
|
t.Fatalf("second Cancel(): %v", err)
|
||||||
}
|
}
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||||
t.Fatalf("idempotent second Release(): %v", err)
|
t.Fatalf("idempotent second Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
waitRunner(t, done)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisLeaderSessionPreservesFetchQuotaAcrossFailover(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
firstCtx, cancelFirst := context.WithCancel(context.Background())
|
||||||
|
firstSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
firstDone <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||||
|
firstCtx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
firstSessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
firstSession := receiveSession(t, firstSessions)
|
||||||
|
firstPermit, available, err := firstSession.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || firstPermit == nil {
|
||||||
|
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", firstPermit, available, err)
|
||||||
|
}
|
||||||
|
firstFence := firstSession.Fence()
|
||||||
|
cancelFirst()
|
||||||
|
waitRunner(t, firstDone)
|
||||||
|
if err := firstPermit.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("Complete() after leadership loss: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
secondCtx, cancelSecond := context.WithCancel(context.Background())
|
||||||
|
defer cancelSecond()
|
||||||
|
secondSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
secondDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
secondDone <- fixture.coordinator(t, "controller-b").RunLeader(
|
||||||
|
secondCtx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
secondSessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
secondSession := receiveSession(t, secondSessions)
|
||||||
|
secondFence := secondSession.Fence()
|
||||||
|
if secondFence.Generation != firstFence.Generation || secondFence.Epoch <= firstFence.Epoch {
|
||||||
|
t.Fatalf("second fence = %+v, first = %+v", secondFence, firstFence)
|
||||||
|
}
|
||||||
|
|
||||||
|
secondPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || secondPermit == nil {
|
||||||
|
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", secondPermit, available, err)
|
||||||
|
}
|
||||||
|
if err := secondPermit.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("second Complete(): %v", err)
|
||||||
|
}
|
||||||
|
exhaustedPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhaustedPermit != nil {
|
||||||
|
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v), want unavailable", exhaustedPermit, available, err)
|
||||||
|
}
|
||||||
|
cancelSecond()
|
||||||
|
waitRunner(t, secondDone)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisFetchQuotaSettlementIsIdempotentAndCancellationRefundsReservation(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 2, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||||
|
ctx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
sessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
session := receiveSession(t, sessions)
|
||||||
|
|
||||||
|
cancelled, available, err := session.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || cancelled == nil {
|
||||||
|
t.Fatalf("cancelled AcquireFetch() = (%v, %t, %v)", cancelled, available, err)
|
||||||
|
}
|
||||||
|
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("idempotent Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
completed, available, err := session.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || completed == nil {
|
||||||
|
t.Fatalf("completed AcquireFetch() = (%v, %t, %v)", completed, available, err)
|
||||||
|
}
|
||||||
|
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("Complete(): %v", err)
|
||||||
|
}
|
||||||
|
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("idempotent Complete(): %v", err)
|
||||||
|
}
|
||||||
|
if err := completed.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Cancel() after Complete(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
last, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || last == nil {
|
||||||
|
t.Fatalf("last AcquireFetch() = (%v, %t, %v)", last, available, err)
|
||||||
|
}
|
||||||
|
if err := last.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("last Complete(): %v", err)
|
||||||
|
}
|
||||||
|
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhausted != nil {
|
||||||
|
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v)", exhausted, available, err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
waitRunner(t, done)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisExpiredFetchReservationIsConservativelyCharged(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
coordinator, err := New(fixture.client, Options{
|
||||||
|
Namespace: fixture.namespace, HolderID: "controller-a", LeaseTTL: 600 * time.Millisecond,
|
||||||
|
RenewEvery: 150 * time.Millisecond, RetryInterval: 10 * time.Millisecond, PermitGrace: 10 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(): %v", err)
|
||||||
|
}
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: 40 * time.Millisecond, MaxTotal: 1,
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- coordinator.RunLeader(ctx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
sessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
session := receiveSession(t, sessions)
|
||||||
|
abandoned, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || abandoned == nil {
|
||||||
|
t.Fatalf("abandoned AcquireFetch() = (%v, %t, %v)", abandoned, available, err)
|
||||||
|
}
|
||||||
|
time.Sleep(80 * time.Millisecond)
|
||||||
|
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhausted != nil {
|
||||||
|
t.Fatalf("post-expiry AcquireFetch() = (%v, %t, %v), want charged quota", exhausted, available, err)
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
waitRunner(t, done)
|
waitRunner(t, done)
|
||||||
@ -281,6 +446,7 @@ func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResult
|
|||||||
|
|
||||||
type permitResultFixture struct {
|
type permitResultFixture struct {
|
||||||
permit controllerProvider.RequestPermit
|
permit controllerProvider.RequestPermit
|
||||||
|
available bool
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,10 @@ type upstreamKeys struct {
|
|||||||
leader string
|
leader string
|
||||||
next string
|
next string
|
||||||
inflight string
|
inflight string
|
||||||
|
fetchedTotal string
|
||||||
|
pendingTotal string
|
||||||
|
permits string
|
||||||
|
permitExpiry string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
||||||
@ -33,11 +37,18 @@ func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
|||||||
leader: prefix + ":leader",
|
leader: prefix + ":leader",
|
||||||
next: prefix + ":next-request",
|
next: prefix + ":next-request",
|
||||||
inflight: prefix + ":inflight",
|
inflight: prefix + ":inflight",
|
||||||
|
fetchedTotal: prefix + ":fetched-total",
|
||||||
|
pendingTotal: prefix + ":pending-total",
|
||||||
|
permits: prefix + ":permits",
|
||||||
|
permitExpiry: prefix + ":permit-expiry",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (keys upstreamKeys) all() []string {
|
func (keys upstreamKeys) all() []string {
|
||||||
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
|
return []string{
|
||||||
|
keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight,
|
||||||
|
keys.fetchedTotal, keys.pendingTotal, keys.permits, keys.permitExpiry,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func digestParts(values ...string) string {
|
func digestParts(values ...string) string {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
|
local max_safe_integer = 9007199254740991
|
||||||
|
|
||||||
local function now_ms()
|
local function now_ms()
|
||||||
local value = redis.call('TIME')
|
local value = redis.call('TIME')
|
||||||
@ -33,6 +34,63 @@ local function same_leader(value, generation, holder_id, token, epoch)
|
|||||||
value.token == token and value.epoch == tostring(epoch)
|
value.token == token and value.epoch == tostring(epoch)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function read_counter(key)
|
||||||
|
local encoded = redis.call('GET', key)
|
||||||
|
if not encoded then
|
||||||
|
return 0, nil
|
||||||
|
end
|
||||||
|
local value = tonumber(encoded)
|
||||||
|
if not value or value < 0 or value ~= math.floor(value) then
|
||||||
|
return nil, 'invalid'
|
||||||
|
end
|
||||||
|
return value, nil
|
||||||
|
end
|
||||||
|
|
||||||
|
local function read_permit(token)
|
||||||
|
local encoded = redis.call('HGET', KEYS[8], token)
|
||||||
|
if not encoded then
|
||||||
|
return nil, nil
|
||||||
|
end
|
||||||
|
local ok, value = pcall(cjson.decode, encoded)
|
||||||
|
if not ok or type(value) ~= 'table' or type(value.state) ~= 'string' or
|
||||||
|
type(value.expected) ~= 'number' or value.expected <= 0 or
|
||||||
|
value.expected ~= math.floor(value.expected) then
|
||||||
|
return nil, 'invalid'
|
||||||
|
end
|
||||||
|
return value, nil
|
||||||
|
end
|
||||||
|
|
||||||
|
local function cleanup_expired(now)
|
||||||
|
local expired = redis.call('ZRANGEBYSCORE', KEYS[9], '-inf', now, 'LIMIT', 0, 256)
|
||||||
|
if #expired == 0 then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
for _, permit_token in ipairs(expired) do
|
||||||
|
local permit, permit_error = read_permit(permit_token)
|
||||||
|
if permit_error then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
if permit and permit.state == 'reserved' then
|
||||||
|
if pending < permit.expected or fetched > max_safe_integer - permit.expected then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
pending = pending - permit.expected
|
||||||
|
fetched = fetched + permit.expected
|
||||||
|
end
|
||||||
|
redis.call('HDEL', KEYS[8], permit_token)
|
||||||
|
redis.call('ZREM', KEYS[5], permit_token)
|
||||||
|
redis.call('ZREM', KEYS[9], permit_token)
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[6], fetched)
|
||||||
|
redis.call('SET', KEYS[7], pending)
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
|
||||||
if operation == 'acquire_leader' then
|
if operation == 'acquire_leader' then
|
||||||
local generation_candidate = ARGV[2]
|
local generation_candidate = ARGV[2]
|
||||||
local holder_id = ARGV[3]
|
local holder_id = ARGV[3]
|
||||||
@ -41,7 +99,10 @@ if operation == 'acquire_leader' then
|
|||||||
if not lease_ttl or lease_ttl <= 0 then
|
if not lease_ttl or lease_ttl <= 0 then
|
||||||
return reply('invalid', '', 0, 0)
|
return reply('invalid', '', 0, 0)
|
||||||
end
|
end
|
||||||
redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
local created = redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
||||||
|
if created then
|
||||||
|
redis.call('DEL', KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9])
|
||||||
|
end
|
||||||
local generation = redis.call('GET', KEYS[1])
|
local generation = redis.call('GET', KEYS[1])
|
||||||
local current, current_error = read_leader()
|
local current, current_error = read_leader()
|
||||||
if current_error then
|
if current_error then
|
||||||
@ -112,6 +173,13 @@ if operation == 'acquire_fetch' then
|
|||||||
local request_interval = tonumber(ARGV[7])
|
local request_interval = tonumber(ARGV[7])
|
||||||
local max_in_flight = tonumber(ARGV[8])
|
local max_in_flight = tonumber(ARGV[8])
|
||||||
local permit_ttl = tonumber(ARGV[9])
|
local permit_ttl = tonumber(ARGV[9])
|
||||||
|
local expected = tonumber(ARGV[10])
|
||||||
|
local max_total = tonumber(ARGV[11])
|
||||||
|
if not expected or expected <= 0 or expected ~= math.floor(expected) or
|
||||||
|
expected > max_safe_integer or not max_total or max_total < 0 or
|
||||||
|
max_total > max_safe_integer or max_total ~= math.floor(max_total) then
|
||||||
|
return reply('invalid', generation, epoch or 0, 0)
|
||||||
|
end
|
||||||
local current, current_error = read_leader()
|
local current, current_error = read_leader()
|
||||||
if current_error then
|
if current_error then
|
||||||
return reply('unavailable', generation, epoch or 0, 0)
|
return reply('unavailable', generation, epoch or 0, 0)
|
||||||
@ -120,11 +188,27 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('stale', generation, epoch or 0, 0)
|
return reply('stale', generation, epoch or 0, 0)
|
||||||
end
|
end
|
||||||
local now = now_ms()
|
local now = now_ms()
|
||||||
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
|
if cleanup_expired(now) then
|
||||||
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
|
return reply('unavailable', generation, epoch, 0)
|
||||||
if existing then
|
end
|
||||||
|
local existing, existing_error = read_permit(permit_token)
|
||||||
|
if existing_error then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
if existing and existing.state == 'reserved' then
|
||||||
return reply('ok', generation, epoch, 0)
|
return reply('ok', generation, epoch, 0)
|
||||||
end
|
end
|
||||||
|
if existing then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
if max_total > 0 and fetched + pending + expected > max_total then
|
||||||
|
return reply('quota_exhausted', generation, epoch, 0)
|
||||||
|
end
|
||||||
local next_request = redis.call('GET', KEYS[4])
|
local next_request = redis.call('GET', KEYS[4])
|
||||||
if next_request and not tonumber(next_request) then
|
if next_request and not tonumber(next_request) then
|
||||||
return reply('unavailable', generation, epoch, 0)
|
return reply('unavailable', generation, epoch, 0)
|
||||||
@ -141,7 +225,13 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('at_capacity', generation, epoch, wait_ms)
|
return reply('at_capacity', generation, epoch, wait_ms)
|
||||||
end
|
end
|
||||||
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
||||||
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
|
redis.call('ZADD', KEYS[9], now + permit_ttl, permit_token)
|
||||||
|
redis.call('HSET', KEYS[8], permit_token, cjson.encode({
|
||||||
|
version = 1,
|
||||||
|
state = 'reserved',
|
||||||
|
expected = expected
|
||||||
|
}))
|
||||||
|
redis.call('SET', KEYS[7], pending + expected)
|
||||||
if request_interval > 0 then
|
if request_interval > 0 then
|
||||||
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
||||||
else
|
else
|
||||||
@ -150,8 +240,43 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('ok', generation, epoch, 0)
|
return reply('ok', generation, epoch, 0)
|
||||||
end
|
end
|
||||||
|
|
||||||
if operation == 'release_fetch' then
|
if operation == 'complete_fetch' or operation == 'cancel_fetch' then
|
||||||
redis.call('ZREM', KEYS[5], ARGV[2])
|
local permit_token = ARGV[2]
|
||||||
|
local fetched_count = tonumber(ARGV[3])
|
||||||
|
local settlement_ttl = tonumber(ARGV[4])
|
||||||
|
if not fetched_count or fetched_count < 0 or fetched_count ~= math.floor(fetched_count) or
|
||||||
|
fetched_count > max_safe_integer or not settlement_ttl or settlement_ttl <= 0 then
|
||||||
|
return reply('invalid', '', 0, 0)
|
||||||
|
end
|
||||||
|
local now = now_ms()
|
||||||
|
if cleanup_expired(now) then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
local permit, permit_error = read_permit(permit_token)
|
||||||
|
if permit_error then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
if not permit or permit.state ~= 'reserved' then
|
||||||
|
return reply('ok', '', 0, 0)
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error or pending < permit.expected or
|
||||||
|
fetched > max_safe_integer - fetched_count then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
pending = pending - permit.expected
|
||||||
|
if operation == 'complete_fetch' then
|
||||||
|
fetched = fetched + fetched_count
|
||||||
|
permit.state = 'completed'
|
||||||
|
else
|
||||||
|
permit.state = 'cancelled'
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[6], fetched)
|
||||||
|
redis.call('SET', KEYS[7], pending)
|
||||||
|
redis.call('HSET', KEYS[8], permit_token, cjson.encode(permit))
|
||||||
|
redis.call('ZREM', KEYS[5], permit_token)
|
||||||
|
redis.call('ZADD', KEYS[9], now + settlement_ttl, permit_token)
|
||||||
return reply('ok', '', 0, 0)
|
return reply('ok', '', 0, 0)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@ -5,6 +5,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
MaximumPoolSize = 1_000_000
|
||||||
|
MaximumExactCounter = int64(1<<53 - 1)
|
||||||
|
MaximumUpstreams = 4_096
|
||||||
|
)
|
||||||
|
|
||||||
type Duration time.Duration
|
type Duration time.Duration
|
||||||
|
|
||||||
func (d *Duration) UnmarshalText(text []byte) error {
|
func (d *Duration) UnmarshalText(text []byte) error {
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
|||||||
},
|
},
|
||||||
want: "estimatedIPsPerCall",
|
want: "estimatedIPsPerCall",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "pool exceeds runtime scan bound",
|
||||||
|
mutate: func(cfg *Config) {
|
||||||
|
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
|
||||||
|
},
|
||||||
|
want: "pool.maxSize",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "zero refill interval",
|
name: "zero refill interval",
|
||||||
mutate: func(cfg *Config) {
|
mutate: func(cfg *Config) {
|
||||||
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
|
||||||
|
cfg := mustLoadValidConfig(t)
|
||||||
|
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
|
||||||
|
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
|
||||||
|
t.Fatalf("Validate(too many upstreams) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
|
||||||
|
if strconv.IntSize < 64 {
|
||||||
|
t.Skip("64-bit int is required for values above the Redis exact range")
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*Upstream)
|
||||||
|
}{
|
||||||
|
{name: "fetch total", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
|
||||||
|
}},
|
||||||
|
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
|
||||||
|
}},
|
||||||
|
{name: "minimum slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
|
||||||
|
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
|
||||||
|
}},
|
||||||
|
{name: "target slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
|
||||||
|
upstream.Pool.MaxSize = 1
|
||||||
|
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
|
||||||
|
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
|
||||||
|
}},
|
||||||
|
{name: "theoretical slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Pool.MaxSize = 2
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
|
||||||
|
upstream.Refill.MinimumAvailableSlots = 1
|
||||||
|
upstream.Refill.TargetAvailableSlots = 2
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg := mustLoadValidConfig(t)
|
||||||
|
updateUpstream(cfg, test.mutate)
|
||||||
|
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
|
||||||
|
t.Fatalf("Validate(inexact counter) error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
|
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
|
||||||
configured := strings.Replace(validConfig, ` auth:
|
configured := strings.Replace(validConfig, ` auth:
|
||||||
mode: none`, ` auth:
|
mode: none`, ` auth:
|
||||||
|
|||||||
30
internal/config/fingerprint.go
Normal file
30
internal/config/fingerprint.go
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
const MinimumFingerprintKeyBytes = 32
|
||||||
|
|
||||||
|
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
|
||||||
|
|
||||||
|
// Fingerprint returns a keyed digest of one fully resolved configuration.
|
||||||
|
// Only the digest is persisted; the resolved configuration and key remain local.
|
||||||
|
func Fingerprint(configuration *Config, key []byte) (string, error) {
|
||||||
|
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
|
||||||
|
return "", ErrInvalidFingerprint
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(configuration)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||||
|
}
|
||||||
|
digest := hmac.New(sha256.New, key)
|
||||||
|
if _, err := digest.Write(encoded); err != nil {
|
||||||
|
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(digest.Sum(nil)), nil
|
||||||
|
}
|
||||||
68
internal/config/fingerprint_test.go
Normal file
68
internal/config/fingerprint_test.go
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
|
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
|
||||||
|
first := storeTestConfig("provider-a")
|
||||||
|
upstream := first.Upstreams["provider-a"]
|
||||||
|
upstream.ProxyAuth.Password = "secret-a"
|
||||||
|
first.Upstreams["provider-a"] = upstream
|
||||||
|
|
||||||
|
stable, err := Fingerprint(first, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(first): %v", err)
|
||||||
|
}
|
||||||
|
again, err := Fingerprint(first, testFingerprintKey)
|
||||||
|
if err != nil || again != stable {
|
||||||
|
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
|
||||||
|
}
|
||||||
|
|
||||||
|
rotated := storeTestConfig("provider-a")
|
||||||
|
upstream = rotated.Upstreams["provider-a"]
|
||||||
|
upstream.ProxyAuth.Password = "secret-b"
|
||||||
|
rotated.Upstreams["provider-a"] = upstream
|
||||||
|
changed, err := Fingerprint(rotated, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(rotated): %v", err)
|
||||||
|
}
|
||||||
|
if changed == stable {
|
||||||
|
t.Fatal("Fingerprint did not change after secret rotation")
|
||||||
|
}
|
||||||
|
if len(changed) != 64 {
|
||||||
|
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
|
||||||
|
configuration := storeTestConfig("provider-a")
|
||||||
|
first, err := Fingerprint(configuration, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(first key): %v", err)
|
||||||
|
}
|
||||||
|
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(second key): %v", err)
|
||||||
|
}
|
||||||
|
if first == second {
|
||||||
|
t.Fatal("Fingerprint did not change with independent key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
|
||||||
|
for name, test := range map[string]struct {
|
||||||
|
configuration *Config
|
||||||
|
key []byte
|
||||||
|
}{
|
||||||
|
"nil configuration": {key: testFingerprintKey},
|
||||||
|
"missing key": {configuration: storeTestConfig("provider-a")},
|
||||||
|
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := Fingerprint(test.configuration, test.key); err == nil {
|
||||||
|
t.Fatal("Fingerprint() succeeded")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
|
|||||||
|
|
||||||
// Store publishes complete validated configurations with one atomic pointer swap.
|
// Store publishes complete validated configurations with one atomic pointer swap.
|
||||||
type Store struct {
|
type Store struct {
|
||||||
current atomic.Pointer[Config]
|
current atomic.Pointer[publishedConfiguration]
|
||||||
|
}
|
||||||
|
|
||||||
|
type publishedConfiguration struct {
|
||||||
|
value Config
|
||||||
|
revision uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStore(initial *Config) (*Store, error) {
|
func NewStore(initial *Config) (*Store, error) {
|
||||||
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
|
|||||||
return nil, errors.Join(ErrInvalidStore, err)
|
return nil, errors.Join(ErrInvalidStore, err)
|
||||||
}
|
}
|
||||||
store := &Store{}
|
store := &Store{}
|
||||||
store.Publish(initial)
|
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
|
||||||
return store, nil
|
return store, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
|
|||||||
if store == nil {
|
if store == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
current := store.current.Load()
|
published := store.current.Load()
|
||||||
if current == nil {
|
if published == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
cloned := cloneConfig(*current)
|
cloned := cloneConfig(published.value)
|
||||||
return &cloned
|
return &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// Publish accepts a non-nil configuration already validated by the caller.
|
func (store *Store) Revision() uint64 {
|
||||||
func (store *Store) Publish(configuration *Config) {
|
if store == nil {
|
||||||
if store == nil || configuration == nil {
|
return 0
|
||||||
return
|
}
|
||||||
|
published := store.current.Load()
|
||||||
|
if published == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return published.revision
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishRevision publishes only a strictly newer authoritative revision.
|
||||||
|
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
|
||||||
|
if store == nil || configuration == nil || revision == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
current := store.current.Load()
|
||||||
|
if current != nil && revision <= current.revision {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
|
||||||
|
if store.current.CompareAndSwap(current, next) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
cloned := cloneConfig(*configuration)
|
|
||||||
store.current.Store(&cloned)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
next := storeTestConfig("provider-b")
|
next := storeTestConfig("provider-b")
|
||||||
store.Publish(next)
|
if !store.PublishRevision(next, 1) {
|
||||||
|
t.Fatal("PublishRevision() rejected newer configuration")
|
||||||
|
}
|
||||||
next.Routing[0].Upstreams[0] = "mutated"
|
next.Routing[0].Upstreams[0] = "mutated"
|
||||||
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||||
t.Fatalf("Publish() retained caller state: %q", got)
|
t.Fatalf("Publish() retained caller state: %q", got)
|
||||||
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
|||||||
if index%2 == 1 {
|
if index%2 == 1 {
|
||||||
name = "provider-b"
|
name = "provider-b"
|
||||||
}
|
}
|
||||||
store.Publish(storeTestConfig(name))
|
store.PublishRevision(storeTestConfig(name), uint64(index+1))
|
||||||
}(index)
|
}(index)
|
||||||
go func() {
|
go func() {
|
||||||
defer wait.Done()
|
defer wait.Done()
|
||||||
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
|||||||
wait.Wait()
|
wait.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
store, err := NewStore(storeTestConfig("provider-a"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStore() error = %v", err)
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
|
||||||
|
t.Fatal("PublishRevision(newer) rejected")
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
|
||||||
|
t.Fatal("PublishRevision(stale) succeeded")
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
|
||||||
|
t.Fatal("PublishRevision(equal) succeeded")
|
||||||
|
}
|
||||||
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||||
|
t.Fatalf("Current() upstream = %q, want provider-b", got)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 2 {
|
||||||
|
t.Fatalf("Revision() = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func storeTestConfig(upstreamName string) *Config {
|
func storeTestConfig(upstreamName string) *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
Version: 1,
|
Version: 1,
|
||||||
|
|||||||
@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
|
|||||||
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
|
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(cfg.Upstreams) > MaximumUpstreams {
|
||||||
|
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
|
||||||
|
}
|
||||||
enabledUpstreams := 0
|
enabledUpstreams := 0
|
||||||
for name, upstream := range cfg.Upstreams {
|
for name, upstream := range cfg.Upstreams {
|
||||||
if upstream.Enabled {
|
if upstream.Enabled {
|
||||||
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
|
|||||||
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
|
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if upstream.Pool.MaxSize > MaximumPoolSize {
|
||||||
|
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
|
||||||
|
}
|
||||||
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
|
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
||||||
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
|
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
|
||||||
}
|
}
|
||||||
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
||||||
}
|
}
|
||||||
|
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
||||||
}
|
}
|
||||||
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||||
|
if theoreticalSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
||||||
}
|
}
|
||||||
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
|
|||||||
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
|
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(fetch.MaxInFlight) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
|
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,9 +2,6 @@ package admin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
|
|||||||
LoadConfiguration(context.Context) (LoadedConfiguration, error)
|
LoadConfiguration(context.Context) (LoadedConfiguration, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigurationPublisher must atomically publish an already validated configuration.
|
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
|
||||||
type ConfigurationPublisher interface {
|
type ConfigurationPublisher interface {
|
||||||
Publish(*config.Config)
|
PublishRevision(*config.Config, uint64) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type RuntimeController interface {
|
||||||
|
Notify()
|
||||||
|
ValidateConfiguration(context.Context, *config.Config) error
|
||||||
|
ValidateUpstream(context.Context, string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ ConfigurationPublisher = (*config.Store)(nil)
|
var _ ConfigurationPublisher = (*config.Store)(nil)
|
||||||
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
|
|||||||
Operations OperationalStatusReader
|
Operations OperationalStatusReader
|
||||||
Configuration ConfigurationLoader
|
Configuration ConfigurationLoader
|
||||||
Publisher ConfigurationPublisher
|
Publisher ConfigurationPublisher
|
||||||
|
Runtime RuntimeController
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApplicationOptions struct {
|
type ApplicationOptions struct {
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
|
FingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
type OperationalStatus struct {
|
type OperationalStatus struct {
|
||||||
@ -74,14 +79,17 @@ type ApplicationService struct {
|
|||||||
operations OperationalStatusReader
|
operations OperationalStatusReader
|
||||||
configuration ConfigurationLoader
|
configuration ConfigurationLoader
|
||||||
publisher ConfigurationPublisher
|
publisher ConfigurationPublisher
|
||||||
|
runtime RuntimeController
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
|
fingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Service = (*ApplicationService)(nil)
|
var _ Service = (*ApplicationService)(nil)
|
||||||
|
|
||||||
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
|
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
|
||||||
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
|
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
|
||||||
nilInterface(dependencies.Publisher) || options.Now == nil {
|
nilInterface(dependencies.Publisher) || options.Now == nil ||
|
||||||
|
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||||
return nil, ErrInvalidApplicationService
|
return nil, ErrInvalidApplicationService
|
||||||
}
|
}
|
||||||
return &ApplicationService{
|
return &ApplicationService{
|
||||||
@ -89,11 +97,18 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
|
|||||||
operations: dependencies.Operations,
|
operations: dependencies.Operations,
|
||||||
configuration: dependencies.Configuration,
|
configuration: dependencies.Configuration,
|
||||||
publisher: dependencies.Publisher,
|
publisher: dependencies.Publisher,
|
||||||
|
runtime: dependencies.Runtime,
|
||||||
now: options.Now,
|
now: options.Now,
|
||||||
|
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
||||||
|
if command.Enabled && service.runtime != nil {
|
||||||
|
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
|
||||||
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
|
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
|
||||||
RequestID: command.RequestID,
|
RequestID: command.RequestID,
|
||||||
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
|
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
|
||||||
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
|
|||||||
Name: command.Name,
|
Name: command.Name,
|
||||||
Enabled: command.Enabled,
|
Enabled: command.Enabled,
|
||||||
})
|
})
|
||||||
|
if err == nil && service.runtime != nil {
|
||||||
|
service.runtime.Notify()
|
||||||
|
}
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
|
|||||||
if err := config.Validate(loaded.Value); err != nil {
|
if err := config.Validate(loaded.Value); err != nil {
|
||||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
}
|
}
|
||||||
|
if service.runtime != nil {
|
||||||
|
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
|
||||||
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
managementView := loaded.Value.Redacted()
|
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
|
||||||
encoded, err := json.Marshal(managementView)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
}
|
}
|
||||||
digest := sha256.Sum256(encoded)
|
|
||||||
checksum := hex.EncodeToString(digest[:])
|
|
||||||
|
|
||||||
current, err := service.state.Snapshot(ctx)
|
current, err := service.state.Snapshot(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
service.publisher.Publish(loaded.Value)
|
if result.Revision == 0 {
|
||||||
|
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
|
||||||
|
}
|
||||||
|
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
|
||||||
|
if published && service.runtime != nil {
|
||||||
|
service.runtime.Notify()
|
||||||
|
}
|
||||||
return mutationResult(result), nil
|
return mutationResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package admin
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -10,6 +11,12 @@ import (
|
|||||||
"proxy-pool/internal/domain/adminstate"
|
"proxy-pool/internal/domain/adminstate"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
|
func applicationTestOptions(now func() time.Time) ApplicationOptions {
|
||||||
|
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
||||||
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
Message: "enabled",
|
Message: "enabled",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
runtime := &recordingRuntimeNotifier{}
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state,
|
State: state,
|
||||||
Operations: staticOperationalStatusReader{},
|
Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{},
|
Configuration: staticConfigurationLoader{},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
Runtime: runtime,
|
||||||
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
}) {
|
}) {
|
||||||
t.Fatalf("admin state command = %+v", state.lastUpstream)
|
t.Fatalf("admin state command = %+v", state.lastUpstream)
|
||||||
}
|
}
|
||||||
|
if runtime.notifications != 1 {
|
||||||
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
wantErr := errors.New("invalid Provider template")
|
||||||
|
state := &recordingAdminState{}
|
||||||
|
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
|
Runtime: runtime,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
|
||||||
|
RequestID: "req-enable", Name: "provider-a", Enabled: true,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("SetUpstreamEnabled() error = %v", err)
|
||||||
|
}
|
||||||
|
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
|
||||||
|
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
||||||
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
|||||||
dependencies ApplicationDependencies
|
dependencies ApplicationDependencies
|
||||||
options ApplicationOptions
|
options ApplicationOptions
|
||||||
}{
|
}{
|
||||||
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "clock", dependencies: valid},
|
{name: "clock", dependencies: valid},
|
||||||
|
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
test := test
|
test := test
|
||||||
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
|
|||||||
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
|
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
|
||||||
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
|
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
|
||||||
} {
|
} {
|
||||||
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
|
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
|
||||||
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
|
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||||
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
||||||
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
|
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
|
||||||
|
|
||||||
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
||||||
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
||||||
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
|
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
|
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err = NewApplicationService(ApplicationDependencies{
|
service, err = NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err = NewApplicationService(ApplicationDependencies{
|
service, err = NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||||
configuration := validReloadConfiguration()
|
configuration := validReloadConfiguration()
|
||||||
publisher := &recordingConfigurationPublisher{}
|
publisher := &recordingConfigurationPublisher{}
|
||||||
|
runtime := &recordingRuntimeNotifier{}
|
||||||
state := &recordingAdminState{
|
state := &recordingAdminState{
|
||||||
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
||||||
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
||||||
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: configuration, Source: "configs/proxy-pool.yaml",
|
Value: configuration, Source: "configs/proxy-pool.yaml",
|
||||||
}},
|
}},
|
||||||
Publisher: publisher,
|
Publisher: publisher, Runtime: runtime,
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
||||||
t.Fatalf("published configurations = %+v", publisher.published)
|
t.Fatalf("published configurations = %+v", publisher.published)
|
||||||
}
|
}
|
||||||
|
if runtime.notifications != 1 {
|
||||||
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
|
}
|
||||||
command := state.lastConfig
|
command := state.lastConfig
|
||||||
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
||||||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
||||||
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
|
|||||||
Operations: staticOperationalStatusReader{},
|
Operations: staticOperationalStatusReader{},
|
||||||
Configuration: forbiddenConfigurationLoader{},
|
Configuration: forbiddenConfigurationLoader{},
|
||||||
Publisher: publisher,
|
Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state, Operations: staticOperationalStatusReader{},
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
|
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: validReloadConfiguration(), Source: "config.yaml",
|
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||||
}}, Publisher: publisher,
|
}}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
|
||||||
|
publisher := &recordingConfigurationPublisher{}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
|
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||||
|
}}, Publisher: publisher,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
|
||||||
|
if !errors.Is(err, ErrUnavailable) {
|
||||||
|
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
|
||||||
|
}
|
||||||
|
if len(publisher.published) != 0 {
|
||||||
|
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
|
||||||
|
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &orderedCommitState{
|
||||||
|
firstCommitted: make(chan struct{}),
|
||||||
|
releaseFirst: make(chan struct{}),
|
||||||
|
}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{}, Publisher: store,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
|
||||||
|
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
|
||||||
|
})
|
||||||
|
firstDone <- applyErr
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-state.firstCommitted:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("first commit did not reach delayed return")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
|
||||||
|
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("ApplyConfiguration(new): %v", err)
|
||||||
|
}
|
||||||
|
close(state.releaseFirst)
|
||||||
|
if err := <-firstDone; err != nil {
|
||||||
|
t.Fatalf("ApplyConfiguration(old): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
|
||||||
|
t.Fatalf("published upstream = %q, want provider-c", got)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 2 {
|
||||||
|
t.Fatalf("published revision = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{err: context.Canceled},
|
Configuration: staticConfigurationLoader{err: context.Canceled},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
state := &recordingAdminState{}
|
state := &recordingAdminState{}
|
||||||
publisher := &recordingConfigurationPublisher{}
|
publisher := &recordingConfigurationPublisher{}
|
||||||
var commands []adminstate.CommitConfigCommand
|
var commands []adminstate.CommitConfigCommand
|
||||||
state.onCommit = func(command adminstate.CommitConfigCommand) {
|
state.onCommit = func(command adminstate.CommitConfigCommand) {
|
||||||
commands = append(commands, command)
|
commands = append(commands, command)
|
||||||
|
state.mutation = adminstate.MutationResult{
|
||||||
|
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, secret := range []string{"secret-a", "secret-b"} {
|
for _, secret := range []string{"secret-a", "secret-b"} {
|
||||||
configuration := validReloadConfiguration()
|
configuration := validReloadConfiguration()
|
||||||
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: configuration, Source: "config.yaml",
|
Value: configuration, Source: "config.yaml",
|
||||||
}}, Publisher: publisher,
|
}}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
|||||||
t.Fatalf("ReloadConfiguration() error = %v", err)
|
t.Fatalf("ReloadConfiguration() error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
|
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
|
||||||
t.Fatalf("secret rotation changed public management digest: %+v", commands)
|
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
|
||||||
}
|
}
|
||||||
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
|
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
|
||||||
t.Fatalf("secret rotation was not published: %+v", publisher.published)
|
t.Fatalf("secret rotation was not published: %+v", publisher.published)
|
||||||
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func configWithOnlyUpstream(name string) *config.Config {
|
||||||
|
configuration := validReloadConfiguration()
|
||||||
|
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
|
||||||
|
configuration.Routing = []config.Routing{{
|
||||||
|
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
|
||||||
|
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
|
||||||
|
}}
|
||||||
|
return configuration
|
||||||
|
}
|
||||||
|
|
||||||
func validReloadUpstream(secret string) config.Upstream {
|
func validReloadUpstream(secret string) config.Upstream {
|
||||||
return config.Upstream{
|
return config.Upstream{
|
||||||
Enabled: true, Exposure: []string{"gateway"},
|
Enabled: true, Exposure: []string{"gateway"},
|
||||||
@ -505,6 +629,33 @@ type recordingAdminState struct {
|
|||||||
onCommit func(adminstate.CommitConfigCommand)
|
onCommit func(adminstate.CommitConfigCommand)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type orderedCommitState struct {
|
||||||
|
next atomic.Uint64
|
||||||
|
firstCommitted chan struct{}
|
||||||
|
releaseFirst chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||||
|
return adminstate.MutationResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
|
||||||
|
return adminstate.MutationResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||||
|
revision := state.next.Add(1)
|
||||||
|
if revision == 1 {
|
||||||
|
close(state.firstCommitted)
|
||||||
|
<-state.releaseFirst
|
||||||
|
}
|
||||||
|
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||||
|
return adminstate.Snapshot{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||||
state.lastUpstream = command
|
state.lastUpstream = command
|
||||||
return state.mutation, state.err
|
return state.mutation, state.err
|
||||||
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
|
|||||||
|
|
||||||
type recordingConfigurationPublisher struct {
|
type recordingConfigurationPublisher struct {
|
||||||
published []*config.Config
|
published []*config.Config
|
||||||
|
revisions []uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
|
type recordingRuntimeNotifier struct {
|
||||||
publisher.published = append(publisher.published, configuration)
|
notifications int
|
||||||
|
validationErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
||||||
|
return notifier.validationErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
|
||||||
|
return notifier.validationErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
|
||||||
|
publisher.published = append(publisher.published, configuration)
|
||||||
|
publisher.revisions = append(publisher.revisions, revision)
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -15,11 +16,15 @@ import (
|
|||||||
"proxy-pool/internal/controller/distribution"
|
"proxy-pool/internal/controller/distribution"
|
||||||
"proxy-pool/internal/controller/extraction"
|
"proxy-pool/internal/controller/extraction"
|
||||||
"proxy-pool/internal/controller/operations"
|
"proxy-pool/internal/controller/operations"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
"proxy-pool/internal/platform/admission"
|
"proxy-pool/internal/platform/admission"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
"proxy-pool/internal/platform/httpserver"
|
"proxy-pool/internal/platform/httpserver"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -33,10 +38,15 @@ type Options struct {
|
|||||||
Resolver config.Resolver
|
Resolver config.Resolver
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
HTTP httpserver.Options
|
HTTP httpserver.Options
|
||||||
|
HolderID string
|
||||||
|
RedisNamespace string
|
||||||
|
FingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
type activityStore interface {
|
type activityStore interface {
|
||||||
extractionDomain.Store
|
extractionDomain.Store
|
||||||
|
activitypool.Upserter
|
||||||
|
pool.InventoryReader
|
||||||
activitypool.StateInventoryReader
|
activitypool.StateInventoryReader
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -45,6 +55,9 @@ type ports struct {
|
|||||||
activity activityStore
|
activity activityStore
|
||||||
readiness distribution.ReadinessChecker
|
readiness distribution.ReadinessChecker
|
||||||
metricsReadiness platformMetrics.ReadinessChecker
|
metricsReadiness platformMetrics.ReadinessChecker
|
||||||
|
coordinator provider.Coordinator
|
||||||
|
credentials credentials.Store
|
||||||
|
providerResults provider.ResultRecorder
|
||||||
close func() error
|
close func() error
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -61,7 +74,9 @@ type runtimeFactory interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Run(ctx context.Context, options Options) error {
|
func Run(ctx context.Context, options Options) error {
|
||||||
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
|
return run(ctx, options, &productionInfrastructure{
|
||||||
|
holderID: options.HolderID, namespace: options.RedisNamespace,
|
||||||
|
}, productionRuntimeFactory{})
|
||||||
}
|
}
|
||||||
|
|
||||||
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
||||||
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
|
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||||
|
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
|
||||||
|
}
|
||||||
configurationStore, err := config.NewStore(loaded.Value)
|
configurationStore, err := config.NewStore(loaded.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
||||||
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
defer func() {
|
defer func() {
|
||||||
resultErr = errors.Join(resultErr, opened.close())
|
resultErr = errors.Join(resultErr, opened.close())
|
||||||
}()
|
}()
|
||||||
|
var providerState providerStateReader
|
||||||
|
if loaded.Value.Admin.Enabled {
|
||||||
|
providerState = opened.state
|
||||||
|
}
|
||||||
|
supervisor, err := newProviderSupervisor(
|
||||||
|
configurationStore,
|
||||||
|
providerState,
|
||||||
|
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
buildRuntime, err := providerRuntimeBuilder(opened)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return buildRuntime(name, upstream)
|
||||||
|
},
|
||||||
|
func(ctx context.Context, configuration *config.Config) error {
|
||||||
|
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
|
||||||
|
},
|
||||||
|
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
|
||||||
|
loader,
|
||||||
|
options.FingerprintKey,
|
||||||
|
providerSupervisorInterval,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
|
|
||||||
dependencies := controllerRuntime.Dependencies{}
|
dependencies := controllerRuntime.Dependencies{}
|
||||||
if loaded.Value.Distribution.Enabled {
|
if loaded.Value.Distribution.Enabled {
|
||||||
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
}
|
}
|
||||||
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
|
var providerStats []provider.StatsReader
|
||||||
|
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
||||||
|
providerStats = append(providerStats, stats)
|
||||||
|
}
|
||||||
|
statusReader, statusErr := operations.NewReader(
|
||||||
|
configurationStore,
|
||||||
|
opened.activity,
|
||||||
|
options.Now,
|
||||||
|
providerStats...,
|
||||||
|
)
|
||||||
if statusErr != nil {
|
if statusErr != nil {
|
||||||
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
||||||
}
|
}
|
||||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
||||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
||||||
}, admin.ApplicationOptions{Now: options.Now})
|
Runtime: supervisor,
|
||||||
|
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
||||||
if serviceErr != nil {
|
if serviceErr != nil {
|
||||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
||||||
}
|
}
|
||||||
@ -146,6 +199,8 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
dependencies.MetricsHandler = handler
|
dependencies.MetricsHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runners := make([]lifecycle.Runner, 0, 2)
|
||||||
|
if hasHTTPRuntime(loaded.Value) {
|
||||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
||||||
@ -153,7 +208,59 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if nilInterface(runner) {
|
if nilInterface(runner) {
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
}
|
}
|
||||||
return runner.Run(ctx)
|
runners = append(runners, runner)
|
||||||
|
}
|
||||||
|
runners = append(runners, supervisor)
|
||||||
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
|
return group.Run(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareProviderConfiguration(
|
||||||
|
ctx context.Context,
|
||||||
|
configuration *config.Config,
|
||||||
|
credentialStore credentials.Store,
|
||||||
|
) error {
|
||||||
|
if ctx == nil || configuration == nil {
|
||||||
|
return ErrProviderRuntime
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
|
||||||
|
if !ok || nilInterface(credentialStore) {
|
||||||
|
return ErrProviderRuntime
|
||||||
|
}
|
||||||
|
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
|
||||||
|
if configuration == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
retainer, ok := results.(provider.StatsRetainer)
|
||||||
|
if !ok || nilInterface(retainer) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name := range configuration.Upstreams {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
retainer.RetainProviderStats(names)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasHTTPRuntime(configuration *config.Config) bool {
|
||||||
|
return configuration != nil &&
|
||||||
|
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
||||||
|
|||||||
@ -4,16 +4,25 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"proxy-pool/internal/adapters/redisactivity"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controller/admin"
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
||||||
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
|||||||
if postgresURL == "" || redisURL == "" {
|
if postgresURL == "" || redisURL == "" {
|
||||||
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
||||||
}
|
}
|
||||||
|
var providerCalls atomic.Int64
|
||||||
|
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||||
|
providerCalls.Add(1)
|
||||||
|
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
|
||||||
|
}))
|
||||||
|
defer providerServer.Close()
|
||||||
|
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||||
|
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
|
||||||
|
|
||||||
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
||||||
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
||||||
|
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
||||||
factory := &integrationRuntimeFactory{}
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
providerResults := make(chan provider.Result, 8)
|
||||||
|
factory := &integrationRuntimeFactory{
|
||||||
|
cancel: cancel, inventory: inventory, providerResults: providerResults,
|
||||||
|
}
|
||||||
|
infrastructure := &integrationInfrastructure{
|
||||||
|
productionInfrastructure: productionInfrastructure{namespace: namespace},
|
||||||
|
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
|
||||||
|
}
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
err := run(ctx, Options{
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
||||||
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
||||||
},
|
},
|
||||||
}, &productionInfrastructure{}, factory)
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
if err != nil {
|
}, infrastructure, factory)
|
||||||
t.Fatalf("run() error = %v", err)
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("run() error = %v, want context cancellation", err)
|
||||||
}
|
}
|
||||||
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
||||||
t.Fatalf("Admin Status = %+v", factory.status)
|
t.Fatalf("Admin Status = %+v", factory.status)
|
||||||
@ -46,6 +75,9 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
|||||||
!strings.Contains(factory.metricsBody, "go_") {
|
!strings.Contains(factory.metricsBody, "go_") {
|
||||||
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
||||||
}
|
}
|
||||||
|
if providerCalls.Load() == 0 {
|
||||||
|
t.Fatal("production Provider HTTP adapter was not called")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type integrationRuntimeFactory struct {
|
type integrationRuntimeFactory struct {
|
||||||
@ -53,6 +85,9 @@ type integrationRuntimeFactory struct {
|
|||||||
readyStatus int
|
readyStatus int
|
||||||
metricsStatus int
|
metricsStatus int
|
||||||
metricsBody string
|
metricsBody string
|
||||||
|
cancel context.CancelFunc
|
||||||
|
inventory pool.InventoryReader
|
||||||
|
providerResults <-chan provider.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (factory *integrationRuntimeFactory) New(
|
func (factory *integrationRuntimeFactory) New(
|
||||||
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
|
|||||||
factory.metricsBody = metrics.Body.String()
|
factory.metricsBody = metrics.Body.String()
|
||||||
status, err := dependencies.AdminService.Status(ctx)
|
status, err := dependencies.AdminService.Status(ctx)
|
||||||
factory.status = status
|
factory.status = status
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
|
if err := waitForProviderInventory(
|
||||||
|
ctx,
|
||||||
|
factory.inventory,
|
||||||
|
factory.providerResults,
|
||||||
|
[]string{"provider-a", "provider-b"},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
factory.cancel()
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
}}, nil
|
}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
|
||||||
|
t.Helper()
|
||||||
|
options, err := redis.ParseURL(redisURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("redis.ParseURL(): %v", err)
|
||||||
|
}
|
||||||
|
client := redis.NewClient(options)
|
||||||
|
t.Cleanup(func() { _ = client.Close() })
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("credentials.NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
reader, err := redisactivity.New(client, redisactivity.Options{
|
||||||
|
Namespace: namespace, Credentials: credentialStore,
|
||||||
|
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
|
||||||
|
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("redisactivity.New(): %v", err)
|
||||||
|
}
|
||||||
|
return reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForProviderInventory(
|
||||||
|
ctx context.Context,
|
||||||
|
inventory pool.InventoryReader,
|
||||||
|
results <-chan provider.Result,
|
||||||
|
upstreamIDs []string,
|
||||||
|
) error {
|
||||||
|
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
ticker := time.NewTicker(10 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
var latestResult provider.Result
|
||||||
|
for {
|
||||||
|
var inventoryErr error
|
||||||
|
for _, upstreamID := range upstreamIDs {
|
||||||
|
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
|
||||||
|
inventoryErr = errors.Join(inventoryErr, err)
|
||||||
|
if err == nil && snapshot.Managed > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline.Done():
|
||||||
|
return errors.Join(
|
||||||
|
errors.New("wait for Provider Redis inventory"),
|
||||||
|
deadline.Err(),
|
||||||
|
inventoryErr,
|
||||||
|
latestResult.Err,
|
||||||
|
)
|
||||||
|
case result := <-results:
|
||||||
|
latestResult = result
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type integrationInfrastructure struct {
|
||||||
|
productionInfrastructure
|
||||||
|
results provider.ResultRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (infrastructure *integrationInfrastructure) Open(
|
||||||
|
ctx context.Context,
|
||||||
|
configuration *config.Config,
|
||||||
|
) (ports, error) {
|
||||||
|
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
|
||||||
|
opened.providerResults = infrastructure.results
|
||||||
|
return opened, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerResultRecorder func(provider.Result)
|
||||||
|
|
||||||
|
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
|
||||||
|
|
||||||
type integrationRunner struct {
|
type integrationRunner struct {
|
||||||
run func(context.Context) error
|
run func(context.Context) error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,32 +3,48 @@ package bootstrap
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
"proxy-pool/internal/domain/adminstate"
|
"proxy-pool/internal/domain/adminstate"
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
||||||
state := adminstate.NewMemoryStore()
|
state := adminstate.NewMemoryStore()
|
||||||
activity := &stubActivityStore{}
|
activity := &stubActivityStore{}
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
closeErr := errors.New("close failed")
|
closeErr := errors.New("close failed")
|
||||||
infrastructure := &stubInfrastructure{ports: ports{
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||||
|
coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
close: func() error { return closeErr },
|
close: func() error { return closeErr },
|
||||||
}}
|
}}
|
||||||
runErr := errors.New("runtime failed")
|
runErr := errors.New("runtime failed")
|
||||||
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
||||||
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
err = run(ctx, Options{
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
||||||
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
}, infrastructure, factory)
|
}, infrastructure, factory)
|
||||||
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
||||||
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
||||||
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
|
||||||
|
for name, key := range map[string][]byte{
|
||||||
|
"missing": nil,
|
||||||
|
"short": []byte("too-short"),
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
infrastructure := &stubInfrastructure{}
|
||||||
|
err := run(context.Background(), Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||||
|
Now: time.Now,
|
||||||
|
FingerprintKey: key,
|
||||||
|
}, infrastructure, &recordingRuntimeFactory{})
|
||||||
|
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
|
||||||
|
t.Fatalf("run() error = %v", err)
|
||||||
|
}
|
||||||
|
if infrastructure.opens != 0 {
|
||||||
|
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
|
||||||
|
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
|
||||||
|
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
|
||||||
|
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
|
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
|
close: func() error { return nil },
|
||||||
|
}}
|
||||||
|
factory := &recordingRuntimeFactory{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
time.AfterFunc(20*time.Millisecond, cancel)
|
||||||
|
err = run(ctx, Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
|
||||||
|
Now: time.Now,
|
||||||
|
}, infrastructure, factory)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if factory.configuration != nil {
|
||||||
|
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
|
||||||
|
state := adminstate.NewMemoryStore()
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
started := make(chan string, 2)
|
||||||
|
stopped := make(chan string, 2)
|
||||||
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
|
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||||
|
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
|
||||||
|
started <- upstreamID
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- upstreamID
|
||||||
|
return ctx.Err()
|
||||||
|
}),
|
||||||
|
credentials: credentialStore,
|
||||||
|
close: func() error { return nil },
|
||||||
|
}}
|
||||||
|
wantErr := errors.New("test HTTP runtime stopped")
|
||||||
|
factory := runtimeFactoryFunc(func(
|
||||||
|
_ *config.Config,
|
||||||
|
dependencies controllerRuntime.Dependencies,
|
||||||
|
_ controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error) {
|
||||||
|
return runnerFunc(func(ctx context.Context) error {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case upstreamID := <-started:
|
||||||
|
if upstreamID != "provider-a" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
|
||||||
|
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case stoppedID := <-stopped:
|
||||||
|
if stoppedID == "provider-a" {
|
||||||
|
return wantErr
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}), nil
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
err = run(ctx, Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||||
|
Now: time.Now,
|
||||||
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
|
}, infrastructure, factory)
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
disabled := configuration.Upstreams["provider-b"]
|
||||||
|
disabled.Enabled = false
|
||||||
|
configuration.Upstreams["provider-b"] = disabled
|
||||||
|
stats, err := provider.NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("provider.NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
|
||||||
|
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
retainProviderStats(configuration, stats)
|
||||||
|
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
|
||||||
|
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
|
||||||
|
t.Fatalf("Provider stats after retention = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type memoryResolver struct {
|
type memoryResolver struct {
|
||||||
files map[string][]byte
|
files map[string][]byte
|
||||||
reads int
|
reads int
|
||||||
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
|
|||||||
|
|
||||||
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
||||||
|
|
||||||
|
type runnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||||
|
|
||||||
|
type runtimeFactoryFunc func(
|
||||||
|
*config.Config,
|
||||||
|
controllerRuntime.Dependencies,
|
||||||
|
controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error)
|
||||||
|
|
||||||
|
func (factory runtimeFactoryFunc) New(
|
||||||
|
configuration *config.Config,
|
||||||
|
dependencies controllerRuntime.Dependencies,
|
||||||
|
options controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error) {
|
||||||
|
return factory(configuration, dependencies, options)
|
||||||
|
}
|
||||||
|
|
||||||
type readyStub struct{}
|
type readyStub struct{}
|
||||||
|
|
||||||
func (readyStub) Ready(context.Context) error { return nil }
|
func (readyStub) Ready(context.Context) error { return nil }
|
||||||
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*stubActivityStore) UpsertFetched(
|
||||||
|
_ context.Context,
|
||||||
|
_ string,
|
||||||
|
batch activitypool.FetchedBatch,
|
||||||
|
) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*stubActivityStore) ReadInventory(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
time.Duration,
|
||||||
|
) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorStub struct{}
|
||||||
|
|
||||||
|
func (coordinatorStub) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
_ provider.CoordinationLimits,
|
||||||
|
_ func(context.Context, provider.LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorFunc func(context.Context, string) error
|
||||||
|
|
||||||
|
func (run coordinatorFunc) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
_ provider.CoordinationLimits,
|
||||||
|
_ func(context.Context, provider.LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return run(ctx, upstreamID)
|
||||||
|
}
|
||||||
|
|
||||||
const bootstrapTestConfig = `
|
const bootstrapTestConfig = `
|
||||||
version: 1
|
version: 1
|
||||||
security:
|
security:
|
||||||
|
|||||||
@ -2,6 +2,8 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@ -11,7 +13,9 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/adapters/postgresadmin"
|
"proxy-pool/internal/adapters/postgresadmin"
|
||||||
"proxy-pool/internal/adapters/redisactivity"
|
"proxy-pool/internal/adapters/redisactivity"
|
||||||
|
"proxy-pool/internal/adapters/redisprovider"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
controllerProvider "proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/platform/credentials"
|
"proxy-pool/internal/platform/credentials"
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||||
)
|
)
|
||||||
@ -20,7 +24,12 @@ const (
|
|||||||
redisNamespace = "controller"
|
redisNamespace = "controller"
|
||||||
redisOperationTTL = 30 * time.Second
|
redisOperationTTL = 30 * time.Second
|
||||||
redisMinimumScan = 4_096
|
redisMinimumScan = 4_096
|
||||||
|
redisMaximumScan = config.MaximumPoolSize
|
||||||
redisCleanupLimit = 1_024
|
redisCleanupLimit = 1_024
|
||||||
|
providerLeaseTTL = 15 * time.Second
|
||||||
|
providerRenewEvery = 3 * time.Second
|
||||||
|
providerRetryInterval = 100 * time.Millisecond
|
||||||
|
providerPermitGrace = 5 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -30,15 +39,22 @@ var (
|
|||||||
ErrRedisUnavailable = errors.New("Redis unavailable")
|
ErrRedisUnavailable = errors.New("Redis unavailable")
|
||||||
)
|
)
|
||||||
|
|
||||||
type productionInfrastructure struct{}
|
type productionInfrastructure struct {
|
||||||
|
holderID string
|
||||||
|
namespace string
|
||||||
|
}
|
||||||
|
|
||||||
func (*productionInfrastructure) Open(
|
func (infrastructure *productionInfrastructure) Open(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
configuration *config.Config,
|
configuration *config.Config,
|
||||||
) (_ ports, resultErr error) {
|
) (_ ports, resultErr error) {
|
||||||
if ctx == nil || configuration == nil {
|
if ctx == nil || configuration == nil {
|
||||||
return ports{}, ErrInvalidOptions
|
return ports{}, ErrInvalidOptions
|
||||||
}
|
}
|
||||||
|
namespace, err := resolveRedisNamespace(infrastructure.namespace)
|
||||||
|
if err != nil {
|
||||||
|
return ports{}, err
|
||||||
|
}
|
||||||
var postgresPool *pgxpool.Pool
|
var postgresPool *pgxpool.Pool
|
||||||
var redisClient *redis.Client
|
var redisClient *redis.Client
|
||||||
closeResources := func() error {
|
closeResources := func() error {
|
||||||
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
|
providersEnabled := hasEnabledUpstream(configuration)
|
||||||
|
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
|
||||||
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
||||||
return ports{}, ErrRedisConfiguration
|
return ports{}, ErrRedisConfiguration
|
||||||
}
|
}
|
||||||
@ -95,15 +112,17 @@ func (*productionInfrastructure) Open(
|
|||||||
if err = redisClient.Ping(ctx).Err(); err != nil {
|
if err = redisClient.Ping(ctx).Err(); err != nil {
|
||||||
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
||||||
}
|
}
|
||||||
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
|
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ports{}, err
|
return ports{}, err
|
||||||
}
|
}
|
||||||
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
||||||
Namespace: redisNamespace,
|
Namespace: namespace,
|
||||||
Credentials: credentialStore,
|
Credentials: credentialStore,
|
||||||
OperationTTL: redisOperationTTL,
|
OperationTTL: redisOperationTTL,
|
||||||
MaxCandidateScan: candidateScan(configuration),
|
MaxCandidateScan: candidateScan(configuration),
|
||||||
|
MaxRuntimeCounters: credentialCapacity(configuration),
|
||||||
|
MaxInventoryScan: maxInventoryScan(configuration),
|
||||||
CleanupLimit: redisCleanupLimit,
|
CleanupLimit: redisCleanupLimit,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -111,6 +130,26 @@ func (*productionInfrastructure) Open(
|
|||||||
}
|
}
|
||||||
opened.activity = adapter
|
opened.activity = adapter
|
||||||
opened.readiness = redisReadiness{client: redisClient}
|
opened.readiness = redisReadiness{client: redisClient}
|
||||||
|
opened.credentials = credentialStore
|
||||||
|
if providersEnabled {
|
||||||
|
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
|
||||||
|
if statsErr != nil {
|
||||||
|
return ports{}, statsErr
|
||||||
|
}
|
||||||
|
opened.providerResults = stats
|
||||||
|
holderID, holderErr := resolveHolderID(infrastructure.holderID)
|
||||||
|
if holderErr != nil {
|
||||||
|
return ports{}, holderErr
|
||||||
|
}
|
||||||
|
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
|
||||||
|
Namespace: namespace, HolderID: holderID,
|
||||||
|
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
|
||||||
|
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ports{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if configuration.Metrics.Enabled {
|
if configuration.Metrics.Enabled {
|
||||||
opened.metricsReadiness = selectMetricsReadiness(
|
opened.metricsReadiness = selectMetricsReadiness(
|
||||||
@ -122,11 +161,21 @@ func (*productionInfrastructure) Open(
|
|||||||
return opened, nil
|
return opened, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveRedisNamespace(configured string) (string, error) {
|
||||||
|
if strings.TrimSpace(configured) != configured {
|
||||||
|
return "", ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if configured == "" {
|
||||||
|
return redisNamespace, nil
|
||||||
|
}
|
||||||
|
return configured, nil
|
||||||
|
}
|
||||||
|
|
||||||
func selectMetricsReadiness(
|
func selectMetricsReadiness(
|
||||||
configuration *config.Config,
|
configuration *config.Config,
|
||||||
admin, activity platformMetrics.ReadinessChecker,
|
admin, activity platformMetrics.ReadinessChecker,
|
||||||
) platformMetrics.ReadinessChecker {
|
) platformMetrics.ReadinessChecker {
|
||||||
if configuration.Distribution.Enabled {
|
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
|
||||||
return activity
|
return activity
|
||||||
}
|
}
|
||||||
if configuration.Admin.Enabled {
|
if configuration.Admin.Enabled {
|
||||||
@ -184,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
|
|||||||
capacity := 0
|
capacity := 0
|
||||||
maximum := int(^uint(0) >> 1)
|
maximum := int(^uint(0) >> 1)
|
||||||
for _, upstream := range configuration.Upstreams {
|
for _, upstream := range configuration.Upstreams {
|
||||||
if upstream.Pool.MaxSize <= 0 {
|
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if capacity > maximum-upstream.Pool.MaxSize {
|
if capacity > maximum-upstream.Pool.MaxSize {
|
||||||
@ -198,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
|
|||||||
return capacity
|
return capacity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func providerCredentialCapacity(configuration *config.Config) int {
|
||||||
|
capacity := 0
|
||||||
|
maximum := int(^uint(0) >> 1)
|
||||||
|
for _, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Pool.MaxSize <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
maxInFlight := upstream.Fetch.MaxInFlight
|
||||||
|
if maxInFlight <= 0 {
|
||||||
|
maxInFlight = 1
|
||||||
|
}
|
||||||
|
if upstream.Pool.MaxSize > maximum/maxInFlight {
|
||||||
|
return maximum
|
||||||
|
}
|
||||||
|
leases := upstream.Pool.MaxSize * maxInFlight
|
||||||
|
if capacity > maximum-leases {
|
||||||
|
return maximum
|
||||||
|
}
|
||||||
|
capacity += leases
|
||||||
|
}
|
||||||
|
if capacity == 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return capacity
|
||||||
|
}
|
||||||
|
|
||||||
|
func maxInventoryScan(_ *config.Config) int {
|
||||||
|
return config.MaximumPoolSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasEnabledUpstream(configuration *config.Config) bool {
|
||||||
|
if configuration == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveHolderID(configured string) (string, error) {
|
||||||
|
if strings.TrimSpace(configured) != configured {
|
||||||
|
return "", ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if configured != "" {
|
||||||
|
return configured, nil
|
||||||
|
}
|
||||||
|
var entropy [16]byte
|
||||||
|
if _, err := rand.Read(entropy[:]); err != nil {
|
||||||
|
return "", errors.Join(ErrStartup, err)
|
||||||
|
}
|
||||||
|
return "controller-" + hex.EncodeToString(entropy[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
func candidateScan(configuration *config.Config) int {
|
func candidateScan(configuration *config.Config) int {
|
||||||
configured := configuration.Distribution.Extraction
|
configured := configuration.Distribution.Extraction
|
||||||
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
||||||
|
|||||||
@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
|||||||
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
||||||
}},
|
}},
|
||||||
Upstreams: map[string]config.Upstream{
|
Upstreams: map[string]config.Upstream{
|
||||||
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
|
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
|
||||||
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
|
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
|
||||||
|
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if got := credentialCapacity(configuration); got != 5_000 {
|
if got := credentialCapacity(configuration); got != 5_000 {
|
||||||
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
||||||
}
|
}
|
||||||
|
if got := providerCredentialCapacity(configuration); got != 158_000 {
|
||||||
|
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
|
||||||
|
}
|
||||||
if got := candidateScan(configuration); got != 5_100 {
|
if got := candidateScan(configuration); got != 5_100 {
|
||||||
t.Fatalf("candidateScan() = %d, want 5100", got)
|
t.Fatalf("candidateScan() = %d, want 5100", got)
|
||||||
}
|
}
|
||||||
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
|||||||
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
||||||
|
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
|
||||||
|
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
126
internal/controller/bootstrap/provider.go
Normal file
126
internal/controller/bootstrap/provider.go
Normal file
@ -0,0 +1,126 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/adapters/providerapi"
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
controllerProvider "proxy-pool/internal/controller/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
|
||||||
|
|
||||||
|
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
|
||||||
|
if configuration == nil {
|
||||||
|
return nil, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
builder, err := providerRuntimeBuilder(opened)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
runtime, err := builder(name, configuration.Upstreams[name])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runtimes = append(runtimes, runtime)
|
||||||
|
}
|
||||||
|
return controllerProvider.NewFleet(runtimes...)
|
||||||
|
}
|
||||||
|
|
||||||
|
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
|
||||||
|
|
||||||
|
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
|
||||||
|
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
|
||||||
|
return nil, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
results := opened.providerResults
|
||||||
|
if nilInterface(results) {
|
||||||
|
results = discardProviderResults{}
|
||||||
|
}
|
||||||
|
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
|
||||||
|
upstream.Enabled = true
|
||||||
|
mapped, err := providerRuntimeConfig(name, upstream)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: opened.coordinator,
|
||||||
|
Inventory: opened.activity,
|
||||||
|
Adapter: adapter,
|
||||||
|
Parser: parser,
|
||||||
|
Activity: opened.activity,
|
||||||
|
Results: results,
|
||||||
|
})
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func providerRuntimeConfig(
|
||||||
|
upstreamID string,
|
||||||
|
upstream config.Upstream,
|
||||||
|
) (controllerProvider.UpstreamRuntimeConfig, error) {
|
||||||
|
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
|
||||||
|
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
|
||||||
|
upstream.Pool.MaxSize > config.MaximumPoolSize ||
|
||||||
|
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||||
|
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||||
|
return controllerProvider.UpstreamRuntimeConfig{
|
||||||
|
Provider: controllerProvider.Config{
|
||||||
|
UpstreamID: upstreamID,
|
||||||
|
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
|
||||||
|
Timeout: time.Duration(upstream.Fetch.Timeout),
|
||||||
|
MaxAttempts: upstream.Fetch.MaxAttempts,
|
||||||
|
MaxInFlight: upstream.Fetch.MaxInFlight,
|
||||||
|
MaxTotal: int64(upstream.Fetch.MaxTotal),
|
||||||
|
MaxSize: upstream.Pool.MaxSize,
|
||||||
|
TTL: time.Duration(upstream.Lifecycle.TTL),
|
||||||
|
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||||
|
Retry: controllerProvider.RetryConfig{
|
||||||
|
Initial: time.Duration(upstream.Fetch.Retry.Initial),
|
||||||
|
Max: time.Duration(upstream.Fetch.Retry.Max),
|
||||||
|
Jitter: upstream.Fetch.Retry.Jitter,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ReconcilePolicy: pool.ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
|
||||||
|
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
|
||||||
|
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
|
||||||
|
ExpectedSlotsPerFetch: expectedSlots,
|
||||||
|
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||||
|
},
|
||||||
|
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type discardProviderResults struct{}
|
||||||
|
|
||||||
|
func (discardProviderResults) Record(controllerProvider.Result) {}
|
||||||
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
@ -0,0 +1,375 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
const providerSupervisorInterval = time.Second
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
|
||||||
|
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
|
||||||
|
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
|
||||||
|
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
|
||||||
|
)
|
||||||
|
|
||||||
|
type providerConfigurationStore interface {
|
||||||
|
Current() *config.Config
|
||||||
|
Revision() uint64
|
||||||
|
PublishRevision(*config.Config, uint64) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerConfigurationSource interface {
|
||||||
|
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerStateReader interface {
|
||||||
|
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
|
||||||
|
type providerConfigurationPreparer func(context.Context, *config.Config) error
|
||||||
|
type providerConfigurationObserver func(*config.Config)
|
||||||
|
|
||||||
|
type providerSupervisor struct {
|
||||||
|
configuration providerConfigurationStore
|
||||||
|
state providerStateReader
|
||||||
|
build providerRunnerBuilder
|
||||||
|
prepare providerConfigurationPreparer
|
||||||
|
observe providerConfigurationObserver
|
||||||
|
source providerConfigurationSource
|
||||||
|
fingerprintKey []byte
|
||||||
|
interval time.Duration
|
||||||
|
notify chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runningProvider struct {
|
||||||
|
configuration config.Upstream
|
||||||
|
cancel context.CancelFunc
|
||||||
|
done chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProviderSupervisor(
|
||||||
|
configuration providerConfigurationStore,
|
||||||
|
state providerStateReader,
|
||||||
|
build providerRunnerBuilder,
|
||||||
|
prepare providerConfigurationPreparer,
|
||||||
|
observe providerConfigurationObserver,
|
||||||
|
source providerConfigurationSource,
|
||||||
|
fingerprintKey []byte,
|
||||||
|
interval time.Duration,
|
||||||
|
) (*providerSupervisor, error) {
|
||||||
|
if nilInterface(configuration) || build == nil || interval <= 0 ||
|
||||||
|
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
|
||||||
|
return nil, ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
return &providerSupervisor{
|
||||||
|
configuration: configuration,
|
||||||
|
state: state,
|
||||||
|
build: build,
|
||||||
|
prepare: prepare,
|
||||||
|
observe: observe,
|
||||||
|
source: source,
|
||||||
|
fingerprintKey: append([]byte(nil), fingerprintKey...),
|
||||||
|
interval: interval,
|
||||||
|
notify: make(chan struct{}, 1),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) Notify() {
|
||||||
|
if supervisor == nil || supervisor.notify == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case supervisor.notify <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||||
|
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := config.Validate(configuration); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
|
||||||
|
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
configuration := supervisor.configuration.Current()
|
||||||
|
if configuration == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
upstream, exists := configuration.Upstreams[name]
|
||||||
|
if !exists {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
upstream.Enabled = true
|
||||||
|
if _, err := supervisor.build(name, upstream); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
|
||||||
|
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
|
||||||
|
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
active := make(map[string]*runningProvider)
|
||||||
|
failures := make(chan error, 1)
|
||||||
|
defer stopAllProviders(active)
|
||||||
|
ticker := time.NewTicker(supervisor.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
if err := supervisor.reconcile(ctx, active, failures); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case err := <-failures:
|
||||||
|
return err
|
||||||
|
case <-supervisor.notify:
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) reconcile(
|
||||||
|
ctx context.Context,
|
||||||
|
active map[string]*runningProvider,
|
||||||
|
failures chan<- error,
|
||||||
|
) error {
|
||||||
|
desired, err := supervisor.desired(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, errProviderManagementStateUnavailable) ||
|
||||||
|
errors.Is(err, errProviderManagementSnapshotStale) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, errProviderConfigurationPending) {
|
||||||
|
stopAllProviders(active)
|
||||||
|
clear(active)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(desired))
|
||||||
|
prepared := make(map[string]lifecycle.Runner)
|
||||||
|
for name, upstream := range desired {
|
||||||
|
names = append(names, name)
|
||||||
|
current := active[name]
|
||||||
|
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
runner, buildErr := supervisor.build(name, upstream)
|
||||||
|
if buildErr != nil || nilInterface(runner) {
|
||||||
|
return errors.Join(ErrProviderSupervisor, buildErr)
|
||||||
|
}
|
||||||
|
prepared[name] = runner
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
for name, current := range active {
|
||||||
|
if _, keep := desired[name]; !keep {
|
||||||
|
stopProvider(current)
|
||||||
|
delete(active, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range names {
|
||||||
|
runner := prepared[name]
|
||||||
|
if runner == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if current := active[name]; current != nil {
|
||||||
|
stopProvider(current)
|
||||||
|
}
|
||||||
|
active[name] = startProvider(ctx, name, desired[name], runner, failures)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
|
||||||
|
configuration := supervisor.configuration.Current()
|
||||||
|
if configuration == nil {
|
||||||
|
return nil, ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
enabled := make(map[string]bool, len(configuration.Upstreams))
|
||||||
|
var snapshot adminstate.Snapshot
|
||||||
|
if !nilInterface(supervisor.state) {
|
||||||
|
var err error
|
||||||
|
snapshot, err = supervisor.state.Snapshot(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(errProviderManagementStateUnavailable, err)
|
||||||
|
}
|
||||||
|
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, upstream := range snapshot.Upstreams {
|
||||||
|
enabled[upstream.Name] = upstream.Enabled
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if supervisor.observe != nil {
|
||||||
|
supervisor.observe(configuration)
|
||||||
|
}
|
||||||
|
desired := make(map[string]config.Upstream)
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
isEnabled := upstream.Enabled
|
||||||
|
if !nilInterface(supervisor.state) {
|
||||||
|
isEnabled = enabled[name]
|
||||||
|
}
|
||||||
|
if isEnabled {
|
||||||
|
upstream.Enabled = true
|
||||||
|
desired[name] = upstream
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return desired, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) synchronizeConfiguration(
|
||||||
|
ctx context.Context,
|
||||||
|
current *config.Config,
|
||||||
|
snapshot adminstate.Snapshot,
|
||||||
|
) (*config.Config, error) {
|
||||||
|
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
|
||||||
|
return current, nil
|
||||||
|
}
|
||||||
|
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||||
|
return nil, errProviderManagementSnapshotStale
|
||||||
|
}
|
||||||
|
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
if checksum == snapshot.Config.Checksum {
|
||||||
|
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
|
||||||
|
return current, nil
|
||||||
|
}
|
||||||
|
if nilInterface(supervisor.source) {
|
||||||
|
return nil, errProviderConfigurationPending
|
||||||
|
}
|
||||||
|
loaded, err := supervisor.source.LoadConfiguration(ctx)
|
||||||
|
if err != nil || loaded.Value == nil {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
|
||||||
|
if err != nil || checksum != snapshot.Config.Checksum {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
|
||||||
|
return loaded.Value, nil
|
||||||
|
}
|
||||||
|
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||||
|
return nil, errProviderManagementSnapshotStale
|
||||||
|
}
|
||||||
|
return nil, errProviderConfigurationPending
|
||||||
|
}
|
||||||
|
|
||||||
|
func startProvider(
|
||||||
|
ctx context.Context,
|
||||||
|
name string,
|
||||||
|
configuration config.Upstream,
|
||||||
|
runner lifecycle.Runner,
|
||||||
|
failures chan<- error,
|
||||||
|
) *runningProvider {
|
||||||
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
|
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
|
||||||
|
go func() {
|
||||||
|
defer close(running.done)
|
||||||
|
err := runner.Run(runCtx)
|
||||||
|
if runCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = lifecycle.ErrRunnerStopped
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopProvider(running *runningProvider) {
|
||||||
|
if running == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running.cancel()
|
||||||
|
<-running.done
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopAllProviders(active map[string]*runningProvider) {
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for _, running := range active {
|
||||||
|
wait.Add(1)
|
||||||
|
go func(current *runningProvider) {
|
||||||
|
defer wait.Done()
|
||||||
|
stopProvider(current)
|
||||||
|
}(running)
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
}
|
||||||
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
@ -0,0 +1,376 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||||
|
started := make(chan providerRuntimeEvent, 4)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 4)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
state.set("provider-a", false)
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||||
|
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated := store.Current()
|
||||||
|
providerB := updated.Upstreams["provider-b"]
|
||||||
|
providerB.API.URL = "https://replacement.invalid/proxies"
|
||||||
|
updated.Upstreams["provider-b"] = providerB
|
||||||
|
if !store.PublishRevision(updated, 1) {
|
||||||
|
t.Fatal("PublishRevision() rejected updated configuration")
|
||||||
|
}
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
|
||||||
|
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
|
||||||
|
}
|
||||||
|
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
|
||||||
|
t.Fatalf("replacement Provider = %+v", event)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
wantErr := errors.New("coordination stopped")
|
||||||
|
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||||
|
started := make(chan providerRuntimeEvent, 2)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 2)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
state.setError(errors.New("PostgreSQL temporarily unavailable"))
|
||||||
|
supervisor.Notify()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
|
||||||
|
case event := <-stopped:
|
||||||
|
t.Fatalf("Provider stopped during transient state failure: %+v", event)
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
|
||||||
|
state.setError(nil)
|
||||||
|
state.set("provider-a", false)
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||||
|
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
|
||||||
|
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(initial)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(initial): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{
|
||||||
|
enabled: map[string]bool{"provider-a": true, "provider-b": true},
|
||||||
|
checksum: initialChecksum,
|
||||||
|
revision: 1,
|
||||||
|
}
|
||||||
|
source := &mutableProviderConfigurationSource{configuration: initial}
|
||||||
|
started := make(chan providerRuntimeEvent, 4)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 4)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
updated := store.Current()
|
||||||
|
providerA := updated.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://replacement.invalid/proxies"
|
||||||
|
updated.Upstreams["provider-a"] = providerA
|
||||||
|
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(updated): %v", err)
|
||||||
|
}
|
||||||
|
state.setChecksum(updatedChecksum)
|
||||||
|
supervisor.Notify()
|
||||||
|
waitForRuntimeEvents(t, stopped, 2)
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
|
||||||
|
source.set(updated)
|
||||||
|
supervisor.Notify()
|
||||||
|
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
|
||||||
|
foundReplacement := false
|
||||||
|
for _, event := range events {
|
||||||
|
if event.name == "provider-a" && event.url == providerA.API.URL {
|
||||||
|
foundReplacement = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundReplacement {
|
||||||
|
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
|
||||||
|
}
|
||||||
|
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
|
||||||
|
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
|
||||||
|
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(initial)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
store.PublishRevision(initial, 1)
|
||||||
|
|
||||||
|
candidate := store.Current()
|
||||||
|
providerA := candidate.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://candidate.invalid/proxies"
|
||||||
|
candidate.Upstreams["provider-a"] = providerA
|
||||||
|
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(candidate): %v", err)
|
||||||
|
}
|
||||||
|
newest := store.Current()
|
||||||
|
providerA = newest.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://newest.invalid/proxies"
|
||||||
|
newest.Upstreams["provider-a"] = providerA
|
||||||
|
|
||||||
|
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
if !store.PublishRevision(newest, 3) {
|
||||||
|
t.Fatal("failed to publish simulated concurrent revision")
|
||||||
|
}
|
||||||
|
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
|
||||||
|
})
|
||||||
|
supervisor, err := newProviderSupervisor(
|
||||||
|
store,
|
||||||
|
&mutableProviderState{},
|
||||||
|
func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
source,
|
||||||
|
bootstrapTestFingerprintKey,
|
||||||
|
time.Hour,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
|
||||||
|
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errProviderManagementSnapshotStale) {
|
||||||
|
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 3 {
|
||||||
|
t.Fatalf("configuration revision = %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerRuntimeEvent struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
type supervisorRunnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||||
|
|
||||||
|
type mutableProviderState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
enabled map[string]bool
|
||||||
|
err error
|
||||||
|
checksum string
|
||||||
|
revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
if state.err != nil {
|
||||||
|
return adminstate.Snapshot{}, state.err
|
||||||
|
}
|
||||||
|
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
|
||||||
|
if state.checksum != "" {
|
||||||
|
snapshot.Config = &adminstate.ConfigRevision{
|
||||||
|
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, enabled := range state.enabled {
|
||||||
|
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
|
||||||
|
}
|
||||||
|
return snapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) setError(err error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.err = err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) setChecksum(checksum string) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.checksum = checksum
|
||||||
|
state.revision++
|
||||||
|
}
|
||||||
|
|
||||||
|
type mutableProviderConfigurationSource struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
configuration *config.Config
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
|
||||||
|
|
||||||
|
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
return source(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
source.configuration = configuration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) set(name string, enabled bool) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.enabled[name] = enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
|
||||||
|
t.Helper()
|
||||||
|
for range count {
|
||||||
|
_ = waitForRuntimeEvent(t, events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case event := <-events:
|
||||||
|
return event
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for Provider runtime event")
|
||||||
|
return providerRuntimeEvent{}
|
||||||
|
}
|
||||||
|
}
|
||||||
104
internal/controller/bootstrap/provider_test.go
Normal file
104
internal/controller/bootstrap/provider_test.go
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
upstream := configuration.Upstreams["provider-a"]
|
||||||
|
mapped, err := providerRuntimeConfig("provider-a", upstream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("providerRuntimeConfig(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if mapped.Provider.UpstreamID != "provider-a" ||
|
||||||
|
mapped.Provider.RequestInterval != time.Second ||
|
||||||
|
mapped.Provider.Timeout != 3*time.Second ||
|
||||||
|
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
|
||||||
|
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
|
||||||
|
mapped.Provider.TTL != 2*time.Minute ||
|
||||||
|
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
|
||||||
|
t.Fatalf("Provider config = %+v", mapped.Provider)
|
||||||
|
}
|
||||||
|
if mapped.ReconcileInterval != time.Second ||
|
||||||
|
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
|
||||||
|
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
|
||||||
|
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
|
||||||
|
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
|
||||||
|
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
|
||||||
|
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
configuration.Upstreams["disabled"] = config.Upstream{}
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
fleet, err := newProviderFleet(configuration, ports{
|
||||||
|
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderFleet(): %v", err)
|
||||||
|
}
|
||||||
|
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
|
||||||
|
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
configuration.Upstreams["provider-a"] = config.Upstream{}
|
||||||
|
configuration.Upstreams["provider-b"] = config.Upstream{}
|
||||||
|
fleet, err = newProviderFleet(configuration, ports{})
|
||||||
|
if err != nil || fleet != nil {
|
||||||
|
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(disabled) error = nil")
|
||||||
|
}
|
||||||
|
overflowing := config.Upstream{
|
||||||
|
Enabled: true,
|
||||||
|
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
|
||||||
|
Capacity: config.Capacity{
|
||||||
|
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
|
||||||
|
},
|
||||||
|
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
|
||||||
|
}
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(overflow) error = nil")
|
||||||
|
}
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
tooLarge := configuration.Upstreams["provider-a"]
|
||||||
|
tooLarge.Pool.MaxSize = redisMaximumScan + 1
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
|
||||||
|
}
|
||||||
|
tooLarge = configuration.Upstreams["provider-a"]
|
||||||
|
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
|
||||||
|
if strconv.IntSize == 64 {
|
||||||
|
tooLarge.Fetch.MaxTotal++
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controller/admin"
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -24,6 +25,7 @@ type ConfigurationReader interface {
|
|||||||
type Reader struct {
|
type Reader struct {
|
||||||
configuration ConfigurationReader
|
configuration ConfigurationReader
|
||||||
inventory activitypool.StateInventoryReader
|
inventory activitypool.StateInventoryReader
|
||||||
|
providerStats provider.StatsReader
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -33,11 +35,17 @@ func NewReader(
|
|||||||
configuration ConfigurationReader,
|
configuration ConfigurationReader,
|
||||||
inventory activitypool.StateInventoryReader,
|
inventory activitypool.StateInventoryReader,
|
||||||
now func() time.Time,
|
now func() time.Time,
|
||||||
|
stats ...provider.StatsReader,
|
||||||
) (*Reader, error) {
|
) (*Reader, error) {
|
||||||
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
|
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
|
||||||
|
(len(stats) == 1 && nilInterface(stats[0])) {
|
||||||
return nil, ErrInvalidReader
|
return nil, ErrInvalidReader
|
||||||
}
|
}
|
||||||
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
|
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
|
||||||
|
if len(stats) == 1 {
|
||||||
|
reader.providerStats = stats[0]
|
||||||
|
}
|
||||||
|
return reader, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
||||||
@ -69,8 +77,16 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
|
|||||||
}
|
}
|
||||||
|
|
||||||
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
||||||
|
providerStats := make([]provider.Stats, len(upstreamIDs))
|
||||||
|
if reader.providerStats != nil {
|
||||||
|
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
|
||||||
|
if len(providerStats) != len(upstreamIDs) {
|
||||||
|
return admin.OperationalStatus{}, ErrUnavailable
|
||||||
|
}
|
||||||
|
}
|
||||||
for index, inventory := range inventories {
|
for index, inventory := range inventories {
|
||||||
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
|
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
|
||||||
|
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
|
||||||
return admin.OperationalStatus{}, ErrUnavailable
|
return admin.OperationalStatus{}, ErrUnavailable
|
||||||
}
|
}
|
||||||
status.Upstreams[index] = admin.UpstreamActivity{
|
status.Upstreams[index] = admin.UpstreamActivity{
|
||||||
@ -80,6 +96,8 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
|
|||||||
Suspect: inventory.Suspect,
|
Suspect: inventory.Suspect,
|
||||||
Draining: inventory.Draining,
|
Draining: inventory.Draining,
|
||||||
Extracted: inventory.Extracted,
|
Extracted: inventory.Extracted,
|
||||||
|
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
|
||||||
|
FetchErrorCount: providerStats[index].FetchErrorCount,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return status, nil
|
return status, nil
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
|||||||
}}
|
}}
|
||||||
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
||||||
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
||||||
}}, inventory, func() time.Time { return now })
|
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
|
||||||
|
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
|
||||||
|
{UpstreamID: "provider-b", FetchErrorCount: 1},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReader() error = %v", err)
|
t.Fatalf("NewReader() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
first := status.Upstreams[0]
|
first := status.Upstreams[0]
|
||||||
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
||||||
first.Draining != 4 || first.Extracted != 8 {
|
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
|
||||||
t.Fatalf("first upstream = %+v", first)
|
t.Fatalf("first upstream = %+v", first)
|
||||||
}
|
}
|
||||||
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
||||||
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
|
|||||||
now time.Time
|
now time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type staticProviderStatsReader struct {
|
||||||
|
result []provider.Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
|
||||||
|
return append([]provider.Stats(nil), reader.result...)
|
||||||
|
}
|
||||||
|
|
||||||
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
upstreamIDs []string,
|
upstreamIDs []string,
|
||||||
|
|||||||
@ -20,48 +20,38 @@ var (
|
|||||||
type FetchBudgetConfig struct {
|
type FetchBudgetConfig struct {
|
||||||
UpstreamID string
|
UpstreamID string
|
||||||
MaxSize int
|
MaxSize int
|
||||||
MaxTotal int64
|
|
||||||
ExpectedPerFetch int
|
ExpectedPerFetch int
|
||||||
Managed int
|
Managed int
|
||||||
FetchedTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type FetchBudgetSnapshot struct {
|
type FetchBudgetSnapshot struct {
|
||||||
Managed int
|
Managed int
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FetchBudget owns both current-inventory and cumulative-fetch accounting for
|
// FetchBudget owns current-inventory accounting for one upstream. Reserving
|
||||||
// one upstream. Reserving the expected response before I/O closes the race
|
// the expected response before I/O closes the race between concurrent calls;
|
||||||
// between concurrent provider calls.
|
// distributed cumulative quota belongs to the Provider coordination permit.
|
||||||
type FetchBudget struct {
|
type FetchBudget struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
|
||||||
upstreamID string
|
upstreamID string
|
||||||
maxSize int
|
maxSize int
|
||||||
maxTotal int64
|
|
||||||
expected int
|
expected int
|
||||||
usage FetchBudgetSnapshot
|
usage FetchBudgetSnapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
|
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
|
||||||
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
|
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
|
||||||
config.ExpectedPerFetch > config.MaxSize || config.MaxTotal < 0 ||
|
config.ExpectedPerFetch > config.MaxSize || config.Managed < 0 {
|
||||||
config.Managed < 0 || config.FetchedTotal < 0 {
|
|
||||||
return nil, ErrInvalidFetchBudget
|
|
||||||
}
|
|
||||||
if config.MaxTotal > 0 && int64(config.ExpectedPerFetch) > config.MaxTotal {
|
|
||||||
return nil, ErrInvalidFetchBudget
|
return nil, ErrInvalidFetchBudget
|
||||||
}
|
}
|
||||||
return &FetchBudget{
|
return &FetchBudget{
|
||||||
upstreamID: config.UpstreamID,
|
upstreamID: config.UpstreamID,
|
||||||
maxSize: config.MaxSize,
|
maxSize: config.MaxSize,
|
||||||
maxTotal: config.MaxTotal,
|
|
||||||
expected: config.ExpectedPerFetch,
|
expected: config.ExpectedPerFetch,
|
||||||
usage: FetchBudgetSnapshot{
|
usage: FetchBudgetSnapshot{
|
||||||
Managed: config.Managed,
|
Managed: config.Managed,
|
||||||
FetchedTotal: config.FetchedTotal,
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -95,14 +85,7 @@ func (b *FetchBudget) FetchAllowance() int {
|
|||||||
|
|
||||||
func (b *FetchBudget) canReserveLocked() bool {
|
func (b *FetchBudget) canReserveLocked() bool {
|
||||||
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
|
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
|
||||||
if poolRoom < b.expected {
|
return poolRoom >= b.expected
|
||||||
return false
|
|
||||||
}
|
|
||||||
if b.maxTotal > 0 {
|
|
||||||
totalRoom := b.maxTotal - b.usage.FetchedTotal - int64(b.usage.PendingExpected)
|
|
||||||
return totalRoom >= int64(b.expected)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
||||||
@ -115,8 +98,8 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SynchronizeManaged replaces the local current-inventory count with the
|
// SynchronizeManaged replaces the local current-inventory count with the
|
||||||
// authoritative activity-store observation. Pending requests and cumulative
|
// authoritative activity-store observation. Pending reservations remain owned
|
||||||
// fetch usage remain owned by this budget.
|
// by this budget.
|
||||||
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
||||||
if b == nil || managed < 0 {
|
if b == nil || managed < 0 {
|
||||||
return ErrInvalidManagedSynchronization
|
return ErrInvalidManagedSynchronization
|
||||||
@ -158,11 +141,11 @@ func (p *fetchPermit) Expected() int {
|
|||||||
return p.expected
|
return p.expected
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fetchPermit) Complete(fetched, retained int) error {
|
func (p *fetchPermit) Complete(retained int) error {
|
||||||
if p == nil || p.budget == nil {
|
if p == nil || p.budget == nil {
|
||||||
return ErrFetchPermitFinished
|
return ErrFetchPermitFinished
|
||||||
}
|
}
|
||||||
if fetched < 0 || retained < 0 || retained > fetched || retained > p.expected {
|
if retained < 0 || retained > p.expected {
|
||||||
return ErrInvalidFetchCompletion
|
return ErrInvalidFetchCompletion
|
||||||
}
|
}
|
||||||
p.budget.mu.Lock()
|
p.budget.mu.Lock()
|
||||||
@ -173,7 +156,6 @@ func (p *fetchPermit) Complete(fetched, retained int) error {
|
|||||||
p.finished = true
|
p.finished = true
|
||||||
p.budget.usage.PendingExpected -= p.expected
|
p.budget.usage.PendingExpected -= p.expected
|
||||||
p.budget.usage.Managed += retained
|
p.budget.usage.Managed += retained
|
||||||
p.budget.usage.FetchedTotal += int64(fetched)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -6,14 +6,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
func TestFetchBudgetReservesExpectedPoolCapacity(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a",
|
UpstreamID: "provider-a",
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 20,
|
|
||||||
ExpectedPerFetch: 4,
|
ExpectedPerFetch: 4,
|
||||||
Managed: 2,
|
Managed: 2,
|
||||||
FetchedTotal: 3,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -31,21 +29,21 @@ func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
|||||||
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := first.Complete(4, 3); err != nil {
|
if err := first.Complete(3); err != nil {
|
||||||
t.Fatalf("first.Complete(): %v", err)
|
t.Fatalf("first.Complete(): %v", err)
|
||||||
}
|
}
|
||||||
if err := second.Cancel(); err != nil {
|
if err := second.Cancel(); err != nil {
|
||||||
t.Fatalf("second.Cancel(): %v", err)
|
t.Fatalf("second.Cancel(): %v", err)
|
||||||
}
|
}
|
||||||
usage := budget.Snapshot()
|
usage := budget.Snapshot()
|
||||||
if usage.Managed != 5 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
if usage.Managed != 5 || usage.PendingExpected != 0 {
|
||||||
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0 fetched=7", usage)
|
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0", usage)
|
||||||
}
|
}
|
||||||
if err := budget.ReleaseManaged(2); err != nil {
|
if err := budget.ReleaseManaged(2); err != nil {
|
||||||
t.Fatalf("ReleaseManaged(): %v", err)
|
t.Fatalf("ReleaseManaged(): %v", err)
|
||||||
}
|
}
|
||||||
if usage := budget.Snapshot(); usage.Managed != 3 || usage.FetchedTotal != 7 {
|
if usage := budget.Snapshot(); usage.Managed != 3 {
|
||||||
t.Fatalf("Snapshot() after release = %+v, want managed=3 fetched=7", usage)
|
t.Fatalf("Snapshot() after release = %+v, want managed=3", usage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -63,7 +61,7 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
|
|||||||
|
|
||||||
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
|
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -77,8 +75,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
|||||||
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
||||||
}
|
}
|
||||||
usage := budget.Snapshot()
|
usage := budget.Snapshot()
|
||||||
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
|
if usage.Managed != 3 || usage.PendingExpected != 2 {
|
||||||
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
|
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2", usage)
|
||||||
}
|
}
|
||||||
if err := permit.Cancel(); err != nil {
|
if err := permit.Cancel(); err != nil {
|
||||||
t.Fatalf("Cancel(): %v", err)
|
t.Fatalf("Cancel(): %v", err)
|
||||||
@ -86,8 +84,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
|||||||
if err := budget.SynchronizeManaged(1); err != nil {
|
if err := budget.SynchronizeManaged(1); err != nil {
|
||||||
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
||||||
}
|
}
|
||||||
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 {
|
||||||
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
|
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0", usage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -103,36 +101,16 @@ func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
|
func TestFetchBudgetRequiresWholeExpectedBatchToFitPool(t *testing.T) {
|
||||||
tests := []struct {
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
name string
|
|
||||||
config FetchBudgetConfig
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "pool size",
|
|
||||||
config: FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
||||||
},
|
})
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cumulative total",
|
|
||||||
config: FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 10, MaxTotal: 5, ExpectedPerFetch: 4, FetchedTotal: 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(tt.config)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
}
|
}
|
||||||
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchBudgetConcurrentReservationsNeverExceedMaxSize(t *testing.T) {
|
func TestFetchBudgetConcurrentReservationsNeverExceedMaxSize(t *testing.T) {
|
||||||
@ -185,7 +163,7 @@ func TestFetchPermitRejectsDoubleFinishAndInvalidCounts(t *testing.T) {
|
|||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
||||||
}
|
}
|
||||||
if err := permit.Complete(1, 2); !errors.Is(err, ErrInvalidFetchCompletion) {
|
if err := permit.Complete(3); !errors.Is(err, ErrInvalidFetchCompletion) {
|
||||||
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
|
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
|
||||||
}
|
}
|
||||||
if err := permit.Cancel(); err != nil {
|
if err := permit.Cancel(); err != nil {
|
||||||
|
|||||||
17
internal/controller/pool/inventory.go
Normal file
17
internal/controller/pool/inventory.go
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
package pool
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type InventorySnapshot struct {
|
||||||
|
Managed int
|
||||||
|
AvailableSlots int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type InventoryReader interface {
|
||||||
|
// ReadInventory returns a bounded, authoritative aggregate. Unknown Worker
|
||||||
|
// runtime must reduce capacity rather than being treated as idle.
|
||||||
|
ReadInventory(context.Context, string, time.Duration) (InventorySnapshot, error)
|
||||||
|
}
|
||||||
@ -26,7 +26,6 @@ type FetchNotifier interface {
|
|||||||
type ReconcileDecision struct {
|
type ReconcileDecision struct {
|
||||||
AvailableSlots int64
|
AvailableSlots int64
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
FetchAllowance int
|
FetchAllowance int
|
||||||
EffectiveSlots int64
|
EffectiveSlots int64
|
||||||
Triggered bool
|
Triggered bool
|
||||||
@ -62,15 +61,28 @@ func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNo
|
|||||||
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
|
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
|
||||||
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
|
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
|
||||||
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
|
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
|
||||||
|
return r.reconcileSnapshot(
|
||||||
|
InventorySnapshot{AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin)},
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) ReconcileSnapshot(inventory InventorySnapshot) ReconcileDecision {
|
||||||
|
return r.reconcileSnapshot(inventory, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) reconcileSnapshot(inventory InventorySnapshot, synchronizeManaged bool) ReconcileDecision {
|
||||||
usage := r.budget.Snapshot()
|
usage := r.budget.Snapshot()
|
||||||
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
|
if synchronizeManaged && inventory.Managed >= 0 && usage.PendingExpected == 0 {
|
||||||
|
_ = r.budget.SynchronizeManaged(inventory.Managed)
|
||||||
|
usage = r.budget.Snapshot()
|
||||||
|
}
|
||||||
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
|
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
|
||||||
decision := ReconcileDecision{
|
decision := ReconcileDecision{
|
||||||
AvailableSlots: availableSlots,
|
AvailableSlots: inventory.AvailableSlots,
|
||||||
PendingExpected: usage.PendingExpected,
|
PendingExpected: usage.PendingExpected,
|
||||||
FetchedTotal: usage.FetchedTotal,
|
|
||||||
FetchAllowance: r.budget.FetchAllowance(),
|
FetchAllowance: r.budget.FetchAllowance(),
|
||||||
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
|
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
|
||||||
}
|
}
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
if r.refilling {
|
if r.refilling {
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
|
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a", MaxSize: 10, MaxTotal: 20, ExpectedPerFetch: 2,
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -32,7 +32,6 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
|||||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
|
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
|
||||||
}},
|
}},
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 20,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := reconciler.Reconcile(now, inventory)
|
decision := reconciler.Reconcile(now, inventory)
|
||||||
@ -44,9 +43,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
func TestPoolReconcilerUsesPendingPoolReservation(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a", MaxSize: 2, MaxTotal: 2, ExpectedPerFetch: 2,
|
UpstreamID: "provider-a", MaxSize: 2, ExpectedPerFetch: 2,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -65,7 +64,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2, MaxTotal: 2})
|
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2})
|
||||||
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
|
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
|
||||||
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
|
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
|
||||||
}
|
}
|
||||||
@ -133,7 +132,7 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
|
|||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
||||||
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
||||||
}
|
}
|
||||||
if err := permit.Complete(2, 1); err != nil {
|
if err := permit.Complete(1); err != nil {
|
||||||
t.Fatalf("Complete(): %v", err)
|
t.Fatalf("Complete(): %v", err)
|
||||||
}
|
}
|
||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
||||||
@ -141,6 +140,53 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPoolReconcilerConsumesAuthoritativeInventorySnapshot(t *testing.T) {
|
||||||
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
|
}
|
||||||
|
notifier := &recordingFetchNotifier{}
|
||||||
|
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
|
||||||
|
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
|
||||||
|
}, budget, notifier)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
decision := reconciler.ReconcileSnapshot(InventorySnapshot{Managed: 9, AvailableSlots: 2})
|
||||||
|
if decision.Triggered || decision.AvailableSlots != 2 || decision.FetchAllowance != 0 {
|
||||||
|
t.Fatalf("ReconcileSnapshot() = %+v, want authoritative managed inventory to close budget", decision)
|
||||||
|
}
|
||||||
|
if usage := budget.Snapshot(); usage.Managed != 9 {
|
||||||
|
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolReconcilerLegacyInventoryDoesNotClearManagedBudget(t *testing.T) {
|
||||||
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2, Managed: 9,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
|
}
|
||||||
|
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
|
||||||
|
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
|
||||||
|
}, budget, &recordingFetchNotifier{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{})
|
||||||
|
if decision.Triggered || decision.FetchAllowance != 0 {
|
||||||
|
t.Fatalf("Reconcile(legacy) = %+v, want preserved managed budget", decision)
|
||||||
|
}
|
||||||
|
if usage := budget.Snapshot(); usage.Managed != 9 {
|
||||||
|
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
|
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
|
||||||
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
|
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
|
||||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,
|
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,
|
||||||
|
|||||||
@ -13,10 +13,13 @@ var (
|
|||||||
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const MaximumCoordinationInteger = int64(1<<53 - 1)
|
||||||
|
|
||||||
type CoordinationLimits struct {
|
type CoordinationLimits struct {
|
||||||
RequestInterval time.Duration
|
RequestInterval time.Duration
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
MaxAttemptDuration time.Duration
|
MaxAttemptDuration time.Duration
|
||||||
|
MaxTotal int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type Fence struct {
|
type Fence struct {
|
||||||
@ -32,10 +35,14 @@ type Coordinator interface {
|
|||||||
|
|
||||||
type LeaderSession interface {
|
type LeaderSession interface {
|
||||||
Fence() Fence
|
Fence() Fence
|
||||||
AcquireFetch(context.Context) (RequestPermit, error)
|
AcquireFetch(context.Context, int) (RequestPermit, bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type RequestPermit interface {
|
type RequestPermit interface {
|
||||||
// Release is idempotent. A failed release expires automatically in storage.
|
// Complete atomically releases in-flight capacity and charges the actual
|
||||||
Release(context.Context) error
|
// successful candidate count. Repeated calls are idempotent.
|
||||||
|
Complete(context.Context, int) error
|
||||||
|
// Cancel releases a reservation that is known not to have consumed Provider
|
||||||
|
// quota. A crashed or abandoned reservation is conservatively charged.
|
||||||
|
Cancel(context.Context) error
|
||||||
}
|
}
|
||||||
|
|||||||
64
internal/controller/provider/fleet.go
Normal file
64
internal/controller/provider/fleet.go
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidFleet = errors.New("invalid Provider fleet")
|
||||||
|
|
||||||
|
type Fleet struct {
|
||||||
|
runtimes []*UpstreamRuntime
|
||||||
|
group *lifecycle.Group
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
|
||||||
|
if len(runtimes) == 0 {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(runtimes))
|
||||||
|
owned := make([]*UpstreamRuntime, len(runtimes))
|
||||||
|
for index, runtime := range runtimes {
|
||||||
|
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
seen[runtime.config.Provider.UpstreamID] = struct{}{}
|
||||||
|
owned[index] = runtime
|
||||||
|
}
|
||||||
|
sort.Slice(owned, func(left, right int) bool {
|
||||||
|
return owned[left].ID() < owned[right].ID()
|
||||||
|
})
|
||||||
|
runners := make([]lifecycle.Runner, len(owned))
|
||||||
|
for index, runtime := range owned {
|
||||||
|
runners[index] = runtime
|
||||||
|
}
|
||||||
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(ErrInvalidFleet, err)
|
||||||
|
}
|
||||||
|
return &Fleet{runtimes: owned, group: group}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fleet *Fleet) Run(ctx context.Context) error {
|
||||||
|
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
|
||||||
|
return ErrInvalidFleet
|
||||||
|
}
|
||||||
|
return fleet.group.Run(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fleet *Fleet) IDs() []string {
|
||||||
|
if fleet == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ids := make([]string, len(fleet.runtimes))
|
||||||
|
for index, runtime := range fleet.runtimes {
|
||||||
|
ids[index] = runtime.ID()
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
101
internal/controller/provider/fleet_test.go
Normal file
101
internal/controller/provider/fleet_test.go
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
|
||||||
|
started := make(chan string, 2)
|
||||||
|
siblingCancelled := make(chan struct{}, 1)
|
||||||
|
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||||
|
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
started <- "provider-a"
|
||||||
|
return errors.New("provider-a stopped")
|
||||||
|
}))
|
||||||
|
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
|
||||||
|
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
started <- "provider-b"
|
||||||
|
<-ctx.Done()
|
||||||
|
siblingCancelled <- struct{}{}
|
||||||
|
return ctx.Err()
|
||||||
|
}))
|
||||||
|
fleet, err := NewFleet(first, second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFleet(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = fleet.Run(context.Background())
|
||||||
|
if err == nil || err.Error() != "provider-a stopped" {
|
||||||
|
t.Fatalf("Run() error = %v, want provider-a failure", err)
|
||||||
|
}
|
||||||
|
seen := map[string]bool{<-started: true, <-started: true}
|
||||||
|
if !seen["provider-a"] || !seen["provider-b"] {
|
||||||
|
t.Fatalf("started upstreams = %v", seen)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-siblingCancelled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("sibling runtime was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
|
||||||
|
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||||
|
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return ctx.Err()
|
||||||
|
}))
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
runtimes []*UpstreamRuntime
|
||||||
|
}{
|
||||||
|
{name: "empty"},
|
||||||
|
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
|
||||||
|
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
fleet, err := NewFleet(test.runtimes...)
|
||||||
|
if err == nil || fleet != nil {
|
||||||
|
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
|
||||||
|
t.Helper()
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig(upstreamID),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: time.Second,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinator,
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
|
||||||
|
}
|
||||||
|
return runtime
|
||||||
|
}
|
||||||
@ -27,9 +27,11 @@ type RetryableError interface {
|
|||||||
Retryable() bool
|
Retryable() bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parser must be safe for concurrent calls and must honor context cancellation.
|
// Parser owns any transient resources attached to parsed candidates. It must
|
||||||
|
// be safe for concurrent calls and must honor context cancellation.
|
||||||
type Parser interface {
|
type Parser interface {
|
||||||
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
||||||
|
ReleaseCandidates([]proxyDomain.Proxy)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Result struct {
|
type Result struct {
|
||||||
@ -42,7 +44,8 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResultRecorder interface {
|
type ResultRecorder interface {
|
||||||
// Record may be called concurrently and must not retain mutable result data.
|
// Record may be called concurrently, must return promptly, and must not
|
||||||
|
// retain mutable result data.
|
||||||
Record(Result)
|
Record(Result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,7 @@ type Config struct {
|
|||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
|
MaxTotal int64
|
||||||
MaxSize int
|
MaxSize int
|
||||||
TTL time.Duration
|
TTL time.Duration
|
||||||
AllocationSafetyMargin time.Duration
|
AllocationSafetyMargin time.Duration
|
||||||
@ -31,6 +32,8 @@ type RetryConfig struct {
|
|||||||
Jitter int
|
Jitter int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const requestPermitSettlementTimeout = 5 * time.Second
|
||||||
|
|
||||||
type Reconciler struct {
|
type Reconciler struct {
|
||||||
config Config
|
config Config
|
||||||
ports Ports
|
ports Ports
|
||||||
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
|
|||||||
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
||||||
}
|
}
|
||||||
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
||||||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
|
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
|
||||||
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
||||||
}
|
}
|
||||||
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
||||||
@ -112,7 +115,10 @@ func (r *Reconciler) Notify() {
|
|||||||
r.signal.Notify()
|
r.signal.Notify()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) Run(ctx context.Context) error {
|
func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error {
|
||||||
|
if ctx == nil || session == nil {
|
||||||
|
return ErrInvalidCoordination
|
||||||
|
}
|
||||||
var workers sync.WaitGroup
|
var workers sync.WaitGroup
|
||||||
defer workers.Wait()
|
defer workers.Wait()
|
||||||
for {
|
for {
|
||||||
@ -131,17 +137,57 @@ func (r *Reconciler) Run(ctx context.Context) error {
|
|||||||
go func() {
|
go func() {
|
||||||
defer workers.Done()
|
defer workers.Done()
|
||||||
defer func() { <-r.inFlight }()
|
defer func() { <-r.inFlight }()
|
||||||
r.reconcile(ctx)
|
r.reconcile(ctx, session)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) reconcile(ctx context.Context) {
|
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
||||||
|
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
||||||
|
if err != nil {
|
||||||
|
r.ports.Results.Record(Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("reserve fetch capacity: %w", err),
|
||||||
|
Attempt: 1,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !available {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if permit == nil || permit.Expected() <= 0 {
|
||||||
|
if permit != nil {
|
||||||
|
_ = permit.Cancel()
|
||||||
|
}
|
||||||
|
r.ports.Results.Record(Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
||||||
|
Attempt: 1,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
permitFinished := false
|
||||||
|
defer func() {
|
||||||
|
if !permitFinished {
|
||||||
|
_ = permit.Cancel()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
||||||
response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
|
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
|
||||||
|
result.Class = upstream.FetchError
|
||||||
|
result.Err = errors.Join(result.Err, capacityErr)
|
||||||
|
} else {
|
||||||
|
permitFinished = true
|
||||||
|
}
|
||||||
|
}
|
||||||
r.ports.Results.Record(result)
|
r.ports.Results.Record(result)
|
||||||
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
|
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
|
||||||
return
|
return
|
||||||
@ -157,38 +203,42 @@ func (r *Reconciler) reconcile(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
|
func (r *Reconciler) fetchAttempt(
|
||||||
|
ctx context.Context,
|
||||||
|
session LeaderSession,
|
||||||
|
permit upstream.FetchPermit,
|
||||||
|
attempt int,
|
||||||
|
) (FetchResponse, Result, bool, bool) {
|
||||||
if err := r.waitForRequestSlot(ctx); err != nil {
|
if err := r.waitForRequestSlot(ctx); err != nil {
|
||||||
return FetchResponse{}, Result{}, false, false
|
return FetchResponse{}, Result{}, false, false
|
||||||
}
|
}
|
||||||
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
|
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
|
||||||
|
return FetchResponse{}, Result{}, false, false
|
||||||
|
}
|
||||||
return FetchResponse{}, Result{
|
return FetchResponse{}, Result{
|
||||||
UpstreamID: r.config.UpstreamID,
|
UpstreamID: r.config.UpstreamID,
|
||||||
Class: upstream.FetchError,
|
Class: upstream.FetchError,
|
||||||
Err: resultErr,
|
Err: fmt.Errorf("acquire distributed fetch capacity: %w", err),
|
||||||
Attempt: attempt,
|
Attempt: attempt,
|
||||||
}, false, true
|
}, false, true
|
||||||
}
|
}
|
||||||
if !available {
|
if !available {
|
||||||
return FetchResponse{}, Result{}, false, false
|
return FetchResponse{}, Result{}, false, false
|
||||||
}
|
}
|
||||||
if permit == nil || permit.Expected() <= 0 {
|
if requestPermit == nil {
|
||||||
if permit != nil {
|
|
||||||
_ = permit.Cancel()
|
|
||||||
}
|
|
||||||
return FetchResponse{}, Result{
|
return FetchResponse{}, Result{
|
||||||
UpstreamID: r.config.UpstreamID,
|
UpstreamID: r.config.UpstreamID,
|
||||||
Class: upstream.FetchError,
|
Class: upstream.FetchError,
|
||||||
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"),
|
||||||
Attempt: attempt,
|
Attempt: attempt,
|
||||||
}, false, true
|
}, false, true
|
||||||
}
|
}
|
||||||
permitFinished := false
|
requestSettlementAttempted := false
|
||||||
defer func() {
|
defer func() {
|
||||||
if !permitFinished {
|
if !requestSettlementAttempted {
|
||||||
_ = permit.Cancel()
|
_ = r.settleRequestPermit(requestPermit, false, 0)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@ -200,13 +250,23 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
||||||
|
|
||||||
var parseErr, candidateErr, capacityErr error
|
var parseErr, candidateErr, coordinationErr error
|
||||||
var validCount, newCount int
|
var validCount, newCount int
|
||||||
if callErr == nil {
|
if callErr != nil {
|
||||||
|
requestSettlementAttempted = true
|
||||||
|
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
|
||||||
|
} else {
|
||||||
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
||||||
|
defer r.ports.Parser.ReleaseCandidates(candidates)
|
||||||
parseErr = err
|
parseErr = err
|
||||||
validCount = len(candidates)
|
validCount = len(candidates)
|
||||||
if parseErr == nil && validCount > 0 {
|
charged := validCount
|
||||||
|
if parseErr != nil {
|
||||||
|
charged = permit.Expected()
|
||||||
|
}
|
||||||
|
requestSettlementAttempted = true
|
||||||
|
coordinationErr = r.settleRequestPermit(requestPermit, true, charged)
|
||||||
|
if parseErr == nil && coordinationErr == nil && validCount > 0 {
|
||||||
retained := candidates
|
retained := candidates
|
||||||
if expected := permit.Expected(); expected < len(retained) {
|
if expected := permit.Expected(); expected < len(retained) {
|
||||||
retained = retained[:expected]
|
retained = retained[:expected]
|
||||||
@ -222,12 +282,13 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
newCount = upserted.Inserted
|
newCount = upserted.Inserted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if callErr == nil && parseErr == nil && candidateErr == nil {
|
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
|
||||||
capacityErr = permit.Complete(validCount, newCount)
|
class := upstream.ClassifyFetchResult(
|
||||||
permitFinished = capacityErr == nil
|
callErr,
|
||||||
}
|
errors.Join(parseErr, candidateErr, coordinationErr),
|
||||||
resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
|
validCount,
|
||||||
class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
|
newCount,
|
||||||
|
)
|
||||||
return response, Result{
|
return response, Result{
|
||||||
UpstreamID: r.config.UpstreamID,
|
UpstreamID: r.config.UpstreamID,
|
||||||
Class: class,
|
Class: class,
|
||||||
@ -238,6 +299,15 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
}, isRetryable(callErr) || parseErr != nil, true
|
}, isRetryable(callErr) || parseErr != nil, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if complete {
|
||||||
|
return permit.Complete(ctx, fetched)
|
||||||
|
}
|
||||||
|
return permit.Cancel(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
func isRetryable(err error) bool {
|
func isRetryable(err error) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
controllerPool "proxy-pool/internal/controller/pool"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
"proxy-pool/internal/domain/upstream"
|
"proxy-pool/internal/domain/upstream"
|
||||||
@ -89,7 +90,7 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case got := <-result:
|
case got := <-result:
|
||||||
@ -130,7 +131,7 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
@ -183,7 +184,7 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
|
|
||||||
wantClasses := []upstream.FetchClass{
|
wantClasses := []upstream.FetchClass{
|
||||||
@ -291,7 +292,7 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
<-results
|
<-results
|
||||||
@ -332,7 +333,7 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
<-results
|
<-results
|
||||||
@ -417,7 +418,7 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
|
|
||||||
@ -473,8 +474,8 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 2)
|
done := make(chan error, 2)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
@ -519,7 +520,7 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
@ -545,6 +546,7 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|||||||
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
||||||
sleeper := &fakeSleeper{clock: clock}
|
sleeper := &fakeSleeper{clock: clock}
|
||||||
results := make(chan Result, 3)
|
results := make(chan Result, 3)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
var calls atomic.Int64
|
var calls atomic.Int64
|
||||||
ports := successfulPorts(func() { calls.Add(1) }, results)
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||||
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
@ -562,13 +564,20 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := runSingleReconcile(t, reconciler, results)
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||||
|
func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
},
|
||||||
|
))
|
||||||
if result.Class != upstream.FetchError {
|
if result.Class != upstream.FetchError {
|
||||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
}
|
}
|
||||||
if got := calls.Load(); got != 1 {
|
if got := calls.Load(); got != 1 {
|
||||||
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
||||||
}
|
}
|
||||||
|
if got := <-globalCompleted; got != 1 {
|
||||||
|
t.Fatalf("global charged count = %d, want fetched=1", got)
|
||||||
|
}
|
||||||
if got := sleeper.Durations(); len(got) != 0 {
|
if got := sleeper.Durations(); len(got) != 0 {
|
||||||
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
||||||
}
|
}
|
||||||
@ -623,7 +632,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
time.Sleep(20 * time.Millisecond)
|
time.Sleep(20 * time.Millisecond)
|
||||||
cancel()
|
cancel()
|
||||||
@ -640,13 +649,116 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T) {
|
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
|
||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
completed := make(chan fetchCompletion, 1)
|
var calls atomic.Int64
|
||||||
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||||
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
|
return &recordingFetchPermit{expected: 2}, true, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
if expected != 2 {
|
||||||
|
t.Errorf("distributed expected = %d, want 2", expected)
|
||||||
|
}
|
||||||
|
return nil, false, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||||
|
reconciler.Notify()
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("RunLeader(): %v", err)
|
||||||
|
}
|
||||||
|
if got := calls.Load(); got != 0 {
|
||||||
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case result := <-results:
|
||||||
|
t.Fatalf("unexpected fetch result: %+v", result)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
|
localCancelled := make(chan struct{}, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
return nil, errors.New("invalid provider payload")
|
||||||
})
|
})
|
||||||
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
|
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
result := runSingleReconcile(t, reconciler, results, session)
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
|
}
|
||||||
|
if got := <-globalCompleted; got != 2 {
|
||||||
|
t.Fatalf("global charged count = %d, want expected=2", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-localCancelled:
|
||||||
|
default:
|
||||||
|
t.Fatal("local pool reservation was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
return FetchResponse{}, errors.New("provider connection failed")
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
result := runSingleReconcile(t, reconciler, results, session)
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
|
}
|
||||||
|
if got := <-globalCompleted; got <= 0 {
|
||||||
|
t.Fatalf("global charged count = %d, want conservative expected count", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
localCompleted := make(chan fetchCompletion, 1)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
|
released := make(chan []proxyDomain.Proxy, 1)
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Parser = &recordingCandidateParser{
|
||||||
|
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
|
||||||
|
released: released,
|
||||||
|
}
|
||||||
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
if len(batch.Proxies) != 2 {
|
if len(batch.Proxies) != 2 {
|
||||||
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
||||||
@ -654,7 +766,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
|||||||
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
||||||
})
|
})
|
||||||
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
|
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
|
||||||
})
|
})
|
||||||
reconciler, err := NewReconciler(Config{
|
reconciler, err := NewReconciler(Config{
|
||||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
@ -663,12 +775,30 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := runSingleReconcile(t, reconciler, results)
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||||
|
func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
if expected != 2 {
|
||||||
|
t.Errorf("distributed expected = %d, want 2", expected)
|
||||||
|
}
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
},
|
||||||
|
))
|
||||||
if result.ValidCount != 3 || result.NewCount != 1 {
|
if result.ValidCount != 3 || result.NewCount != 1 {
|
||||||
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
||||||
}
|
}
|
||||||
if got := <-completed; got.fetched != 3 || got.retained != 1 {
|
if got := <-globalCompleted; got != 3 {
|
||||||
t.Fatalf("fetch completion = %+v, want fetched=3 retained=1", got)
|
t.Fatalf("global fetch completion = %d, want fetched=3", got)
|
||||||
|
}
|
||||||
|
if got := <-localCompleted; got.retained != 1 {
|
||||||
|
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case got := <-released:
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("parsed candidates were not released")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -699,7 +829,7 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
first, second := <-results, <-results
|
first, second := <-results, <-results
|
||||||
cancel()
|
cancel()
|
||||||
@ -717,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
|
||||||
|
results := make(chan Result, 2)
|
||||||
|
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
|
||||||
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
|
}
|
||||||
|
var parses atomic.Int64
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Capacity = budget
|
||||||
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
if parses.Add(1) == 1 {
|
||||||
|
return nil, errors.New("temporary parser failure")
|
||||||
|
}
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
|
||||||
|
MaxInFlight: 1, MaxSize: 10,
|
||||||
|
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
|
||||||
|
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
|
||||||
|
if got := budget.Snapshot().PendingExpected; got != 1 {
|
||||||
|
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
reconciler.Notify()
|
||||||
|
<-results
|
||||||
|
<-results
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("RunLeader(): %v", err)
|
||||||
|
}
|
||||||
|
usage := budget.Snapshot()
|
||||||
|
if usage.PendingExpected != 0 || usage.Managed != 1 {
|
||||||
|
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
@ -744,11 +921,23 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func runSingleReconcile(t *testing.T, reconciler *Reconciler, results <-chan Result) Result {
|
func runSingleReconcile(
|
||||||
|
t *testing.T,
|
||||||
|
reconciler *Reconciler,
|
||||||
|
results <-chan Result,
|
||||||
|
sessions ...LeaderSession,
|
||||||
|
) Result {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
session := LeaderSession(unlimitedLeaderSession{})
|
||||||
|
if len(sessions) > 1 {
|
||||||
|
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
|
||||||
|
}
|
||||||
|
if len(sessions) == 1 {
|
||||||
|
session = sessions[0]
|
||||||
|
}
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
var result Result
|
var result Result
|
||||||
select {
|
select {
|
||||||
@ -821,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
|
|||||||
return errors.New("unexpected sleep")
|
return errors.New("unexpected sleep")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type sleeperFunc func(context.Context, time.Duration) error
|
||||||
|
|
||||||
|
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
|
||||||
|
return f(ctx, duration)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -848,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
|
|||||||
return f(ctx, body)
|
return f(ctx, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
|
||||||
|
|
||||||
|
type recordingCandidateParser struct {
|
||||||
|
candidates []proxyDomain.Proxy
|
||||||
|
released chan<- []proxyDomain.Proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||||
|
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
|
||||||
|
}
|
||||||
|
|
||||||
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
||||||
|
|
||||||
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
@ -867,26 +1077,66 @@ func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool,
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fetchCompletion struct {
|
type fetchCompletion struct {
|
||||||
fetched int
|
|
||||||
retained int
|
retained int
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingFetchPermit struct {
|
type recordingFetchPermit struct {
|
||||||
expected int
|
expected int
|
||||||
completed chan<- fetchCompletion
|
completed chan<- fetchCompletion
|
||||||
|
cancelled chan<- struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
||||||
|
|
||||||
func (p *recordingFetchPermit) Complete(fetched, retained int) error {
|
func (p *recordingFetchPermit) Complete(retained int) error {
|
||||||
if p.completed != nil {
|
if p.completed != nil {
|
||||||
p.completed <- fetchCompletion{fetched: fetched, retained: retained}
|
p.completed <- fetchCompletion{retained: retained}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*recordingFetchPermit) Cancel() error { return nil }
|
func (p *recordingFetchPermit) Cancel() error {
|
||||||
|
if p.cancelled != nil {
|
||||||
|
p.cancelled <- struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type resultRecorderFunc func(Result)
|
type resultRecorderFunc func(Result)
|
||||||
|
|
||||||
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
||||||
|
|
||||||
|
type unlimitedLeaderSession struct{}
|
||||||
|
|
||||||
|
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||||
|
|
||||||
|
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
|
||||||
|
|
||||||
|
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||||
|
|
||||||
|
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
return f(ctx, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingRequestPermit struct {
|
||||||
|
completed chan<- int
|
||||||
|
cancelled chan<- struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
|
||||||
|
if p.completed != nil {
|
||||||
|
p.completed <- fetched
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingRequestPermit) Cancel(context.Context) error {
|
||||||
|
if p.cancelled != nil {
|
||||||
|
p.cancelled <- struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
103
internal/controller/provider/stats.go
Normal file
103
internal/controller/provider/stats.go
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
|
||||||
|
|
||||||
|
type Stats struct {
|
||||||
|
UpstreamID string
|
||||||
|
ConsecutiveEmptyFetch int64
|
||||||
|
FetchErrorCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsReader interface {
|
||||||
|
ReadProviderStats([]string) []Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsRetainer interface {
|
||||||
|
RetainProviderStats([]string)
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsRecorder struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
maximum int
|
||||||
|
byID map[string]Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
|
||||||
|
if maximum <= 0 {
|
||||||
|
return nil, ErrInvalidStatsRecorder
|
||||||
|
}
|
||||||
|
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (recorder *StatsRecorder) Record(result Result) {
|
||||||
|
if recorder == nil || result.UpstreamID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
stats, exists := recorder.byID[result.UpstreamID]
|
||||||
|
if !exists {
|
||||||
|
if len(recorder.byID) >= recorder.maximum {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stats.UpstreamID = result.UpstreamID
|
||||||
|
}
|
||||||
|
switch result.Class {
|
||||||
|
case upstream.FetchEmpty:
|
||||||
|
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
|
||||||
|
stats.ConsecutiveEmptyFetch++
|
||||||
|
}
|
||||||
|
case upstream.FetchValid, upstream.FetchDuplicateOnly:
|
||||||
|
stats.ConsecutiveEmptyFetch = 0
|
||||||
|
case upstream.FetchError:
|
||||||
|
if stats.FetchErrorCount < math.MaxInt64 {
|
||||||
|
stats.FetchErrorCount++
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
recorder.byID[result.UpstreamID] = stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
|
||||||
|
result := make([]Stats, len(upstreamIDs))
|
||||||
|
if recorder == nil {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
for index, upstreamID := range upstreamIDs {
|
||||||
|
result[index] = recorder.byID[upstreamID]
|
||||||
|
result[index].UpstreamID = upstreamID
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetainProviderStats removes observations for upstreams no longer present in
|
||||||
|
// the complete configuration. Disabled but configured upstreams must be kept.
|
||||||
|
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
|
||||||
|
if recorder == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
retained := make(map[string]struct{}, len(upstreamIDs))
|
||||||
|
for _, upstreamID := range upstreamIDs {
|
||||||
|
if upstreamID != "" {
|
||||||
|
retained[upstreamID] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
for upstreamID := range recorder.byID {
|
||||||
|
if _, keep := retained[upstreamID]; !keep {
|
||||||
|
delete(recorder.byID, upstreamID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
83
internal/controller/provider/stats_test.go
Normal file
83
internal/controller/provider/stats_test.go
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
for _, class := range []upstream.FetchClass{
|
||||||
|
upstream.FetchEmpty,
|
||||||
|
upstream.FetchEmpty,
|
||||||
|
upstream.FetchError,
|
||||||
|
} {
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
|
||||||
|
}
|
||||||
|
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
|
||||||
|
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
|
||||||
|
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
|
||||||
|
}
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
|
||||||
|
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
|
||||||
|
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
const workers = 100
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for range workers {
|
||||||
|
wait.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
|
||||||
|
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
const workers = 100
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for range workers {
|
||||||
|
wait.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
|
||||||
|
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
|
||||||
|
t.Fatalf("stats after retention = %+v", stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
219
internal/controller/provider/upstream_runtime.go
Normal file
219
internal/controller/provider/upstream_runtime.go
Normal file
@ -0,0 +1,219 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"hash/fnv"
|
||||||
|
"reflect"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
|
||||||
|
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
|
||||||
|
)
|
||||||
|
|
||||||
|
type UpstreamRuntimeConfig struct {
|
||||||
|
Provider Config
|
||||||
|
ReconcilePolicy pool.ReconcilePolicy
|
||||||
|
ReconcileInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type UpstreamRuntimeDependencies struct {
|
||||||
|
Coordinator Coordinator
|
||||||
|
Inventory pool.InventoryReader
|
||||||
|
Adapter ProviderAdapter
|
||||||
|
Parser Parser
|
||||||
|
Activity activitypool.Upserter
|
||||||
|
Results ResultRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
|
||||||
|
// local budget and coalescing signal are created for each leadership term.
|
||||||
|
type UpstreamRuntime struct {
|
||||||
|
config UpstreamRuntimeConfig
|
||||||
|
dependencies UpstreamRuntimeDependencies
|
||||||
|
sleeper Sleeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewUpstreamRuntime(
|
||||||
|
config UpstreamRuntimeConfig,
|
||||||
|
dependencies UpstreamRuntimeDependencies,
|
||||||
|
) (*UpstreamRuntime, error) {
|
||||||
|
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
|
||||||
|
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
|
||||||
|
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
|
||||||
|
nilRuntimeDependency(dependencies.Results) {
|
||||||
|
return nil, ErrInvalidUpstreamRuntime
|
||||||
|
}
|
||||||
|
runtime := &UpstreamRuntime{
|
||||||
|
config: config, dependencies: dependencies, sleeper: timerSleeper{},
|
||||||
|
}
|
||||||
|
if _, err := runtime.newLeaderTerm(); err != nil {
|
||||||
|
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
|
||||||
|
}
|
||||||
|
return runtime, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
|
||||||
|
if runtime == nil || ctx == nil {
|
||||||
|
return ErrInvalidUpstreamRuntime
|
||||||
|
}
|
||||||
|
limits := CoordinationLimits{
|
||||||
|
RequestInterval: runtime.config.Provider.RequestInterval,
|
||||||
|
MaxInFlight: runtime.config.Provider.MaxInFlight,
|
||||||
|
MaxAttemptDuration: runtime.config.Provider.Timeout,
|
||||||
|
MaxTotal: runtime.config.Provider.MaxTotal,
|
||||||
|
}
|
||||||
|
err := runtime.dependencies.Coordinator.RunLeader(
|
||||||
|
ctx,
|
||||||
|
runtime.config.Provider.UpstreamID,
|
||||||
|
limits,
|
||||||
|
func(leaderCtx context.Context, session LeaderSession) error {
|
||||||
|
term, buildErr := runtime.newLeaderTerm()
|
||||||
|
if buildErr != nil {
|
||||||
|
return buildErr
|
||||||
|
}
|
||||||
|
return term.run(leaderCtx, session)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
return ErrUpstreamRuntimeStopped
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) ID() string {
|
||||||
|
if runtime == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return runtime.config.Provider.UpstreamID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
|
||||||
|
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
|
||||||
|
UpstreamID: runtime.config.Provider.UpstreamID,
|
||||||
|
MaxSize: runtime.config.Provider.MaxSize,
|
||||||
|
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
providerConfig := runtime.config.Provider
|
||||||
|
providerConfig.RequestInterval = 0
|
||||||
|
providerReconciler, err := NewReconciler(providerConfig, Ports{
|
||||||
|
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
|
||||||
|
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
|
||||||
|
Capacity: budget,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
poolReconciler, err := pool.NewReconciler(
|
||||||
|
runtime.config.ReconcilePolicy,
|
||||||
|
budget,
|
||||||
|
providerReconciler,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &upstreamLeaderTerm{
|
||||||
|
upstreamID: runtime.config.Provider.UpstreamID,
|
||||||
|
interval: runtime.config.ReconcileInterval,
|
||||||
|
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
|
||||||
|
inventory: runtime.dependencies.Inventory,
|
||||||
|
pool: poolReconciler,
|
||||||
|
provider: providerReconciler,
|
||||||
|
sleeper: runtime.sleeper,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type upstreamLeaderTerm struct {
|
||||||
|
upstreamID string
|
||||||
|
interval time.Duration
|
||||||
|
safetyMargin time.Duration
|
||||||
|
inventory pool.InventoryReader
|
||||||
|
pool *pool.Reconciler
|
||||||
|
provider *Reconciler
|
||||||
|
sleeper Sleeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
|
||||||
|
termCtx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 2)
|
||||||
|
go func() { done <- term.provider.RunLeader(termCtx, session) }()
|
||||||
|
go func() { done <- term.reconcileInventory(termCtx) }()
|
||||||
|
|
||||||
|
first := <-done
|
||||||
|
cancel()
|
||||||
|
second := <-done
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
first = ErrUpstreamRuntimeStopped
|
||||||
|
}
|
||||||
|
if second != nil && !errors.Is(second, context.Canceled) {
|
||||||
|
return errors.Join(first, second)
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
|
||||||
|
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
|
||||||
|
if err := term.sleeper.Sleep(ctx, delay); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
inventory, err := term.inventory.ReadInventory(
|
||||||
|
ctx,
|
||||||
|
term.upstreamID,
|
||||||
|
term.safetyMargin,
|
||||||
|
)
|
||||||
|
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
|
||||||
|
term.pool.ReconcileSnapshot(inventory)
|
||||||
|
}
|
||||||
|
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
|
||||||
|
window := min(interval/4, 250*time.Millisecond)
|
||||||
|
if window <= 1 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
digest := fnv.New64a()
|
||||||
|
_, _ = digest.Write([]byte(upstreamID))
|
||||||
|
return time.Duration(digest.Sum64() % uint64(window))
|
||||||
|
}
|
||||||
|
|
||||||
|
func nilRuntimeDependency(value any) bool {
|
||||||
|
if value == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
reflected := reflect.ValueOf(value)
|
||||||
|
switch reflected.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return reflected.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
247
internal/controller/provider/upstream_runtime_test.go
Normal file
247
internal/controller/provider/upstream_runtime_test.go
Normal file
@ -0,0 +1,247 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
|
||||||
|
var inventoryReads atomic.Int64
|
||||||
|
fetched := make(chan struct{}, 1)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 10 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
|
||||||
|
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
|
||||||
|
}
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
inventoryReads.Add(1)
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for leader fetch")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if got := inventoryReads.Load(); got == 0 {
|
||||||
|
t.Fatal("inventory was not read inside leader term")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
|
||||||
|
var inventoryReads atomic.Int64
|
||||||
|
var providerCalls atomic.Int64
|
||||||
|
fetched := make(chan struct{}, 1)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 10 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
_ CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
if inventoryReads.Add(1) == 1 {
|
||||||
|
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
|
||||||
|
}
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
providerCalls.Add(1)
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for recovered inventory fetch")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
if got := inventoryReads.Load(); got < 2 {
|
||||||
|
t.Fatalf("inventory reads = %d, want recovery retry", got)
|
||||||
|
}
|
||||||
|
if got := providerCalls.Load(); got != 1 {
|
||||||
|
t.Fatalf("provider calls = %d, want one call after recovery", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
|
||||||
|
providerConfig := runtimeProviderConfig("provider-a")
|
||||||
|
providerConfig.RequestInterval = 500 * time.Millisecond
|
||||||
|
fetched := make(chan struct{}, 2)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: providerConfig,
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 5 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
if limits.RequestInterval != 500*time.Millisecond {
|
||||||
|
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
|
||||||
|
}
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
for range 2 {
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(150 * time.Millisecond):
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
t.Fatal("local Provider reconciler duplicated the distributed request interval")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
|
||||||
|
validConfig := UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: time.Second,
|
||||||
|
}
|
||||||
|
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
|
||||||
|
const interval = time.Second
|
||||||
|
first := initialReconcileDelay("provider-a", interval)
|
||||||
|
if first != initialReconcileDelay("provider-a", interval) {
|
||||||
|
t.Fatal("initial reconcile delay is not stable")
|
||||||
|
}
|
||||||
|
if first < 0 || first >= 250*time.Millisecond {
|
||||||
|
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
|
||||||
|
}
|
||||||
|
if other := initialReconcileDelay("provider-b", interval); other == first {
|
||||||
|
t.Fatalf("different upstreams have the same initial delay %s", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runtimeProviderConfig(upstreamID string) Config {
|
||||||
|
return Config{
|
||||||
|
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
|
||||||
|
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runtimeReconcilePolicy() pool.ReconcilePolicy {
|
||||||
|
return pool.ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
||||||
|
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorFunc func(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
CoordinationLimits,
|
||||||
|
func(context.Context, LeaderSession) error,
|
||||||
|
) error
|
||||||
|
|
||||||
|
func (f coordinatorFunc) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return f(ctx, upstreamID, limits, work)
|
||||||
|
}
|
||||||
|
|
||||||
|
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
|
||||||
|
|
||||||
|
func (f inventoryReaderFunc) ReadInventory(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
safetyMargin time.Duration,
|
||||||
|
) (pool.InventorySnapshot, error) {
|
||||||
|
return f(ctx, upstreamID, safetyMargin)
|
||||||
|
}
|
||||||
@ -15,17 +15,44 @@ var (
|
|||||||
type Capacity struct {
|
type Capacity struct {
|
||||||
max atomic.Uint32
|
max atomic.Uint32
|
||||||
counters atomic.Uint64
|
counters atomic.Uint64
|
||||||
|
configuredObserver *activityObserver
|
||||||
|
observer atomic.Pointer[activityObserver]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type activityObserver struct{ notify func(bool) }
|
||||||
|
|
||||||
func NewCapacity(max int64) *Capacity {
|
func NewCapacity(max int64) *Capacity {
|
||||||
|
return NewCapacityWithActivityObserver(max, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCapacityWithActivityObserver reports successful zero-to-nonzero and
|
||||||
|
// nonzero-to-zero transitions. The observer must tolerate concurrent calls.
|
||||||
|
func NewCapacityWithActivityObserver(max int64, observer func(nonzero bool)) *Capacity {
|
||||||
capacity := &Capacity{}
|
capacity := &Capacity{}
|
||||||
if max < 0 || max > int64(counterMask) {
|
if max < 0 || max > int64(counterMask) {
|
||||||
max = 0
|
max = 0
|
||||||
}
|
}
|
||||||
capacity.max.Store(uint32(max))
|
capacity.max.Store(uint32(max))
|
||||||
|
if observer != nil {
|
||||||
|
capacity.configuredObserver = &activityObserver{notify: observer}
|
||||||
|
capacity.observer.Store(capacity.configuredObserver)
|
||||||
|
}
|
||||||
return capacity
|
return capacity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetActivityObservationEnabled lets snapshot ownership disable callbacks for
|
||||||
|
// current Proxies and enable them only while a runtime is retired and draining.
|
||||||
|
func (c *Capacity) SetActivityObservationEnabled(enabled bool) {
|
||||||
|
if c == nil || c.configuredObserver == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if enabled {
|
||||||
|
c.observer.Store(c.configuredObserver)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.observer.Store(nil)
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Capacity) SetMax(max int64) bool {
|
func (c *Capacity) SetMax(max int64) bool {
|
||||||
if max < 0 || max > int64(counterMask) {
|
if max < 0 || max > int64(counterMask) {
|
||||||
return false
|
return false
|
||||||
@ -36,6 +63,15 @@ func (c *Capacity) SetMax(max int64) bool {
|
|||||||
|
|
||||||
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
|
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
|
||||||
|
|
||||||
|
func (c *Capacity) Counters() (active, reserved, maximum int64) {
|
||||||
|
if c == nil {
|
||||||
|
return 0, 0, 0
|
||||||
|
}
|
||||||
|
packed := c.counters.Load()
|
||||||
|
activeCounter, reservedCounter := unpack(packed)
|
||||||
|
return int64(activeCounter), int64(reservedCounter), int64(c.max.Load())
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Capacity) Reserve() (*Reservation, bool) {
|
func (c *Capacity) Reserve() (*Reservation, bool) {
|
||||||
for {
|
for {
|
||||||
current := c.counters.Load()
|
current := c.counters.Load()
|
||||||
@ -45,6 +81,11 @@ func (c *Capacity) Reserve() (*Reservation, bool) {
|
|||||||
}
|
}
|
||||||
next := pack(active, reserved+1)
|
next := pack(active, reserved+1)
|
||||||
if c.counters.CompareAndSwap(current, next) {
|
if c.counters.CompareAndSwap(current, next) {
|
||||||
|
if active+reserved == 0 {
|
||||||
|
if observer := c.observer.Load(); observer != nil {
|
||||||
|
observer.notify(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
return &Reservation{capacity: c}, true
|
return &Reservation{capacity: c}, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -77,7 +118,16 @@ func (c *Capacity) cancel() {
|
|||||||
for {
|
for {
|
||||||
current := c.counters.Load()
|
current := c.counters.Load()
|
||||||
active, reserved := unpack(current)
|
active, reserved := unpack(current)
|
||||||
if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) {
|
if reserved == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next := pack(active, reserved-1)
|
||||||
|
if c.counters.CompareAndSwap(current, next) {
|
||||||
|
if active+reserved == 1 {
|
||||||
|
if observer := c.observer.Load(); observer != nil {
|
||||||
|
observer.notify(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -87,7 +137,16 @@ func (c *Capacity) release() {
|
|||||||
for {
|
for {
|
||||||
current := c.counters.Load()
|
current := c.counters.Load()
|
||||||
active, reserved := unpack(current)
|
active, reserved := unpack(current)
|
||||||
if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) {
|
if active == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next := pack(active-1, reserved)
|
||||||
|
if c.counters.CompareAndSwap(current, next) {
|
||||||
|
if active+reserved == 1 {
|
||||||
|
if observer := c.observer.Load(); observer != nil {
|
||||||
|
observer.notify(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package proxy
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"reflect"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@ -38,6 +39,56 @@ func TestReservationCancelReleasesReservedCapacity(t *testing.T) {
|
|||||||
assertCapacityCounters(t, capacity, 0, 0)
|
assertCapacityCounters(t, capacity, 0, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCapacityCountersReadsOnePackedSnapshot(t *testing.T) {
|
||||||
|
capacity := NewCapacity(3)
|
||||||
|
first, ok := capacity.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("first Reserve() = false")
|
||||||
|
}
|
||||||
|
second, ok := capacity.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("second Reserve() = false")
|
||||||
|
}
|
||||||
|
if err := first.Commit(); err != nil {
|
||||||
|
t.Fatalf("Commit(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
active, reserved, maximum := capacity.Counters()
|
||||||
|
if active != 1 || reserved != 1 || maximum != 3 {
|
||||||
|
t.Fatalf("Counters() = (%d, %d, %d), want (1, 1, 3)", active, reserved, maximum)
|
||||||
|
}
|
||||||
|
if err := first.Release(); err != nil {
|
||||||
|
t.Fatalf("Release(): %v", err)
|
||||||
|
}
|
||||||
|
if err := second.Cancel(); err != nil {
|
||||||
|
t.Fatalf("Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCapacityActivityObserverTracksOnlyNonzeroTransitions(t *testing.T) {
|
||||||
|
var transitions []bool
|
||||||
|
capacity := NewCapacityWithActivityObserver(2, func(nonzero bool) {
|
||||||
|
transitions = append(transitions, nonzero)
|
||||||
|
})
|
||||||
|
first, ok := capacity.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Reserve(first) = false")
|
||||||
|
}
|
||||||
|
second, ok := capacity.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Reserve(second) = false")
|
||||||
|
}
|
||||||
|
if err := first.Cancel(); err != nil {
|
||||||
|
t.Fatalf("Cancel(first): %v", err)
|
||||||
|
}
|
||||||
|
if err := second.Cancel(); err != nil {
|
||||||
|
t.Fatalf("Cancel(second): %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(transitions, []bool{true, false}) {
|
||||||
|
t.Fatalf("transitions = %v, want [true false]", transitions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestReservationCommitAndReleaseAreSingleUse(t *testing.T) {
|
func TestReservationCommitAndReleaseAreSingleUse(t *testing.T) {
|
||||||
capacity := NewCapacity(1)
|
capacity := NewCapacity(1)
|
||||||
reservation, ok := capacity.Reserve()
|
reservation, ok := capacity.Reserve()
|
||||||
|
|||||||
@ -8,6 +8,6 @@ type FetchCapacity interface {
|
|||||||
|
|
||||||
type FetchPermit interface {
|
type FetchPermit interface {
|
||||||
Expected() int
|
Expected() int
|
||||||
Complete(fetched, retained int) error
|
Complete(retained int) error
|
||||||
Cancel() error
|
Cancel() error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,9 +17,7 @@ type ProxyCapacity struct {
|
|||||||
type Inventory struct {
|
type Inventory struct {
|
||||||
Proxies []ProxyCapacity
|
Proxies []ProxyCapacity
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
MaxSize int
|
MaxSize int
|
||||||
MaxTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
|
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
|
||||||
@ -55,15 +53,6 @@ func (i Inventory) FetchAllowance(requested int) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
|
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
|
||||||
if i.MaxTotal > 0 {
|
|
||||||
remaining := i.MaxTotal - i.FetchedTotal
|
|
||||||
if remaining <= 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if int64(allowed) > remaining {
|
|
||||||
allowed = int(remaining)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return allowed
|
return allowed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -21,7 +21,7 @@ func TestInventoryAvailableSlotsUsesOnlyAllocatableCapacity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
func TestInventoryFetchAllowanceUsesManagedPoolCapacity(t *testing.T) {
|
||||||
inventory := Inventory{
|
inventory := Inventory{
|
||||||
Proxies: []ProxyCapacity{
|
Proxies: []ProxyCapacity{
|
||||||
{State: proxyDomain.StateFetched},
|
{State: proxyDomain.StateFetched},
|
||||||
@ -32,20 +32,13 @@ func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
|||||||
{State: proxyDomain.StateExtracted},
|
{State: proxyDomain.StateExtracted},
|
||||||
},
|
},
|
||||||
PendingExpected: 2,
|
PendingExpected: 2,
|
||||||
FetchedTotal: 98,
|
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 100,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := inventory.ManagedCount(); got != 7 {
|
if got := inventory.ManagedCount(); got != 7 {
|
||||||
t.Fatalf("ManagedCount() = %d, want 7", got)
|
t.Fatalf("ManagedCount() = %d, want 7", got)
|
||||||
}
|
}
|
||||||
if got := inventory.FetchAllowance(10); got != 2 {
|
|
||||||
t.Fatalf("FetchAllowance() = %d, want 2 from cumulative quota", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
inventory.MaxTotal = 0
|
|
||||||
if got := inventory.FetchAllowance(10); got != 3 {
|
if got := inventory.FetchAllowance(10); got != 3 {
|
||||||
t.Fatalf("FetchAllowance() with unlimited cumulative quota = %d, want 3 from pool size", got)
|
t.Fatalf("FetchAllowance() = %d, want 3 from pool size", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
211
internal/domain/workerruntime/memory.go
Normal file
211
internal/domain/workerruntime/memory.go
Normal file
@ -0,0 +1,211 @@
|
|||||||
|
package workerruntime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MemoryStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
now func() time.Time
|
||||||
|
sessions map[string]memorySession
|
||||||
|
reports map[string]memoryReport
|
||||||
|
}
|
||||||
|
|
||||||
|
type memorySession struct {
|
||||||
|
value Session
|
||||||
|
expiresAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type memoryReport struct {
|
||||||
|
value Report
|
||||||
|
digest [sha256.Size]byte
|
||||||
|
expiresAt time.Time
|
||||||
|
counters map[string]Counter
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
_ SessionWriter = (*MemoryStore)(nil)
|
||||||
|
_ ReportWriter = (*MemoryStore)(nil)
|
||||||
|
_ RuntimeReader = (*MemoryStore)(nil)
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewMemoryStore(now func() time.Time) (*MemoryStore, error) {
|
||||||
|
if now == nil {
|
||||||
|
return nil, ErrInvalidStore
|
||||||
|
}
|
||||||
|
return &MemoryStore{
|
||||||
|
now: now, sessions: make(map[string]memorySession), reports: make(map[string]memoryReport),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *MemoryStore) ReplaceSession(ctx context.Context, session Session, ttl time.Duration) error {
|
||||||
|
if ctx == nil || store == nil || !validSession(session) || ttl <= 0 {
|
||||||
|
return ErrInvalidSession
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
now := store.now().UTC()
|
||||||
|
if now.IsZero() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
store.mu.Lock()
|
||||||
|
defer store.mu.Unlock()
|
||||||
|
current, exists := store.sessions[session.WorkerID]
|
||||||
|
identityChanged := exists && (current.value.SessionID != session.SessionID || current.value.InstanceID != session.InstanceID)
|
||||||
|
expired := exists && !current.expiresAt.After(now)
|
||||||
|
if exists && !identityChanged && !expired && sessionBefore(session, current.value) {
|
||||||
|
return ErrStaleSession
|
||||||
|
}
|
||||||
|
ackAdvanced := exists && !identityChanged && !expired && sessionAfter(session, current.value)
|
||||||
|
if identityChanged || expired || ackAdvanced {
|
||||||
|
delete(store.reports, session.WorkerID)
|
||||||
|
}
|
||||||
|
store.sessions[session.WorkerID] = memorySession{value: session, expiresAt: now.Add(ttl)}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *MemoryStore) ReplaceRuntime(ctx context.Context, report Report, ttl time.Duration) error {
|
||||||
|
if ctx == nil || store == nil || ttl <= 0 {
|
||||||
|
return ErrInvalidReport
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
normalized, counterIndex, err := normalizeReport(report)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(normalized)
|
||||||
|
if err != nil {
|
||||||
|
return ErrInvalidReport
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(payload)
|
||||||
|
now := store.now().UTC()
|
||||||
|
if now.IsZero() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
store.mu.Lock()
|
||||||
|
defer store.mu.Unlock()
|
||||||
|
session, exists := store.sessions[report.WorkerID]
|
||||||
|
if !exists || !session.expiresAt.After(now) || session.value.SessionID != report.SessionID {
|
||||||
|
return ErrStaleSession
|
||||||
|
}
|
||||||
|
if report.SnapshotVersion != session.value.AckedSnapshotVersion ||
|
||||||
|
report.OwnershipEpoch != session.value.AckedOwnershipEpoch {
|
||||||
|
return ErrStaleReport
|
||||||
|
}
|
||||||
|
if current, exists := store.reports[report.WorkerID]; exists && current.value.SessionID == report.SessionID {
|
||||||
|
switch {
|
||||||
|
case normalized.Sequence < current.value.Sequence:
|
||||||
|
return ErrStaleReport
|
||||||
|
case normalized.Sequence == current.value.Sequence && digest != current.digest:
|
||||||
|
return ErrConflictingReport
|
||||||
|
case normalized.Sequence == current.value.Sequence:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
store.reports[report.WorkerID] = memoryReport{
|
||||||
|
value: normalized, digest: digest, expiresAt: now.Add(ttl), counters: counterIndex,
|
||||||
|
}
|
||||||
|
session.expiresAt = now.Add(ttl)
|
||||||
|
store.sessions[report.WorkerID] = session
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *MemoryStore) ReadRuntime(ctx context.Context, proxies []OwnedProxy) ([]Snapshot, error) {
|
||||||
|
if ctx == nil || store == nil {
|
||||||
|
return nil, ErrInvalidQuery
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(proxies))
|
||||||
|
for _, proxy := range proxies {
|
||||||
|
if !clean(proxy.ProxyID) || !clean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
|
||||||
|
return nil, ErrInvalidQuery
|
||||||
|
}
|
||||||
|
key := proxy.WorkerID + "\x00" + proxy.ProxyID
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return nil, ErrInvalidQuery
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
now := store.now().UTC()
|
||||||
|
if now.IsZero() {
|
||||||
|
return nil, ErrInvalidStore
|
||||||
|
}
|
||||||
|
store.mu.Lock()
|
||||||
|
defer store.mu.Unlock()
|
||||||
|
result := make([]Snapshot, len(proxies))
|
||||||
|
for index, proxy := range proxies {
|
||||||
|
result[index].ProxyID = proxy.ProxyID
|
||||||
|
session, sessionExists := store.sessions[proxy.WorkerID]
|
||||||
|
report, reportExists := store.reports[proxy.WorkerID]
|
||||||
|
if !sessionExists || !reportExists || !session.expiresAt.After(now) || !report.expiresAt.After(now) ||
|
||||||
|
report.value.SessionID != session.value.SessionID ||
|
||||||
|
report.value.SnapshotVersion != session.value.AckedSnapshotVersion ||
|
||||||
|
report.value.OwnershipEpoch != session.value.AckedOwnershipEpoch ||
|
||||||
|
report.value.OwnershipEpoch < proxy.OwnershipEpoch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result[index].Fresh = true
|
||||||
|
if counter, exists := report.counters[proxy.ProxyID]; exists {
|
||||||
|
result[index].Active = counter.Active
|
||||||
|
result[index].Reserved = counter.Reserved
|
||||||
|
result[index].Draining = counter.Draining
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeReport(report Report) (Report, map[string]Counter, error) {
|
||||||
|
if !clean(report.WorkerID) || !clean(report.SessionID) || report.Sequence == 0 ||
|
||||||
|
report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() {
|
||||||
|
return Report{}, nil, ErrInvalidReport
|
||||||
|
}
|
||||||
|
normalized := report
|
||||||
|
normalized.ObservedAt = report.ObservedAt.UTC()
|
||||||
|
normalized.Counters = append([]Counter(nil), report.Counters...)
|
||||||
|
sort.Slice(normalized.Counters, func(left, right int) bool {
|
||||||
|
return normalized.Counters[left].ProxyID < normalized.Counters[right].ProxyID
|
||||||
|
})
|
||||||
|
index := make(map[string]Counter, len(normalized.Counters))
|
||||||
|
for _, counter := range normalized.Counters {
|
||||||
|
if !clean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 {
|
||||||
|
return Report{}, nil, ErrInvalidReport
|
||||||
|
}
|
||||||
|
if _, exists := index[counter.ProxyID]; exists {
|
||||||
|
return Report{}, nil, ErrInvalidReport
|
||||||
|
}
|
||||||
|
index[counter.ProxyID] = counter
|
||||||
|
}
|
||||||
|
return normalized, index, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validSession(session Session) bool {
|
||||||
|
return clean(session.WorkerID) && clean(session.InstanceID) && clean(session.SessionID) &&
|
||||||
|
session.AckedSnapshotVersion > 0 && session.AckedOwnershipEpoch > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func sessionBefore(left, right Session) bool {
|
||||||
|
return left.AckedOwnershipEpoch < right.AckedOwnershipEpoch ||
|
||||||
|
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
|
||||||
|
left.AckedSnapshotVersion < right.AckedSnapshotVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sessionAfter(left, right Session) bool {
|
||||||
|
return left.AckedOwnershipEpoch > right.AckedOwnershipEpoch ||
|
||||||
|
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
|
||||||
|
left.AckedSnapshotVersion > right.AckedSnapshotVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func clean(value string) bool {
|
||||||
|
return value != "" && strings.TrimSpace(value) == value
|
||||||
|
}
|
||||||
156
internal/domain/workerruntime/memory_test.go
Normal file
156
internal/domain/workerruntime/memory_test.go
Normal file
@ -0,0 +1,156 @@
|
|||||||
|
package workerruntime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
ctx := context.Background()
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||||
|
report := Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
Counters: []Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
|
||||||
|
}
|
||||||
|
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||||
|
}
|
||||||
|
got, err := store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
|
||||||
|
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true}) {
|
||||||
|
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
report.Sequence = 2
|
||||||
|
report.ObservedAt = now.Add(time.Second)
|
||||||
|
report.Counters = nil
|
||||||
|
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(empty): %v", err)
|
||||||
|
}
|
||||||
|
got, err = store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
|
||||||
|
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Fresh: true}) {
|
||||||
|
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreFencesSessionsAndReportSequence(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
ctx := context.Background()
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||||
|
report := Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
Counters: []Counter{{ProxyID: "proxy-a", Active: 1}},
|
||||||
|
}
|
||||||
|
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(idempotent): %v", err)
|
||||||
|
}
|
||||||
|
conflict := report
|
||||||
|
conflict.Counters = []Counter{{ProxyID: "proxy-a", Active: 2}}
|
||||||
|
if err := store.ReplaceRuntime(ctx, conflict, time.Minute); !errors.Is(err, ErrConflictingReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
|
||||||
|
}
|
||||||
|
stale := report
|
||||||
|
stale.Sequence = 1
|
||||||
|
if err := store.ReplaceRuntime(ctx, stale, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
|
||||||
|
}
|
||||||
|
registerRuntimeSession(t, store, "session-b", time.Minute)
|
||||||
|
if err := store.ReplaceRuntime(ctx, report, time.Minute); !errors.Is(err, ErrStaleSession) {
|
||||||
|
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreFailsClosedForExpiredOrOlderOwnershipReport(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
ctx := context.Background()
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||||
|
if err := store.ReplaceRuntime(ctx, Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
queries := []OwnedProxy{
|
||||||
|
{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 10},
|
||||||
|
{ProxyID: "proxy-b", WorkerID: "worker-a", OwnershipEpoch: 9},
|
||||||
|
}
|
||||||
|
got, err := store.ReadRuntime(ctx, queries)
|
||||||
|
if err != nil || got[0].Fresh || !got[1].Fresh {
|
||||||
|
t.Fatalf("ReadRuntime(ownership fence) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
now = now.Add(time.Minute)
|
||||||
|
got, err = store.ReadRuntime(ctx, queries[1:])
|
||||||
|
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||||
|
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreRejectsRuntimeBeyondAcknowledgedSnapshot(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||||
|
report := Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
}
|
||||||
|
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(ahead snapshot) error = %v, want ErrStaleReport", err)
|
||||||
|
}
|
||||||
|
report.SnapshotVersion = 3
|
||||||
|
report.OwnershipEpoch = 10
|
||||||
|
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||||
|
t.Fatalf("ReplaceRuntime(ahead epoch) error = %v, want ErrStaleReport", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreExpiredSameIdentitySessionDoesNotReactivateOldReport(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Second)
|
||||||
|
if err := store.ReplaceRuntime(context.Background(), Report{
|
||||||
|
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||||
|
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||||
|
}, time.Minute); err != nil {
|
||||||
|
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
session := store.sessions["worker-a"]
|
||||||
|
session.expiresAt = now.Add(time.Second)
|
||||||
|
store.sessions["worker-a"] = session
|
||||||
|
now = now.Add(2 * time.Second)
|
||||||
|
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||||
|
got, err := store.ReadRuntime(context.Background(), []OwnedProxy{{
|
||||||
|
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9,
|
||||||
|
}})
|
||||||
|
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||||
|
t.Fatalf("ReadRuntime(after re-register) = %+v, %v; want stale", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRuntimeStore(t *testing.T, now *time.Time) *MemoryStore {
|
||||||
|
t.Helper()
|
||||||
|
store, err := NewMemoryStore(func() time.Time { return *now })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
return store
|
||||||
|
}
|
||||||
|
|
||||||
|
func registerRuntimeSession(t *testing.T, store *MemoryStore, sessionID string, ttl time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
if err := store.ReplaceSession(context.Background(), Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: sessionID,
|
||||||
|
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
|
||||||
|
}, ttl); err != nil {
|
||||||
|
t.Fatalf("ReplaceSession(): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
70
internal/domain/workerruntime/runtime.go
Normal file
70
internal/domain/workerruntime/runtime.go
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
package workerruntime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidStore = errors.New("invalid worker runtime store")
|
||||||
|
ErrInvalidSession = errors.New("invalid worker runtime session")
|
||||||
|
ErrInvalidReport = errors.New("invalid worker runtime report")
|
||||||
|
ErrInvalidQuery = errors.New("invalid worker runtime query")
|
||||||
|
ErrStaleSession = errors.New("stale worker runtime session")
|
||||||
|
ErrStaleReport = errors.New("stale worker runtime report")
|
||||||
|
ErrConflictingReport = errors.New("conflicting worker runtime report")
|
||||||
|
)
|
||||||
|
|
||||||
|
type Session struct {
|
||||||
|
WorkerID string
|
||||||
|
InstanceID string
|
||||||
|
SessionID string
|
||||||
|
AckedSnapshotVersion uint64
|
||||||
|
AckedOwnershipEpoch uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
type Counter struct {
|
||||||
|
ProxyID string
|
||||||
|
Active int64
|
||||||
|
Reserved int64
|
||||||
|
Draining bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Report is a complete sparse replacement. Missing counters are zero for the
|
||||||
|
// reported Worker snapshot; callers must increase Sequence for every update.
|
||||||
|
type Report struct {
|
||||||
|
WorkerID string
|
||||||
|
SessionID string
|
||||||
|
Sequence uint64
|
||||||
|
SnapshotVersion uint64
|
||||||
|
OwnershipEpoch uint64
|
||||||
|
ObservedAt time.Time
|
||||||
|
Counters []Counter
|
||||||
|
}
|
||||||
|
|
||||||
|
type OwnedProxy struct {
|
||||||
|
ProxyID string
|
||||||
|
WorkerID string
|
||||||
|
OwnershipEpoch uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
ProxyID string
|
||||||
|
Active int64
|
||||||
|
Reserved int64
|
||||||
|
Draining bool
|
||||||
|
Fresh bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type SessionWriter interface {
|
||||||
|
ReplaceSession(context.Context, Session, time.Duration) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReportWriter interface {
|
||||||
|
ReplaceRuntime(context.Context, Report, time.Duration) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type RuntimeReader interface {
|
||||||
|
ReadRuntime(context.Context, []OwnedProxy) ([]Snapshot, error)
|
||||||
|
}
|
||||||
@ -7,19 +7,90 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
|
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
|
||||||
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
|
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
|
||||||
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
|
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
|
||||||
|
ErrInvalidRuntimeReport = errors.New("invalid worker runtime report")
|
||||||
|
ErrInvalidRuntimeLimit = errors.New("invalid snapshot runtime limit")
|
||||||
|
ErrRuntimeLimitExceeded = errors.New("snapshot runtime limit exceeded")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const defaultRuntimeLimit = 1_000_000
|
||||||
|
|
||||||
|
const activeRuntimeShardCount = 64
|
||||||
|
|
||||||
|
type activeRuntimeShard struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
entries map[string]*proxyDomain.Capacity
|
||||||
|
}
|
||||||
|
|
||||||
|
type activeRuntimeIndex [activeRuntimeShardCount]activeRuntimeShard
|
||||||
|
|
||||||
|
func (index *activeRuntimeIndex) track(proxyID string, runtime *proxyDomain.Capacity, nonzero bool) {
|
||||||
|
shard := &index[activeRuntimeShardIndex(proxyID)]
|
||||||
|
shard.mu.Lock()
|
||||||
|
defer shard.mu.Unlock()
|
||||||
|
if nonzero {
|
||||||
|
if shard.entries == nil {
|
||||||
|
shard.entries = make(map[string]*proxyDomain.Capacity)
|
||||||
|
}
|
||||||
|
shard.entries[proxyID] = runtime
|
||||||
|
return
|
||||||
|
}
|
||||||
|
active, reserved, _ := runtime.Counters()
|
||||||
|
if active == 0 && reserved == 0 && shard.entries[proxyID] == runtime {
|
||||||
|
delete(shard.entries, proxyID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (index *activeRuntimeIndex) remove(proxyID string, runtime *proxyDomain.Capacity) {
|
||||||
|
shard := &index[activeRuntimeShardIndex(proxyID)]
|
||||||
|
shard.mu.Lock()
|
||||||
|
defer shard.mu.Unlock()
|
||||||
|
if shard.entries[proxyID] == runtime {
|
||||||
|
delete(shard.entries, proxyID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (index *activeRuntimeIndex) rangeEntries(visit func(string, *proxyDomain.Capacity)) {
|
||||||
|
for shardIndex := range index {
|
||||||
|
shard := &index[shardIndex]
|
||||||
|
shard.mu.Lock()
|
||||||
|
for proxyID, runtime := range shard.entries {
|
||||||
|
visit(proxyID, runtime)
|
||||||
|
}
|
||||||
|
shard.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func activeRuntimeShardIndex(proxyID string) uint64 {
|
||||||
|
const (
|
||||||
|
offset = uint64(14695981039346656037)
|
||||||
|
prime = uint64(1099511628211)
|
||||||
|
)
|
||||||
|
hash := offset
|
||||||
|
for index := 0; index < len(proxyID); index++ {
|
||||||
|
hash ^= uint64(proxyID[index])
|
||||||
|
hash *= prime
|
||||||
|
}
|
||||||
|
return hash % activeRuntimeShardCount
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeRegistration struct {
|
||||||
|
capacity *proxyDomain.Capacity
|
||||||
|
current atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
type Envelope struct {
|
type Envelope struct {
|
||||||
ClusterID string
|
ClusterID string
|
||||||
WorkerID string
|
WorkerID string
|
||||||
@ -70,17 +141,29 @@ type Store struct {
|
|||||||
current atomic.Pointer[View]
|
current atomic.Pointer[View]
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
runtimes map[string]*proxyDomain.Capacity
|
runtimes map[string]*runtimeRegistration
|
||||||
|
active activeRuntimeIndex
|
||||||
|
limit int
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStore(clusterID, workerID string) *Store {
|
func NewStore(clusterID, workerID string) *Store {
|
||||||
return &Store{
|
return &Store{
|
||||||
clusterID: clusterID,
|
clusterID: clusterID,
|
||||||
workerID: workerID,
|
workerID: workerID,
|
||||||
runtimes: make(map[string]*proxyDomain.Capacity),
|
runtimes: make(map[string]*runtimeRegistration),
|
||||||
|
limit: defaultRuntimeLimit,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewStoreWithRuntimeLimit(clusterID, workerID string, limit int) (*Store, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, ErrInvalidRuntimeLimit
|
||||||
|
}
|
||||||
|
store := NewStore(clusterID, workerID)
|
||||||
|
store.limit = limit
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) Current() *View {
|
func (s *Store) Current() *View {
|
||||||
if s == nil {
|
if s == nil {
|
||||||
return nil
|
return nil
|
||||||
@ -88,6 +171,49 @@ func (s *Store) Current() *View {
|
|||||||
return s.current.Load()
|
return s.current.Load()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Store) RuntimeReport(sessionID string, sequence uint64, observedAt time.Time) (workerruntime.Report, error) {
|
||||||
|
if s == nil || strings.TrimSpace(sessionID) != sessionID || sessionID == "" || sequence == 0 || observedAt.IsZero() {
|
||||||
|
return workerruntime.Report{}, ErrInvalidRuntimeReport
|
||||||
|
}
|
||||||
|
current := s.current.Load()
|
||||||
|
if current == nil {
|
||||||
|
return workerruntime.Report{}, ErrInvalidRuntimeReport
|
||||||
|
}
|
||||||
|
visible := make(map[string]struct{}, len(current.Entries))
|
||||||
|
counters := make([]workerruntime.Counter, 0)
|
||||||
|
for _, entry := range current.Entries {
|
||||||
|
visible[entry.Proxy.ID] = struct{}{}
|
||||||
|
active, reserved, _ := entry.Runtime.Counters()
|
||||||
|
if active == 0 && reserved == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
counters = append(counters, workerruntime.Counter{
|
||||||
|
ProxyID: entry.Proxy.ID, Active: active, Reserved: reserved,
|
||||||
|
Draining: entry.Proxy.State == proxyDomain.StateDraining,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
s.active.rangeEntries(func(proxyID string, runtime *proxyDomain.Capacity) {
|
||||||
|
if _, currentProxy := visible[proxyID]; currentProxy {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
active, reserved, _ := runtime.Counters()
|
||||||
|
if active == 0 && reserved == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
counters = append(counters, workerruntime.Counter{
|
||||||
|
ProxyID: proxyID, Active: active, Reserved: reserved, Draining: true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
sort.Slice(counters, func(left, right int) bool {
|
||||||
|
return counters[left].ProxyID < counters[right].ProxyID
|
||||||
|
})
|
||||||
|
return workerruntime.Report{
|
||||||
|
WorkerID: s.workerID, SessionID: sessionID, Sequence: sequence,
|
||||||
|
SnapshotVersion: current.Version, OwnershipEpoch: current.Epoch,
|
||||||
|
ObservedAt: observedAt.UTC(), Counters: counters,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) Apply(envelope Envelope) error {
|
func (s *Store) Apply(envelope Envelope) error {
|
||||||
if s == nil {
|
if s == nil {
|
||||||
return fmt.Errorf("apply snapshot: nil store")
|
return fmt.Errorf("apply snapshot: nil store")
|
||||||
@ -118,19 +244,47 @@ func (s *Store) Apply(envelope Envelope) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
proxies := cloneAndSort(envelope.Proxies)
|
proxies := cloneAndSort(envelope.Proxies)
|
||||||
|
newRuntimeCount := 0
|
||||||
|
for _, descriptor := range proxies {
|
||||||
|
if s.runtimes[descriptor.ID] == nil {
|
||||||
|
newRuntimeCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(s.runtimes)+newRuntimeCount > s.limit {
|
||||||
|
return ErrRuntimeLimitExceeded
|
||||||
|
}
|
||||||
|
if current != nil {
|
||||||
|
for _, entry := range current.Entries {
|
||||||
|
registration := s.runtimes[entry.Proxy.ID]
|
||||||
|
registration.capacity.SetActivityObservationEnabled(true)
|
||||||
|
registration.current.Store(false)
|
||||||
|
active, reserved, _ := registration.capacity.Counters()
|
||||||
|
s.active.track(entry.Proxy.ID, registration.capacity, active+reserved > 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
entries := make([]Entry, 0, len(proxies))
|
entries := make([]Entry, 0, len(proxies))
|
||||||
for _, descriptor := range proxies {
|
for _, descriptor := range proxies {
|
||||||
runtime := s.runtimes[descriptor.ID]
|
registration := s.runtimes[descriptor.ID]
|
||||||
if runtime == nil {
|
if registration == nil {
|
||||||
runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency)
|
proxyID := descriptor.ID
|
||||||
} else {
|
registration = &runtimeRegistration{}
|
||||||
runtime.SetMax(descriptor.MaxConcurrency)
|
registration.capacity = proxyDomain.NewCapacityWithActivityObserver(descriptor.MaxConcurrency, func(nonzero bool) {
|
||||||
|
if registration.current.Load() {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
s.active.track(proxyID, registration.capacity, nonzero)
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
registration.capacity.SetMax(descriptor.MaxConcurrency)
|
||||||
|
}
|
||||||
|
registration.current.Store(true)
|
||||||
|
registration.capacity.SetActivityObservationEnabled(false)
|
||||||
|
s.active.remove(descriptor.ID, registration.capacity)
|
||||||
// Keep runtimes for temporarily absent IDs. Old immutable views may still
|
// Keep runtimes for temporarily absent IDs. Old immutable views may still
|
||||||
// hold in-flight leases, so reclaiming here could reset active capacity if
|
// hold in-flight leases, so reclaiming here could reset active capacity if
|
||||||
// the same Proxy reappears in a later snapshot.
|
// the same Proxy reappears in a later snapshot.
|
||||||
s.runtimes[descriptor.ID] = runtime
|
s.runtimes[descriptor.ID] = registration
|
||||||
entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime})
|
entries = append(entries, Entry{Proxy: descriptor, Runtime: registration.capacity})
|
||||||
}
|
}
|
||||||
|
|
||||||
next := &View{
|
next := &View{
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
|
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
|
||||||
@ -266,6 +267,165 @@ func TestStoreReusesRuntimeWhenProxyDisappearsAndReappears(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStoreRuntimeReportKeepsRemovedActiveProxyUntilRelease(t *testing.T) {
|
||||||
|
store := NewStore("cluster-a", "worker-a")
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
proxy := proxyDomain.Proxy{
|
||||||
|
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "provider-a",
|
||||||
|
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||||
|
}
|
||||||
|
initial := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
||||||
|
Proxies: []proxyDomain.Proxy{proxy},
|
||||||
|
}
|
||||||
|
initial.Checksum = Checksum(initial.Proxies)
|
||||||
|
if err := store.Apply(initial); err != nil {
|
||||||
|
t.Fatalf("Apply(initial): %v", err)
|
||||||
|
}
|
||||||
|
runtime := store.Current().Entries[0].Runtime
|
||||||
|
reservation, ok := runtime.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Reserve() = false")
|
||||||
|
}
|
||||||
|
if err := reservation.Commit(); err != nil {
|
||||||
|
t.Fatalf("Commit(): %v", err)
|
||||||
|
}
|
||||||
|
removed := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true}
|
||||||
|
removed.Checksum = Checksum(nil)
|
||||||
|
if err := store.Apply(removed); err != nil {
|
||||||
|
t.Fatalf("Apply(remove): %v", err)
|
||||||
|
}
|
||||||
|
if got := store.activeRuntimeCount(); got != 1 {
|
||||||
|
t.Fatalf("activeRuntimeCount(removed) = %d, want 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := store.RuntimeReport("session-a", 7, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RuntimeReport(): %v", err)
|
||||||
|
}
|
||||||
|
if report.WorkerID != "worker-a" || report.SessionID != "session-a" || report.Sequence != 7 ||
|
||||||
|
report.SnapshotVersion != 2 || report.OwnershipEpoch != 1 || !report.ObservedAt.Equal(now) ||
|
||||||
|
len(report.Counters) != 1 || report.Counters[0] != (workerruntime.Counter{
|
||||||
|
ProxyID: "proxy-a", Active: 1, Draining: true,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("RuntimeReport() = %+v", report)
|
||||||
|
}
|
||||||
|
if err := reservation.Release(); err != nil {
|
||||||
|
t.Fatalf("Release(): %v", err)
|
||||||
|
}
|
||||||
|
if got := store.activeRuntimeCount(); got != 0 {
|
||||||
|
t.Fatalf("activeRuntimeCount(released) = %d, want 0", got)
|
||||||
|
}
|
||||||
|
report, err = store.RuntimeReport("session-a", 8, now.Add(time.Second))
|
||||||
|
if err != nil || len(report.Counters) != 0 {
|
||||||
|
t.Fatalf("RuntimeReport(after release) = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreRuntimeReportMarksCurrentDrainingProxy(t *testing.T) {
|
||||||
|
store := NewStore("cluster-a", "worker-a")
|
||||||
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
|
proxy := proxyDomain.Proxy{
|
||||||
|
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||||
|
State: proxyDomain.StateDraining, MaxConcurrency: 1,
|
||||||
|
}
|
||||||
|
envelope := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
||||||
|
Proxies: []proxyDomain.Proxy{proxy},
|
||||||
|
}
|
||||||
|
envelope.Checksum = Checksum(envelope.Proxies)
|
||||||
|
if err := store.Apply(envelope); err != nil {
|
||||||
|
t.Fatalf("Apply(): %v", err)
|
||||||
|
}
|
||||||
|
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Reserve() = false")
|
||||||
|
}
|
||||||
|
report, err := store.RuntimeReport("session-a", 1, now)
|
||||||
|
if err != nil || len(report.Counters) != 1 || !report.Counters[0].Draining {
|
||||||
|
t.Fatalf("RuntimeReport() = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if err := reservation.Cancel(); err != nil {
|
||||||
|
t.Fatalf("Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreBoundsHistoricalRuntimeRegistryAndKeepsApplyTransactional(t *testing.T) {
|
||||||
|
store, err := NewStoreWithRuntimeLimit("cluster-a", "worker-a", 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStoreWithRuntimeLimit(): %v", err)
|
||||||
|
}
|
||||||
|
first := proxyDomain.Proxy{
|
||||||
|
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||||
|
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||||
|
}
|
||||||
|
envelope := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
|
||||||
|
Full: true, Proxies: []proxyDomain.Proxy{first},
|
||||||
|
}
|
||||||
|
envelope.Checksum = Checksum(envelope.Proxies)
|
||||||
|
if err := store.Apply(envelope); err != nil {
|
||||||
|
t.Fatalf("Apply(first): %v", err)
|
||||||
|
}
|
||||||
|
removed := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true,
|
||||||
|
}
|
||||||
|
removed.Checksum = Checksum(nil)
|
||||||
|
if err := store.Apply(removed); err != nil {
|
||||||
|
t.Fatalf("Apply(removed): %v", err)
|
||||||
|
}
|
||||||
|
second := first
|
||||||
|
second.ID = "proxy-b"
|
||||||
|
overLimit := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 3,
|
||||||
|
Full: true, Proxies: []proxyDomain.Proxy{second},
|
||||||
|
}
|
||||||
|
overLimit.Checksum = Checksum(overLimit.Proxies)
|
||||||
|
if err := store.Apply(overLimit); !errors.Is(err, ErrRuntimeLimitExceeded) {
|
||||||
|
t.Fatalf("Apply(over limit) error = %v, want ErrRuntimeLimitExceeded", err)
|
||||||
|
}
|
||||||
|
if current := store.Current(); current.Version != 2 || len(current.Entries) != 0 {
|
||||||
|
t.Fatalf("Current() after rejected apply = version %d entries %d", current.Version, len(current.Entries))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreRuntimeActiveIndexDropsZeroCounters(t *testing.T) {
|
||||||
|
store := NewStore("cluster-a", "worker-a")
|
||||||
|
proxy := proxyDomain.Proxy{
|
||||||
|
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||||
|
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||||
|
}
|
||||||
|
envelope := Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
|
||||||
|
Full: true, Proxies: []proxyDomain.Proxy{proxy},
|
||||||
|
}
|
||||||
|
envelope.Checksum = Checksum(envelope.Proxies)
|
||||||
|
if err := store.Apply(envelope); err != nil {
|
||||||
|
t.Fatalf("Apply(): %v", err)
|
||||||
|
}
|
||||||
|
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Reserve() = false")
|
||||||
|
}
|
||||||
|
if got := store.activeRuntimeCount(); got != 0 {
|
||||||
|
t.Fatalf("activeRuntimeCount(current) = %d, want 0", got)
|
||||||
|
}
|
||||||
|
if err := reservation.Cancel(); err != nil {
|
||||||
|
t.Fatalf("Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
if got := store.activeRuntimeCount(); got != 0 {
|
||||||
|
t.Fatalf("activeRuntimeCount() = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) activeRuntimeCount() int {
|
||||||
|
count := 0
|
||||||
|
s.active.rangeEntries(func(_ string, _ *proxyDomain.Capacity) {
|
||||||
|
count++
|
||||||
|
})
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
func collectSelectionIDs(selection Selection) []string {
|
func collectSelectionIDs(selection Selection) []string {
|
||||||
ids := make([]string, 0, selection.Len())
|
ids := make([]string, 0, selection.Len())
|
||||||
for index := 0; index < selection.Len(); index++ {
|
for index := 0; index < selection.Len(); index++ {
|
||||||
|
|||||||
@ -48,10 +48,27 @@ type Store interface {
|
|||||||
Resolve(context.Context, Reference) (Value, error)
|
Resolve(context.Context, Reference) (Value, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Releaser removes transient credential material after the consumer has copied
|
||||||
|
// it into its authoritative storage. Release is idempotent and version fenced.
|
||||||
|
type Releaser interface {
|
||||||
|
Release(context.Context, Reference) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type CapacityEnsurer interface {
|
||||||
|
EnsureCapacity(context.Context, int) error
|
||||||
|
}
|
||||||
|
|
||||||
type entry struct {
|
type entry struct {
|
||||||
|
scope *scopeState
|
||||||
|
value Value
|
||||||
|
reference Reference
|
||||||
|
}
|
||||||
|
|
||||||
|
type scopeState struct {
|
||||||
|
name string
|
||||||
value Value
|
value Value
|
||||||
version uint64
|
version uint64
|
||||||
reference Reference
|
leases int
|
||||||
}
|
}
|
||||||
|
|
||||||
// MemoryStore keeps credentials in process memory and serializes access with a
|
// MemoryStore keeps credentials in process memory and serializes access with a
|
||||||
@ -59,7 +76,7 @@ type entry struct {
|
|||||||
type MemoryStore struct {
|
type MemoryStore struct {
|
||||||
lock chan struct{}
|
lock chan struct{}
|
||||||
capacity int
|
capacity int
|
||||||
byScope map[string]*entry
|
byScope map[string]*scopeState
|
||||||
byRef map[string]*entry
|
byRef map[string]*entry
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
|
|||||||
return &MemoryStore{
|
return &MemoryStore{
|
||||||
lock: lock,
|
lock: lock,
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
byScope: make(map[string]*entry),
|
byScope: make(map[string]*scopeState),
|
||||||
byRef: make(map[string]*entry),
|
byRef: make(map[string]*entry),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
|
|||||||
if err := s.acquire(ctx); err != nil {
|
if err := s.acquire(ctx); err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
defer s.release()
|
defer s.unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
|
if len(s.byRef) >= s.capacity {
|
||||||
|
return Reference{}, ErrCapacityExceeded
|
||||||
|
}
|
||||||
|
|
||||||
if current, ok := s.byScope[scope]; ok {
|
current, exists := s.byScope[scope]
|
||||||
if current.value == value {
|
if !exists && len(s.byScope) >= s.capacity {
|
||||||
return current.reference, nil
|
|
||||||
}
|
|
||||||
current.value = value
|
|
||||||
current.version++
|
|
||||||
current.reference.CredentialVersion = versionString(current.version)
|
|
||||||
return current.reference, nil
|
|
||||||
}
|
|
||||||
if len(s.byScope) >= s.capacity {
|
|
||||||
return Reference{}, ErrCapacityExceeded
|
return Reference{}, ErrCapacityExceeded
|
||||||
}
|
}
|
||||||
secretRef, err := s.newUniqueSecretRef()
|
secretRef, err := s.newUniqueSecretRef()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
|
if !exists {
|
||||||
|
current = &scopeState{name: scope, value: value, version: 1}
|
||||||
|
s.byScope[scope] = current
|
||||||
|
} else if current.value != value {
|
||||||
|
current.value = value
|
||||||
|
current.version++
|
||||||
|
}
|
||||||
created := &entry{
|
created := &entry{
|
||||||
|
scope: current,
|
||||||
value: value,
|
value: value,
|
||||||
version: 1,
|
|
||||||
reference: Reference{
|
reference: Reference{
|
||||||
SecretRef: secretRef,
|
SecretRef: secretRef,
|
||||||
CredentialVersion: versionString(1),
|
CredentialVersion: versionString(current.version),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
s.byScope[scope] = created
|
current.leases++
|
||||||
s.byRef[secretRef] = created
|
s.byRef[secretRef] = created
|
||||||
return created.reference, nil
|
return created.reference, nil
|
||||||
}
|
}
|
||||||
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
|||||||
if err := s.acquire(ctx); err != nil {
|
if err := s.acquire(ctx); err != nil {
|
||||||
return Value{}, err
|
return Value{}, err
|
||||||
}
|
}
|
||||||
defer s.release()
|
defer s.unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return Value{}, err
|
return Value{}, err
|
||||||
}
|
}
|
||||||
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
|||||||
return current.value, nil
|
return current.value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
|
||||||
|
if err := contextError(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !s.valid() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
|
||||||
|
return ErrInvalidReference
|
||||||
|
}
|
||||||
|
if err := s.acquire(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.unlock()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current, ok := s.byRef[reference.SecretRef]
|
||||||
|
if !ok || current.reference != reference {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
delete(s.byRef, reference.SecretRef)
|
||||||
|
current.scope.leases--
|
||||||
|
if current.scope.leases == 0 {
|
||||||
|
delete(s.byScope, current.scope.name)
|
||||||
|
current.scope.value = Value{}
|
||||||
|
}
|
||||||
|
current.value = Value{}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
|
||||||
|
if err := contextError(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !s.valid() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
if minimum <= 0 {
|
||||||
|
return ErrInvalidCapacity
|
||||||
|
}
|
||||||
|
if err := s.acquire(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.unlock()
|
||||||
|
if minimum > s.capacity {
|
||||||
|
s.capacity = minimum
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MemoryStore) acquire(ctx context.Context) error {
|
func (s *MemoryStore) acquire(ctx context.Context) error {
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MemoryStore) release() {
|
func (s *MemoryStore) unlock() {
|
||||||
s.lock <- struct{}{}
|
s.lock <- struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
|
||||||
store, err := NewMemoryStore(2)
|
store, err := NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Put(second): %v", err)
|
t.Fatalf("Put(second): %v", err)
|
||||||
}
|
}
|
||||||
if first != second {
|
if first == second || first.SecretRef == second.SecretRef {
|
||||||
t.Fatalf("second reference = %#v, want %#v", second, first)
|
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
|
||||||
}
|
}
|
||||||
if first.SecretRef == "" || first.CredentialVersion != "v1" {
|
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
|
||||||
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
|
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
|
||||||
}
|
}
|
||||||
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
|
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
|
||||||
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
|||||||
if got != value {
|
if got != value {
|
||||||
t.Fatalf("Resolve() = %#v, want %#v", got, value)
|
t.Fatalf("Resolve() = %#v, want %#v", got, value)
|
||||||
}
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(first lease): %v", err)
|
||||||
|
}
|
||||||
|
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
|
||||||
|
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
||||||
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
store, err := NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Put(new): %v", err)
|
t.Fatalf("Put(new): %v", err)
|
||||||
}
|
}
|
||||||
if newReference.SecretRef != oldReference.SecretRef {
|
if newReference.SecretRef == oldReference.SecretRef {
|
||||||
t.Fatalf("new SecretRef changed across versions")
|
t.Fatalf("new SecretRef reused an active lease")
|
||||||
}
|
}
|
||||||
if newReference.CredentialVersion != "v2" {
|
if newReference.CredentialVersion != "v2" {
|
||||||
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
|
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
|
||||||
}
|
}
|
||||||
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
|
old, err := store.Resolve(context.Background(), oldReference)
|
||||||
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
|
if err != nil || old.Password != "old-password" {
|
||||||
|
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
|
||||||
}
|
}
|
||||||
got, err := store.Resolve(context.Background(), newReference)
|
got, err := store.Resolve(context.Background(), newReference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
|
|||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("Resolve(existing) returned changed credentials")
|
t.Fatalf("Resolve(existing) returned changed credentials")
|
||||||
}
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), reference); err != nil {
|
||||||
|
t.Fatalf("Release(first lease): %v", err)
|
||||||
|
}
|
||||||
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
|
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
|
||||||
t.Fatalf("Put(idempotent at capacity): %v", err)
|
t.Fatalf("Put(after lease release): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(first): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(first): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
|
||||||
|
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
|
||||||
|
t.Fatalf("Put(after release): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(idempotent): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(old): %v", err)
|
||||||
|
}
|
||||||
|
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(new): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), stale); err != nil {
|
||||||
|
t.Fatalf("Release(stale): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Resolve(context.Background(), current); err != nil {
|
||||||
|
t.Fatalf("Resolve(current): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
|
||||||
|
t.Fatalf("EnsureCapacity(2): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
|
||||||
|
t.Fatalf("Put(provider-a): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
|
||||||
|
t.Fatalf("Put(provider-b): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("EnsureCapacity(shrink request): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
const workers = 100
|
||||||
|
store, err := NewMemoryStore(workers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
want := Value{Username: "alice", Password: "shared-password"}
|
want := Value{Username: "alice", Password: "shared-password"}
|
||||||
const workers = 100
|
|
||||||
references := make(chan Reference, workers)
|
references := make(chan Reference, workers)
|
||||||
errorsSeen := make(chan error, workers)
|
errorsSeen := make(chan error, workers)
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
|||||||
for err := range errorsSeen {
|
for err := range errorsSeen {
|
||||||
t.Errorf("concurrent Put(): %v", err)
|
t.Errorf("concurrent Put(): %v", err)
|
||||||
}
|
}
|
||||||
var first Reference
|
secretRefs := make(map[string]struct{}, workers)
|
||||||
for reference := range references {
|
for reference := range references {
|
||||||
if first == (Reference{}) {
|
if reference.CredentialVersion != "v1" {
|
||||||
first = reference
|
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
|
||||||
}
|
}
|
||||||
if reference != first {
|
secretRefs[reference.SecretRef] = struct{}{}
|
||||||
t.Errorf("concurrent Put() reference differs from first")
|
|
||||||
}
|
}
|
||||||
|
if len(secretRefs) != workers {
|
||||||
|
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
|
||||||
}
|
}
|
||||||
if first.CredentialVersion != "v1" {
|
for secretRef := range secretRefs {
|
||||||
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
|
got, err := store.Resolve(context.Background(), Reference{
|
||||||
|
SecretRef: secretRef, CredentialVersion: "v1",
|
||||||
|
})
|
||||||
|
if err != nil || got != want {
|
||||||
|
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
|
||||||
}
|
}
|
||||||
got, err := store.Resolve(context.Background(), first)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve(): %v", err)
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("Resolve() returned unexpected credentials")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
const workers = 100
|
||||||
|
store, err := NewMemoryStore(workers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
reference Reference
|
reference Reference
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
const workers = 100
|
|
||||||
results := make(chan result, workers)
|
results := make(chan result, workers)
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
for index := range workers {
|
for index := range workers {
|
||||||
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
close(results)
|
close(results)
|
||||||
|
|
||||||
versions := make(map[string]struct{}, workers)
|
versions := make(map[string]struct{}, workers)
|
||||||
var secretRef string
|
secretRefs := make(map[string]struct{}, workers)
|
||||||
var latest result
|
var latest result
|
||||||
for current := range results {
|
for current := range results {
|
||||||
if current.err != nil {
|
if current.err != nil {
|
||||||
t.Fatalf("concurrent Put(): %v", current.err)
|
t.Fatalf("concurrent Put(): %v", current.err)
|
||||||
}
|
}
|
||||||
if secretRef == "" {
|
secretRefs[current.reference.SecretRef] = struct{}{}
|
||||||
secretRef = current.reference.SecretRef
|
|
||||||
}
|
|
||||||
if current.reference.SecretRef != secretRef {
|
|
||||||
t.Fatal("SecretRef changed across concurrent updates")
|
|
||||||
}
|
|
||||||
versions[current.reference.CredentialVersion] = struct{}{}
|
versions[current.reference.CredentialVersion] = struct{}{}
|
||||||
if current.reference.CredentialVersion == "v100" {
|
if current.reference.CredentialVersion == "v100" {
|
||||||
latest = current
|
latest = current
|
||||||
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
if len(versions) != workers {
|
if len(versions) != workers {
|
||||||
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
|
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
|
||||||
}
|
}
|
||||||
|
if len(secretRefs) != workers {
|
||||||
|
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
|
||||||
|
}
|
||||||
if latest.reference == (Reference{}) {
|
if latest.reference == (Reference{}) {
|
||||||
t.Fatal("highest version v100 was not returned")
|
t.Fatal("highest version v100 was not returned")
|
||||||
}
|
}
|
||||||
|
|||||||
72
internal/platform/lifecycle/group.go
Normal file
72
internal/platform/lifecycle/group.go
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidGroup = errors.New("invalid lifecycle group")
|
||||||
|
ErrRunnerStopped = errors.New("lifecycle runner stopped")
|
||||||
|
)
|
||||||
|
|
||||||
|
type Runner interface {
|
||||||
|
Run(context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type Group struct {
|
||||||
|
runners []Runner
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewGroup(runners ...Runner) (*Group, error) {
|
||||||
|
if len(runners) == 0 {
|
||||||
|
return nil, ErrInvalidGroup
|
||||||
|
}
|
||||||
|
owned := make([]Runner, len(runners))
|
||||||
|
for index, runner := range runners {
|
||||||
|
if isNilRunner(runner) {
|
||||||
|
return nil, ErrInvalidGroup
|
||||||
|
}
|
||||||
|
owned[index] = runner
|
||||||
|
}
|
||||||
|
return &Group{runners: owned}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (group *Group) Run(ctx context.Context) error {
|
||||||
|
if group == nil || ctx == nil || len(group.runners) == 0 {
|
||||||
|
return ErrInvalidGroup
|
||||||
|
}
|
||||||
|
groupCtx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, len(group.runners))
|
||||||
|
for _, runner := range group.runners {
|
||||||
|
go func() { done <- runner.Run(groupCtx) }()
|
||||||
|
}
|
||||||
|
|
||||||
|
first := <-done
|
||||||
|
cancel()
|
||||||
|
for range len(group.runners) - 1 {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
return ErrRunnerStopped
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
func isNilRunner(runner Runner) bool {
|
||||||
|
if runner == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
reflected := reflect.ValueOf(runner)
|
||||||
|
switch reflected.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return reflected.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
87
internal/platform/lifecycle/group_test.go
Normal file
87
internal/platform/lifecycle/group_test.go
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
|
||||||
|
started := make(chan struct{}, 2)
|
||||||
|
cancelled := make(chan struct{}, 1)
|
||||||
|
wantErr := errors.New("runner failed")
|
||||||
|
group, err := NewGroup(
|
||||||
|
runnerFunc(func(context.Context) error {
|
||||||
|
started <- struct{}{}
|
||||||
|
return wantErr
|
||||||
|
}),
|
||||||
|
runnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- struct{}{}
|
||||||
|
<-ctx.Done()
|
||||||
|
cancelled <- struct{}{}
|
||||||
|
return ctx.Err()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGroup(): %v", err)
|
||||||
|
}
|
||||||
|
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
<-started
|
||||||
|
<-started
|
||||||
|
select {
|
||||||
|
case <-cancelled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("sibling runner was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
var exited atomic.Bool
|
||||||
|
group, err := NewGroup(
|
||||||
|
runnerFunc(func(context.Context) error { return errors.New("failed") }),
|
||||||
|
runnerFunc(func(ctx context.Context) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
<-release
|
||||||
|
exited.Store(true)
|
||||||
|
return nil
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGroup(): %v", err)
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- group.Run(context.Background()) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatal("Run() returned before sibling exited")
|
||||||
|
case <-time.After(20 * time.Millisecond):
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
<-done
|
||||||
|
if !exited.Load() {
|
||||||
|
t.Fatal("sibling exit was not observed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
|
||||||
|
var typedNil *nilRunner
|
||||||
|
for _, runners := range [][]Runner{nil, {typedNil}} {
|
||||||
|
group, err := NewGroup(runners...)
|
||||||
|
if err == nil || group != nil {
|
||||||
|
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
|
||||||
|
|
||||||
|
type nilRunner struct{}
|
||||||
|
|
||||||
|
func (*nilRunner) Run(context.Context) error { return nil }
|
||||||
20
progress.md
20
progress.md
@ -2,6 +2,26 @@
|
|||||||
|
|
||||||
## 2026-07-30
|
## 2026-07-30
|
||||||
|
|
||||||
|
- Gateway `Capacity` 新增一次打包原子读取,`snapshot.Store` 可生成完整稀疏
|
||||||
|
Active/Reserved 运行态报告;当前快照已移除但仍有连接的 Proxy 会持续以
|
||||||
|
draining 上报,归零后从后续报告消失。
|
||||||
|
- 新增公用 `workerruntime` session/report/read seam 与并发安全 MemoryStore;
|
||||||
|
完整替换、空报告清零、session fencing、单调 sequence、同内容幂等重放、
|
||||||
|
冲突/倒序拒绝和 TTL fail-closed 均已有单测。
|
||||||
|
- 生产 `redisactivity` Adapter 新增 Worker session/运行态 Lua 和权威
|
||||||
|
`pool.InventoryReader`;Managed/Available Slots 原子计入状态、TTL safety、
|
||||||
|
MaxConcurrency、ownership 及 Active/Reserved,未知或过期运行态贡献零容量。
|
||||||
|
- 真实 Redis 8.2 已覆盖 ACK snapshot/epoch、旧 session、空报告、超前 epoch、
|
||||||
|
报告过期、扫描预算耗尽及容量聚合。Managed 使用已有权威计数,Available
|
||||||
|
Slots 只扫描目标 Upstream 的未分配/已分配可用索引;Gateway 报告扫描当前
|
||||||
|
Snapshot,并用分片索引补充已移除但仍非零的 runtime,历史注册表设置硬上限。
|
||||||
|
WorkerControlPlane 接收端、Provider
|
||||||
|
Fleet/bootstrap、全局 `fetch.maxTotal` 和健康执行链仍待完成,总验收计数保持
|
||||||
|
51/73。
|
||||||
|
- Gateway 当前 Proxy 禁用活跃索引回调,只有移出 Snapshot 后才开启分片追踪;
|
||||||
|
本机 100k Proxy `Acquire` 三轮 1 秒基准为 893.9-1047 ns/op、256 B/op、
|
||||||
|
2 allocs/op。该数据只证明本地调度微基准,不代表 100k QPS 集群验收。
|
||||||
|
|
||||||
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
|
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
|
||||||
`redisprovider` Adapter;Redis Lua 原子维护 generation、epoch、Leader 租约、
|
`redisprovider` Adapter;Redis Lua 原子维护 generation、epoch、Leader 租约、
|
||||||
全局 requestInterval 与带 TTL 的 maxInFlight Permit,异常时 fail-closed。
|
全局 requestInterval 与带 TTL 的 maxInFlight Permit,异常时 fail-closed。
|
||||||
|
|||||||
@ -35,7 +35,8 @@
|
|||||||
机器契约和文档类滞后勾选已按仓库证据校正
|
机器契约和文档类滞后勾选已按仓库证据校正
|
||||||
13. [进行中] 落地 `proxy-controller` 进程装配;配置单次加载、PostgreSQL 迁移、
|
13. [进行中] 落地 `proxy-controller` 进程装配;配置单次加载、PostgreSQL 迁移、
|
||||||
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
|
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
|
||||||
双存储 bootstrap 和探针集成已通过,Provider、业务指标与完整容器进程链仍待实现
|
双存储 bootstrap 和探针集成已通过;Worker 运行态存储与权威容量读取原语已
|
||||||
|
完成,WorkerControlPlane、Provider、业务指标与完整容器进程链仍待实现
|
||||||
|
|
||||||
## 串并行关系
|
## 串并行关系
|
||||||
|
|
||||||
@ -57,7 +58,8 @@
|
|||||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;Redis 8.2
|
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;Redis 8.2
|
||||||
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
|
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
|
||||||
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
|
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
|
||||||
Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
|
Provider 分布式协调和 Worker 运行态 Redis 原语已完成,但 WorkerControlPlane
|
||||||
故障转移验证与代表性集群压测属于后续实施范围。
|
接收端、Provider Fleet、Gateway、Checker、Loadgen、业务指标、Redis 故障
|
||||||
|
转移验证与代表性集群压测属于后续实施范围。
|
||||||
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
|
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
|
||||||
验收项完成率约 69.9%,不等同于生产就绪度。
|
验收项完成率约 69.9%,不等同于生产就绪度。
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user