Compare commits
No commits in common. "63bc56be275eb8d1a728465506e849bba8d3bda8" and "4de3ffb85f994d38a434acd14ebbbfd908dce2cd" have entirely different histories.
63bc56be27
...
4de3ffb85f
@ -1,91 +0,0 @@
|
|||||||
package deploy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.yaml.in/yaml/v4"
|
|
||||||
)
|
|
||||||
|
|
||||||
type composeDocument struct {
|
|
||||||
Services map[string]composeService `yaml:"services"`
|
|
||||||
Volumes map[string]any `yaml:"volumes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type composeService struct {
|
|
||||||
Command []string `yaml:"command"`
|
|
||||||
Volumes []string `yaml:"volumes"`
|
|
||||||
DependsOn any `yaml:"depends_on"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) {
|
|
||||||
document := loadComposeDocument(t)
|
|
||||||
redis, ok := document.Services["redis"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("docker-compose.yml has no redis service")
|
|
||||||
}
|
|
||||||
if value, ok := commandFlag(redis.Command, "--appendonly"); !ok || value != "no" {
|
|
||||||
t.Fatalf("redis --appendonly = %q, %t; want no", value, ok)
|
|
||||||
}
|
|
||||||
if value, ok := commandFlag(redis.Command, "--save"); !ok || value != "" {
|
|
||||||
t.Fatalf("redis --save = %q, %t; want empty schedule", value, ok)
|
|
||||||
}
|
|
||||||
if len(redis.Volumes) != 0 {
|
|
||||||
t.Fatalf("redis volumes = %v; want no persistent mount", redis.Volumes)
|
|
||||||
}
|
|
||||||
if _, exists := document.Volumes["redis-data"]; exists {
|
|
||||||
t.Fatal("docker-compose.yml still declares redis-data")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalGatewaysDoNotDependOnControlPlaneStorage(t *testing.T) {
|
|
||||||
document := loadComposeDocument(t)
|
|
||||||
for _, name := range []string{"gateway-a", "gateway-b"} {
|
|
||||||
gateway, ok := document.Services[name]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("docker-compose.yml has no %s service", name)
|
|
||||||
}
|
|
||||||
for _, storage := range []string{"postgres", "redis"} {
|
|
||||||
if composeDependsOn(gateway.DependsOn, storage) {
|
|
||||||
t.Errorf("%s depends on %s; gateway startup must be storage-independent", name, storage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadComposeDocument(t *testing.T) composeDocument {
|
|
||||||
t.Helper()
|
|
||||||
payload, err := os.ReadFile("docker-compose.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read docker-compose.yml: %v", err)
|
|
||||||
}
|
|
||||||
var document composeDocument
|
|
||||||
if err := yaml.Unmarshal(payload, &document); err != nil {
|
|
||||||
t.Fatalf("parse docker-compose.yml: %v", err)
|
|
||||||
}
|
|
||||||
return document
|
|
||||||
}
|
|
||||||
|
|
||||||
func composeDependsOn(value any, service string) bool {
|
|
||||||
switch dependencies := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
_, exists := dependencies[service]
|
|
||||||
return exists
|
|
||||||
case []any:
|
|
||||||
for _, dependency := range dependencies {
|
|
||||||
if dependency == service {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func commandFlag(command []string, name string) (string, bool) {
|
|
||||||
for index := 0; index+1 < len(command); index++ {
|
|
||||||
if command[index] == name {
|
|
||||||
return command[index+1], true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
name: proxy-pool-test
|
|
||||||
|
|
||||||
services:
|
|
||||||
redis:
|
|
||||||
image: redis:8.2-alpine
|
|
||||||
command: ["redis-server", "--appendonly", "no", "--save", ""]
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:16379:6379"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
|
||||||
interval: 1s
|
|
||||||
timeout: 1s
|
|
||||||
retries: 30
|
|
||||||
start_period: 1s
|
|
||||||
@ -17,6 +17,11 @@ x-app: &app
|
|||||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||||
stop_grace_period: 45s
|
stop_grace_period: 45s
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
services:
|
services:
|
||||||
gateway-a:
|
gateway-a:
|
||||||
@ -44,11 +49,6 @@ services:
|
|||||||
controller:
|
controller:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-controller"]
|
command: ["proxy-controller"]
|
||||||
depends_on:
|
|
||||||
postgres:
|
|
||||||
condition: service_healthy
|
|
||||||
redis:
|
|
||||||
condition: service_healthy
|
|
||||||
expose: ["8081", "8082", "9090"]
|
expose: ["8081", "8082", "9090"]
|
||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:8081:8081"
|
- "127.0.0.1:8081:8081"
|
||||||
@ -63,11 +63,6 @@ services:
|
|||||||
checker:
|
checker:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-checker"]
|
command: ["proxy-checker"]
|
||||||
depends_on:
|
|
||||||
postgres:
|
|
||||||
condition: service_healthy
|
|
||||||
redis:
|
|
||||||
condition: service_healthy
|
|
||||||
expose: ["9090"]
|
expose: ["9090"]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||||
@ -116,8 +111,10 @@ services:
|
|||||||
redis:
|
redis:
|
||||||
image: redis:8.2-alpine
|
image: redis:8.2-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: ["redis-server", "--appendonly", "no", "--save", ""]
|
command: ["redis-server", "--appendonly", "yes", "--save", "60", "1"]
|
||||||
networks: [backend]
|
networks: [backend]
|
||||||
|
volumes:
|
||||||
|
- redis-data:/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
interval: 5s
|
interval: 5s
|
||||||
@ -163,5 +160,6 @@ networks:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres-data: {}
|
postgres-data: {}
|
||||||
|
redis-data: {}
|
||||||
prometheus-data: {}
|
prometheus-data: {}
|
||||||
grafana-data: {}
|
grafana-data: {}
|
||||||
|
|||||||
@ -1,278 +0,0 @@
|
|||||||
# ADR-005：Redis 活动池采用单实例原子深模块
|
|
||||||
|
|
||||||
## 状态
|
|
||||||
|
|
||||||
接受并已实现，2026-07-29。
|
|
||||||
|
|
||||||
## 背景
|
|
||||||
|
|
||||||
Proxy Pool 的 Gateway 峰值目标是每秒 100,000 个请求。Gateway 请求热路径必须
|
|
||||||
只读取 Worker 本地不可变 Snapshot 和本地容量计数，不得同步查询 Redis、
|
|
||||||
PostgreSQL 或 Provider。
|
|
||||||
|
|
||||||
Redis 只承载控制面中的可重建短效状态：Proxy 活动池、健康状态、Worker
|
|
||||||
所有权、独占提取、短期幂等结果和库存计数。供应商 Proxy 的有效期可能只有
|
|
||||||
30 秒，因此数据结构必须支持高频刷新、有界清理和硬过期，不能把逐个 Proxy
|
|
||||||
或逐次提取记录写入 PostgreSQL。
|
|
||||||
|
|
||||||
`activitypool.MemoryPool` 定义 Provider Upsert、Distribution Extract 和 Worker
|
|
||||||
Ownership 的参考语义，生产 Redis Adapter 已按同一套公用契约实现：
|
|
||||||
|
|
||||||
- 所有远程存储端口接收 `context.Context` 并返回存储错误。
|
|
||||||
- Provider Upsert、健康更新、提取、所有权和维护能力通过窄接口复用。
|
|
||||||
- 原子 Lua 在提交前完成记录解码和候选校验，提取结果不依赖提交后的凭据解析。
|
|
||||||
- 全局 ownership epoch 使用 Redis `INCR`，库存读取与过期清理均采用有界扫描。
|
|
||||||
- 真实 Redis 8.2 fixture 覆盖并发提取、所有权竞争、幂等硬过期和键 TTL。
|
|
||||||
|
|
||||||
## 决策
|
|
||||||
|
|
||||||
### 部署边界
|
|
||||||
|
|
||||||
首版支持 Redis 单实例或 Sentinel，不实现 Redis Cluster 多分片。所有活动池键
|
|
||||||
仍使用固定 `{activity}` hash tag，使未来迁移到 Cluster 单槽时不需要改变业务
|
|
||||||
键名和原子边界。
|
|
||||||
|
|
||||||
Redis 不进入 Gateway 请求热路径：
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart LR
|
|
||||||
Provider[Provider Reconciler] -->|UpsertFetched| Adapter[Redis Activity Adapter]
|
|
||||||
Checker[Checker] -->|ApplyHealth| Adapter
|
|
||||||
Distribution[Distribution Service] -->|Extract| Adapter
|
|
||||||
Ownership[Ownership Manager] -->|Assign / Drain / Expire| Adapter
|
|
||||||
Adapter --> Redis[(Redis Activity Pool)]
|
|
||||||
Adapter --> Snapshot[Snapshot Publisher]
|
|
||||||
Snapshot --> Worker[Gateway Worker]
|
|
||||||
Client[Gateway Client] --> Worker
|
|
||||||
Worker -->|本地快照与本地计数| Upstream[Upstream Proxy]
|
|
||||||
```
|
|
||||||
|
|
||||||
### 模块边界
|
|
||||||
|
|
||||||
生产实现是一个深模块，对外只暴露一个构造器和窄领域端口：
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Adapter struct {
|
|
||||||
// Redis client、键构造、脚本、编解码和指标均为私有实现。
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(client RedisClient, options Options) (*Adapter, error)
|
|
||||||
|
|
||||||
var _ activitypool.Upserter = (*Adapter)(nil)
|
|
||||||
var _ activitypool.HealthStore = (*Adapter)(nil)
|
|
||||||
var _ activitypool.InventoryReader = (*Adapter)(nil)
|
|
||||||
var _ extraction.Store = (*Adapter)(nil)
|
|
||||||
var _ ownership.Repository = (*Adapter)(nil)
|
|
||||||
```
|
|
||||||
|
|
||||||
Provider、Checker、Distribution 和 Ownership 只依赖各自需要的端口，不直接
|
|
||||||
依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
|
|
||||||
|
|
||||||
`ownership.Repository` 改为适合远程存储的上下文感知接口：
|
|
||||||
|
|
||||||
```go
|
|
||||||
Assign(context.Context, time.Time, string, string, time.Duration) (Assignment, error)
|
|
||||||
Renew(context.Context, time.Time, string, string, uint64, time.Duration) (Assignment, error)
|
|
||||||
BeginDrain(context.Context, string, string, uint64) (Assignment, error)
|
|
||||||
AcknowledgeDrain(context.Context, string, string, uint64, int64, int64) error
|
|
||||||
Get(context.Context, string) (Assignment, bool, error)
|
|
||||||
Expire(context.Context, time.Time, int) ([]Assignment, error)
|
|
||||||
```
|
|
||||||
|
|
||||||
不保留旧签名。内存参考实现、Ownership Manager 和测试调用方同步迁移。
|
|
||||||
|
|
||||||
### 键空间
|
|
||||||
|
|
||||||
```text
|
|
||||||
pp:{activity}:records HASH proxyID -> 短期 Proxy 记录
|
|
||||||
pp:{activity}:unique HASH uniqueKey digest -> proxyID
|
|
||||||
pp:{activity}:idkeys HASH proxyID -> uniqueKey digest
|
|
||||||
pp:{activity}:expiry ZSET proxyID -> hard expiry milliseconds
|
|
||||||
pp:{activity}:available ZSET proxyID -> usableUntil milliseconds
|
|
||||||
pp:{activity}:protocol:<value> ZSET 协议候选索引
|
|
||||||
pp:{activity}:region:<value> ZSET 地区候选索引
|
|
||||||
pp:{activity}:carrier:<value> ZSET 运营商候选索引
|
|
||||||
pp:{activity}:upstream:<value> ZSET 供应商候选索引
|
|
||||||
pp:{activity}:owners HASH proxyID -> ownership assignment
|
|
||||||
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
|
|
||||||
pp:{activity}:epoch STRING ownership 全局递增代次
|
|
||||||
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
|
|
||||||
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
|
|
||||||
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
|
|
||||||
```
|
|
||||||
|
|
||||||
Proxy 记录包含地址、状态、健康信息、硬过期时间、`usableUntil`、供应商、标签、
|
|
||||||
所有权引用及 Distribution 返回所需的短期凭据。Redis 键、日志、指标和错误不得
|
|
||||||
包含密码或原始 `SecretRef`。
|
|
||||||
|
|
||||||
### Provider Upsert
|
|
||||||
|
|
||||||
Go 层先完成上下文检查、批次校验、唯一键摘要、TTL/安全余量计算、凭据解析和
|
|
||||||
批内 ID 冲突检查。无效候选计入 `Dropped`，批级非法输入在写 Redis 前失败。
|
|
||||||
|
|
||||||
Lua 原子执行以下操作：
|
|
||||||
|
|
||||||
1. 有界清理已过期 incumbent。
|
|
||||||
2. 校验 `proxyID` 与唯一键映射。
|
|
||||||
3. 保持当前生命周期的 incumbent upstream；其他供应商的重复项不得覆盖。
|
|
||||||
4. EXTRACTED 条目不得通过刷新重新进入活动池。
|
|
||||||
5. 原子维护记录、唯一键、过期索引、过滤索引和 upstream 库存。
|
|
||||||
6. 使用 `FetchedBatch.MaxSize` 对当前未提取库存执行最终硬限制。
|
|
||||||
|
|
||||||
单次脚本批量有固定上限。超大 Provider 响应在 Go 层分块，Redis 库存计数始终
|
|
||||||
作为 `pool.maxSize` 的最终保护；本地 FetchBudget 只负责调用前的成本控制。
|
|
||||||
分块写入使用内部 `operationID`，连接中断后的底层重试不会改变结果计数。
|
|
||||||
|
|
||||||
### 健康状态
|
|
||||||
|
|
||||||
`HealthStore.ApplyHealth` 原子更新 Proxy 状态、检查时间、成功时间、延迟和失败
|
|
||||||
信息。只有满足下列条件的 Proxy 才进入 AVAILABLE 索引：
|
|
||||||
|
|
||||||
- 状态是 `AVAILABLE`。
|
|
||||||
- 没有 Worker 所有权。
|
|
||||||
- 当前时间早于 `usableUntil`。
|
|
||||||
|
|
||||||
`SUSPECT`、`UNHEALTHY`、`EXTRACTED`、`EXPIRED` 和 `REMOVED` 必须退出所有
|
|
||||||
AVAILABLE 索引。Checker 不直接拼接 Redis 命令。
|
|
||||||
|
|
||||||
### 独占提取
|
|
||||||
|
|
||||||
一个 Lua 操作完成：
|
|
||||||
|
|
||||||
1. 检查 Client ID、幂等键和请求摘要。
|
|
||||||
2. 选择候选数量最小的可用过滤索引作为驱动索引。
|
|
||||||
3. 有界复核状态、硬 TTL、`usableUntil`、健康新鲜度、所有权和全部过滤条件。
|
|
||||||
4. 为 Gateway 保留 `reserveForGateway` 个符合条件的候选。
|
|
||||||
5. 按 `partial` 或 `allOrNothing` 判断结果。
|
|
||||||
6. 将选中条目从 `AVAILABLE` 原子迁移到 `EXTRACTED`。
|
|
||||||
7. 原子减少 upstream 库存并写入短期幂等响应。
|
|
||||||
|
|
||||||
脚本扫描达到内部上限但仍不能确认结果时返回临时不可用，不得把未完成扫描
|
|
||||||
错误报告为库存不足。`allOrNothing` 在库存不足、扫描未完成或脚本异常时均为
|
|
||||||
零状态变更。
|
|
||||||
|
|
||||||
幂等结果过期时间为配置 `idempotencyTTL` 和本次结果最早 Proxy 硬过期时间中的
|
|
||||||
较早者。没有客户端幂等键时，Request ID 仍作为单次底层重试的内部操作 ID，
|
|
||||||
但不承诺不同 HTTP 请求之间的业务幂等。
|
|
||||||
|
|
||||||
### Worker 所有权
|
|
||||||
|
|
||||||
Assign、Renew、BeginDrain 和 AcknowledgeDrain 分别使用有界小脚本，与 Extract
|
|
||||||
共享 Proxy 记录和 AVAILABLE 索引。
|
|
||||||
|
|
||||||
- Assign 只接受 AVAILABLE、无 owner 且未到 `usableUntil` 的 Proxy。
|
|
||||||
- Renew 必须匹配 worker、epoch，并把 lease 截断到 `usableUntil`。
|
|
||||||
- BeginDrain 对同一 assignment 幂等。
|
|
||||||
- AcknowledgeDrain 仅在 Active 和 Reserved 都为零时释放所有权。
|
|
||||||
- Assign 与 Extract 并发竞争同一 Proxy 时，只允许一个操作成功。
|
|
||||||
- Expire 使用 `limit` 分批回收过期 assignment，禁止无界返回。
|
|
||||||
|
|
||||||
### 短 TTL 清理
|
|
||||||
|
|
||||||
Redis Hash 字段没有独立 TTL，因此使用三层有界清理：
|
|
||||||
|
|
||||||
1. 幂等和内部操作结果使用 Redis 原生键 TTL。
|
|
||||||
2. Upsert、Health、Extract 和 Ownership 脚本机会式清理少量过期记录。
|
|
||||||
3. 公用维护循环按 `limit` 从 expiry ZSET 分批清理记录、唯一键、过滤索引、
|
|
||||||
ownership 和库存计数。
|
|
||||||
|
|
||||||
主活动键的过期时间始终延伸到当前最晚 Proxy 硬过期时间。活动池停止写入后，
|
|
||||||
整个命名空间最终自动释放；持续写入时由有界维护循环阻止旧字段累积。
|
|
||||||
|
|
||||||
### 凭据
|
|
||||||
|
|
||||||
Provider Parser 继续通过 `credentials.Store` 生成 `SecretRef` 和
|
|
||||||
`CredentialVersion`。Redis Adapter 在 Upsert 写入前解析凭据，使解析失败发生
|
|
||||||
在活动池状态提交之前。Distribution 所需凭据只保存在 Proxy 硬 TTL 和幂等 TTL
|
|
||||||
约束内，Extract 脚本可原子保存完整重放响应。
|
|
||||||
|
|
||||||
Gateway Snapshot 继续只携带凭据引用；Gateway 通过控制面下发到节点内存的
|
|
||||||
凭据材料解析引用，不在请求热路径查询 Redis。凭据分发与轮换属于独立后续
|
|
||||||
实现，不改变本 ADR 的活动池边界。
|
|
||||||
|
|
||||||
### 库存真值
|
|
||||||
|
|
||||||
Redis `inventory` 是当前未提取 Proxy 数量的运行时真值：
|
|
||||||
|
|
||||||
- 插入新的当前生命周期时增加。
|
|
||||||
- EXTRACTED、EXPIRED 或 REMOVED 时减少。
|
|
||||||
- 重复刷新和其他供应商重复上报不改变。
|
|
||||||
- Redis 丢失后归零，由 Provider 重新获取并重建。
|
|
||||||
|
|
||||||
`InventoryReader` 为控制面提供低频校准。PostgreSQL 不保存 Proxy 明细，也不
|
|
||||||
参与每秒库存读取；可选长期指标只能保存无 Proxy 明细的聚合值。
|
|
||||||
|
|
||||||
## 故障语义
|
|
||||||
|
|
||||||
- Redis 不可用时停止 Provider 入池、Extract 和所有权变更。
|
|
||||||
- Distribution 将存储不可用和扫描预算耗尽映射为 503。
|
|
||||||
- PostgreSQL 不可用不阻断 Redis 中能够完成的 Extract。
|
|
||||||
- Worker 在控制面故障时继续使用未过期本地 Snapshot，超过最大陈旧时间后
|
|
||||||
停止接收新流量。
|
|
||||||
- 写脚本通过内部 operation ID 抵御连接中断后的重复执行。
|
|
||||||
- Redis 整体丢失代表活动池代次终止；Provider 重建是新代次，不从 PostgreSQL
|
|
||||||
恢复旧 Proxy，也不延续已丢失代次的排他状态。
|
|
||||||
|
|
||||||
## 测试与验收
|
|
||||||
|
|
||||||
实现必须先建立可复用行为契约，并让 MemoryPool 与 Redis Adapter 运行相同
|
|
||||||
测试向量：
|
|
||||||
|
|
||||||
- 供应商 TTL、安全余量、MaxSize、重复刷新和跨供应商 incumbent。
|
|
||||||
- FETCHED 到 AVAILABLE 及不健康状态退出索引。
|
|
||||||
- partial、allOrNothing、过滤、健康新鲜度和 Gateway 预留。
|
|
||||||
- 幂等重放、摘要冲突和最早 Proxy 过期时间上限。
|
|
||||||
- 100 轮并发 Extract 的返回集合无交集。
|
|
||||||
- Assign 与 Extract 并发互斥，以及 renew/drain/ACK/expire。
|
|
||||||
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
|
|
||||||
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
|
|
||||||
|
|
||||||
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒，并执行
|
|
||||||
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由
|
|
||||||
后续代表性集群压测证明，本 ADR 不把设计目标表述为已验证吞吐。
|
|
||||||
|
|
||||||
## 数据持久化
|
|
||||||
|
|
||||||
本地 Compose 的 Redis 关闭 AOF 和 RDB，因为活动池是可重建短效状态，避免
|
|
||||||
将代理地址、凭据和幂等响应持续写入开发机磁盘。生产 Redis 是否启用受保护的
|
|
||||||
磁盘持久化由部署策略决定，但不得把 Redis 备份当作 Proxy 恢复来源。
|
|
||||||
|
|
||||||
## 备选方案
|
|
||||||
|
|
||||||
### Redis Cluster 单槽
|
|
||||||
|
|
||||||
可以提供 Cluster 故障转移，但活动池仍集中在一个 slot，不能获得水平吞吐
|
|
||||||
扩展。首版使用 Sentinel 已满足当前部署边界，因此暂不承担 Cluster 运维成本。
|
|
||||||
|
|
||||||
### Redis Cluster 多分片
|
|
||||||
|
|
||||||
可以分摊控制面吞吐，但会破坏全局唯一键、Gateway 预留和跨分片
|
|
||||||
`allOrNothing` 原子性，需要 reservation/commit/rollback 两阶段协议。当前
|
|
||||||
Distribution 频率远低于 Gateway 流量，不采用该复杂度。
|
|
||||||
|
|
||||||
### 每个 Proxy 一个带 TTL 的 Redis Key
|
|
||||||
|
|
||||||
硬 TTL 直观，但原子 Extract 需要先发现候选再访问动态 key，键声明、索引清理
|
|
||||||
和批量脚本复杂度更高。固定 Hash 与 ZSET 组合更适合当前单实例原子边界，并用
|
|
||||||
有界清理保证内存回收。
|
|
||||||
|
|
||||||
### 在 PostgreSQL 保存 Proxy 或提取记录
|
|
||||||
|
|
||||||
会引入高频写入、过期清理和不必要存储，并让 PostgreSQL 进入运行时数据路径，
|
|
||||||
与已确认的数据最小化边界冲突，因此不采用。
|
|
||||||
|
|
||||||
## 后果
|
|
||||||
|
|
||||||
收益：
|
|
||||||
|
|
||||||
- Redis 复杂性集中在一个深模块，业务调用方只依赖窄端口。
|
|
||||||
- 独占提取、所有权、库存和幂等共享明确原子边界。
|
|
||||||
- 30 秒短 TTL、过期风暴和扫描工作量具有明确上限。
|
|
||||||
- 10 万 QPS Gateway 路径继续完全本地化。
|
|
||||||
|
|
||||||
代价：
|
|
||||||
|
|
||||||
- 单个活动池主节点是控制面吞吐上限，需要监控脚本 p95/p99 和 CPU。
|
|
||||||
- Hash 字段 TTL 需要 ZSET 和维护循环配合。
|
|
||||||
- Redis Adapter 需要真实 Redis 集成测试，纯内存替身不足以证明 Lua 原子性。
|
|
||||||
- Gateway 凭据安全下发仍需单独实现，但不得改变热路径无 Redis 的约束。
|
|
||||||
@ -35,15 +35,3 @@ Provider 重新获取并重建，不从 PostgreSQL 恢复原 Proxy。
|
|||||||
|
|
||||||
PostgreSQL 只持久化配置版本、Upstream/Routing 管理状态、Admin 审计与 Outbox，
|
PostgreSQL 只持久化配置版本、Upstream/Routing 管理状态、Admin 审计与 Outbox，
|
||||||
以及可选的无 Proxy 明细聚合指标。两类存储不双写 Proxy，也不建立跨存储事务。
|
以及可选的无 Proxy 明细聚合指标。两类存储不双写 Proxy，也不建立跨存储事务。
|
||||||
|
|
||||||
## ADR-005：Redis 活动池采用单实例原子深模块
|
|
||||||
|
|
||||||
**状态：** 接受。
|
|
||||||
|
|
||||||
首版 Redis 活动池部署在单实例或 Sentinel 主节点，通过一个深 Adapter 统一实现
|
|
||||||
Provider 入池、健康状态、Distribution 独占提取、Worker 所有权、库存读取和
|
|
||||||
有界过期清理。所有键使用固定 hash tag，为未来 Redis Cluster 单槽迁移保留
|
|
||||||
兼容性，但首版不引入跨分片事务。
|
|
||||||
|
|
||||||
完整决策、键空间、原子操作和测试门禁见
|
|
||||||
[ADR-005](005-redis-activity-pool.md)。
|
|
||||||
|
|||||||
@ -127,7 +127,7 @@ test/{fixtures,integration,e2e,load}/
|
|||||||
- [x] Enforce minRemainingTTL, maxHealthCheckAge, maxCount, client limits, and
|
- [x] Enforce minRemainingTTL, maxHealthCheckAge, maxCount, client limits, and
|
||||||
reserveForGateway.
|
reserveForGateway.
|
||||||
- [x] Atomically remove selected AVAILABLE entries from the allocatable set and return
|
- [x] Atomically remove selected AVAILABLE entries from the allocatable set and return
|
||||||
the result; MemoryPool and the production Redis Adapter run the same shared contract.
|
the result; the current memory Store models the production Redis atomic boundary.
|
||||||
- [x] Implement partial and allOrNothing without Lease, release, or renewal concepts.
|
- [x] Implement partial and allOrNothing without Lease, release, or renewal concepts.
|
||||||
- [x] Run 1,000 concurrent claim attempts and prove every Proxy ID appears at most once.
|
- [x] Run 1,000 concurrent claim attempts and prove every Proxy ID appears at most once.
|
||||||
|
|
||||||
@ -161,18 +161,14 @@ test/{fixtures,integration,e2e,load}/
|
|||||||
- [ ] Define PostgreSQL ports for ConfigVersion, Upstream/Routing management state,
|
- [ ] Define PostgreSQL ports for ConfigVersion, Upstream/Routing management state,
|
||||||
AdminAudit, Outbox, and optional aggregate metrics; never persist Proxy details or
|
AdminAudit, Outbox, and optional aggregate metrics; never persist Proxy details or
|
||||||
per-extraction records.
|
per-extraction records.
|
||||||
- [x] Implement the Redis TTL activity pool and one atomic extraction operation covering
|
- [ ] Implement the Redis TTL activity pool and one atomic extraction operation covering
|
||||||
candidate eligibility, Gateway reserve, ownership, removal, and short-lived idempotency.
|
candidate eligibility, Gateway reserve, ownership, removal, and short-lived idempotency.
|
||||||
- [x] Implement Redis Worker ownership, drain/ACK, expiry reclaim, inventory and bounded
|
- [ ] Implement Redis Provider leader, distributed rate, Client limit, and Worker
|
||||||
sweep primitives with a monotonic global epoch.
|
heartbeat/ownership; rebuild short-lived Proxy inventory from Providers after loss.
|
||||||
- [ ] Implement Redis Provider leader, distributed rate, Client limit and Worker
|
- [ ] Keep Provider output in Redis TTL activity state and node memory only; keep the
|
||||||
heartbeat; wire automatic Provider inventory rebuild after Redis loss.
|
|
||||||
- [x] Keep Provider output in Redis TTL activity state and node memory only; keep the
|
|
||||||
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
|
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
|
||||||
- [x] Expose Distribution extraction/status and Admin status/enable/disable/switch/reload
|
- [ ] Expose Distribution extraction/status and Admin status/enable/disable/switch/reload.
|
||||||
HTTP handlers and contracts.
|
- [ ] Add integration tests using Compose-backed PostgreSQL/Redis.
|
||||||
- [x] Add Compose-backed Redis 8.2 integration and shared Adapter contract tests.
|
|
||||||
- [ ] Add PostgreSQL management Adapter and Compose-backed integration tests.
|
|
||||||
|
|
||||||
当前进度（2026-07-29）：已实现共享 `platform/httpapi`、Distribution
|
当前进度（2026-07-29）：已实现共享 `platform/httpapi`、Distribution
|
||||||
extract/live/ready Handler 与 Admin status/enable/disable/switch/reload Handler；
|
extract/live/ready Handler 与 Admin status/enable/disable/switch/reload Handler；
|
||||||
@ -181,15 +177,14 @@ extract/live/ready Handler 与 Admin status/enable/disable/switch/reload Handler
|
|||||||
Bearer/CIDR、可信代理、Client ID、本地准入和 API 401/Gateway 407 差异，并作为
|
Bearer/CIDR、可信代理、Client ID、本地准入和 API 401/Gateway 407 差异，并作为
|
||||||
Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
|
Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
|
||||||
`controller/runtime` 已完成 Distribution/Admin 独立监听器、首错联动关闭和
|
`controller/runtime` 已完成 Distribution/Admin 独立监听器、首错联动关闭和
|
||||||
有界优雅停机。生产命令入口、PostgreSQL 管理面 Adapter 及其 Compose 集成测试
|
有界优雅停机；端点正式勾选仍等待 Redis 活动池/原子提取 Adapter、PostgreSQL
|
||||||
仍待实现。
|
管理面 Adapter、命令入口与 Compose 集成测试。
|
||||||
|
|
||||||
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现，Provider
|
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现，Provider
|
||||||
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次；已覆盖
|
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次；已覆盖
|
||||||
`usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、
|
`usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、
|
||||||
重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis
|
重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis
|
||||||
Adapter 已通过真实 Redis 8.2 运行同一套公用契约；原子 Lua 覆盖提取、所有权和
|
Lua/Function Adapter 和多节点集成测试仍待实现。
|
||||||
有界清理。Redis Sentinel/故障转移验证与代表性多节点压测仍待实施。
|
|
||||||
|
|
||||||
## Task 11: Checker and Health Reducer
|
## Task 11: Checker and Health Reducer
|
||||||
|
|
||||||
|
|||||||
@ -48,12 +48,6 @@ docker compose -f deploy/docker-compose.yml config
|
|||||||
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
本地 Compose 的 Redis 只作为可重建短效状态 fixture，固定使用
|
|
||||||
`--appendonly no --save ""`，且不挂载 `/data` 或命名卷。真实 Redis 8.2 契约可
|
|
||||||
通过 `.\scripts\test-redis.ps1` 执行；脚本使用唯一命名空间并在结束时定向清理，
|
|
||||||
不执行 `FLUSHDB`。生产环境的 Redis 高可用与持久化策略必须独立评审，不能照搬
|
|
||||||
本地 fixture。
|
|
||||||
|
|
||||||
目标拓扑入口：
|
目标拓扑入口：
|
||||||
|
|
||||||
- Gateway：`127.0.0.1:8080`
|
- Gateway：`127.0.0.1:8080`
|
||||||
@ -249,8 +243,7 @@ Prometheus 标签禁止包含 Proxy IP、Client ID、Session、完整 URL、requ
|
|||||||
- PostgreSQL：每日全量、连续 WAL/PITR，保护配置版本、Upstream/Routing 管理
|
- PostgreSQL：每日全量、连续 WAL/PITR，保护配置版本、Upstream/Routing 管理
|
||||||
状态、Admin 审计与 outbox；至少每季度做恢复演练。
|
状态、Admin 审计与 outbox；至少每季度做恢复演练。
|
||||||
- Redis：保存可由 Provider 重建的 TTL 活动池、所有权/Leader 协调和短期幂等
|
- Redis：保存可由 Provider 重建的 TTL 活动池、所有权/Leader 协调和短期幂等
|
||||||
结果；生产环境可使用高可用与受控持久化降低窗口丢失风险，但不把它当长期
|
结果；使用高可用与持久化降低窗口丢失风险，但不把它当长期业务档案。
|
||||||
业务档案或备份源。本地 Compose 刻意关闭持久化并且不挂载数据卷。
|
|
||||||
- 配置：版本化保存校验通过的不可变 Revision 与校验和。
|
- 配置：版本化保存校验通过的不可变 Revision 与校验和。
|
||||||
- Secret：由密钥平台版本化，日志和备份中不得出现明文。
|
- Secret：由密钥平台版本化，日志和备份中不得出现明文。
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,7 @@
|
|||||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | 依赖规则、测试、性能剖析 |
|
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | 依赖规则、测试、性能剖析 |
|
||||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | 配置、监听装配、端口测试 |
|
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | 配置、监听装配、端口测试 |
|
||||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Leader、singleflight、集成测试 |
|
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Leader、singleflight、集成测试 |
|
||||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标；待代表性集群负载报告 |
|
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 容量公式、负载场景、部署清单 |
|
||||||
|
|
||||||
## Routing 与 Upstream
|
## Routing 与 Upstream
|
||||||
|
|
||||||
@ -49,7 +49,7 @@
|
|||||||
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `Inventory.AvailableSlots` 与 Pool Reconciler 测试 |
|
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `Inventory.AvailableSlots` 与 Pool Reconciler 测试 |
|
||||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | 依赖审计与压测 |
|
||||||
|
|
||||||
## Gateway
|
## Gateway
|
||||||
|
|
||||||
@ -66,9 +66,9 @@
|
|||||||
| ID | 最终需求 | 来源 | 验证证据 |
|
| ID | 最终需求 | 来源 | 验证证据 |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| DIST-001 | API 提取固定为一次性独占发放，不使用 Lease | 9083-9404 | Domain 状态机与 API 测试 |
|
| DIST-001 | API 提取固定为一次性独占发放，不使用 Lease | 9083-9404 | Domain 状态机与 API 测试 |
|
||||||
| DIST-002 | AVAILABLE -> EXTRACTED 必须原子完成后才能返回 | 9083-9189 | Memory/Redis 公用契约与真实 Redis 100 轮竞态测试 |
|
| DIST-002 | AVAILABLE -> EXTRACTED 必须原子完成后才能返回 | 9083-9189 | 共享 Repository 所有权/提取 100 轮竞态测试 |
|
||||||
| DIST-003 | 支持 partial 与 allOrNothing，默认 partial | 9190-9215 | API 契约测试 |
|
| DIST-003 | 支持 partial 与 allOrNothing，默认 partial | 9190-9215 | API 契约测试 |
|
||||||
| DIST-004 | 不保存逐代理/逐次提取审计记录，不提供释放接口；仅保留短期幂等结果 | 9216-9252 | Redis 幂等 TTL 契约、OpenAPI 与 PostgreSQL 边界审计 |
|
| DIST-004 | 保存审计记录，不提供释放接口 | 9216-9252 | 原子审计、幂等测试与 OpenAPI |
|
||||||
| DIST-005 | 返回 expiresAt 与 remainingTtlSeconds | 9334-9360 | `extraction/service_test.go` |
|
| DIST-005 | 返回 expiresAt 与 remainingTtlSeconds | 9334-9360 | `extraction/service_test.go` |
|
||||||
| DIST-006 | 提取前校验 minRemainingTTL 与 maxHealthCheckAge | 9334-9369 | 过滤测试 |
|
| DIST-006 | 提取前校验 minRemainingTTL 与 maxHealthCheckAge | 9334-9369 | 过滤测试 |
|
||||||
| DIST-007 | reserveForGateway 防止 Extract 清空共享池 | 9281-9333 | 共享池测试 |
|
| DIST-007 | reserveForGateway 防止 Extract 清空共享池 | 9281-9333 | 共享池测试 |
|
||||||
@ -86,4 +86,4 @@
|
|||||||
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
||||||
| OPS-002 | 优雅停机停止新请求/Fetch，等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
| OPS-002 | 优雅停机停止新请求/Fetch，等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
||||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | 指标描述符测试 |
|
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | 指标描述符测试 |
|
||||||
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单；Redis 活动池由 Memory/Redis 公用契约覆盖，跨进程故障场景仍按清单推进 |
|
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | CI 测试清单 |
|
||||||
|
|||||||
@ -1,761 +0,0 @@
|
|||||||
# Redis Activity Pool Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Build the production Redis activity pool that atomically owns short-lived Proxy inventory, health availability, exclusive extraction, Worker ownership, bounded idempotency, and expiry cleanup without putting Redis on the Gateway request path.
|
|
||||||
|
|
||||||
**Architecture:** One deep `redisactivity.Adapter` implements the existing narrow domain ports plus health, inventory, and maintenance ports. The first release targets Redis standalone/Sentinel; all keys share `{activity}` for future single-slot Cluster compatibility, and bounded Lua scripts serialize every state transition that can race.
|
|
||||||
|
|
||||||
**Tech Stack:** Go 1.26, `github.com/redis/go-redis/v9`, embedded Redis Lua scripts, Redis 8.2 integration tests, Docker Compose, PowerShell verification.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Public Test Seams
|
|
||||||
|
|
||||||
The approved public seams are:
|
|
||||||
|
|
||||||
- `activitypool.Upserter.UpsertFetched`
|
|
||||||
- `activitypool.HealthStore.ApplyHealth`
|
|
||||||
- `activitypool.InventoryReader.Inventory`
|
|
||||||
- `extraction.Store.Extract`
|
|
||||||
- `ownership.Repository` context-aware methods
|
|
||||||
- `activitypool.Maintainer.SweepExpired`
|
|
||||||
|
|
||||||
Tests assert behavior only through these seams. Lua source, Redis key contents, codec fields,
|
|
||||||
script SHA values, and private runner calls are not test seams.
|
|
||||||
|
|
||||||
## File Map
|
|
||||||
|
|
||||||
- Modify `internal/domain/activitypool/pool.go`: public activity contracts and memory reference implementation.
|
|
||||||
- Modify `internal/domain/activitypool/pool_test.go`: shared behavioral expectations for max size, health, inventory, and maintenance.
|
|
||||||
- Modify `internal/domain/ownership/ownership.go`: context-aware production repository contract.
|
|
||||||
- Modify `internal/controller/pool/ownership.go`: pass contexts and return storage failures.
|
|
||||||
- Modify `internal/controller/pool/ownership_test.go`: public manager contract after the signature migration.
|
|
||||||
- Modify `internal/controller/provider/reconciler.go`: pass `pool.maxSize` into authoritative Upsert.
|
|
||||||
- Modify `internal/controller/provider/reconciler_test.go`: verify max-size propagation.
|
|
||||||
- Modify `internal/controller/extraction/service.go`: classify Redis availability failures as 503-safe service errors.
|
|
||||||
- Modify `internal/controller/extraction/service_test.go`: verify error classification.
|
|
||||||
- Create `internal/adapters/redisactivity/adapter.go`: constructor, dependencies, options, interface assertions.
|
|
||||||
- Create `internal/adapters/redisactivity/keys.go`: normalized key construction and hash tag enforcement.
|
|
||||||
- Create `internal/adapters/redisactivity/codec.go`: Proxy, ownership, result, and script reply encoding.
|
|
||||||
- Create `internal/adapters/redisactivity/scripts.go`: embedded script declarations.
|
|
||||||
- Create `internal/adapters/redisactivity/upsert.go`: credential materialization and bounded Upsert calls.
|
|
||||||
- Create `internal/adapters/redisactivity/health.go`: health transition adapter.
|
|
||||||
- Create `internal/adapters/redisactivity/extract.go`: extraction command digest, reply mapping, and URL construction.
|
|
||||||
- Create `internal/adapters/redisactivity/ownership.go`: context-aware ownership methods.
|
|
||||||
- Create `internal/adapters/redisactivity/maintenance.go`: bounded expiry cleanup and inventory reads.
|
|
||||||
- Create `internal/adapters/redisactivity/scripts/*.lua`: atomic Redis state transitions.
|
|
||||||
- Create `internal/adapters/redisactivity/contract_test.go`: real Redis public-seam contract suite.
|
|
||||||
- Create `internal/adapters/redisactivity/testredis_test.go`: isolated integration client and namespace helpers.
|
|
||||||
- Create `deploy/docker-compose.test.yml`: non-persistent Redis 8.2 test fixture.
|
|
||||||
- Create `deploy/compose_test.go`: structured local Redis persistence-policy assertion.
|
|
||||||
- Create `scripts/test-redis.ps1`: bounded integration-test runner.
|
|
||||||
- Modify `deploy/docker-compose.yml`: make local runtime Redis explicitly non-persistent.
|
|
||||||
- Modify `docs/development/implementation-plan.md`: record the delivered Adapter boundary.
|
|
||||||
- Modify `docs/testing/test-strategy.md`: record Redis contract and failure tests.
|
|
||||||
|
|
||||||
### Task 1: Make Ownership Storage Context-Aware
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/domain/ownership/ownership.go`
|
|
||||||
- Modify: `internal/controller/pool/ownership.go`
|
|
||||||
- Modify: `internal/controller/pool/ownership_test.go`
|
|
||||||
- Modify: `internal/domain/activitypool/pool.go`
|
|
||||||
- Modify: `internal/domain/activitypool/pool_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing manager tests for cancellation, storage errors, and bounded expiry**
|
|
||||||
|
|
||||||
Add a recording repository implementing the new intended contract and tests with these calls:
|
|
||||||
|
|
||||||
```go
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
if _, err := manager.Assign(ctx, now, "proxy-1", "worker-1", time.Minute); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Assign() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
expired, err := manager.Expire(context.Background(), now, 32)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Expire(): %v", err)
|
|
||||||
}
|
|
||||||
if repository.expireLimit != 32 || len(expired) != 1 {
|
|
||||||
t.Fatalf("Expire() = %+v, limit=%d", expired, repository.expireLimit)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run the focused tests and verify RED**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/controller/pool ./internal/domain/activitypool
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: compilation fails because the current manager methods do not accept a context and
|
|
||||||
`Expire` has neither a limit nor an error result.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Replace the ownership port and manager signatures**
|
|
||||||
|
|
||||||
Use this exact repository contract:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Repository interface {
|
|
||||||
Assign(context.Context, time.Time, string, string, time.Duration) (Assignment, error)
|
|
||||||
Renew(context.Context, time.Time, string, string, uint64, time.Duration) (Assignment, error)
|
|
||||||
BeginDrain(context.Context, string, string, uint64) (Assignment, error)
|
|
||||||
AcknowledgeDrain(context.Context, string, string, uint64, int64, int64) error
|
|
||||||
Get(context.Context, string) (Assignment, bool, error)
|
|
||||||
Expire(context.Context, time.Time, int) ([]Assignment, error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Update `OwnershipManager` to validate `ctx != nil`, pass the context unchanged, require
|
|
||||||
`limit > 0`, and return repository errors without swallowing them. Update `MemoryPool` to
|
|
||||||
check `ctx.Err()` before and after locking, return `(Assignment, bool, error)` from `Get`,
|
|
||||||
and stop `Expire` after `limit` assignments.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Update all ownership call sites and run GREEN**
|
|
||||||
|
|
||||||
Pass `context.Background()` from existing tests that do not exercise cancellation. Run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/controller/pool ./internal/domain/activitypool
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit the interface migration**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
git add internal/domain/ownership internal/domain/activitypool internal/controller/pool
|
|
||||||
git commit -m "refactor: make ownership repository context aware"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Task 2: Add Health, Inventory, MaxSize, and Maintenance Contracts
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/domain/activitypool/pool.go`
|
|
||||||
- Modify: `internal/domain/activitypool/pool_test.go`
|
|
||||||
- Modify: `internal/controller/provider/reconciler.go`
|
|
||||||
- Modify: `internal/controller/provider/reconciler_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing public-seam tests**
|
|
||||||
|
|
||||||
Add tests covering a Fetched -> Checking -> Available transition, stale health rejection,
|
|
||||||
per-upstream MaxSize, current inventory, and bounded expiry:
|
|
||||||
|
|
||||||
```go
|
|
||||||
updated, err := pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: proxyID, CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
})
|
|
||||||
if err != nil || updated.State != proxyDomain.StateChecking {
|
|
||||||
t.Fatalf("ApplyHealth(checking) = %+v, %v", updated, err)
|
|
||||||
}
|
|
||||||
updated, err = pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: proxyID, CheckedAt: now.Add(2 * time.Second),
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: 25 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil || updated.State != proxyDomain.StateAvailable {
|
|
||||||
t.Fatalf("ApplyHealth(available) = %+v, %v", updated, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
inventory, err := pool.Inventory(context.Background(), "provider-a", now.Add(2*time.Second))
|
|
||||||
if err != nil || inventory.Managed != 1 {
|
|
||||||
t.Fatalf("Inventory() = %+v, %v", inventory, err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Insert two distinct candidates with `FetchedBatch.MaxSize = 1` and assert one insert plus one
|
|
||||||
capacity drop. Call `SweepExpired(ctx, afterExpiry, 1)` twice and assert each call removes at
|
|
||||||
most one record.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run the tests and verify RED**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/domain/activitypool ./internal/controller/provider
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: compilation fails because the new ports and `MaxSize` do not exist.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add the approved activity contracts**
|
|
||||||
|
|
||||||
Add these public types:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type HealthUpdate struct {
|
|
||||||
ProxyID string
|
|
||||||
CheckedAt time.Time
|
|
||||||
NextState proxyDomain.State
|
|
||||||
Latency time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type Inventory struct {
|
|
||||||
UpstreamID string
|
|
||||||
Managed int
|
|
||||||
}
|
|
||||||
|
|
||||||
type HealthStore interface {
|
|
||||||
ApplyHealth(context.Context, HealthUpdate) (Entry, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type InventoryReader interface {
|
|
||||||
Inventory(context.Context, string, time.Time) (Inventory, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Maintainer interface {
|
|
||||||
SweepExpired(context.Context, time.Time, int) (int, error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Add `MaxSize int` to `FetchedBatch`. Reject non-positive MaxSize as an invalid batch. Count
|
|
||||||
only managed states belonging to the incumbent upstream. Capacity-rejected candidates count
|
|
||||||
as `Dropped` and never create unique-key mappings.
|
|
||||||
|
|
||||||
`ApplyHealth` must reject missing IDs, zero time, negative latency, missing entries, invalid
|
|
||||||
state transitions, and observations older than `LastCheckedAt`. Replaying the same state and
|
|
||||||
timestamp is idempotent. A transition to AVAILABLE updates `LastSuccessAt`.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Pass MaxSize from Provider Reconciler**
|
|
||||||
|
|
||||||
Add `MaxSize int` to `provider.Config`, require it to be positive, and populate:
|
|
||||||
|
|
||||||
```go
|
|
||||||
activitypool.FetchedBatch{
|
|
||||||
ObservedAt: r.runtime.Clock.Now().UTC(),
|
|
||||||
ConfiguredTTL: r.config.TTL,
|
|
||||||
AllocationSafetyMargin: r.config.AllocationSafetyMargin,
|
|
||||||
MaxSize: r.config.MaxSize,
|
|
||||||
Proxies: retained,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Update constructor tests to use a positive MaxSize and assert the activity sink receives it.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run GREEN and commit**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/domain/activitypool ./internal/controller/provider ./internal/controller/pool
|
|
||||||
git add internal/domain/activitypool internal/controller/provider
|
|
||||||
git commit -m "feat: add activity health and inventory contracts"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS, then a commit containing only this slice.
|
|
||||||
|
|
||||||
### Task 3: Classify Activity Store Availability Failures
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `internal/domain/extraction/extraction.go`
|
|
||||||
- Modify: `internal/controller/extraction/service.go`
|
|
||||||
- Modify: `internal/controller/extraction/service_test.go`
|
|
||||||
- Modify: `internal/controller/distribution/handler_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing service and HTTP error tests**
|
|
||||||
|
|
||||||
Configure a recording Store to return `domain.ErrStoreUnavailable`. Assert:
|
|
||||||
|
|
||||||
```go
|
|
||||||
_, err := service.Extract(context.Background(), validRequest)
|
|
||||||
if !errors.Is(err, ErrUnavailable) || !errors.Is(err, domain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("Extract() error = %v, want unavailable classification", err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
At the Handler seam, assert the same error becomes HTTP 503 with code
|
|
||||||
`SERVICE_UNAVAILABLE` and no underlying Redis text in the response body.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run RED**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/controller/extraction ./internal/controller/distribution
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: compilation fails because `ErrStoreUnavailable` is not defined.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add the stable storage error and mapping**
|
|
||||||
|
|
||||||
Add:
|
|
||||||
|
|
||||||
```go
|
|
||||||
var ErrStoreUnavailable = errors.New("extraction store unavailable")
|
|
||||||
```
|
|
||||||
|
|
||||||
In `Service.Extract`, preserve `ErrInsufficientProxies`, `ErrIdempotencyConflict`, and
|
|
||||||
`ErrInvalidCommand`; wrap any error matching `ErrStoreUnavailable` with `ErrUnavailable`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, domain.ErrStoreUnavailable) {
|
|
||||||
return response, errors.Join(ErrUnavailable, err)
|
|
||||||
}
|
|
||||||
return response, err
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run GREEN and commit**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/controller/extraction ./internal/controller/distribution
|
|
||||||
git add internal/domain/extraction internal/controller/extraction internal/controller/distribution
|
|
||||||
git commit -m "feat: classify extraction store failures"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
### Task 4: Add Redis Adapter Foundation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `go.mod`
|
|
||||||
- Modify: `go.sum`
|
|
||||||
- Create: `internal/adapters/redisactivity/adapter.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/keys.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/codec.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/adapter_test.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/testredis_test.go`
|
|
||||||
- Create: `deploy/docker-compose.test.yml`
|
|
||||||
- Create: `scripts/test-redis.ps1`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add failing constructor and key-safety tests**
|
|
||||||
|
|
||||||
Test that nil clients, nil credential stores, empty namespaces, braces in namespaces, zero
|
|
||||||
operation TTL, and non-positive scan/cleanup limits fail. Test that valid options build keys
|
|
||||||
with exactly one fixed hash tag:
|
|
||||||
|
|
||||||
```go
|
|
||||||
adapter, err := New(client, Options{
|
|
||||||
Namespace: "test-a", Credentials: credentialStore,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 2048, CleanupLimit: 128,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(): %v", err)
|
|
||||||
}
|
|
||||||
if got := adapter.keys.records; got != "pp:{activity}:test-a:records" {
|
|
||||||
t.Fatalf("records key = %q", got)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run RED**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/adapters/redisactivity
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: package does not exist.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Pin go-redis and implement the narrow constructor**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go get github.com/redis/go-redis/v9@v9.19.0
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `redis.Scripter` rather than exposing `redis.UniversalClient` throughout the package:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Options struct {
|
|
||||||
Namespace string
|
|
||||||
Credentials credentials.Store
|
|
||||||
OperationTTL time.Duration
|
|
||||||
MaxCandidateScan int
|
|
||||||
CleanupLimit int
|
|
||||||
}
|
|
||||||
|
|
||||||
type Adapter struct {
|
|
||||||
client redis.Scripter
|
|
||||||
credentials credentials.Store
|
|
||||||
keys keyspace
|
|
||||||
options Options
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(client redis.Scripter, options Options) (*Adapter, error)
|
|
||||||
```
|
|
||||||
|
|
||||||
Embed immutable scripts once and reuse `redis.Script`, which provides EVALSHA with EVAL
|
|
||||||
fallback when the server script cache is empty:
|
|
||||||
|
|
||||||
```go
|
|
||||||
//go:embed scripts/upsert.lua
|
|
||||||
var upsertSource string
|
|
||||||
|
|
||||||
var upsertScript = redis.NewScript(upsertSource)
|
|
||||||
```
|
|
||||||
|
|
||||||
Normalize the namespace with a strict `[A-Za-z0-9._-]+` policy. Hash user-controlled
|
|
||||||
Client IDs, idempotency keys, proxy unique keys, and operation IDs with SHA-256 before using
|
|
||||||
them in Redis key names or fields.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Implement deterministic codecs with redacted formatting**
|
|
||||||
|
|
||||||
Use JSON only at the Redis boundary. Define private `proxyRecord`, `ownershipRecord`,
|
|
||||||
`idempotencyRecord`, and typed script reply structures. Store times as Unix milliseconds and
|
|
||||||
durations as integer nanoseconds. Decode with strict state and integer validation. Any
|
|
||||||
formatting method for structures containing passwords writes `<redacted>`.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run GREEN and commit**
|
|
||||||
|
|
||||||
Create the Redis 8.2 test fixture before the first integration slice. It binds only
|
|
||||||
`127.0.0.1:16379`, has no data volume, runs `--appendonly no --save ""`, and exposes a PING
|
|
||||||
health check. Add `testredis_test.go` that skips integration-tagged tests when
|
|
||||||
`PROXY_POOL_TEST_REDIS_URL` is absent and otherwise creates a unique namespace without
|
|
||||||
flushing shared databases. Add `scripts/test-redis.ps1` with `try/finally` fixture shutdown.
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
gofmt -w internal/adapters/redisactivity
|
|
||||||
go test -timeout 60s ./internal/adapters/redisactivity
|
|
||||||
git add go.mod go.sum internal/adapters/redisactivity deploy/docker-compose.test.yml scripts/test-redis.ps1
|
|
||||||
git commit -m "feat: add redis activity adapter foundation"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
### Task 5: Implement Atomic Upsert and Health Transitions
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/adapters/redisactivity/upsert.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/health.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts/upsert.lua`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts/health.lua`
|
|
||||||
- Create: `internal/adapters/redisactivity/upsert_integration_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing real-Redis tests at the approved seams**
|
|
||||||
|
|
||||||
Put `//go:build integration` at the top of every real-Redis test file and use a unique
|
|
||||||
namespace per test. Cover:
|
|
||||||
|
|
||||||
```go
|
|
||||||
result, err := store.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second,
|
|
||||||
AllocationSafetyMargin: 3 * time.Second, MaxSize: 1,
|
|
||||||
Proxies: []proxyDomain.Proxy{first, second},
|
|
||||||
})
|
|
||||||
if err != nil || result.Inserted != 1 || result.Dropped != 1 {
|
|
||||||
t.Fatalf("UpsertFetched() = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Also verify cross-provider incumbent preservation, refresh preserving runtime health, expired
|
|
||||||
incumbent replacement, credential resolution before commit, Fetched -> Checking -> Available,
|
|
||||||
and stale health observations not replacing newer state.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Start Redis and verify RED**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
docker compose -f deploy/docker-compose.test.yml up -d --wait
|
|
||||||
$env:PROXY_POOL_TEST_REDIS_URL='redis://127.0.0.1:16379/15'
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*(Upsert|Health)'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: tests fail because Upsert and Health return no production behavior.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Implement bounded Upsert**
|
|
||||||
|
|
||||||
Resolve every non-empty credential reference before running Lua:
|
|
||||||
|
|
||||||
```go
|
|
||||||
value, err := a.credentials.Resolve(ctx, credentials.Reference{
|
|
||||||
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.UpsertResult{}, fmt.Errorf("resolve proxy credential: %w", err)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The script receives validated records and atomically maintains `records`, `unique`, `idkeys`,
|
|
||||||
`expiry`, `available`, facet indexes, and `inventory`. It performs a cleanup batch first,
|
|
||||||
preserves incumbent upstream and runtime state, refuses EXTRACTED refresh, enforces MaxSize,
|
|
||||||
and stores an operation result keyed by one generated operation ID so EVALSHA retry returns
|
|
||||||
the original counters.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Implement Health state changes**
|
|
||||||
|
|
||||||
The health script loads one record, rejects missing/expired entries, rejects invalid or stale
|
|
||||||
transitions, updates health fields, and adds or removes every availability index in the same
|
|
||||||
atomic call. Replaying the same `CheckedAt` and state returns the stored result without
|
|
||||||
changing indexes.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run GREEN and commit**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*(Upsert|Health)'
|
|
||||||
git add internal/adapters/redisactivity
|
|
||||||
git commit -m "feat: add redis activity upsert and health"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
### Task 6: Implement Atomic Exclusive Extraction
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/adapters/redisactivity/extract.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts/extract.lua`
|
|
||||||
- Create: `internal/adapters/redisactivity/extract_integration_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing extraction contract tests**
|
|
||||||
|
|
||||||
Seed AVAILABLE records through Upsert/ApplyHealth and test partial, allOrNothing, every filter,
|
|
||||||
MinRemainingTTL, MaxHealthCheckAge, ReserveForGateway, same-key replay, conflicting digest,
|
|
||||||
and bounded idempotency expiry. Add a 100-round concurrent race:
|
|
||||||
|
|
||||||
```go
|
|
||||||
for iteration := 0; iteration < 100; iteration++ {
|
|
||||||
// Two goroutines call Extract for the same one-item namespace.
|
|
||||||
// Exactly one returned result must contain the Proxy ID.
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Test that a selective query exhausting `MaxCandidateScan` returns
|
|
||||||
`extraction.ErrStoreUnavailable` and leaves every Proxy AVAILABLE.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run RED against Redis 8.2**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*Extract'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: FAIL because `Adapter.Extract` is absent.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Implement request digest and typed reply mapping**
|
|
||||||
|
|
||||||
Canonicalize each filter as a sorted, duplicate-free list and hash this payload:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type digestInput struct {
|
|
||||||
Requested int `json:"requested"`
|
|
||||||
Fulfillment extraction.Fulfillment `json:"fulfillment"`
|
|
||||||
Protocols []string `json:"protocols"`
|
|
||||||
Regions []string `json:"regions"`
|
|
||||||
Carriers []string `json:"carriers"`
|
|
||||||
Upstreams []string `json:"upstreams"`
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Exclude RequestID, SourceIP, Now, and operational TTLs from the business digest. Use Client ID
|
|
||||||
plus Idempotency-Key for business replay; use RequestID only for an internal operation key.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Implement one bounded extraction script**
|
|
||||||
|
|
||||||
The script must:
|
|
||||||
|
|
||||||
1. Return a committed result when the idempotency digest matches.
|
|
||||||
2. Return conflict without mutation when it differs.
|
|
||||||
3. Choose the smallest `ZCARD` among filter dimensions containing exactly one requested
|
|
||||||
value; otherwise use global available. This keeps the driver a complete superset when a
|
|
||||||
dimension contains OR values.
|
|
||||||
4. Scan from longest `usableUntil`, validate complete records, and stop after
|
|
||||||
`requested + reserveForGateway` matches or `MaxCandidateScan` records.
|
|
||||||
5. Return scan-budget exhaustion without mutation when the result cannot be decided.
|
|
||||||
6. Return insufficient without mutation for allOrNothing.
|
|
||||||
7. Mark selected records EXTRACTED, remove every availability index, decrement inventory,
|
|
||||||
and store the response in the same script.
|
|
||||||
8. Set idempotency TTL to the earlier of configured TTL and earliest selected hard expiry.
|
|
||||||
|
|
||||||
Build URL values in Go with `net/url.URL` and `net.JoinHostPort`; never concatenate user or
|
|
||||||
password fields manually.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run GREEN and commit**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*Extract'
|
|
||||||
git add internal/adapters/redisactivity
|
|
||||||
git commit -m "feat: add atomic redis proxy extraction"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
### Task 7: Implement Ownership, Inventory, and Expiry Maintenance
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/adapters/redisactivity/ownership.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/maintenance.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts/ownership.lua`
|
|
||||||
- Create: `internal/adapters/redisactivity/scripts/sweep.lua`
|
|
||||||
- Create: `internal/adapters/redisactivity/ownership_integration_test.go`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing public-seam tests**
|
|
||||||
|
|
||||||
Cover Assign, expired-lease takeover, Renew version increments, stale epoch rejection,
|
|
||||||
BeginDrain idempotency, ACK active/reserved rejection, successful ACK, Get, limited Expire,
|
|
||||||
Inventory decrement, and limited SweepExpired. Add 100 Assign-vs-Extract races and assert
|
|
||||||
exactly one winner each time.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run RED**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*(Ownership|Inventory|Sweep)'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: FAIL because these Adapter methods are absent.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Implement constant-work ownership scripts**
|
|
||||||
|
|
||||||
Use one operation selector with separate validated argument shapes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
assign | renew | begin_drain | acknowledge_drain | get | expire
|
|
||||||
```
|
|
||||||
|
|
||||||
Assign removes the Proxy from AVAILABLE indexes. Renew caps lease expiry at `usableUntil`.
|
|
||||||
ACK restores indexes only when the Proxy remains AVAILABLE and usable. Each mutating call
|
|
||||||
uses an internal operation ID so a transport retry does not increment epoch/version twice.
|
|
||||||
Expire processes no more than the caller-provided limit.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Implement inventory and bounded sweep**
|
|
||||||
|
|
||||||
`Inventory` first runs the same small cleanup budget, then returns the upstream counter.
|
|
||||||
`SweepExpired` pops no more than `limit` hard-expired Proxy IDs, removes record/unique/id/facet/
|
|
||||||
ownership indexes, and decrements only the incumbent upstream counter. Repeated sweep calls
|
|
||||||
are idempotent and counters never go below zero.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run GREEN and commit**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -tags=integration -timeout 60s ./internal/adapters/redisactivity -run 'TestRedis.*(Ownership|Inventory|Sweep)'
|
|
||||||
git add internal/adapters/redisactivity
|
|
||||||
git commit -m "feat: add redis ownership and expiry maintenance"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
### Task 8: Reuse One Contract Suite for Memory and Redis
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `internal/domain/activitypool/contracttest/contract.go`
|
|
||||||
- Create: `internal/domain/activitypool/contract_external_test.go`
|
|
||||||
- Create: `internal/adapters/redisactivity/contract_integration_test.go`
|
|
||||||
- Modify: `internal/adapters/redisactivity/testredis_test.go`
|
|
||||||
- Modify: `scripts/test-redis.ps1`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Extract behavior-only contract cases**
|
|
||||||
|
|
||||||
Define a factory that returns the approved narrow seams and cleanup:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Store interface {
|
|
||||||
activitypool.Upserter
|
|
||||||
activitypool.HealthStore
|
|
||||||
activitypool.InventoryReader
|
|
||||||
activitypool.Maintainer
|
|
||||||
extraction.Store
|
|
||||||
ownership.Repository
|
|
||||||
}
|
|
||||||
|
|
||||||
type Factory func(*testing.T) (Store, func())
|
|
||||||
|
|
||||||
func Run(t *testing.T, factory Factory)
|
|
||||||
```
|
|
||||||
|
|
||||||
Create `contract_external_test.go` with package `activitypool_test` and run the shared suite
|
|
||||||
against `activitypool.NewMemoryPool`. Retain the existing package-internal Snapshot tests in
|
|
||||||
`pool_test.go`; do not create an import cycle and do not inspect Redis keys from the shared
|
|
||||||
contract.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run the contract against MemoryPool**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go test -timeout 60s ./internal/domain/activitypool/...
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Complete the isolated Redis fixture runner**
|
|
||||||
|
|
||||||
Extend `scripts/test-redis.ps1` to run the complete integration contract. It must use
|
|
||||||
`try/finally`, wait for health, set `PROXY_POOL_TEST_REDIS_URL`, run only integration-tagged
|
|
||||||
tests with a 60-second Go timeout, and stop the fixture in `finally`.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run the same contract against Redis**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\test-redis.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Redis 8.2 becomes healthy and all integration tests PASS in at most 60 seconds per
|
|
||||||
Go test invocation.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit the shared contract and fixture**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
git add internal/domain/activitypool internal/adapters/redisactivity scripts/test-redis.ps1
|
|
||||||
git commit -m "test: add redis activity pool contract suite"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Task 9: Finalize Local Runtime Policy and Documentation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `deploy/docker-compose.yml`
|
|
||||||
- Create: `deploy/compose_test.go`
|
|
||||||
- Modify: `docs/development/implementation-plan.md`
|
|
||||||
- Modify: `docs/testing/test-strategy.md`
|
|
||||||
- Modify: `docs/operations/runbook.md`
|
|
||||||
- Modify: `docs/requirements/traceability.md`
|
|
||||||
- Modify: `progress.md`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write a failing Compose policy check**
|
|
||||||
|
|
||||||
Add `deploy/compose_test.go` using `go.yaml.in/yaml/v4` to parse `docker-compose.yml` and assert
|
|
||||||
that the local Redis service uses `--appendonly no`, `--save ""`, and has no persistent data
|
|
||||||
volume. Run `go test -timeout 60s ./deploy` before changing Compose and confirm it fails.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Make local Redis explicitly ephemeral**
|
|
||||||
|
|
||||||
Change only the Redis service command and volume mount. Keep health checks and the backend
|
|
||||||
network unchanged. Do not remove or manipulate any existing Docker volume on the machine.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Update authoritative documents**
|
|
||||||
|
|
||||||
Record:
|
|
||||||
|
|
||||||
- Redis Adapter and shared contract as complete.
|
|
||||||
- Gateway hot path still has no Redis access.
|
|
||||||
- PostgreSQL still contains no Proxy details or extraction records.
|
|
||||||
- Redis Activity Pool is runtime truth and local Redis is intentionally non-persistent.
|
|
||||||
- `docs/testing/strategy.md` legacy PostgreSQL extraction wording is superseded.
|
|
||||||
- `docs/requirements/traceability.md` no longer claims per-proxy extraction audit storage.
|
|
||||||
- 100,000 QPS remains an unverified end-to-end target.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run full verification**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\verify.ps1
|
|
||||||
.\scripts\test-redis.ps1
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: gofmt, go vet, all unit tests, build, and Redis integration tests PASS. Race tests
|
|
||||||
run only when `CGO_ENABLED=1`; otherwise the script prints the existing explicit skip reason.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Perform independent review**
|
|
||||||
|
|
||||||
Review public contracts, Lua atomicity, retry behavior, TTL bounds, credential redaction,
|
|
||||||
inventory counters, and the absence of Redis calls in `internal/gateway`. Fix every blocking
|
|
||||||
finding and rerun both verification scripts.
|
|
||||||
|
|
||||||
- [ ] **Step 6: Commit the runtime policy and documentation**
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
git add deploy/docker-compose.yml deploy/compose_test.go docs/development/implementation-plan.md docs/testing/test-strategy.md docs/operations/runbook.md docs/requirements/traceability.md progress.md
|
|
||||||
git commit -m "docs: finalize redis activity pool delivery"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 7: Push the completed branch**
|
|
||||||
|
|
||||||
Verify the diff contains no user-owned deletion, then push:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
git status --short
|
|
||||||
git push origin build/proxy-pool-architecture
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: the remote branch advances through all Redis Activity Pool commits while
|
|
||||||
`proxy-pool-docs-v1.0.zip` remains untracked by the commits.
|
|
||||||
|
|
||||||
## Self-Review Result
|
|
||||||
|
|
||||||
- ADR-005 deployment, module, keyspace, Upsert, Health, Extract, Ownership, cleanup,
|
|
||||||
credentials, inventory, failure, test, and persistence sections each map to a task above.
|
|
||||||
- Every public method used by later tasks is defined in Tasks 1-4.
|
|
||||||
- Memory and Redis implementations share behavior tests only through approved public seams.
|
|
||||||
- No PostgreSQL Proxy storage, Gateway Redis hot-path access, Cluster sharding, or production
|
|
||||||
throughput claim is introduced.
|
|
||||||
@ -1,15 +1,10 @@
|
|||||||
# 测试策略
|
# 测试策略
|
||||||
|
|
||||||
> 本文保留为早期领域与容量检查清单。存储边界和当前执行命令以
|
|
||||||
> `test-strategy.md`、ADR-005 为准：Proxy 明细与逐次提取记录不写 PostgreSQL，
|
|
||||||
> 独占提取只在 Redis TTL 活动池保留短期幂等结果。
|
|
||||||
|
|
||||||
## 1. 分层
|
## 1. 分层
|
||||||
|
|
||||||
- **领域单测**：状态机、TTL、路由、容量、Fetch 分类和 Extraction 原子性。
|
- **领域单测**：状态机、TTL、路由、容量、Fetch 分类和 Extraction 原子性。
|
||||||
- **契约测试**：配置、OpenAPI、Protobuf 和 Provider Adapter fixture。
|
- **契约测试**：配置、OpenAPI、Protobuf 和 Provider Adapter fixture。
|
||||||
- **集成测试**：Redis 活动池原子契约、PostgreSQL 管理事务、Leader/限流、
|
- **集成测试**：PostgreSQL 事务、Redis Leader/限流、Outbox 与重建。
|
||||||
Outbox 与重建。
|
|
||||||
- **端到端测试**：HTTP、CONNECT、Admin、Distribution 和优雅停机。
|
- **端到端测试**：HTTP、CONNECT、Admin、Distribution 和优雅停机。
|
||||||
- **负载测试**：Worker 调度微基准、50k 隧道 soak、集群 100k QPS 场景。
|
- **负载测试**：Worker 调度微基准、50k 隧道 soak、集群 100k QPS 场景。
|
||||||
|
|
||||||
@ -37,15 +32,6 @@ go test -race ./internal/...
|
|||||||
go build ./...
|
go build ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
真实 Redis 8.2 活动池契约使用独立 Compose fixture：
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\test-redis.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
该 fixture 使用唯一命名空间，不执行 `FLUSHDB`，并关闭 AOF、RDB 与数据卷；
|
|
||||||
测试结束后按命名空间清理活动池、所有权和幂等键。
|
|
||||||
|
|
||||||
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock；集成和
|
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock；集成和
|
||||||
soak 测试单独标记，不混入快速单测。
|
soak 测试单独标记，不混入快速单测。
|
||||||
|
|
||||||
|
|||||||
@ -76,18 +76,6 @@ go vet ./...
|
|||||||
go build ./...
|
go build ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
Redis 活动池 Adapter 与内存参考实现共享同一套公用行为契约。真实 Redis 8.2
|
|
||||||
fixture 的执行命令是：
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\test-redis.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
契约覆盖 Upsert/去重/容量、健康更新、partial/allOrNothing 提取、Gateway 保留、
|
|
||||||
零数量与非零数量幂等、幂等硬过期、Worker 所有权/Drain/ACK、库存、过期清理、
|
|
||||||
100 轮并发提取和 100 轮所有权竞争。fixture 使用唯一命名空间，不执行
|
|
||||||
`FLUSHDB`；本地 Redis 关闭 AOF、RDB 和数据卷，避免短效 Proxy 与凭据落盘。
|
|
||||||
|
|
||||||
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器，不复用开发数据。
|
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器，不复用开发数据。
|
||||||
测试结束后验证没有残留 Worker ownership、Leader 租约、活动池条目或幂等键，
|
测试结束后验证没有残留 Worker ownership、Leader 租约、活动池条目或幂等键，
|
||||||
并检查 PostgreSQL 中不存在 Proxy 明细和逐次提取记录。
|
并检查 PostgreSQL 中不存在 Proxy 明细和逐次提取记录。
|
||||||
|
|||||||
10
go.mod
10
go.mod
@ -2,12 +2,4 @@ module proxy-pool
|
|||||||
|
|
||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||||
github.com/redis/go-redis/v9 v9.19.0
|
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
|
||||||
go.uber.org/atomic v1.11.0 // indirect
|
|
||||||
)
|
|
||||||
|
|||||||
22
go.sum
22
go.sum
@ -1,24 +1,2 @@
|
|||||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
|
||||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
|
||||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
|
|
||||||
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
|
||||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
|
||||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
|
||||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
|
||||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
|
||||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
|
||||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
|
||||||
|
|||||||
@ -1,70 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidOptions = errors.New("invalid redis activity adapter options")
|
|
||||||
namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
|
||||||
)
|
|
||||||
|
|
||||||
type Options struct {
|
|
||||||
Namespace string
|
|
||||||
Credentials credentials.Store
|
|
||||||
OperationTTL time.Duration
|
|
||||||
MaxCandidateScan int
|
|
||||||
CleanupLimit int
|
|
||||||
}
|
|
||||||
|
|
||||||
type Adapter struct {
|
|
||||||
client redis.Scripter
|
|
||||||
credentials credentials.Store
|
|
||||||
keys keyspace
|
|
||||||
options Options
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(client redis.Scripter, options Options) (*Adapter, error) {
|
|
||||||
options.Namespace = strings.TrimSpace(options.Namespace)
|
|
||||||
if nilInterface(client) || nilInterface(options.Credentials) ||
|
|
||||||
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
|
|
||||||
options.MaxCandidateScan <= 0 || options.CleanupLimit <= 0 {
|
|
||||||
return nil, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
return &Adapter{
|
|
||||||
client: client,
|
|
||||||
credentials: options.Credentials,
|
|
||||||
keys: newKeyspace(options.Namespace),
|
|
||||||
options: options,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) Format(state fmt.State, _ rune) {
|
|
||||||
if a == nil {
|
|
||||||
_, _ = state.Write([]byte("redisactivity.Adapter<nil>"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(state, "redisactivity.Adapter{Namespace:%q}", a.options.Namespace)
|
|
||||||
}
|
|
||||||
|
|
||||||
func nilInterface(value any) bool {
|
|
||||||
if value == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
reflected := reflect.ValueOf(value)
|
|
||||||
switch reflected.Kind() {
|
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
|
||||||
return reflected.IsNil()
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,232 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRunScriptPreservesContextCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
||||||
t.Cleanup(func() { _ = client.Close() })
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
_, err := runScript(ctx, client, redis.NewScript("return 1"), nil)
|
|
||||||
if !errors.Is(err, context.Canceled) || errors.Is(err, extractionDomain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("runScript() error = %v, want only context cancellation", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
||||||
t.Cleanup(func() { _ = client.Close() })
|
|
||||||
credentialStore, err := credentials.NewMemoryStore(10)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
|
||||||
}
|
|
||||||
valid := Options{
|
|
||||||
Namespace: "test-a", Credentials: credentialStore,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 2048, CleanupLimit: 128,
|
|
||||||
}
|
|
||||||
var typedNilClient *redis.Client
|
|
||||||
var typedNilCredentials *credentials.MemoryStore
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
client redis.Scripter
|
|
||||||
options Options
|
|
||||||
}{
|
|
||||||
{name: "nil client", options: valid},
|
|
||||||
{name: "typed nil client", client: typedNilClient, options: valid},
|
|
||||||
{name: "nil credentials", client: client, options: withCredentials(valid, nil)},
|
|
||||||
{name: "typed nil credentials", client: client, options: withCredentials(valid, typedNilCredentials)},
|
|
||||||
{name: "empty namespace", client: client, options: withNamespace(valid, "")},
|
|
||||||
{name: "braces in namespace", client: client, options: withNamespace(valid, "tenant{other}")},
|
|
||||||
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
|
|
||||||
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
|
|
||||||
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
|
|
||||||
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
if adapter, err := New(tt.client, tt.options); err == nil || adapter != nil {
|
|
||||||
t.Fatalf("New() = (%v, %v), want nil adapter and error", adapter, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
||||||
t.Cleanup(func() { _ = client.Close() })
|
|
||||||
credentialStore, err := credentials.NewMemoryStore(10)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
|
||||||
}
|
|
||||||
adapter, err := New(client, Options{
|
|
||||||
Namespace: " test-a ", Credentials: credentialStore,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 2048, CleanupLimit: 128,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := adapter.keys.records; got != "pp:{activity}:test-a:records" {
|
|
||||||
t.Fatalf("records key = %q", got)
|
|
||||||
}
|
|
||||||
staticKeys := []string{
|
|
||||||
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
|
||||||
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
|
||||||
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
|
||||||
}
|
|
||||||
for _, key := range staticKeys {
|
|
||||||
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
|
||||||
t.Fatalf("key %q does not contain exactly one fixed hash tag", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
raw := "tenant:{unsafe}:TOKEN"
|
|
||||||
dynamicKeys := []string{
|
|
||||||
adapter.keys.idempotency(raw, raw),
|
|
||||||
adapter.keys.operation(raw),
|
|
||||||
adapter.keys.protocol(raw),
|
|
||||||
adapter.keys.region(raw),
|
|
||||||
adapter.keys.carrier(raw),
|
|
||||||
adapter.keys.upstream(raw),
|
|
||||||
}
|
|
||||||
for _, key := range dynamicKeys {
|
|
||||||
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
|
|
||||||
t.Fatalf("dynamic key is unsafe: %q", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := digestToken(raw); len(got) != 64 || got != digestToken(raw) || strings.Contains(got, raw) {
|
|
||||||
t.Fatalf("digestToken() = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
record := proxyRecord{
|
|
||||||
Version: 1, ID: "proxy-a", Scheme: string(proxyDomain.SchemeHTTP),
|
|
||||||
Host: "192.0.2.10", Port: 8080, Username: "user", Password: "top-secret",
|
|
||||||
SourceUpstream: "provider-a", CreatedAtMS: 1_000, ExpiresAtMS: 61_000,
|
|
||||||
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
|
|
||||||
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
|
|
||||||
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
|
||||||
}
|
|
||||||
first, err := encodeProxyRecord(record)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("encodeProxyRecord(): %v", err)
|
|
||||||
}
|
|
||||||
second, err := encodeProxyRecord(record)
|
|
||||||
if err != nil || first != second {
|
|
||||||
t.Fatalf("deterministic encode = (%q, %q, %v)", first, second, err)
|
|
||||||
}
|
|
||||||
decoded, err := decodeProxyRecord(first)
|
|
||||||
if err != nil || decoded.Password != "top-secret" || decoded.State != string(proxyDomain.StateAvailable) {
|
|
||||||
t.Fatalf("decodeProxyRecord() = (%+v, %v)", decoded, err)
|
|
||||||
}
|
|
||||||
if formatted := fmt.Sprintf("%+v", record); strings.Contains(formatted, "top-secret") || !strings.Contains(formatted, "<redacted>") {
|
|
||||||
t.Fatalf("proxyRecord formatting leaked password: %s", formatted)
|
|
||||||
}
|
|
||||||
|
|
||||||
invalid := []string{
|
|
||||||
strings.Replace(first, `"state":"AVAILABLE"`, `"state":"UNKNOWN"`, 1),
|
|
||||||
strings.Replace(first, `"latencyNs":25000000`, `"latencyNs":-1`, 1),
|
|
||||||
strings.TrimSuffix(first, "}") + `,"unexpected":true}`,
|
|
||||||
}
|
|
||||||
for _, payload := range invalid {
|
|
||||||
if _, err := decodeProxyRecord(payload); err == nil {
|
|
||||||
t.Fatalf("decodeProxyRecord(%s) error = nil", payload)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOwnershipAndIdempotencyCodecsValidateAndRedact(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
assignment := ownershipRecord{
|
|
||||||
Version: 1, ProxyID: "proxy-a", WorkerID: "worker-a",
|
|
||||||
Epoch: 2, AssignmentVersion: 3, ExpiresAtMS: 5_000, Draining: true,
|
|
||||||
}
|
|
||||||
payload, err := encodeOwnershipRecord(assignment)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("encodeOwnershipRecord(): %v", err)
|
|
||||||
}
|
|
||||||
if decoded, err := decodeOwnershipRecord(payload); err != nil || decoded != assignment {
|
|
||||||
t.Fatalf("decodeOwnershipRecord() = (%+v, %v)", decoded, err)
|
|
||||||
}
|
|
||||||
if _, err := decodeOwnershipRecord(strings.Replace(payload, `"epoch":2`, `"epoch":0`, 1)); err == nil {
|
|
||||||
t.Fatal("decodeOwnershipRecord() accepted zero epoch")
|
|
||||||
}
|
|
||||||
|
|
||||||
idempotency := idempotencyRecord{
|
|
||||||
Version: 1, RequestDigest: digestToken("request"), ExpiresAtMS: 10_000,
|
|
||||||
Result: extractionDomain.Result{
|
|
||||||
Requested: 1, Returned: 1, ExtractedAt: time.UnixMilli(1_000).UTC(),
|
|
||||||
Items: []extractionDomain.Candidate{{
|
|
||||||
ID: "proxy-a", Protocol: "http", Host: "192.0.2.10", Port: 8080,
|
|
||||||
Username: "user", Password: "top-secret", Upstream: "provider-a", State: extractionDomain.Extracted,
|
|
||||||
ExpiresAt: time.UnixMilli(10_000).UTC(),
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
payload, err = encodeIdempotencyRecord(idempotency)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("encodeIdempotencyRecord(): %v", err)
|
|
||||||
}
|
|
||||||
decodedID, err := decodeIdempotencyRecord(payload)
|
|
||||||
if err != nil || decodedID.Result.Items[0].Password != "top-secret" {
|
|
||||||
t.Fatalf("decodeIdempotencyRecord() = (%+v, %v)", decodedID, err)
|
|
||||||
}
|
|
||||||
invalidPayloads := []string{
|
|
||||||
strings.Replace(payload, `"port":8080`, `"port":65537`, 1),
|
|
||||||
strings.Replace(payload, `"state":"EXTRACTED","expiresAtMs":10000`, `"state":"EXTRACTED","expiresAtMs":10000,"lastCheckedAtMs":-1`, 1),
|
|
||||||
}
|
|
||||||
for _, invalidPayload := range invalidPayloads {
|
|
||||||
if _, err := decodeIdempotencyRecord(invalidPayload); err == nil {
|
|
||||||
t.Fatalf("decodeIdempotencyRecord(%s) error = nil", invalidPayload)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if formatted := fmt.Sprintf("%+v", idempotency); strings.Contains(formatted, "top-secret") || !strings.Contains(formatted, "<redacted>") {
|
|
||||||
t.Fatalf("idempotencyRecord formatting leaked password: %s", formatted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func withCredentials(options Options, store credentials.Store) Options {
|
|
||||||
options.Credentials = store
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withNamespace(options Options, namespace string) Options {
|
|
||||||
options.Namespace = namespace
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withOperationTTL(options Options, ttl time.Duration) Options {
|
|
||||||
options.OperationTTL = ttl
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withMaxCandidateScan(options Options, limit int) Options {
|
|
||||||
options.MaxCandidateScan = limit
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withCleanupLimit(options Options, limit int) Options {
|
|
||||||
options.CleanupLimit = limit
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
@ -1,357 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrInvalidRecord = errors.New("invalid redis activity record")
|
|
||||||
|
|
||||||
const recordVersion = 1
|
|
||||||
|
|
||||||
type proxyRecord struct {
|
|
||||||
Version int `json:"version"`
|
|
||||||
ID string `json:"id"`
|
|
||||||
Scheme string `json:"scheme"`
|
|
||||||
Host string `json:"host"`
|
|
||||||
Port int64 `json:"port"`
|
|
||||||
Username string `json:"username,omitempty"`
|
|
||||||
Password string `json:"password,omitempty"`
|
|
||||||
CredentialVersion string `json:"credentialVersion,omitempty"`
|
|
||||||
SourceUpstream string `json:"sourceUpstream"`
|
|
||||||
CreatedAtMS int64 `json:"createdAtMs"`
|
|
||||||
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
||||||
UsableUntilMS int64 `json:"usableUntilMs"`
|
|
||||||
LastCheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"`
|
|
||||||
LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"`
|
|
||||||
LatencyNS int64 `json:"latencyNs"`
|
|
||||||
MaxConcurrency int64 `json:"maxConcurrency"`
|
|
||||||
State string `json:"state"`
|
|
||||||
Tags map[string]string `json:"tags,omitempty"`
|
|
||||||
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
|
||||||
IndexKeys []string `json:"indexKeys,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ownershipRecord struct {
|
|
||||||
Version int `json:"version"`
|
|
||||||
ProxyID string `json:"proxyId"`
|
|
||||||
WorkerID string `json:"workerId"`
|
|
||||||
Epoch uint64 `json:"epoch"`
|
|
||||||
AssignmentVersion uint64 `json:"assignmentVersion"`
|
|
||||||
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
||||||
Draining bool `json:"draining"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type idempotencyRecord struct {
|
|
||||||
Version int
|
|
||||||
RequestDigest string
|
|
||||||
ExpiresAtMS int64
|
|
||||||
Result extractionDomain.Result
|
|
||||||
}
|
|
||||||
|
|
||||||
type idempotencyWire struct {
|
|
||||||
Version int `json:"version"`
|
|
||||||
RequestDigest string `json:"requestDigest"`
|
|
||||||
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
||||||
Result extractionWire `json:"result"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type extractionWire struct {
|
|
||||||
Requested int `json:"requested"`
|
|
||||||
Returned int `json:"returned"`
|
|
||||||
ExtractedAtMS int64 `json:"extractedAtMs,omitempty"`
|
|
||||||
Items []candidateWire `json:"items"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type candidateWire struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Protocol string `json:"protocol"`
|
|
||||||
Host string `json:"host"`
|
|
||||||
Port int64 `json:"port"`
|
|
||||||
Username string `json:"username,omitempty"`
|
|
||||||
Password string `json:"password,omitempty"`
|
|
||||||
Region string `json:"region,omitempty"`
|
|
||||||
Carrier string `json:"carrier,omitempty"`
|
|
||||||
Upstream string `json:"upstream"`
|
|
||||||
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
State string `json:"state"`
|
|
||||||
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
||||||
CheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (record proxyRecord) Format(state fmt.State, _ rune) {
|
|
||||||
_, _ = fmt.Fprintf(state,
|
|
||||||
"redisactivity.proxyRecord{ID:%q, Scheme:%q, Host:%q, Port:%d, Username:%q, Password:<redacted>, State:%q}",
|
|
||||||
record.ID, record.Scheme, record.Host, record.Port, record.Username, record.State,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (record ownershipRecord) Format(state fmt.State, _ rune) {
|
|
||||||
_, _ = fmt.Fprintf(state,
|
|
||||||
"redisactivity.ownershipRecord{ProxyID:%q, WorkerID:%q, Epoch:%d, Version:%d, Draining:%t}",
|
|
||||||
record.ProxyID, record.WorkerID, record.Epoch, record.AssignmentVersion, record.Draining,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (record idempotencyRecord) Format(state fmt.State, _ rune) {
|
|
||||||
_, _ = fmt.Fprintf(state,
|
|
||||||
"redisactivity.idempotencyRecord{RequestDigest:%q, Returned:%d, Credentials:<redacted>}",
|
|
||||||
record.RequestDigest, record.Result.Returned,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeProxyRecord(record proxyRecord) (string, error) {
|
|
||||||
if err := validateProxyRecord(record); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return encodeJSON(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeProxyRecord(payload string) (proxyRecord, error) {
|
|
||||||
var record proxyRecord
|
|
||||||
if err := decodeJSON(payload, &record); err != nil {
|
|
||||||
return proxyRecord{}, err
|
|
||||||
}
|
|
||||||
if err := validateProxyRecord(record); err != nil {
|
|
||||||
return proxyRecord{}, err
|
|
||||||
}
|
|
||||||
return record, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeOwnershipRecord(record ownershipRecord) (string, error) {
|
|
||||||
if err := validateOwnershipRecord(record); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return encodeJSON(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeOwnershipRecord(payload string) (ownershipRecord, error) {
|
|
||||||
var record ownershipRecord
|
|
||||||
if err := decodeJSON(payload, &record); err != nil {
|
|
||||||
return ownershipRecord{}, err
|
|
||||||
}
|
|
||||||
if err := validateOwnershipRecord(record); err != nil {
|
|
||||||
return ownershipRecord{}, err
|
|
||||||
}
|
|
||||||
return record, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeIdempotencyRecord(record idempotencyRecord) (string, error) {
|
|
||||||
if err := validateIdempotencyRecord(record); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
wire := idempotencyWire{
|
|
||||||
Version: record.Version, RequestDigest: record.RequestDigest, ExpiresAtMS: record.ExpiresAtMS,
|
|
||||||
Result: extractionToWire(record.Result),
|
|
||||||
}
|
|
||||||
return encodeJSON(wire)
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeIdempotencyRecord(payload string) (idempotencyRecord, error) {
|
|
||||||
var wire idempotencyWire
|
|
||||||
if err := decodeJSON(payload, &wire); err != nil {
|
|
||||||
return idempotencyRecord{}, err
|
|
||||||
}
|
|
||||||
if err := validateIdempotencyWire(wire); err != nil {
|
|
||||||
return idempotencyRecord{}, err
|
|
||||||
}
|
|
||||||
record := idempotencyRecord{
|
|
||||||
Version: wire.Version, RequestDigest: wire.RequestDigest, ExpiresAtMS: wire.ExpiresAtMS,
|
|
||||||
Result: extractionFromWire(wire.Result),
|
|
||||||
}
|
|
||||||
if err := validateIdempotencyRecord(record); err != nil {
|
|
||||||
return idempotencyRecord{}, err
|
|
||||||
}
|
|
||||||
return record, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodeJSON(value any) (string, error) {
|
|
||||||
payload, err := json.Marshal(value)
|
|
||||||
if err != nil {
|
|
||||||
return "", errors.Join(ErrInvalidRecord, err)
|
|
||||||
}
|
|
||||||
return string(payload), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeJSON(payload string, destination any) error {
|
|
||||||
decoder := json.NewDecoder(strings.NewReader(payload))
|
|
||||||
decoder.DisallowUnknownFields()
|
|
||||||
if err := decoder.Decode(destination); err != nil {
|
|
||||||
return errors.Join(ErrInvalidRecord, err)
|
|
||||||
}
|
|
||||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
|
||||||
if err == nil {
|
|
||||||
err = errors.New("multiple JSON values")
|
|
||||||
}
|
|
||||||
return errors.Join(ErrInvalidRecord, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateProxyRecord(record proxyRecord) error {
|
|
||||||
if record.Version != recordVersion || record.ID == "" || record.Host == "" ||
|
|
||||||
record.Port <= 0 || record.Port > 65_535 || record.SourceUpstream == "" ||
|
|
||||||
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
|
|
||||||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
|
|
||||||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
|
|
||||||
!validScheme(record.Scheme) || !validProxyState(record.State) {
|
|
||||||
return ErrInvalidRecord
|
|
||||||
}
|
|
||||||
for _, key := range record.IndexKeys {
|
|
||||||
if key == "" || !strings.Contains(key, "{activity}") {
|
|
||||||
return ErrInvalidRecord
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOwnershipRecord(record ownershipRecord) error {
|
|
||||||
if record.Version != recordVersion || record.ProxyID == "" || record.WorkerID == "" ||
|
|
||||||
record.Epoch == 0 || record.AssignmentVersion == 0 || record.ExpiresAtMS <= 0 {
|
|
||||||
return ErrInvalidRecord
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateIdempotencyRecord(record idempotencyRecord) error {
|
|
||||||
if record.Version != recordVersion {
|
|
||||||
return invalidRecord("unsupported idempotency version")
|
|
||||||
}
|
|
||||||
if !validDigest(record.RequestDigest) {
|
|
||||||
return invalidRecord("invalid request digest")
|
|
||||||
}
|
|
||||||
if record.ExpiresAtMS <= 0 {
|
|
||||||
return invalidRecord("invalid idempotency expiry")
|
|
||||||
}
|
|
||||||
if record.Result.Requested < 0 || record.Result.Returned < 0 ||
|
|
||||||
record.Result.Returned != len(record.Result.Items) || record.Result.Returned > record.Result.Requested {
|
|
||||||
return invalidRecord("invalid extraction counters")
|
|
||||||
}
|
|
||||||
if record.Result.Returned > 0 && record.Result.ExtractedAt.IsZero() {
|
|
||||||
return invalidRecord("missing extraction time")
|
|
||||||
}
|
|
||||||
for _, candidate := range record.Result.Items {
|
|
||||||
if candidate.ID == "" || candidate.Host == "" || candidate.Port == 0 || candidate.Upstream == "" {
|
|
||||||
return invalidRecord("incomplete extraction candidate identity")
|
|
||||||
}
|
|
||||||
if !validScheme(candidate.Protocol) || candidate.State != extractionDomain.Extracted {
|
|
||||||
return invalidRecord("invalid extraction candidate state")
|
|
||||||
}
|
|
||||||
if candidate.ExpiresAt.IsZero() || candidate.ExpiresAt.UnixMilli() <= 0 ||
|
|
||||||
(!candidate.LastCheckedAt.IsZero() && candidate.LastCheckedAt.UnixMilli() <= 0) {
|
|
||||||
return invalidRecord("invalid extraction candidate time")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateIdempotencyWire(wire idempotencyWire) error {
|
|
||||||
if wire.Version != recordVersion || !validDigest(wire.RequestDigest) || wire.ExpiresAtMS <= 0 {
|
|
||||||
return invalidRecord("invalid idempotency wire header")
|
|
||||||
}
|
|
||||||
if wire.Result.Requested < 0 || wire.Result.Returned < 0 ||
|
|
||||||
wire.Result.Returned != len(wire.Result.Items) || wire.Result.Returned > wire.Result.Requested {
|
|
||||||
return invalidRecord("invalid extraction wire counters")
|
|
||||||
}
|
|
||||||
if wire.Result.Returned > 0 && wire.Result.ExtractedAtMS <= 0 {
|
|
||||||
return invalidRecord("invalid extraction wire time")
|
|
||||||
}
|
|
||||||
for _, candidate := range wire.Result.Items {
|
|
||||||
if candidate.ID == "" || candidate.Host == "" || candidate.Port <= 0 || candidate.Port > 65_535 ||
|
|
||||||
candidate.Upstream == "" || !validScheme(candidate.Protocol) ||
|
|
||||||
extractionDomain.State(candidate.State) != extractionDomain.Extracted ||
|
|
||||||
candidate.ExpiresAtMS <= 0 || candidate.CheckedAtMS < 0 {
|
|
||||||
return invalidRecord("invalid extraction wire candidate")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func invalidRecord(reason string) error {
|
|
||||||
return fmt.Errorf("%w: %s", ErrInvalidRecord, reason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractionToWire(result extractionDomain.Result) extractionWire {
|
|
||||||
wire := extractionWire{
|
|
||||||
Requested: result.Requested, Returned: result.Returned,
|
|
||||||
Items: make([]candidateWire, 0, len(result.Items)),
|
|
||||||
}
|
|
||||||
if !result.ExtractedAt.IsZero() {
|
|
||||||
wire.ExtractedAtMS = result.ExtractedAt.UnixMilli()
|
|
||||||
}
|
|
||||||
for _, candidate := range result.Items {
|
|
||||||
item := candidateWire{
|
|
||||||
ID: candidate.ID, Protocol: candidate.Protocol, Host: candidate.Host, Port: int64(candidate.Port),
|
|
||||||
Username: candidate.Username, Password: candidate.Password, Region: candidate.Region,
|
|
||||||
Carrier: candidate.Carrier, Upstream: candidate.Upstream, OwnerWorkerID: candidate.OwnerWorkerID,
|
|
||||||
URL: candidate.URL, State: string(candidate.State), ExpiresAtMS: candidate.ExpiresAt.UnixMilli(),
|
|
||||||
}
|
|
||||||
if !candidate.LastCheckedAt.IsZero() {
|
|
||||||
item.CheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
|
||||||
}
|
|
||||||
wire.Items = append(wire.Items, item)
|
|
||||||
}
|
|
||||||
return wire
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractionFromWire(wire extractionWire) extractionDomain.Result {
|
|
||||||
result := extractionDomain.Result{
|
|
||||||
Requested: wire.Requested, Returned: wire.Returned,
|
|
||||||
Items: make([]extractionDomain.Candidate, 0, len(wire.Items)),
|
|
||||||
}
|
|
||||||
if wire.ExtractedAtMS > 0 {
|
|
||||||
result.ExtractedAt = time.UnixMilli(wire.ExtractedAtMS).UTC()
|
|
||||||
}
|
|
||||||
for _, item := range wire.Items {
|
|
||||||
candidate := extractionDomain.Candidate{
|
|
||||||
ID: item.ID, Protocol: item.Protocol, Host: item.Host, Port: uint16(item.Port),
|
|
||||||
Username: item.Username, Password: item.Password, Region: item.Region,
|
|
||||||
Carrier: item.Carrier, Upstream: item.Upstream, OwnerWorkerID: item.OwnerWorkerID,
|
|
||||||
URL: item.URL, State: extractionDomain.State(item.State), ExpiresAt: time.UnixMilli(item.ExpiresAtMS).UTC(),
|
|
||||||
}
|
|
||||||
if item.CheckedAtMS > 0 {
|
|
||||||
candidate.LastCheckedAt = time.UnixMilli(item.CheckedAtMS).UTC()
|
|
||||||
}
|
|
||||||
result.Items = append(result.Items, candidate)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func validDigest(value string) bool {
|
|
||||||
if len(value) != sha256HexSize {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, err := hex.DecodeString(value)
|
|
||||||
return err == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validScheme(value string) bool {
|
|
||||||
switch proxyDomain.Scheme(value) {
|
|
||||||
case proxyDomain.SchemeHTTP, proxyDomain.SchemeHTTPS, proxyDomain.SchemeSOCKS5:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validProxyState(value string) bool {
|
|
||||||
switch proxyDomain.State(value) {
|
|
||||||
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable,
|
|
||||||
proxyDomain.StateSuspect, proxyDomain.StateDraining, proxyDomain.StateUnhealthy,
|
|
||||||
proxyDomain.StateExtracted, proxyDomain.StateExpired, proxyDomain.StateRemoved:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const sha256HexSize = 64
|
|
||||||
@ -1,16 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool/contracttest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRedisActivityPoolContract(t *testing.T) {
|
|
||||||
contracttest.Run(t, func(t *testing.T) (contracttest.Store, func()) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
return fixture.Adapter, fixture.Cleanup
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@ -1,208 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/url"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
)
|
|
||||||
|
|
||||||
const defaultRedisIdempotencyTTL = 5 * time.Minute
|
|
||||||
|
|
||||||
type extractionDigestInput struct {
|
|
||||||
Requested int `json:"requested"`
|
|
||||||
Fulfillment extractionDomain.Fulfillment `json:"fulfillment"`
|
|
||||||
Protocols []string `json:"protocols"`
|
|
||||||
Regions []string `json:"regions"`
|
|
||||||
Carriers []string `json:"carriers"`
|
|
||||||
Upstreams []string `json:"upstreams"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type extractionFilterWire struct {
|
|
||||||
Protocols []string `json:"protocols"`
|
|
||||||
Regions []string `json:"regions"`
|
|
||||||
Carriers []string `json:"carriers"`
|
|
||||||
Upstreams []string `json:"upstreams"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ extractionDomain.Store = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) Extract(ctx context.Context, command extractionDomain.Command) (extractionDomain.Result, error) {
|
|
||||||
result := extractionDomain.Result{Requested: command.Requested}
|
|
||||||
if ctx == nil {
|
|
||||||
return result, extractionDomain.ErrInvalidCommand
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if a == nil || command.Now.IsZero() || command.Requested < 0 || command.ReserveForGateway < 0 ||
|
|
||||||
command.MinRemainingTTL < 0 || command.MaxHealthCheckAge < 0 || command.IdempotencyTTL < 0 ||
|
|
||||||
(command.IdempotencyKey != "" && command.ClientID == "") ||
|
|
||||||
(command.Fulfillment != extractionDomain.Partial && command.Fulfillment != extractionDomain.AllOrNothing) {
|
|
||||||
return result, extractionDomain.ErrInvalidCommand
|
|
||||||
}
|
|
||||||
|
|
||||||
digestInput := extractionDigestInput{
|
|
||||||
Requested: command.Requested, Fulfillment: command.Fulfillment,
|
|
||||||
Protocols: canonicalFilter(command.Protocols), Regions: canonicalFilter(command.Regions),
|
|
||||||
Carriers: canonicalFilter(command.Carriers), Upstreams: canonicalFilter(command.Upstreams),
|
|
||||||
}
|
|
||||||
requestDigest, err := extractionRequestDigest(digestInput)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if command.Requested == 0 && command.IdempotencyKey == "" {
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
operationID := command.RequestID
|
|
||||||
if operationID == "" {
|
|
||||||
operationID, err = newOperationID()
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
operationKey := a.keys.operation(digestParts(command.ClientID, operationID))
|
|
||||||
idempotencyKey := operationKey
|
|
||||||
hasIdempotency := 0
|
|
||||||
if command.IdempotencyKey != "" {
|
|
||||||
hasIdempotency = 1
|
|
||||||
idempotencyKey = a.keys.idempotency(command.ClientID, command.IdempotencyKey)
|
|
||||||
}
|
|
||||||
filterPayload, err := json.Marshal(extractionFilterWire{
|
|
||||||
Protocols: digestInput.Protocols, Regions: digestInput.Regions,
|
|
||||||
Carriers: digestInput.Carriers, Upstreams: digestInput.Upstreams,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return result, fmt.Errorf("encode Redis extraction filters: %w", err)
|
|
||||||
}
|
|
||||||
keys := []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, operationKey, idempotencyKey,
|
|
||||||
}
|
|
||||||
keys = append(keys, a.extractionDriverKeys(digestInput)...)
|
|
||||||
idempotencyTTL := command.IdempotencyTTL
|
|
||||||
if idempotencyTTL == 0 {
|
|
||||||
idempotencyTTL = defaultRedisIdempotencyTTL
|
|
||||||
}
|
|
||||||
scriptResult, err := runScript(ctx, a.client, extractScript, keys,
|
|
||||||
command.Now.UnixMilli(), command.Requested, string(command.Fulfillment), command.ReserveForGateway,
|
|
||||||
durationMillis(command.MinRemainingTTL), durationMillis(command.MaxHealthCheckAge),
|
|
||||||
a.options.MaxCandidateScan, a.options.CleanupLimit, durationMillis(idempotencyTTL),
|
|
||||||
operationTTLMillis(a.options.OperationTTL), requestDigest, hasIdempotency, string(filterPayload))
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
var reply extractScriptReply
|
|
||||||
if err := decodeScriptResult(scriptResult, &reply); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if reply.RequestDigest != requestDigest {
|
|
||||||
return result, invalidScriptReply("extraction reply digest mismatch")
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptConflict:
|
|
||||||
return result, extractionDomain.ErrIdempotencyConflict
|
|
||||||
case scriptInsufficient:
|
|
||||||
return result, extractionDomain.ErrInsufficientProxies
|
|
||||||
case scriptUnavailable:
|
|
||||||
return result, extractionDomain.ErrStoreUnavailable
|
|
||||||
case scriptInvalid:
|
|
||||||
return result, extractionDomain.ErrInvalidCommand
|
|
||||||
case scriptOK:
|
|
||||||
if reply.Record == "" {
|
|
||||||
return result, invalidScriptReply("extraction reply omitted record")
|
|
||||||
}
|
|
||||||
committed, err := decodeIdempotencyRecord(reply.Record)
|
|
||||||
if err != nil {
|
|
||||||
return result, errors.Join(
|
|
||||||
invalidScriptReply("extraction reply contained an invalid record"),
|
|
||||||
fmt.Errorf("decode extraction record: %w", err),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
if committed.RequestDigest != requestDigest {
|
|
||||||
return result, invalidScriptReply("extraction reply contained an invalid record")
|
|
||||||
}
|
|
||||||
return buildExtractionResult(committed.Result), nil
|
|
||||||
default:
|
|
||||||
return result, invalidScriptReply("unexpected extraction status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) extractionDriverKeys(input extractionDigestInput) []string {
|
|
||||||
keys := make([]string, 0, 4)
|
|
||||||
if len(input.Protocols) == 1 {
|
|
||||||
keys = append(keys, a.keys.protocol(input.Protocols[0]))
|
|
||||||
}
|
|
||||||
if len(input.Regions) == 1 {
|
|
||||||
keys = append(keys, a.keys.region(input.Regions[0]))
|
|
||||||
}
|
|
||||||
if len(input.Carriers) == 1 {
|
|
||||||
keys = append(keys, a.keys.carrier(input.Carriers[0]))
|
|
||||||
}
|
|
||||||
if len(input.Upstreams) == 1 {
|
|
||||||
keys = append(keys, a.keys.upstream(input.Upstreams[0]))
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractionRequestDigest(input extractionDigestInput) (string, error) {
|
|
||||||
payload, err := json.Marshal(input)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("encode extraction request digest: %w", err)
|
|
||||||
}
|
|
||||||
digest := sha256.Sum256(payload)
|
|
||||||
return hex.EncodeToString(digest[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalFilter(values []string) []string {
|
|
||||||
if len(values) == 0 {
|
|
||||||
return []string{}
|
|
||||||
}
|
|
||||||
unique := make(map[string]struct{}, len(values))
|
|
||||||
for _, value := range values {
|
|
||||||
unique[value] = struct{}{}
|
|
||||||
}
|
|
||||||
result := make([]string, 0, len(unique))
|
|
||||||
for value := range unique {
|
|
||||||
result = append(result, value)
|
|
||||||
}
|
|
||||||
sort.Strings(result)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func durationMillis(duration time.Duration) int64 {
|
|
||||||
if duration <= 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return operationTTLMillis(duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildExtractionResult(result extractionDomain.Result) extractionDomain.Result {
|
|
||||||
result.Items = append([]extractionDomain.Candidate(nil), result.Items...)
|
|
||||||
for index := range result.Items {
|
|
||||||
result.Items[index].URL = proxyURL(result.Items[index])
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func proxyURL(candidate extractionDomain.Candidate) string {
|
|
||||||
parsed := url.URL{
|
|
||||||
Scheme: candidate.Protocol,
|
|
||||||
Host: net.JoinHostPort(candidate.Host, strconv.FormatUint(uint64(candidate.Port), 10)),
|
|
||||||
}
|
|
||||||
if candidate.Password != "" {
|
|
||||||
parsed.User = url.UserPassword(candidate.Username, candidate.Password)
|
|
||||||
} else if candidate.Username != "" {
|
|
||||||
parsed.User = url.User(candidate.Username)
|
|
||||||
}
|
|
||||||
return parsed.String()
|
|
||||||
}
|
|
||||||
@ -1,455 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRedisExtractAppliesEveryFilterAndBuildsCredentialURL(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
reference, err := fixture.Credentials.Put(context.Background(), "target", credentials.Value{
|
|
||||||
Username: "user@tenant", Password: "p:/@ss",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Credentials.Put(): %v", err)
|
|
||||||
}
|
|
||||||
target := testProxy("target", "192.0.2.10")
|
|
||||||
target.Username = "user@tenant"
|
|
||||||
target.SecretRef = reference.SecretRef
|
|
||||||
target.CredentialVersion = reference.CredentialVersion
|
|
||||||
target.Tags = map[string]string{"region": "cn", "carrier": "ct"}
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(2*time.Second), time.Minute, target)
|
|
||||||
|
|
||||||
other := testProxy("other", "192.0.2.11")
|
|
||||||
other.Scheme = proxyDomain.SchemeSOCKS5
|
|
||||||
other.Tags = map[string]string{"region": "us", "carrier": "cu"}
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(2*time.Second), time.Minute, other)
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-filter", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second),
|
|
||||||
Protocols: []string{"http"}, Regions: []string{"cn"}, Carriers: []string{"ct"},
|
|
||||||
Upstreams: []string{"provider-a"},
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 1 || result.Items[0].ID != "target" {
|
|
||||||
t.Fatalf("Extract(filtered) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
parsed, err := url.Parse(result.Items[0].URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse extracted URL: %v", err)
|
|
||||||
}
|
|
||||||
password, hasPassword := parsed.User.Password()
|
|
||||||
if parsed.User.Username() != "user@tenant" || !hasPassword || password != "p:/@ss" {
|
|
||||||
t.Fatalf("URL credentials = (%q, %q, %t)", parsed.User.Username(), password, hasPassword)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractSupportsPartialAndAllOrNothing(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
candidate := testProxy("proxy-a", "192.0.2.10")
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(2*time.Second), time.Minute, candidate)
|
|
||||||
|
|
||||||
_, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-all", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.AllOrNothing, Now: now.Add(3 * time.Second),
|
|
||||||
})
|
|
||||||
if !errors.Is(err, extractionDomain.ErrInsufficientProxies) {
|
|
||||||
t.Fatalf("Extract(allOrNothing) error = %v", err)
|
|
||||||
}
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-partial", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 1 || result.Items[0].ID != "proxy-a" {
|
|
||||||
t.Fatalf("Extract(partial) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractReturnsEmptyItemsForValidZeroResults(t *testing.T) {
|
|
||||||
t.Run("gateway reserve", func(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-empty-reserve", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second), ReserveForGateway: 1,
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 0 || len(result.Items) != 0 {
|
|
||||||
t.Fatalf("Extract(reserved empty) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("zero requested with idempotency", func(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-empty-zero", ClientID: "client-a", IdempotencyKey: "idem-empty-zero",
|
|
||||||
Requested: 0, Fulfillment: extractionDomain.Partial, Now: now,
|
|
||||||
}
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || result.Requested != 0 || result.Returned != 0 || len(result.Items) != 0 {
|
|
||||||
t.Fatalf("Extract(zero requested) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-empty-zero-replay"
|
|
||||||
command.Now = now.Add(time.Second)
|
|
||||||
replayed, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || replayed.Requested != 0 || replayed.Returned != 0 || len(replayed.Items) != 0 {
|
|
||||||
t.Fatalf("Extract(zero requested replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-empty-zero-conflict"
|
|
||||||
command.Requested = 1
|
|
||||||
if _, err := fixture.Adapter.Extract(context.Background(), command); !errors.Is(err, extractionDomain.ErrIdempotencyConflict) {
|
|
||||||
t.Fatalf("Extract(zero requested conflict) error = %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractAppliesTTLHealthAgeAndGatewayReserve(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
observedAt := now.Add(-30 * time.Second)
|
|
||||||
|
|
||||||
short := testProxy("short", "192.0.2.10")
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", observedAt, now.Add(-time.Second), 35*time.Second, short)
|
|
||||||
stale := testProxy("stale", "192.0.2.11")
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", observedAt, now.Add(-10*time.Second), 2*time.Minute, stale)
|
|
||||||
fresh := testProxy("fresh", "192.0.2.12")
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", observedAt, now.Add(-time.Second), 2*time.Minute, fresh)
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-policy", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now,
|
|
||||||
MinRemainingTTL: 30 * time.Second, MaxHealthCheckAge: 5 * time.Second,
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 1 || result.Items[0].ID != "fresh" {
|
|
||||||
t.Fatalf("Extract(policy) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reserveFixture := newRedisTestFixture(t)
|
|
||||||
for index := range 3 {
|
|
||||||
candidate := testProxy(fmt.Sprintf("reserve-%d", index), fmt.Sprintf("192.0.2.%d", index+20))
|
|
||||||
seedRedisAvailable(t, reserveFixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute, candidate)
|
|
||||||
}
|
|
||||||
reserved, err := reserveFixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-reserve", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second), ReserveForGateway: 2,
|
|
||||||
})
|
|
||||||
if err != nil || reserved.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(reserve) = %+v, %v", reserved, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractIdempotencyReplayConflictAndExpiryBound(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-first", ClientID: "client-a", IdempotencyKey: "idem-12345678",
|
|
||||||
IdempotencyTTL: 5 * time.Minute, Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
}
|
|
||||||
first, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || first.Returned != 1 {
|
|
||||||
t.Fatalf("first Extract() = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-replay"
|
|
||||||
command.Now = now.Add(10 * time.Second)
|
|
||||||
replayed, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || replayed.Returned != 1 || !replayed.ExtractedAt.Equal(first.ExtractedAt) {
|
|
||||||
t.Fatalf("replayed Extract() = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-conflict"
|
|
||||||
command.Requested = 2
|
|
||||||
if _, err := fixture.Adapter.Extract(context.Background(), command); !errors.Is(err, extractionDomain.ErrIdempotencyConflict) {
|
|
||||||
t.Fatalf("conflicting Extract() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
expiryFixture := newRedisTestFixture(t)
|
|
||||||
realNow := time.Now().UTC().Truncate(time.Millisecond)
|
|
||||||
shortLived := testProxy("short-lived", "192.0.2.30")
|
|
||||||
shortLived.State = proxyDomain.StateAvailable
|
|
||||||
if _, err := expiryFixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: realNow, ConfiguredTTL: 2 * time.Second, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{shortLived},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(short-lived): %v", err)
|
|
||||||
}
|
|
||||||
expiryCommand := extractionDomain.Command{
|
|
||||||
RequestID: "req-expiry-first", ClientID: "client-a", IdempotencyKey: "idem-expiry",
|
|
||||||
IdempotencyTTL: time.Minute, Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: realNow.Add(time.Millisecond),
|
|
||||||
}
|
|
||||||
original, err := expiryFixture.Adapter.Extract(context.Background(), expiryCommand)
|
|
||||||
if err != nil || original.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(short-lived) = %+v, %v", original, err)
|
|
||||||
}
|
|
||||||
time.Sleep(2200 * time.Millisecond)
|
|
||||||
replacementAt := time.Now().UTC().Truncate(time.Millisecond)
|
|
||||||
if _, err := expiryFixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: replacementAt, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{shortLived},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(replacement): %v", err)
|
|
||||||
}
|
|
||||||
expiryCommand.RequestID = "req-expiry-second"
|
|
||||||
expiryCommand.Now = replacementAt.Add(time.Millisecond)
|
|
||||||
again, err := expiryFixture.Adapter.Extract(context.Background(), expiryCommand)
|
|
||||||
if err != nil || again.Returned != 1 || again.ExtractedAt.Equal(original.ExtractedAt) {
|
|
||||||
t.Fatalf("Extract(after idempotency expiry) = %+v, %v", again, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractIdempotencyDigestCanonicalizesBusinessFilters(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
candidate := testProxy("proxy-a", "192.0.2.10")
|
|
||||||
candidate.Tags = map[string]string{"region": "cn", "carrier": "ct"}
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute, candidate)
|
|
||||||
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-canonical-first", ClientID: "client-a", SourceIP: "192.0.2.100",
|
|
||||||
IdempotencyKey: "idem-canonical", IdempotencyTTL: time.Minute,
|
|
||||||
Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
MinRemainingTTL: time.Second, MaxHealthCheckAge: time.Minute,
|
|
||||||
Protocols: []string{"http", "http"}, Regions: []string{"cn", "cn"},
|
|
||||||
Carriers: []string{"ct", "ct"}, Upstreams: []string{"provider-a", "provider-a"},
|
|
||||||
}
|
|
||||||
first, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || first.Returned != 1 {
|
|
||||||
t.Fatalf("first Extract() = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
command.RequestID = "req-canonical-replay"
|
|
||||||
command.SourceIP = "198.51.100.200"
|
|
||||||
command.Now = now.Add(10 * time.Second)
|
|
||||||
command.IdempotencyTTL = 2 * time.Minute
|
|
||||||
command.MinRemainingTTL = 2 * time.Second
|
|
||||||
command.MaxHealthCheckAge = 2 * time.Minute
|
|
||||||
command.Protocols = []string{"http"}
|
|
||||||
command.Regions = []string{"cn"}
|
|
||||||
command.Carriers = []string{"ct"}
|
|
||||||
command.Upstreams = []string{"provider-a"}
|
|
||||||
replayed, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || replayed.Returned != 1 || !replayed.ExtractedAt.Equal(first.ExtractedAt) ||
|
|
||||||
replayed.Items[0].ID != first.Items[0].ID {
|
|
||||||
t.Fatalf("replayed Extract() = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractReplaysTheCommittedRequestOperation(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-operation-replay", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
}
|
|
||||||
|
|
||||||
first, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || first.Returned != 1 {
|
|
||||||
t.Fatalf("first Extract() = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
command.Now = now.Add(3 * time.Second)
|
|
||||||
replayed, err := fixture.Adapter.Extract(context.Background(), command)
|
|
||||||
if err != nil || replayed.Returned != 1 || !replayed.ExtractedAt.Equal(first.ExtractedAt) ||
|
|
||||||
replayed.Items[0].ID != first.Items[0].ID {
|
|
||||||
t.Fatalf("replayed Extract() = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractIsExclusiveUnderConcurrentRaces(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for iteration := range 100 {
|
|
||||||
candidate := testProxy(fmt.Sprintf("race-%d", iteration), fmt.Sprintf("198.51.100.%d", iteration+1))
|
|
||||||
candidate.State = proxyDomain.StateAvailable
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("iteration %d UpsertFetched(): %v", iteration, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
results := make(chan extractionDomain.Result, 2)
|
|
||||||
errorsCh := make(chan error, 2)
|
|
||||||
var workers sync.WaitGroup
|
|
||||||
for worker := range 2 {
|
|
||||||
workers.Add(1)
|
|
||||||
go func(worker int) {
|
|
||||||
defer workers.Done()
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: fmt.Sprintf("req-race-%d-%d", iteration, worker),
|
|
||||||
ClientID: fmt.Sprintf("client-%d", worker), Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
})
|
|
||||||
results <- result
|
|
||||||
errorsCh <- err
|
|
||||||
}(worker)
|
|
||||||
}
|
|
||||||
workers.Wait()
|
|
||||||
close(results)
|
|
||||||
close(errorsCh)
|
|
||||||
for err := range errorsCh {
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("iteration %d Extract(): %v", iteration, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
returned := 0
|
|
||||||
for result := range results {
|
|
||||||
returned += result.Returned
|
|
||||||
}
|
|
||||||
if returned != 1 {
|
|
||||||
t.Fatalf("iteration %d total returned = %d, want 1", iteration, returned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractScanBudgetExhaustionDoesNotMutateCandidates(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for index, ttl := range []time.Duration{3 * time.Minute, 2 * time.Minute, time.Minute} {
|
|
||||||
candidate := testProxy(fmt.Sprintf("scan-%d", index), fmt.Sprintf("203.0.113.%d", index+1))
|
|
||||||
candidate.State = proxyDomain.StateAvailable
|
|
||||||
candidate.Tags["region"] = "none"
|
|
||||||
if index == 2 {
|
|
||||||
candidate.Tags["region"] = "target"
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: ttl, MaxSize: 10, Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(scan-%d): %v", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
bounded, err := New(fixture.Client, Options{
|
|
||||||
Namespace: fixture.Namespace, Credentials: fixture.Credentials,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 2, CleanupLimit: 16,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(bounded): %v", err)
|
|
||||||
}
|
|
||||||
_, err = bounded.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-scan", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.AllOrNothing, Now: now.Add(time.Second),
|
|
||||||
Regions: []string{"target", "other"},
|
|
||||||
})
|
|
||||||
if !errors.Is(err, extractionDomain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("Extract(scan exhausted) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
remaining, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-after-scan", ClientID: "client-a", Requested: 3,
|
|
||||||
Fulfillment: extractionDomain.AllOrNothing, Now: now.Add(time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || remaining.Returned != 3 {
|
|
||||||
t.Fatalf("Extract(after scan exhaustion) = %+v, %v", remaining, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisExtractDoesNotConsumeIncompleteRecords(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(map[string]any)
|
|
||||||
}{
|
|
||||||
{name: "empty host", mutate: func(record map[string]any) { record["host"] = "" }},
|
|
||||||
{name: "missing creation time", mutate: func(record map[string]any) { record["createdAtMs"] = float64(0) }},
|
|
||||||
{name: "negative latency", mutate: func(record map[string]any) { record["latencyNs"] = float64(-1) }},
|
|
||||||
{name: "negative concurrency", mutate: func(record map[string]any) { record["maxConcurrency"] = float64(-1) }},
|
|
||||||
{name: "invalid state", mutate: func(record map[string]any) { record["state"] = "BROKEN" }},
|
|
||||||
{name: "invalid credential version", mutate: func(record map[string]any) { record["credentialVersion"] = float64(1) }},
|
|
||||||
{name: "invalid last success", mutate: func(record map[string]any) { record["lastSuccessAtMs"] = float64(-1) }},
|
|
||||||
{name: "foreign index key", mutate: func(record map[string]any) { record["indexKeys"] = []any{"pp:{other}:available"} }},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("corrupt", "192.0.2.10"))
|
|
||||||
|
|
||||||
raw, err := fixture.Client.HGet(context.Background(), fixture.Adapter.keys.records, "corrupt").Result()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HGet(corrupt): %v", err)
|
|
||||||
}
|
|
||||||
var stored map[string]any
|
|
||||||
if err := json.Unmarshal([]byte(raw), &stored); err != nil {
|
|
||||||
t.Fatalf("decode stored record: %v", err)
|
|
||||||
}
|
|
||||||
test.mutate(stored)
|
|
||||||
expectedState := stored["state"]
|
|
||||||
corrupted, err := json.Marshal(stored)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("encode corrupt record: %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Client.HSet(context.Background(), fixture.Adapter.keys.records, "corrupt", corrupted).Err(); err != nil {
|
|
||||||
t.Fatalf("HSet(corrupt): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-corrupt", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 0 || len(result.Items) != 0 {
|
|
||||||
t.Fatalf("Extract(corrupt) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
after, err := fixture.Client.HGet(context.Background(), fixture.Adapter.keys.records, "corrupt").Result()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("HGet(corrupt after extraction): %v", err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(after), &stored); err != nil {
|
|
||||||
t.Fatalf("decode record after extraction: %v", err)
|
|
||||||
}
|
|
||||||
if stored["state"] != expectedState {
|
|
||||||
t.Fatalf("corrupt record state = %q, want unchanged %q", stored["state"], expectedState)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func seedRedisAvailable(
|
|
||||||
t *testing.T,
|
|
||||||
adapter *Adapter,
|
|
||||||
upstreamID string,
|
|
||||||
observedAt time.Time,
|
|
||||||
availableAt time.Time,
|
|
||||||
ttl time.Duration,
|
|
||||||
candidate proxyDomain.Proxy,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
candidate.State = proxyDomain.StateFetched
|
|
||||||
if _, err := adapter.UpsertFetched(context.Background(), upstreamID, activitypool.FetchedBatch{
|
|
||||||
ObservedAt: observedAt, ConfiguredTTL: ttl, MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(%s): %v", candidate.ID, err)
|
|
||||||
}
|
|
||||||
if _, err := adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: candidate.ID, CheckedAt: availableAt.Add(-time.Millisecond), NextState: proxyDomain.StateChecking,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(%s, checking): %v", candidate.ID, err)
|
|
||||||
}
|
|
||||||
if _, err := adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: candidate.ID, CheckedAt: availableAt, NextState: proxyDomain.StateAvailable,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(%s, available): %v", candidate.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ activitypool.HealthStore = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpdate) (activitypool.Entry, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return activitypool.Entry{}, err
|
|
||||||
}
|
|
||||||
if a == nil || update.ProxyID == "" || update.CheckedAt.IsZero() || update.Latency < 0 ||
|
|
||||||
!validProxyState(string(update.NextState)) {
|
|
||||||
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
||||||
}
|
|
||||||
operationID, err := newOperationID()
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.Entry{}, err
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, healthScript, []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
|
||||||
}, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency),
|
|
||||||
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID)
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.Entry{}, err
|
|
||||||
}
|
|
||||||
var reply healthScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return activitypool.Entry{}, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptNotFound:
|
|
||||||
return activitypool.Entry{}, activitypool.ErrActivityNotFound
|
|
||||||
case scriptStale:
|
|
||||||
return activitypool.Entry{}, activitypool.ErrStaleHealthUpdate
|
|
||||||
case scriptInvalid:
|
|
||||||
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
||||||
case scriptOK:
|
|
||||||
if reply.Record == "" {
|
|
||||||
return activitypool.Entry{}, invalidScriptReply("health reply omitted record")
|
|
||||||
}
|
|
||||||
record, err := decodeProxyRecord(reply.Record)
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.Entry{}, invalidScriptReply("health reply contained an invalid record")
|
|
||||||
}
|
|
||||||
return proxyRecordEntry(record), nil
|
|
||||||
default:
|
|
||||||
return activitypool.Entry{}, invalidScriptReply("unexpected health status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,80 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"strconv"
|
|
||||||
)
|
|
||||||
|
|
||||||
const redisKeyPrefix = "pp:{activity}:"
|
|
||||||
|
|
||||||
type keyspace struct {
|
|
||||||
prefix string
|
|
||||||
records string
|
|
||||||
unique string
|
|
||||||
idkeys string
|
|
||||||
expiry string
|
|
||||||
available string
|
|
||||||
owners string
|
|
||||||
ownerExpiry string
|
|
||||||
epoch string
|
|
||||||
inventory string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newKeyspace(namespace string) keyspace {
|
|
||||||
prefix := redisKeyPrefix + namespace
|
|
||||||
return keyspace{
|
|
||||||
prefix: prefix,
|
|
||||||
records: prefix + ":records",
|
|
||||||
unique: prefix + ":unique",
|
|
||||||
idkeys: prefix + ":idkeys",
|
|
||||||
expiry: prefix + ":expiry",
|
|
||||||
available: prefix + ":available",
|
|
||||||
owners: prefix + ":owners",
|
|
||||||
ownerExpiry: prefix + ":owner-expiry",
|
|
||||||
epoch: prefix + ":epoch",
|
|
||||||
inventory: prefix + ":inventory",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) idempotency(clientID, idempotencyKey string) string {
|
|
||||||
return keys.prefix + ":idem:" + digestParts(clientID, idempotencyKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) operation(operationID string) string {
|
|
||||||
return keys.prefix + ":op:" + digestToken(operationID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) protocol(value string) string {
|
|
||||||
return keys.facet("protocol", value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) region(value string) string {
|
|
||||||
return keys.facet("region", value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) carrier(value string) string {
|
|
||||||
return keys.facet("carrier", value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) upstream(value string) string {
|
|
||||||
return keys.facet("upstream", value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) facet(name, value string) string {
|
|
||||||
return keys.prefix + ":" + name + ":" + digestToken(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func digestToken(value string) string {
|
|
||||||
return digestParts(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func digestParts(values ...string) string {
|
|
||||||
digest := sha256.New()
|
|
||||||
for _, value := range values {
|
|
||||||
_, _ = digest.Write([]byte(strconv.Itoa(len(value))))
|
|
||||||
_, _ = digest.Write([]byte{':'})
|
|
||||||
_, _ = digest.Write([]byte(value))
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(digest.Sum(nil))
|
|
||||||
}
|
|
||||||
@ -1,91 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
maintenanceInventory = "inventory"
|
|
||||||
maintenanceSweep = "sweep"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
_ activitypool.InventoryReader = (*Adapter)(nil)
|
|
||||||
_ activitypool.Maintainer = (*Adapter)(nil)
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *Adapter) Inventory(ctx context.Context, upstreamID string, now time.Time) (activitypool.Inventory, error) {
|
|
||||||
result := activitypool.Inventory{UpstreamID: upstreamID}
|
|
||||||
if ctx == nil {
|
|
||||||
return result, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if a == nil || upstreamID == "" || now.IsZero() {
|
|
||||||
return result, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
reply, err := a.runMaintenance(ctx, maintenanceInventory, now, a.options.CleanupLimit, upstreamID)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return result, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK || reply.Count < 0 {
|
|
||||||
return result, invalidScriptReply("unexpected inventory reply")
|
|
||||||
}
|
|
||||||
result.Managed = reply.Count
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return 0, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if a == nil || now.IsZero() || limit <= 0 {
|
|
||||||
return 0, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
reply, err := a.runMaintenance(ctx, maintenanceSweep, now, limit, "")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return 0, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK || reply.Count < 0 || reply.Count > limit {
|
|
||||||
return 0, invalidScriptReply("unexpected expiry sweep reply")
|
|
||||||
}
|
|
||||||
return reply.Count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) runMaintenance(
|
|
||||||
ctx context.Context,
|
|
||||||
operation string,
|
|
||||||
now time.Time,
|
|
||||||
limit int,
|
|
||||||
upstreamID string,
|
|
||||||
) (maintenanceScriptReply, error) {
|
|
||||||
operationID, err := newOperationID()
|
|
||||||
if err != nil {
|
|
||||||
return maintenanceScriptReply{}, err
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, sweepScript, []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
|
||||||
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL))
|
|
||||||
if err != nil {
|
|
||||||
return maintenanceScriptReply{}, err
|
|
||||||
}
|
|
||||||
var reply maintenanceScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return maintenanceScriptReply{}, err
|
|
||||||
}
|
|
||||||
return reply, nil
|
|
||||||
}
|
|
||||||
@ -1,269 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
ownershipDomain "proxy-pool/internal/domain/ownership"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
ownershipAssign = "assign"
|
|
||||||
ownershipRenew = "renew"
|
|
||||||
ownershipBeginDrain = "begin_drain"
|
|
||||||
ownershipAcknowledgeDrain = "acknowledge_drain"
|
|
||||||
ownershipGet = "get"
|
|
||||||
ownershipExpire = "expire"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ ownershipDomain.Repository = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) Assign(
|
|
||||||
ctx context.Context,
|
|
||||||
now time.Time,
|
|
||||||
proxyID string,
|
|
||||||
workerID string,
|
|
||||||
ttl time.Duration,
|
|
||||||
) (ownershipDomain.Assignment, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
if now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
return decodeAssignmentReply(reply)
|
|
||||||
case scriptAlreadyOwned:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrAlreadyOwned
|
|
||||||
case scriptUnavailable, scriptNotFound:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrOwnershipUnavailable
|
|
||||||
case scriptInvalid:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
default:
|
|
||||||
return ownershipDomain.Assignment{}, invalidScriptReply("unexpected ownership assign status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) Renew(
|
|
||||||
ctx context.Context,
|
|
||||||
now time.Time,
|
|
||||||
proxyID string,
|
|
||||||
workerID string,
|
|
||||||
epoch uint64,
|
|
||||||
ttl time.Duration,
|
|
||||||
) (ownershipDomain.Assignment, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
if now.IsZero() || proxyID == "" || workerID == "" || epoch == 0 || ttl <= 0 {
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
return decodeAssignmentReply(reply)
|
|
||||||
case scriptStale, scriptNotFound, scriptUnavailable:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
|
||||||
case scriptInvalid:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
default:
|
|
||||||
return ownershipDomain.Assignment{}, invalidScriptReply("unexpected ownership renew status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) BeginDrain(
|
|
||||||
ctx context.Context,
|
|
||||||
proxyID string,
|
|
||||||
workerID string,
|
|
||||||
epoch uint64,
|
|
||||||
) (ownershipDomain.Assignment, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
if proxyID == "" || workerID == "" || epoch == 0 {
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
return decodeAssignmentReply(reply)
|
|
||||||
case scriptStale, scriptNotFound:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
|
||||||
case scriptInvalid:
|
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
|
||||||
default:
|
|
||||||
return ownershipDomain.Assignment{}, invalidScriptReply("unexpected ownership drain status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) AcknowledgeDrain(
|
|
||||||
ctx context.Context,
|
|
||||||
proxyID string,
|
|
||||||
workerID string,
|
|
||||||
epoch uint64,
|
|
||||||
active int64,
|
|
||||||
reserved int64,
|
|
||||||
) error {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if proxyID == "" || workerID == "" || epoch == 0 || active < 0 || reserved < 0 {
|
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
return nil
|
|
||||||
case scriptStale, scriptNotFound:
|
|
||||||
return ownershipDomain.ErrStaleAssignment
|
|
||||||
case scriptNotDraining:
|
|
||||||
return ownershipDomain.ErrNotDraining
|
|
||||||
case scriptDrainNotReady:
|
|
||||||
return ownershipDomain.ErrDrainNotReady
|
|
||||||
case scriptInvalid:
|
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
|
||||||
default:
|
|
||||||
return invalidScriptReply("unexpected ownership acknowledge status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) Get(ctx context.Context, proxyID string) (ownershipDomain.Assignment, bool, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, false, err
|
|
||||||
}
|
|
||||||
if proxyID == "" {
|
|
||||||
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, false, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptNotFound:
|
|
||||||
return ownershipDomain.Assignment{}, false, nil
|
|
||||||
case scriptOK:
|
|
||||||
assignment, err := decodeAssignmentReply(reply)
|
|
||||||
return assignment, err == nil, err
|
|
||||||
case scriptInvalid:
|
|
||||||
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
|
|
||||||
default:
|
|
||||||
return ownershipDomain.Assignment{}, false, invalidScriptReply("unexpected ownership get status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) Expire(ctx context.Context, now time.Time, limit int) ([]ownershipDomain.Assignment, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if now.IsZero() || limit <= 0 {
|
|
||||||
return nil, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return nil, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK || reply.Record == "" {
|
|
||||||
return nil, invalidScriptReply("unexpected ownership expire reply")
|
|
||||||
}
|
|
||||||
var records []ownershipRecord
|
|
||||||
if err := decodeJSON(reply.Record, &records); err != nil {
|
|
||||||
return nil, errors.Join(invalidScriptReply("ownership expire reply contained invalid records"), err)
|
|
||||||
}
|
|
||||||
assignments := make([]ownershipDomain.Assignment, 0, len(records))
|
|
||||||
for _, record := range records {
|
|
||||||
if err := validateOwnershipRecord(record); err != nil {
|
|
||||||
return nil, errors.Join(invalidScriptReply("ownership expire reply contained invalid records"), err)
|
|
||||||
}
|
|
||||||
assignments = append(assignments, assignmentFromRecord(record))
|
|
||||||
}
|
|
||||||
if len(assignments) > limit {
|
|
||||||
return nil, invalidScriptReply("ownership expire reply exceeded limit")
|
|
||||||
}
|
|
||||||
return assignments, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) runOwnership(
|
|
||||||
ctx context.Context,
|
|
||||||
operation string,
|
|
||||||
mutating bool,
|
|
||||||
nowMS int64,
|
|
||||||
proxyID string,
|
|
||||||
workerID string,
|
|
||||||
epoch uint64,
|
|
||||||
value int64,
|
|
||||||
active int64,
|
|
||||||
reserved int64,
|
|
||||||
) (ownershipScriptReply, error) {
|
|
||||||
operationKey := a.keys.epoch
|
|
||||||
if mutating {
|
|
||||||
operationID, err := newOperationID()
|
|
||||||
if err != nil {
|
|
||||||
return ownershipScriptReply{}, err
|
|
||||||
}
|
|
||||||
operationKey = a.keys.operation(operationID)
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, ownershipScript, []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.epoch, operationKey,
|
|
||||||
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
|
||||||
nowMS, proxyID, workerID, epoch, value, active, reserved)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipScriptReply{}, err
|
|
||||||
}
|
|
||||||
var reply ownershipScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return ownershipScriptReply{}, err
|
|
||||||
}
|
|
||||||
return reply, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeAssignmentReply(reply ownershipScriptReply) (ownershipDomain.Assignment, error) {
|
|
||||||
if reply.Record == "" {
|
|
||||||
return ownershipDomain.Assignment{}, invalidScriptReply("ownership reply omitted assignment")
|
|
||||||
}
|
|
||||||
record, err := decodeOwnershipRecord(reply.Record)
|
|
||||||
if err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, errors.Join(
|
|
||||||
invalidScriptReply("ownership reply contained an invalid assignment"),
|
|
||||||
fmt.Errorf("decode ownership assignment: %w", err),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return assignmentFromRecord(record), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func assignmentFromRecord(record ownershipRecord) ownershipDomain.Assignment {
|
|
||||||
return ownershipDomain.Assignment{
|
|
||||||
ProxyID: record.ProxyID, WorkerID: record.WorkerID, Epoch: record.Epoch,
|
|
||||||
Version: record.AssignmentVersion, ExpiresAt: time.UnixMilli(record.ExpiresAtMS).UTC(),
|
|
||||||
Draining: record.Draining,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOwnershipCall(ctx context.Context, adapter *Adapter) error {
|
|
||||||
if ctx == nil || adapter == nil {
|
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -1,408 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
ownershipDomain "proxy-pool/internal/domain/ownership"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRedisOwnershipLifecycle(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
|
|
||||||
assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
|
|
||||||
if err != nil || assigned.Epoch == 0 || assigned.Version != 1 || assigned.Draining {
|
|
||||||
t.Fatalf("Assign() = %+v, %v", assigned, err)
|
|
||||||
}
|
|
||||||
assertRedisKeysHaveTTL(t, fixture,
|
|
||||||
fixture.Adapter.keys.owners,
|
|
||||||
fixture.Adapter.keys.ownerExpiry,
|
|
||||||
fixture.Adapter.keys.epoch,
|
|
||||||
)
|
|
||||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current != assigned {
|
|
||||||
t.Fatalf("Get(assigned) = %+v, %t, %v", current, ok, err)
|
|
||||||
}
|
|
||||||
blocked, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-owned", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || blocked.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(owned) = %+v, %v", blocked, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
renewed, err := fixture.Adapter.Renew(context.Background(), now.Add(30*time.Second),
|
|
||||||
"proxy-a", "worker-a", assigned.Epoch, 5*time.Minute)
|
|
||||||
if err != nil || renewed.Version != assigned.Version+1 || renewed.Epoch != assigned.Epoch ||
|
|
||||||
!renewed.ExpiresAt.Equal(now.Add(2*time.Minute)) {
|
|
||||||
t.Fatalf("Renew() = %+v, %v", renewed, err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Renew(context.Background(), now.Add(31*time.Second),
|
|
||||||
"proxy-a", "worker-a", assigned.Epoch+1, time.Minute); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
||||||
t.Fatalf("Renew(stale epoch) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
draining, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch)
|
|
||||||
if err != nil || !draining.Draining || draining.Version != renewed.Version+1 {
|
|
||||||
t.Fatalf("BeginDrain() = %+v, %v", draining, err)
|
|
||||||
}
|
|
||||||
replayed, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch)
|
|
||||||
if err != nil || replayed != draining {
|
|
||||||
t.Fatalf("BeginDrain(replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 1, 0); !errors.Is(err, ownershipDomain.ErrDrainNotReady) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(active) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 1); !errors.Is(err, ownershipDomain.ErrDrainNotReady) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(reserved) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); err != nil {
|
|
||||||
t.Fatalf("AcknowledgeDrain(): %v", err)
|
|
||||||
}
|
|
||||||
assertRedisKeysHaveTTL(t, fixture,
|
|
||||||
fixture.Adapter.keys.available,
|
|
||||||
fixture.Adapter.keys.protocol("http"),
|
|
||||||
fixture.Adapter.keys.region("cn"),
|
|
||||||
fixture.Adapter.keys.carrier("ct"),
|
|
||||||
fixture.Adapter.keys.upstream("provider-a"),
|
|
||||||
)
|
|
||||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || ok {
|
|
||||||
t.Fatalf("Get(after ACK) = %+v, %t, %v", current, ok, err)
|
|
||||||
}
|
|
||||||
restored, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-restored", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(40 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || restored.Returned != 1 || restored.Items[0].ID != "proxy-a" {
|
|
||||||
t.Fatalf("Extract(after ACK) = %+v, %v", restored, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisOwnershipRejectsInvalidDrainAndStaleAssignment(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 20*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Assign(): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), "proxy-a", "worker-b", time.Minute); !errors.Is(err, ownershipDomain.ErrAlreadyOwned) {
|
|
||||||
t.Fatalf("Assign(already owned) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); !errors.Is(err, ownershipDomain.ErrNotDraining) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(not draining) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-b", assigned.Epoch); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
||||||
t.Fatalf("BeginDrain(stale worker) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch+1, 0, 0); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(stale epoch) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisOwnershipExpireIsLimitedAndAllowsTakeover(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for index := range 2 {
|
|
||||||
proxyID := fmt.Sprintf("proxy-%d", index)
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
||||||
testProxy(proxyID, fmt.Sprintf("192.0.2.%d", index+10)))
|
|
||||||
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), proxyID, "worker-a", time.Second); err != nil {
|
|
||||||
t.Fatalf("Assign(%s): %v", proxyID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
first, _, err := fixture.Adapter.Get(context.Background(), "proxy-0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get(proxy-0): %v", err)
|
|
||||||
}
|
|
||||||
expired, err := fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
||||||
if err != nil || len(expired) != 1 {
|
|
||||||
t.Fatalf("Expire(first) = %+v, %v", expired, err)
|
|
||||||
}
|
|
||||||
expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
||||||
if err != nil || len(expired) != 1 {
|
|
||||||
t.Fatalf("Expire(second) = %+v, %v", expired, err)
|
|
||||||
}
|
|
||||||
expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
||||||
if err != nil || len(expired) != 0 {
|
|
||||||
t.Fatalf("Expire(empty) = %+v, %v", expired, err)
|
|
||||||
}
|
|
||||||
takeover, err := fixture.Adapter.Assign(context.Background(), now.Add(5*time.Second), "proxy-0", "worker-b", time.Minute)
|
|
||||||
if err != nil || takeover.Epoch <= first.Epoch {
|
|
||||||
t.Fatalf("Assign(takeover) = %+v, %v; old epoch=%d", takeover, err, first.Epoch)
|
|
||||||
}
|
|
||||||
|
|
||||||
directFixture := newRedisTestFixture(t)
|
|
||||||
seedRedisAvailable(t, directFixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
||||||
testProxy("direct", "192.0.2.30"))
|
|
||||||
old, err := directFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "direct", "worker-a", time.Second)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Assign(direct old): %v", err)
|
|
||||||
}
|
|
||||||
direct, err := directFixture.Adapter.Assign(context.Background(), now.Add(4*time.Second), "direct", "worker-b", time.Minute)
|
|
||||||
if err != nil || direct.WorkerID != "worker-b" || direct.Epoch <= old.Epoch {
|
|
||||||
t.Fatalf("Assign(direct takeover) = %+v, %v; old=%+v", direct, err, old)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisOwnershipAndExtractionHaveOneWinner(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for iteration := range 100 {
|
|
||||||
proxyID := fmt.Sprintf("race-%d", iteration)
|
|
||||||
candidate := testProxy(proxyID, fmt.Sprintf("198.51.100.%d", iteration+1))
|
|
||||||
candidate.State = proxyDomain.StateAvailable
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 200,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("iteration %d UpsertFetched(): %v", iteration, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var workers sync.WaitGroup
|
|
||||||
ownershipWon := make(chan bool, 1)
|
|
||||||
extractionWon := make(chan bool, 1)
|
|
||||||
errorsCh := make(chan error, 2)
|
|
||||||
workers.Add(2)
|
|
||||||
go func() {
|
|
||||||
defer workers.Done()
|
|
||||||
_, err := fixture.Adapter.Assign(context.Background(), now.Add(time.Second), proxyID, "worker-a", time.Minute)
|
|
||||||
if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) {
|
|
||||||
errorsCh <- err
|
|
||||||
}
|
|
||||||
ownershipWon <- err == nil
|
|
||||||
}()
|
|
||||||
go func() {
|
|
||||||
defer workers.Done()
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: fmt.Sprintf("req-race-%d", iteration), ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
Upstreams: []string{"provider-a"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
errorsCh <- err
|
|
||||||
}
|
|
||||||
extractionWon <- err == nil && result.Returned == 1 && result.Items[0].ID == proxyID
|
|
||||||
}()
|
|
||||||
workers.Wait()
|
|
||||||
close(errorsCh)
|
|
||||||
for err := range errorsCh {
|
|
||||||
t.Fatalf("iteration %d race error: %v", iteration, err)
|
|
||||||
}
|
|
||||||
winners := 0
|
|
||||||
if <-ownershipWon {
|
|
||||||
winners++
|
|
||||||
}
|
|
||||||
if <-extractionWon {
|
|
||||||
winners++
|
|
||||||
}
|
|
||||||
if winners != 1 {
|
|
||||||
t.Fatalf("iteration %d winners = %d, want 1", iteration, winners)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisOwnedProxyIsNotReintroducedByHealth(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 30*time.Second); err != nil {
|
|
||||||
t.Fatalf("Assign(): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateSuspect,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(suspect): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(4 * time.Second), NextState: proxyDomain.StateAvailable,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(available): %v", err)
|
|
||||||
}
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-owned-health", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(5 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(owned after health) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisInventoryTracksExtractionWithoutOwnershipWrites(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for index := range 2 {
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10)))
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(2*time.Second), 2)
|
|
||||||
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-inventory", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 1 {
|
|
||||||
t.Fatalf("Extract() = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(3*time.Second), 1)
|
|
||||||
remainingID := "proxy-0"
|
|
||||||
if result.Items[0].ID == remainingID {
|
|
||||||
remainingID = "proxy-1"
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), remainingID, "worker-a", 10*time.Second); err != nil {
|
|
||||||
t.Fatalf("Assign(remaining): %v", err)
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(4*time.Second), 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisInventoryPerformsBoundedExpiryCleanup(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for _, candidate := range []struct {
|
|
||||||
proxy proxyDomain.Proxy
|
|
||||||
ttl time.Duration
|
|
||||||
}{
|
|
||||||
{proxy: testProxy("short", "192.0.2.10"), ttl: 5 * time.Second},
|
|
||||||
{proxy: testProxy("long", "192.0.2.11"), ttl: time.Minute},
|
|
||||||
} {
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: candidate.ttl, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate.proxy},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(%s): %v", candidate.proxy.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 1)
|
|
||||||
if exists, err := fixture.Client.HExists(context.Background(), fixture.Adapter.keys.records, "short").Result(); err != nil || exists {
|
|
||||||
t.Fatalf("short record after Inventory cleanup = %t, %v", exists, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisSweepExpiredIsLimitedAndCleansOwnership(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
for index := range 2 {
|
|
||||||
candidate := testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10))
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 5 * time.Second, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(proxy-%d): %v", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(time.Second), 2)
|
|
||||||
removed, err := fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
||||||
if err != nil || removed != 1 {
|
|
||||||
t.Fatalf("SweepExpired(first) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
if remaining, err := fixture.Client.ZCard(context.Background(), fixture.Adapter.keys.expiry).Result(); err != nil || remaining != 1 {
|
|
||||||
t.Fatalf("expiry index after limited sweep = %d, %v; want 1", remaining, err)
|
|
||||||
}
|
|
||||||
removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
||||||
if err != nil || removed != 1 {
|
|
||||||
t.Fatalf("SweepExpired(second) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 0)
|
|
||||||
removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
||||||
if err != nil || removed != 0 {
|
|
||||||
t.Fatalf("SweepExpired(empty) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ownedFixture := newRedisTestFixture(t)
|
|
||||||
seedRedisAvailable(t, ownedFixture.Adapter, "provider-a", now, now.Add(time.Second), 5*time.Second,
|
|
||||||
testProxy("owned", "192.0.2.30"))
|
|
||||||
if _, err := ownedFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "owned", "worker-a", time.Minute); err != nil {
|
|
||||||
t.Fatalf("Assign(owned): %v", err)
|
|
||||||
}
|
|
||||||
if removed, err := ownedFixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 1 {
|
|
||||||
t.Fatalf("SweepExpired(owned) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
if assignment, ok, err := ownedFixture.Adapter.Get(context.Background(), "owned"); err != nil || ok {
|
|
||||||
t.Fatalf("Get(swept owned) = %+v, %t, %v", assignment, ok, err)
|
|
||||||
}
|
|
||||||
assertRedisInventory(t, ownedFixture.Adapter, "provider-a", now.Add(6*time.Second), 0)
|
|
||||||
for _, hashKey := range []string{
|
|
||||||
ownedFixture.Adapter.keys.records,
|
|
||||||
ownedFixture.Adapter.keys.idkeys,
|
|
||||||
ownedFixture.Adapter.keys.owners,
|
|
||||||
} {
|
|
||||||
if exists, err := ownedFixture.Client.HExists(context.Background(), hashKey, "owned").Result(); err != nil || exists {
|
|
||||||
t.Fatalf("hash %s retained owned proxy = %t, %v", hashKey, exists, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if count, err := ownedFixture.Client.HLen(context.Background(), ownedFixture.Adapter.keys.unique).Result(); err != nil || count != 0 {
|
|
||||||
t.Fatalf("unique mappings after sweep = %d, %v", count, err)
|
|
||||||
}
|
|
||||||
for _, sortedSet := range []string{
|
|
||||||
ownedFixture.Adapter.keys.expiry,
|
|
||||||
ownedFixture.Adapter.keys.available,
|
|
||||||
ownedFixture.Adapter.keys.ownerExpiry,
|
|
||||||
ownedFixture.Adapter.keys.protocol("http"),
|
|
||||||
ownedFixture.Adapter.keys.region("cn"),
|
|
||||||
ownedFixture.Adapter.keys.carrier("ct"),
|
|
||||||
ownedFixture.Adapter.keys.upstream("provider-a"),
|
|
||||||
} {
|
|
||||||
if count, err := ownedFixture.Client.ZCard(context.Background(), sortedSet).Result(); err != nil || count != 0 {
|
|
||||||
t.Fatalf("sorted set %s entries after sweep = %d, %v", sortedSet, count, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisOwnershipAndMaintenanceValidateInputs(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
if _, err := fixture.Adapter.Assign(nil, now, "proxy-a", "worker-a", time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Assign(nil context) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Renew(context.Background(), now, "proxy-a", "worker-a", 0, time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Renew(zero epoch) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.BeginDrain(context.Background(), "", "worker-a", 1); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("BeginDrain(empty proxy) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", 1, -1, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(negative active) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, _, err := fixture.Adapter.Get(context.Background(), ""); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Get(empty proxy) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Expire(context.Background(), now, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Expire(zero limit) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.Inventory(context.Background(), "", now); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
||||||
t.Fatalf("Inventory(empty upstream) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.SweepExpired(context.Background(), now, 0); !errors.Is(err, activitypool.ErrInvalidMaintenance) {
|
|
||||||
t.Fatalf("SweepExpired(zero limit) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertRedisInventory(t *testing.T, adapter *Adapter, upstreamID string, now time.Time, want int) {
|
|
||||||
t.Helper()
|
|
||||||
inventory, err := adapter.Inventory(context.Background(), upstreamID, now)
|
|
||||||
if err != nil || inventory.UpstreamID != upstreamID || inventory.Managed != want {
|
|
||||||
t.Fatalf("Inventory(%s) = %+v, %v; want %d", upstreamID, inventory, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertRedisKeysHaveTTL(t *testing.T, fixture redisTestFixture, keys ...string) {
|
|
||||||
t.Helper()
|
|
||||||
for _, key := range keys {
|
|
||||||
ttl, err := fixture.Client.PTTL(context.Background(), key).Result()
|
|
||||||
if err != nil || ttl <= 0 {
|
|
||||||
t.Fatalf("PTTL(%s) = %s, %v; want positive TTL", key, ttl, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,128 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
_ "embed"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
)
|
|
||||||
|
|
||||||
type scriptStatus string
|
|
||||||
|
|
||||||
const (
|
|
||||||
scriptOK scriptStatus = "ok"
|
|
||||||
scriptInvalid scriptStatus = "invalid"
|
|
||||||
scriptNotFound scriptStatus = "not_found"
|
|
||||||
scriptConflict scriptStatus = "conflict"
|
|
||||||
scriptStale scriptStatus = "stale"
|
|
||||||
scriptUnavailable scriptStatus = "unavailable"
|
|
||||||
scriptInsufficient scriptStatus = "insufficient"
|
|
||||||
scriptAlreadyOwned scriptStatus = "already_owned"
|
|
||||||
scriptNotDraining scriptStatus = "not_draining"
|
|
||||||
scriptDrainNotReady scriptStatus = "drain_not_ready"
|
|
||||||
)
|
|
||||||
|
|
||||||
type upsertScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
Accepted int `json:"accepted"`
|
|
||||||
Inserted int `json:"inserted"`
|
|
||||||
Refreshed int `json:"refreshed"`
|
|
||||||
Dropped int `json:"dropped"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type healthScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
Record string `json:"record,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type extractScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
RequestDigest string `json:"requestDigest"`
|
|
||||||
Record string `json:"record,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ownershipScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
Record string `json:"record,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type maintenanceScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
Count int `json:"count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:embed scripts/upsert.lua
|
|
||||||
var upsertSource string
|
|
||||||
|
|
||||||
//go:embed scripts/health.lua
|
|
||||||
var healthSource string
|
|
||||||
|
|
||||||
//go:embed scripts/extract.lua
|
|
||||||
var extractSource string
|
|
||||||
|
|
||||||
//go:embed scripts/ownership.lua
|
|
||||||
var ownershipSource string
|
|
||||||
|
|
||||||
//go:embed scripts/sweep.lua
|
|
||||||
var sweepSource string
|
|
||||||
|
|
||||||
var (
|
|
||||||
upsertScript = redis.NewScript(upsertSource)
|
|
||||||
healthScript = redis.NewScript(healthSource)
|
|
||||||
extractScript = redis.NewScript(extractSource)
|
|
||||||
ownershipScript = redis.NewScript(ownershipSource)
|
|
||||||
sweepScript = redis.NewScript(sweepSource)
|
|
||||||
)
|
|
||||||
|
|
||||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
|
||||||
result, err := script.Run(ctx, client, keys, args...).Result()
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return nil, errors.Join(
|
|
||||||
extractionDomain.ErrStoreUnavailable,
|
|
||||||
fmt.Errorf("run redis activity script: %w", err),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newOperationID() (string, error) {
|
|
||||||
var value [16]byte
|
|
||||||
if _, err := rand.Read(value[:]); err != nil {
|
|
||||||
return "", fmt.Errorf("create redis activity operation ID: %w", err)
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(value[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func operationTTLMillis(ttl time.Duration) int64 {
|
|
||||||
milliseconds := ttl / time.Millisecond
|
|
||||||
if ttl%time.Millisecond != 0 {
|
|
||||||
milliseconds++
|
|
||||||
}
|
|
||||||
return int64(milliseconds)
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeScriptResult(result any, destination any) error {
|
|
||||||
var payload string
|
|
||||||
switch value := result.(type) {
|
|
||||||
case string:
|
|
||||||
payload = value
|
|
||||||
case []byte:
|
|
||||||
payload = string(value)
|
|
||||||
default:
|
|
||||||
return errors.Join(extractionDomain.ErrStoreUnavailable, errors.New("invalid Redis script reply type"))
|
|
||||||
}
|
|
||||||
if err := decodeJSON(payload, destination); err != nil {
|
|
||||||
return errors.Join(extractionDomain.ErrStoreUnavailable, fmt.Errorf("decode Redis script reply: %w", err))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -1,339 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local owners_key = KEYS[7]
|
|
||||||
local owner_expiry_key = KEYS[8]
|
|
||||||
local operation_key = KEYS[9]
|
|
||||||
local idempotency_key = KEYS[10]
|
|
||||||
|
|
||||||
local now_ms = tonumber(ARGV[1])
|
|
||||||
local requested = tonumber(ARGV[2])
|
|
||||||
local fulfillment = ARGV[3]
|
|
||||||
local reserve = tonumber(ARGV[4])
|
|
||||||
local min_remaining_ttl_ms = tonumber(ARGV[5])
|
|
||||||
local max_health_age_ms = tonumber(ARGV[6])
|
|
||||||
local max_candidate_scan = tonumber(ARGV[7])
|
|
||||||
local cleanup_limit = tonumber(ARGV[8])
|
|
||||||
local idempotency_ttl_ms = tonumber(ARGV[9])
|
|
||||||
local operation_ttl_ms = tonumber(ARGV[10])
|
|
||||||
local request_digest = ARGV[11]
|
|
||||||
local has_idempotency = tonumber(ARGV[12]) == 1
|
|
||||||
local filters = cjson.decode(ARGV[13])
|
|
||||||
|
|
||||||
local function finish(reply, hard_expiry_ms)
|
|
||||||
local encoded = cjson.encode(reply)
|
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
|
||||||
if hard_expiry_ms then
|
|
||||||
local operation_expiry_ms = redis.call('PEXPIRETIME', operation_key)
|
|
||||||
if operation_expiry_ms > hard_expiry_ms then
|
|
||||||
redis.call('PEXPIREAT', operation_key, hard_expiry_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return encoded
|
|
||||||
end
|
|
||||||
|
|
||||||
local committed = redis.call('GET', operation_key)
|
|
||||||
if committed then
|
|
||||||
local reply = cjson.decode(committed)
|
|
||||||
if reply.requestDigest == request_digest then
|
|
||||||
return committed
|
|
||||||
end
|
|
||||||
return cjson.encode({status = 'conflict', requestDigest = request_digest})
|
|
||||||
end
|
|
||||||
|
|
||||||
if has_idempotency then
|
|
||||||
local replay = redis.call('GET', idempotency_key)
|
|
||||||
if replay then
|
|
||||||
local replay_record = cjson.decode(replay)
|
|
||||||
if tonumber(replay_record.expiresAtMs) <= now_ms then
|
|
||||||
redis.call('DEL', idempotency_key)
|
|
||||||
elseif replay_record.requestDigest ~= request_digest then
|
|
||||||
return finish({status = 'conflict', requestDigest = request_digest})
|
|
||||||
else
|
|
||||||
return finish({status = 'ok', requestDigest = request_digest, record = replay}, tonumber(replay_record.expiresAtMs))
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local value = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if value < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
local index_keys = record and record.indexKeys
|
|
||||||
if type(index_keys) == 'table' then
|
|
||||||
for _, index_key in ipairs(index_keys) do
|
|
||||||
if type(index_key) == 'string' and index_key ~= '' then
|
|
||||||
redis.call('ZREM', index_key, proxy_id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
local decoded
|
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, proxy_id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == proxy_id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, proxy_id)
|
|
||||||
redis.call('HDEL', records_key, proxy_id)
|
|
||||||
redis.call('ZREM', expiry_key, proxy_id)
|
|
||||||
redis.call('HDEL', owners_key, proxy_id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', now_ms, 'LIMIT', 0, cleanup_limit)
|
|
||||||
for _, proxy_id in ipairs(expired) do
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function to_set(values)
|
|
||||||
local result = {}
|
|
||||||
for _, value in ipairs(values or {}) do
|
|
||||||
result[value] = true
|
|
||||||
end
|
|
||||||
return result
|
|
||||||
end
|
|
||||||
|
|
||||||
local protocol_filter = to_set(filters.protocols)
|
|
||||||
local region_filter = to_set(filters.regions)
|
|
||||||
local carrier_filter = to_set(filters.carriers)
|
|
||||||
local upstream_filter = to_set(filters.upstreams)
|
|
||||||
|
|
||||||
local function matches(filter_values, filter_set, value)
|
|
||||||
return #filter_values == 0 or filter_set[value] == true
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_scheme(value)
|
|
||||||
return value == 'http' or value == 'https' or value == 'socks5'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_optional_string(value)
|
|
||||||
return value == nil or type(value) == 'string'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_integer(value)
|
|
||||||
return type(value) == 'number' and value == math.floor(value)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_proxy_state(value)
|
|
||||||
return value == 'FETCHED' or value == 'CHECKING' or value == 'AVAILABLE' or
|
|
||||||
value == 'SUSPECT' or value == 'DRAINING' or value == 'UNHEALTHY' or
|
|
||||||
value == 'EXTRACTED' or value == 'EXPIRED' or value == 'REMOVED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_proxy_record(proxy_id, record)
|
|
||||||
if type(record) ~= 'table' or record.version ~= 1 or record.id ~= proxy_id or
|
|
||||||
not valid_scheme(record.scheme) or type(record.host) ~= 'string' or record.host == '' or
|
|
||||||
type(record.sourceUpstream) ~= 'string' or record.sourceUpstream == '' or
|
|
||||||
not valid_optional_string(record.username) or not valid_optional_string(record.password) or
|
|
||||||
not valid_optional_string(record.credentialVersion) or
|
|
||||||
not valid_optional_string(record.ownerWorkerId) or not valid_proxy_state(record.state) or
|
|
||||||
(record.tags ~= nil and type(record.tags) ~= 'table') or
|
|
||||||
(record.indexKeys ~= nil and type(record.indexKeys) ~= 'table') then
|
|
||||||
return false
|
|
||||||
end
|
|
||||||
if not valid_integer(record.port) or record.port <= 0 or record.port > 65535 or
|
|
||||||
not valid_integer(record.createdAtMs) or record.createdAtMs <= 0 or
|
|
||||||
not valid_integer(record.expiresAtMs) or record.expiresAtMs <= 0 or
|
|
||||||
not valid_integer(record.usableUntilMs) or record.usableUntilMs <= 0 or
|
|
||||||
record.usableUntilMs > record.expiresAtMs or
|
|
||||||
not valid_integer(record.latencyNs) or record.latencyNs < 0 or
|
|
||||||
not valid_integer(record.maxConcurrency) or record.maxConcurrency < 0 or
|
|
||||||
(record.lastCheckedAtMs ~= nil and
|
|
||||||
(not valid_integer(record.lastCheckedAtMs) or record.lastCheckedAtMs < 0)) or
|
|
||||||
(record.lastSuccessAtMs ~= nil and
|
|
||||||
(not valid_integer(record.lastSuccessAtMs) or record.lastSuccessAtMs < 0)) then
|
|
||||||
return false
|
|
||||||
end
|
|
||||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
|
||||||
if type(index_key) ~= 'string' or index_key == '' or
|
|
||||||
not string.find(index_key, '{activity}', 1, true) then
|
|
||||||
return false
|
|
||||||
end
|
|
||||||
end
|
|
||||||
for tag_key, tag_value in pairs(record.tags or {}) do
|
|
||||||
if type(tag_key) ~= 'string' or type(tag_value) ~= 'string' then
|
|
||||||
return false
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return true
|
|
||||||
end
|
|
||||||
|
|
||||||
local function empty_result_record(expires_at_ms)
|
|
||||||
return '{"version":1,"requestDigest":' .. cjson.encode(request_digest) ..
|
|
||||||
',"expiresAtMs":' .. tostring(expires_at_ms) ..
|
|
||||||
',"result":{"requested":' .. tostring(requested) .. ',"returned":0,"items":[]}}'
|
|
||||||
end
|
|
||||||
|
|
||||||
if requested == 0 then
|
|
||||||
local expires_at_ms = now_ms + idempotency_ttl_ms
|
|
||||||
local record = empty_result_record(expires_at_ms)
|
|
||||||
if has_idempotency then
|
|
||||||
redis.call('SET', idempotency_key, record)
|
|
||||||
redis.call('PEXPIREAT', idempotency_key, expires_at_ms)
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', requestDigest = request_digest, record = record}, expires_at_ms)
|
|
||||||
end
|
|
||||||
|
|
||||||
local driver_key = available_key
|
|
||||||
local driver_size = redis.call('ZCARD', available_key)
|
|
||||||
for index = 11, #KEYS do
|
|
||||||
local size = redis.call('ZCARD', KEYS[index])
|
|
||||||
if size < driver_size then
|
|
||||||
driver_key = KEYS[index]
|
|
||||||
driver_size = size
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local candidate_ids = redis.call('ZREVRANGE', driver_key, 0, max_candidate_scan - 1)
|
|
||||||
local matches_found = {}
|
|
||||||
local required_matches = requested + reserve
|
|
||||||
local scanned = 0
|
|
||||||
|
|
||||||
for _, proxy_id in ipairs(candidate_ids) do
|
|
||||||
scanned = scanned + 1
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
if not raw then
|
|
||||||
redis.call('ZREM', driver_key, proxy_id)
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
else
|
|
||||||
local decoded, record = pcall(cjson.decode, raw)
|
|
||||||
if not decoded or not valid_proxy_record(proxy_id, record) then
|
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
|
||||||
elseif tonumber(record.expiresAtMs) <= now_ms then
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
else
|
|
||||||
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
|
|
||||||
local usable = record.state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > now_ms
|
|
||||||
if not usable then
|
|
||||||
remove_available(proxy_id, record)
|
|
||||||
else
|
|
||||||
local tags = record.tags or {}
|
|
||||||
local health_fresh = max_health_age_ms == 0 or
|
|
||||||
(record.lastCheckedAtMs and now_ms - tonumber(record.lastCheckedAtMs) <= max_health_age_ms)
|
|
||||||
local ttl_eligible = tonumber(record.expiresAtMs) - now_ms >= min_remaining_ttl_ms
|
|
||||||
if health_fresh and ttl_eligible and
|
|
||||||
matches(filters.protocols, protocol_filter, record.scheme) and
|
|
||||||
matches(filters.regions, region_filter, tags.region or '') and
|
|
||||||
matches(filters.carriers, carrier_filter, tags.carrier or '') and
|
|
||||||
matches(filters.upstreams, upstream_filter, record.sourceUpstream) then
|
|
||||||
matches_found[#matches_found + 1] = {id = proxy_id, record = record}
|
|
||||||
if #matches_found >= required_matches then
|
|
||||||
break
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
if #matches_found < required_matches and driver_size > scanned then
|
|
||||||
return finish({status = 'unavailable', requestDigest = request_digest})
|
|
||||||
end
|
|
||||||
|
|
||||||
local available_count = #matches_found - reserve
|
|
||||||
if available_count < 0 then
|
|
||||||
available_count = 0
|
|
||||||
end
|
|
||||||
if fulfillment == 'allOrNothing' and available_count < requested then
|
|
||||||
return finish({status = 'insufficient', requestDigest = request_digest})
|
|
||||||
end
|
|
||||||
|
|
||||||
local selected_count = requested
|
|
||||||
if selected_count > available_count then
|
|
||||||
selected_count = available_count
|
|
||||||
end
|
|
||||||
local result_items = cjson.decode('[]')
|
|
||||||
local earliest_expiry_ms = nil
|
|
||||||
|
|
||||||
for index = 1, selected_count do
|
|
||||||
local selected = matches_found[index]
|
|
||||||
local record = selected.record
|
|
||||||
remove_available(selected.id, record)
|
|
||||||
if is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
record.state = 'EXTRACTED'
|
|
||||||
local encoded = cjson.encode(record)
|
|
||||||
redis.call('HSET', records_key, selected.id, encoded)
|
|
||||||
|
|
||||||
local tags = record.tags or {}
|
|
||||||
local item = {
|
|
||||||
id = record.id,
|
|
||||||
protocol = record.scheme,
|
|
||||||
host = record.host,
|
|
||||||
port = record.port,
|
|
||||||
username = record.username,
|
|
||||||
password = record.password,
|
|
||||||
region = tags.region,
|
|
||||||
carrier = tags.carrier,
|
|
||||||
upstream = record.sourceUpstream,
|
|
||||||
ownerWorkerId = record.ownerWorkerId,
|
|
||||||
state = 'EXTRACTED',
|
|
||||||
expiresAtMs = record.expiresAtMs,
|
|
||||||
lastCheckedAtMs = record.lastCheckedAtMs,
|
|
||||||
}
|
|
||||||
result_items[#result_items + 1] = item
|
|
||||||
local hard_expiry_ms = tonumber(record.expiresAtMs)
|
|
||||||
if not earliest_expiry_ms or hard_expiry_ms < earliest_expiry_ms then
|
|
||||||
earliest_expiry_ms = hard_expiry_ms
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local result_expiry_ms = now_ms + idempotency_ttl_ms
|
|
||||||
if earliest_expiry_ms and earliest_expiry_ms < result_expiry_ms then
|
|
||||||
result_expiry_ms = earliest_expiry_ms
|
|
||||||
end
|
|
||||||
local result_value = {
|
|
||||||
requested = requested,
|
|
||||||
returned = selected_count,
|
|
||||||
items = result_items,
|
|
||||||
}
|
|
||||||
if selected_count > 0 then
|
|
||||||
result_value.extractedAtMs = now_ms
|
|
||||||
end
|
|
||||||
local result_record
|
|
||||||
if selected_count == 0 then
|
|
||||||
result_record = empty_result_record(result_expiry_ms)
|
|
||||||
else
|
|
||||||
result_record = cjson.encode({
|
|
||||||
version = 1,
|
|
||||||
requestDigest = request_digest,
|
|
||||||
expiresAtMs = result_expiry_ms,
|
|
||||||
result = result_value,
|
|
||||||
})
|
|
||||||
end
|
|
||||||
|
|
||||||
if has_idempotency then
|
|
||||||
redis.call('SET', idempotency_key, result_record)
|
|
||||||
redis.call('PEXPIREAT', idempotency_key, result_expiry_ms)
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', requestDigest = request_digest, record = result_record}, result_expiry_ms)
|
|
||||||
@ -1,169 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local owners_key = KEYS[7]
|
|
||||||
local owner_expiry_key = KEYS[8]
|
|
||||||
local operation_key = KEYS[9]
|
|
||||||
|
|
||||||
local checked_at_ms = tonumber(ARGV[1])
|
|
||||||
local next_state = ARGV[2]
|
|
||||||
local latency_ns = tonumber(ARGV[3])
|
|
||||||
local cleanup_limit = tonumber(ARGV[4])
|
|
||||||
local operation_ttl_ms = tonumber(ARGV[5])
|
|
||||||
local proxy_id = ARGV[6]
|
|
||||||
|
|
||||||
local committed = redis.call('GET', operation_key)
|
|
||||||
if committed then
|
|
||||||
return committed
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if not upstream or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local value = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if value < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(id, record)
|
|
||||||
redis.call('ZREM', available_key, id)
|
|
||||||
for _, index_key in ipairs(record and record.indexKeys or {}) do
|
|
||||||
redis.call('ZREM', index_key, id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(id)
|
|
||||||
local raw = redis.call('HGET', records_key, id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
record = cjson.decode(raw)
|
|
||||||
remove_available(id, record)
|
|
||||||
if is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, id)
|
|
||||||
redis.call('HDEL', records_key, id)
|
|
||||||
redis.call('ZREM', expiry_key, id)
|
|
||||||
redis.call('HDEL', owners_key, id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function cleanup_expired()
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', checked_at_ms, 'LIMIT', 0, cleanup_limit)
|
|
||||||
for _, id in ipairs(expired) do
|
|
||||||
remove_proxy(id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function touch(key, expires_at_ms)
|
|
||||||
if redis.call('EXISTS', key) == 0 then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local current = redis.call('PEXPIRETIME', key)
|
|
||||||
if current < expires_at_ms then
|
|
||||||
redis.call('PEXPIREAT', key, expires_at_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function finish(reply, hard_expiry_ms)
|
|
||||||
local encoded = cjson.encode(reply)
|
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
|
||||||
if hard_expiry_ms then
|
|
||||||
local operation_expiry_ms = redis.call('PEXPIRETIME', operation_key)
|
|
||||||
if operation_expiry_ms > hard_expiry_ms then
|
|
||||||
redis.call('PEXPIREAT', operation_key, hard_expiry_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return encoded
|
|
||||||
end
|
|
||||||
|
|
||||||
local transitions = {
|
|
||||||
FETCHED = {CHECKING = true, EXPIRED = true, REMOVED = true},
|
|
||||||
CHECKING = {AVAILABLE = true, UNHEALTHY = true, EXPIRED = true, REMOVED = true},
|
|
||||||
AVAILABLE = {SUSPECT = true, DRAINING = true, EXTRACTED = true, EXPIRED = true},
|
|
||||||
SUSPECT = {AVAILABLE = true, UNHEALTHY = true, DRAINING = true, EXPIRED = true},
|
|
||||||
DRAINING = {EXPIRED = true, UNHEALTHY = true, REMOVED = true},
|
|
||||||
UNHEALTHY = {CHECKING = true, REMOVED = true, EXPIRED = true},
|
|
||||||
EXTRACTED = {EXPIRED = true, REMOVED = true},
|
|
||||||
EXPIRED = {REMOVED = true},
|
|
||||||
REMOVED = {},
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup_expired()
|
|
||||||
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
if not raw then
|
|
||||||
return finish({status = 'not_found'})
|
|
||||||
end
|
|
||||||
local record = cjson.decode(raw)
|
|
||||||
if tonumber(record.expiresAtMs) <= checked_at_ms then
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
return finish({status = 'not_found'})
|
|
||||||
end
|
|
||||||
|
|
||||||
local last_checked_at_ms = tonumber(record.lastCheckedAtMs or '0')
|
|
||||||
if checked_at_ms < last_checked_at_ms then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
if checked_at_ms == last_checked_at_ms then
|
|
||||||
if record.state ~= next_state then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', record = raw}, tonumber(record.expiresAtMs))
|
|
||||||
end
|
|
||||||
if record.state ~= next_state and not (transitions[record.state] and transitions[record.state][next_state]) then
|
|
||||||
return finish({status = 'invalid'})
|
|
||||||
end
|
|
||||||
|
|
||||||
local was_managed = is_managed(record.state)
|
|
||||||
local will_be_managed = is_managed(next_state)
|
|
||||||
remove_available(proxy_id, record)
|
|
||||||
record.state = next_state
|
|
||||||
record.lastCheckedAtMs = checked_at_ms
|
|
||||||
record.latencyNs = latency_ns
|
|
||||||
if next_state == 'AVAILABLE' then
|
|
||||||
record.lastSuccessAtMs = checked_at_ms
|
|
||||||
end
|
|
||||||
if was_managed and not will_be_managed then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
elseif not was_managed and will_be_managed then
|
|
||||||
redis.call('HINCRBY', inventory_key, record.sourceUpstream, 1)
|
|
||||||
end
|
|
||||||
|
|
||||||
local encoded = cjson.encode(record)
|
|
||||||
redis.call('HSET', records_key, proxy_id, encoded)
|
|
||||||
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
|
|
||||||
if next_state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > checked_at_ms then
|
|
||||||
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
|
||||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
|
||||||
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
|
||||||
touch(index_key, tonumber(record.expiresAtMs))
|
|
||||||
end
|
|
||||||
end
|
|
||||||
touch(records_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(unique_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(idkeys_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(expiry_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(available_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(inventory_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(owners_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(owner_expiry_key, tonumber(record.expiresAtMs))
|
|
||||||
|
|
||||||
return finish({status = 'ok', record = encoded}, tonumber(record.expiresAtMs))
|
|
||||||
@ -1,293 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local owners_key = KEYS[7]
|
|
||||||
local owner_expiry_key = KEYS[8]
|
|
||||||
local epoch_key = KEYS[9]
|
|
||||||
local operation_key = KEYS[10]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
|
||||||
local operation_ttl_ms = tonumber(ARGV[2])
|
|
||||||
local cleanup_limit = tonumber(ARGV[3])
|
|
||||||
local now_ms = tonumber(ARGV[4])
|
|
||||||
local proxy_id = ARGV[5]
|
|
||||||
local worker_id = ARGV[6]
|
|
||||||
local epoch = tonumber(ARGV[7])
|
|
||||||
local value = tonumber(ARGV[8])
|
|
||||||
local active = tonumber(ARGV[9])
|
|
||||||
local reserved = tonumber(ARGV[10])
|
|
||||||
local mutating = operation ~= 'get'
|
|
||||||
|
|
||||||
local function finish(reply)
|
|
||||||
local encoded = cjson.encode(reply)
|
|
||||||
if mutating then
|
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
|
||||||
end
|
|
||||||
return encoded
|
|
||||||
end
|
|
||||||
|
|
||||||
if mutating then
|
|
||||||
local committed = redis.call('GET', operation_key)
|
|
||||||
if committed then
|
|
||||||
return committed
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local count = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if count < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function touch(key, expires_at_ms)
|
|
||||||
if redis.call('EXISTS', key) == 0 then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local current = redis.call('PEXPIRETIME', key)
|
|
||||||
if current < expires_at_ms then
|
|
||||||
redis.call('PEXPIREAT', key, expires_at_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(id, record)
|
|
||||||
redis.call('ZREM', available_key, id)
|
|
||||||
local index_keys = record and record.indexKeys
|
|
||||||
if type(index_keys) == 'table' then
|
|
||||||
for _, index_key in ipairs(index_keys) do
|
|
||||||
if type(index_key) == 'string' and index_key ~= '' then
|
|
||||||
redis.call('ZREM', index_key, id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function add_available(id, record, at_ms)
|
|
||||||
local usable_until_ms = record and tonumber(record.usableUntilMs)
|
|
||||||
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
redis.call('ZADD', available_key, usable_until_ms, id)
|
|
||||||
touch(available_key, tonumber(record.expiresAtMs))
|
|
||||||
if type(record.indexKeys) == 'table' then
|
|
||||||
for _, index_key in ipairs(record.indexKeys) do
|
|
||||||
if type(index_key) == 'string' and index_key ~= '' then
|
|
||||||
redis.call('ZADD', index_key, usable_until_ms, id)
|
|
||||||
touch(index_key, tonumber(record.expiresAtMs))
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(id)
|
|
||||||
local raw = redis.call('HGET', records_key, id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
local decoded
|
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
|
||||||
remove_available(id, decoded and record or nil)
|
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, id)
|
|
||||||
redis.call('HDEL', records_key, id)
|
|
||||||
redis.call('ZREM', expiry_key, id)
|
|
||||||
redis.call('HDEL', owners_key, id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function cleanup_hard_expired(at_ms)
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', at_ms, 'LIMIT', 0, cleanup_limit)
|
|
||||||
for _, id in ipairs(expired) do
|
|
||||||
remove_proxy(id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decode_table(raw)
|
|
||||||
if not raw then
|
|
||||||
return nil
|
|
||||||
end
|
|
||||||
local decoded, value = pcall(cjson.decode, raw)
|
|
||||||
if not decoded or type(value) ~= 'table' then
|
|
||||||
return nil
|
|
||||||
end
|
|
||||||
return value
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_assignment(assignment)
|
|
||||||
return assignment and assignment.version == 1 and type(assignment.proxyId) == 'string' and
|
|
||||||
assignment.proxyId ~= '' and type(assignment.workerId) == 'string' and assignment.workerId ~= '' and
|
|
||||||
tonumber(assignment.epoch) and tonumber(assignment.epoch) > 0 and
|
|
||||||
tonumber(assignment.assignmentVersion) and tonumber(assignment.assignmentVersion) > 0 and
|
|
||||||
tonumber(assignment.expiresAtMs) and tonumber(assignment.expiresAtMs) > 0 and
|
|
||||||
type(assignment.draining) == 'boolean'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function clear_owner(id, assignment, at_ms, restore)
|
|
||||||
local raw_record = redis.call('HGET', records_key, id)
|
|
||||||
local record = decode_table(raw_record)
|
|
||||||
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
|
|
||||||
record.ownerWorkerId = nil
|
|
||||||
redis.call('HSET', records_key, id, cjson.encode(record))
|
|
||||||
if restore then
|
|
||||||
add_available(id, record, at_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
redis.call('HDEL', owners_key, id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, id)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'assign' then
|
|
||||||
cleanup_hard_expired(now_ms)
|
|
||||||
local current_raw = redis.call('HGET', owners_key, proxy_id)
|
|
||||||
if current_raw then
|
|
||||||
local current = decode_table(current_raw)
|
|
||||||
if not valid_assignment(current) then
|
|
||||||
return finish({status = 'unavailable'})
|
|
||||||
end
|
|
||||||
if tonumber(current.expiresAtMs) > now_ms then
|
|
||||||
return finish({status = 'already_owned'})
|
|
||||||
end
|
|
||||||
clear_owner(proxy_id, current, now_ms, true)
|
|
||||||
end
|
|
||||||
|
|
||||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
|
||||||
if not record or record.state ~= 'AVAILABLE' or
|
|
||||||
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
|
|
||||||
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
|
|
||||||
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
|
|
||||||
return finish({status = 'unavailable'})
|
|
||||||
end
|
|
||||||
local expires_at_ms = now_ms + value
|
|
||||||
if tonumber(record.usableUntilMs) < expires_at_ms then
|
|
||||||
expires_at_ms = tonumber(record.usableUntilMs)
|
|
||||||
end
|
|
||||||
local next_epoch = redis.call('INCR', epoch_key)
|
|
||||||
local assignment = {
|
|
||||||
version = 1,
|
|
||||||
proxyId = proxy_id,
|
|
||||||
workerId = worker_id,
|
|
||||||
epoch = next_epoch,
|
|
||||||
assignmentVersion = 1,
|
|
||||||
expiresAtMs = expires_at_ms,
|
|
||||||
draining = false,
|
|
||||||
}
|
|
||||||
local encoded = cjson.encode(assignment)
|
|
||||||
redis.call('HSET', owners_key, proxy_id, encoded)
|
|
||||||
redis.call('ZADD', owner_expiry_key, expires_at_ms, proxy_id)
|
|
||||||
touch(owners_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(owner_expiry_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(epoch_key, tonumber(record.expiresAtMs))
|
|
||||||
record.ownerWorkerId = worker_id
|
|
||||||
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
|
|
||||||
remove_available(proxy_id, record)
|
|
||||||
return finish({status = 'ok', record = encoded})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'renew' then
|
|
||||||
cleanup_hard_expired(now_ms)
|
|
||||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
|
||||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
if tonumber(current.expiresAtMs) <= now_ms then
|
|
||||||
clear_owner(proxy_id, current, now_ms, true)
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
|
||||||
if not record or record.ownerWorkerId ~= worker_id or
|
|
||||||
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
|
|
||||||
clear_owner(proxy_id, current, now_ms, false)
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
local expires_at_ms = now_ms + value
|
|
||||||
if tonumber(record.usableUntilMs) < expires_at_ms then
|
|
||||||
expires_at_ms = tonumber(record.usableUntilMs)
|
|
||||||
end
|
|
||||||
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
|
|
||||||
current.expiresAtMs = expires_at_ms
|
|
||||||
local encoded = cjson.encode(current)
|
|
||||||
redis.call('HSET', owners_key, proxy_id, encoded)
|
|
||||||
redis.call('ZADD', owner_expiry_key, expires_at_ms, proxy_id)
|
|
||||||
touch(owners_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(owner_expiry_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(epoch_key, tonumber(record.expiresAtMs))
|
|
||||||
return finish({status = 'ok', record = encoded})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'begin_drain' then
|
|
||||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
|
||||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
if not current.draining then
|
|
||||||
current.draining = true
|
|
||||||
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
|
|
||||||
local encoded = cjson.encode(current)
|
|
||||||
redis.call('HSET', owners_key, proxy_id, encoded)
|
|
||||||
return finish({status = 'ok', record = encoded})
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', record = cjson.encode(current)})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'acknowledge_drain' then
|
|
||||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
|
||||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
if not current.draining then
|
|
||||||
return finish({status = 'not_draining'})
|
|
||||||
end
|
|
||||||
if active > 0 or reserved > 0 then
|
|
||||||
return finish({status = 'drain_not_ready'})
|
|
||||||
end
|
|
||||||
local server_time = redis.call('TIME')
|
|
||||||
local server_now_ms = tonumber(server_time[1]) * 1000 + math.floor(tonumber(server_time[2]) / 1000)
|
|
||||||
clear_owner(proxy_id, current, server_now_ms, true)
|
|
||||||
return finish({status = 'ok'})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'get' then
|
|
||||||
local raw = redis.call('HGET', owners_key, proxy_id)
|
|
||||||
if not raw then
|
|
||||||
return finish({status = 'not_found'})
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', record = raw})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'expire' then
|
|
||||||
local ids = redis.call('ZRANGEBYSCORE', owner_expiry_key, '-inf', now_ms, 'LIMIT', 0, value)
|
|
||||||
local expired = {}
|
|
||||||
for _, id in ipairs(ids) do
|
|
||||||
local current = decode_table(redis.call('HGET', owners_key, id))
|
|
||||||
if valid_assignment(current) then
|
|
||||||
expired[#expired + 1] = current
|
|
||||||
end
|
|
||||||
clear_owner(id, current, now_ms, true)
|
|
||||||
end
|
|
||||||
local encoded = '[]'
|
|
||||||
if #expired > 0 then
|
|
||||||
encoded = cjson.encode(expired)
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', record = encoded})
|
|
||||||
end
|
|
||||||
|
|
||||||
return finish({status = 'invalid'})
|
|
||||||
@ -1,95 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local owners_key = KEYS[7]
|
|
||||||
local owner_expiry_key = KEYS[8]
|
|
||||||
local operation_key = KEYS[9]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
|
||||||
local now_ms = tonumber(ARGV[2])
|
|
||||||
local limit = tonumber(ARGV[3])
|
|
||||||
local upstream_id = ARGV[4]
|
|
||||||
local operation_ttl_ms = tonumber(ARGV[5])
|
|
||||||
|
|
||||||
local function finish(reply)
|
|
||||||
local encoded = cjson.encode(reply)
|
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
|
||||||
return encoded
|
|
||||||
end
|
|
||||||
|
|
||||||
local committed = redis.call('GET', operation_key)
|
|
||||||
if committed then
|
|
||||||
return committed
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local count = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if count < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
local index_keys = record and record.indexKeys
|
|
||||||
if type(index_keys) == 'table' then
|
|
||||||
for _, index_key in ipairs(index_keys) do
|
|
||||||
if type(index_key) == 'string' and index_key ~= '' then
|
|
||||||
redis.call('ZREM', index_key, proxy_id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
local decoded
|
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
|
||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, proxy_id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == proxy_id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, proxy_id)
|
|
||||||
redis.call('HDEL', records_key, proxy_id)
|
|
||||||
redis.call('ZREM', expiry_key, proxy_id)
|
|
||||||
redis.call('HDEL', owners_key, proxy_id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', now_ms, 'LIMIT', 0, limit)
|
|
||||||
for _, proxy_id in ipairs(expired) do
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'sweep' then
|
|
||||||
return finish({status = 'ok', count = #expired})
|
|
||||||
end
|
|
||||||
if operation == 'inventory' then
|
|
||||||
local count = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
|
||||||
if count < 0 then
|
|
||||||
count = 0
|
|
||||||
redis.call('HSET', inventory_key, upstream_id, 0)
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', count = count})
|
|
||||||
end
|
|
||||||
return finish({status = 'invalid', count = 0})
|
|
||||||
@ -1,192 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local owners_key = KEYS[7]
|
|
||||||
local owner_expiry_key = KEYS[8]
|
|
||||||
local operation_key = KEYS[9]
|
|
||||||
|
|
||||||
local now_ms = tonumber(ARGV[1])
|
|
||||||
local cleanup_limit = tonumber(ARGV[2])
|
|
||||||
local max_size = tonumber(ARGV[3])
|
|
||||||
local operation_ttl_ms = tonumber(ARGV[4])
|
|
||||||
local candidates = cjson.decode(ARGV[5])
|
|
||||||
|
|
||||||
local committed = redis.call('GET', operation_key)
|
|
||||||
if committed then
|
|
||||||
return committed
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if not upstream or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local value = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if value < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
local indexes = record and record.indexKeys or {}
|
|
||||||
for _, index_key in ipairs(indexes) do
|
|
||||||
redis.call('ZREM', index_key, proxy_id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
record = cjson.decode(raw)
|
|
||||||
remove_available(proxy_id, record)
|
|
||||||
if is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, proxy_id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == proxy_id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, proxy_id)
|
|
||||||
redis.call('HDEL', records_key, proxy_id)
|
|
||||||
redis.call('ZREM', expiry_key, proxy_id)
|
|
||||||
redis.call('HDEL', owners_key, proxy_id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function cleanup_expired()
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', now_ms, 'LIMIT', 0, cleanup_limit)
|
|
||||||
for _, proxy_id in ipairs(expired) do
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function touch(key, expires_at_ms)
|
|
||||||
if redis.call('EXISTS', key) == 0 then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local current = redis.call('PEXPIRETIME', key)
|
|
||||||
if current < expires_at_ms then
|
|
||||||
redis.call('PEXPIREAT', key, expires_at_ms)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function add_available(proxy_id, record)
|
|
||||||
if record.state ~= 'AVAILABLE' or (record.ownerWorkerId and record.ownerWorkerId ~= '') or
|
|
||||||
tonumber(record.usableUntilMs) <= now_ms then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
|
||||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
|
||||||
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
|
||||||
touch(index_key, tonumber(record.expiresAtMs))
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function finish(reply)
|
|
||||||
local encoded = cjson.encode(reply)
|
|
||||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
|
||||||
return encoded
|
|
||||||
end
|
|
||||||
|
|
||||||
cleanup_expired()
|
|
||||||
|
|
||||||
for _, candidate in ipairs(candidates) do
|
|
||||||
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
|
|
||||||
if mapped and mapped ~= candidate.uniqueDigest then
|
|
||||||
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local accepted = #candidates
|
|
||||||
local inserted = 0
|
|
||||||
local refreshed = 0
|
|
||||||
local dropped = 0
|
|
||||||
local max_expiry_ms = 0
|
|
||||||
|
|
||||||
for _, candidate in ipairs(candidates) do
|
|
||||||
local incumbent_id = redis.call('HGET', unique_key, candidate.uniqueDigest)
|
|
||||||
local current_raw = incumbent_id and redis.call('HGET', records_key, incumbent_id) or nil
|
|
||||||
if incumbent_id and not current_raw then
|
|
||||||
remove_proxy(incumbent_id)
|
|
||||||
redis.call('HDEL', unique_key, candidate.uniqueDigest)
|
|
||||||
incumbent_id = nil
|
|
||||||
end
|
|
||||||
|
|
||||||
if current_raw then
|
|
||||||
local current = cjson.decode(current_raw)
|
|
||||||
if tonumber(current.expiresAtMs) <= now_ms then
|
|
||||||
remove_proxy(incumbent_id)
|
|
||||||
incumbent_id = nil
|
|
||||||
current_raw = nil
|
|
||||||
elseif current.state == 'EXTRACTED' or current.sourceUpstream ~= candidate.upstream then
|
|
||||||
refreshed = refreshed + 1
|
|
||||||
else
|
|
||||||
local incoming = cjson.decode(candidate.record)
|
|
||||||
remove_available(incumbent_id, current)
|
|
||||||
incoming.id = current.id
|
|
||||||
incoming.createdAtMs = current.createdAtMs
|
|
||||||
incoming.state = current.state
|
|
||||||
incoming.lastCheckedAtMs = current.lastCheckedAtMs
|
|
||||||
incoming.lastSuccessAtMs = current.lastSuccessAtMs
|
|
||||||
incoming.latencyNs = current.latencyNs
|
|
||||||
incoming.ownerWorkerId = current.ownerWorkerId
|
|
||||||
local encoded = cjson.encode(incoming)
|
|
||||||
redis.call('HSET', records_key, incumbent_id, encoded)
|
|
||||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
|
|
||||||
add_available(incumbent_id, incoming)
|
|
||||||
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
|
||||||
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
|
||||||
end
|
|
||||||
refreshed = refreshed + 1
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
if not incumbent_id then
|
|
||||||
local current_size = tonumber(redis.call('HGET', inventory_key, candidate.upstream) or '0')
|
|
||||||
if current_size >= max_size then
|
|
||||||
dropped = dropped + 1
|
|
||||||
else
|
|
||||||
local incoming = cjson.decode(candidate.record)
|
|
||||||
redis.call('HSET', records_key, candidate.proxyId, candidate.record)
|
|
||||||
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
|
|
||||||
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
|
|
||||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
|
|
||||||
if is_managed(incoming.state) then
|
|
||||||
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
|
||||||
end
|
|
||||||
add_available(candidate.proxyId, incoming)
|
|
||||||
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
|
||||||
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
|
||||||
end
|
|
||||||
inserted = inserted + 1
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
if max_expiry_ms > 0 then
|
|
||||||
touch(records_key, max_expiry_ms)
|
|
||||||
touch(unique_key, max_expiry_ms)
|
|
||||||
touch(idkeys_key, max_expiry_ms)
|
|
||||||
touch(expiry_key, max_expiry_ms)
|
|
||||||
touch(available_key, max_expiry_ms)
|
|
||||||
touch(inventory_key, max_expiry_ms)
|
|
||||||
touch(owners_key, max_expiry_ms)
|
|
||||||
touch(owner_expiry_key, max_expiry_ms)
|
|
||||||
end
|
|
||||||
|
|
||||||
return finish({
|
|
||||||
status = 'ok', accepted = accepted, inserted = inserted,
|
|
||||||
refreshed = refreshed, dropped = dropped,
|
|
||||||
})
|
|
||||||
@ -1,105 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
var testNamespaceSequence atomic.Uint64
|
|
||||||
|
|
||||||
type redisTestFixture struct {
|
|
||||||
Adapter *Adapter
|
|
||||||
Client *redis.Client
|
|
||||||
Credentials *credentials.MemoryStore
|
|
||||||
Namespace string
|
|
||||||
Cleanup func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
key := fixture.Adapter.keys.operation("fixture-probe")
|
|
||||||
if err := fixture.Client.Set(t.Context(), key, "ok", time.Minute).Err(); err != nil {
|
|
||||||
t.Fatalf("write isolated fixture key: %v", err)
|
|
||||||
}
|
|
||||||
if value, err := fixture.Client.Get(t.Context(), key).Result(); err != nil || value != "ok" {
|
|
||||||
t.Fatalf("read isolated fixture key = (%q, %v)", value, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
|
||||||
t.Helper()
|
|
||||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
|
||||||
if redisURL == "" {
|
|
||||||
t.Skip("PROXY_POOL_TEST_REDIS_URL is not set")
|
|
||||||
}
|
|
||||||
redisOptions, err := redis.ParseURL(redisURL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse PROXY_POOL_TEST_REDIS_URL: %v", err)
|
|
||||||
}
|
|
||||||
client := redis.NewClient(redisOptions)
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := client.Ping(ctx).Err(); err != nil {
|
|
||||||
_ = client.Close()
|
|
||||||
t.Fatalf("ping test Redis: %v", err)
|
|
||||||
}
|
|
||||||
credentialStore, err := credentials.NewMemoryStore(10_000)
|
|
||||||
if err != nil {
|
|
||||||
_ = client.Close()
|
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
|
||||||
}
|
|
||||||
namespace := fmt.Sprintf("it-%d-%d-%d", os.Getpid(), time.Now().UnixNano(), testNamespaceSequence.Add(1))
|
|
||||||
adapter, err := New(client, Options{
|
|
||||||
Namespace: namespace, Credentials: credentialStore,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 2_048, CleanupLimit: 128,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
_ = client.Close()
|
|
||||||
t.Fatalf("New(): %v", err)
|
|
||||||
}
|
|
||||||
var cleanupOnce sync.Once
|
|
||||||
cleanup := func() {
|
|
||||||
cleanupOnce.Do(func() {
|
|
||||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cleanupCancel()
|
|
||||||
if err := deleteRedisNamespace(cleanupCtx, client, redisKeyPrefix+namespace+":*"); err != nil {
|
|
||||||
t.Errorf("clean Redis test namespace: %v", err)
|
|
||||||
}
|
|
||||||
_ = client.Close()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
t.Cleanup(cleanup)
|
|
||||||
return redisTestFixture{
|
|
||||||
Adapter: adapter, Client: client, Credentials: credentialStore, Namespace: namespace, Cleanup: cleanup,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteRedisNamespace(ctx context.Context, client *redis.Client, pattern string) error {
|
|
||||||
var cursor uint64
|
|
||||||
for {
|
|
||||||
keys, next, err := client.Scan(ctx, cursor, pattern, 128).Result()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(keys) > 0 {
|
|
||||||
if err := client.Unlink(ctx, keys...).Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cursor = next
|
|
||||||
if cursor == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,258 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxUpsertScriptBatch = 256
|
|
||||||
|
|
||||||
type upsertCandidate struct {
|
|
||||||
ProxyID string `json:"proxyId"`
|
|
||||||
UniqueDigest string `json:"uniqueDigest"`
|
|
||||||
Upstream string `json:"upstream"`
|
|
||||||
Record string `json:"record"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ activitypool.Upserter = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
||||||
var result activitypool.UpsertResult
|
|
||||||
if ctx == nil {
|
|
||||||
return result, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if a == nil || upstreamID == "" || batch.ObservedAt.IsZero() || batch.ConfiguredTTL < 0 ||
|
|
||||||
batch.AllocationSafetyMargin < 0 || batch.MaxSize <= 0 ||
|
|
||||||
(batch.ConfiguredTTL > 0 && batch.AllocationSafetyMargin >= batch.ConfiguredTTL) {
|
|
||||||
return result, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
for _, candidate := range batch.Proxies {
|
|
||||||
if candidate.SourceUpstream != "" && candidate.SourceUpstream != upstreamID {
|
|
||||||
return result, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared := make([]upsertCandidate, 0, len(batch.Proxies))
|
|
||||||
seenIDs := make(map[string]string, len(batch.Proxies))
|
|
||||||
for _, candidate := range batch.Proxies {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return activitypool.UpsertResult{}, err
|
|
||||||
}
|
|
||||||
item, accepted, err := a.prepareUpsertCandidate(ctx, upstreamID, batch, candidate)
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.UpsertResult{}, err
|
|
||||||
}
|
|
||||||
if !accepted {
|
|
||||||
result.Dropped++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if digest, exists := seenIDs[item.ProxyID]; exists && digest != item.UniqueDigest {
|
|
||||||
return activitypool.UpsertResult{}, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
seenIDs[item.ProxyID] = item.UniqueDigest
|
|
||||||
prepared = append(prepared, item)
|
|
||||||
}
|
|
||||||
result.Accepted = len(prepared)
|
|
||||||
|
|
||||||
for start := 0; start < len(prepared); start += maxUpsertScriptBatch {
|
|
||||||
end := min(start+maxUpsertScriptBatch, len(prepared))
|
|
||||||
reply, err := a.upsertChunk(ctx, batch.ObservedAt, batch.MaxSize, prepared[start:end])
|
|
||||||
if err != nil {
|
|
||||||
return activitypool.UpsertResult{}, err
|
|
||||||
}
|
|
||||||
if reply.Accepted != end-start || reply.Inserted < 0 || reply.Refreshed < 0 || reply.Dropped < 0 ||
|
|
||||||
reply.Inserted+reply.Refreshed+reply.Dropped != reply.Accepted {
|
|
||||||
return activitypool.UpsertResult{}, invalidScriptReply("invalid upsert counters")
|
|
||||||
}
|
|
||||||
result.Inserted += reply.Inserted
|
|
||||||
result.Refreshed += reply.Refreshed
|
|
||||||
result.Dropped += reply.Dropped
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) prepareUpsertCandidate(
|
|
||||||
ctx context.Context,
|
|
||||||
upstreamID string,
|
|
||||||
batch activitypool.FetchedBatch,
|
|
||||||
candidate proxyDomain.Proxy,
|
|
||||||
) (upsertCandidate, bool, error) {
|
|
||||||
if !validCandidateIdentity(candidate) {
|
|
||||||
return upsertCandidate{}, false, nil
|
|
||||||
}
|
|
||||||
expiresAt := proxyDomain.EffectiveExpiry(batch.ObservedAt, candidate.ExpiresAt, 0, batch.ConfiguredTTL)
|
|
||||||
if expiresAt == nil {
|
|
||||||
return upsertCandidate{}, false, nil
|
|
||||||
}
|
|
||||||
usableUntil := expiresAt.Add(-batch.AllocationSafetyMargin)
|
|
||||||
if !usableUntil.After(batch.ObservedAt) || usableUntil.UnixMilli() <= batch.ObservedAt.UnixMilli() {
|
|
||||||
return upsertCandidate{}, false, nil
|
|
||||||
}
|
|
||||||
if candidate.MaxConcurrency < 0 || (candidate.State != "" && !validProxyState(string(candidate.State))) {
|
|
||||||
return upsertCandidate{}, false, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
if (candidate.SecretRef == "") != (candidate.CredentialVersion == "") {
|
|
||||||
return upsertCandidate{}, false, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
|
|
||||||
password := ""
|
|
||||||
if candidate.SecretRef != "" {
|
|
||||||
value, err := a.credentials.Resolve(ctx, credentials.Reference{
|
|
||||||
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return upsertCandidate{}, false, fmt.Errorf("resolve proxy credential: %w", err)
|
|
||||||
}
|
|
||||||
candidate.Username = value.Username
|
|
||||||
password = value.Password
|
|
||||||
}
|
|
||||||
candidate.SourceUpstream = upstreamID
|
|
||||||
candidate.ExpiresAt = expiresAt
|
|
||||||
candidate.UsableUntil = &usableUntil
|
|
||||||
if candidate.CreatedAt.IsZero() {
|
|
||||||
candidate.CreatedAt = batch.ObservedAt.UTC()
|
|
||||||
}
|
|
||||||
if candidate.State == "" {
|
|
||||||
candidate.State = proxyDomain.StateFetched
|
|
||||||
}
|
|
||||||
uniqueKey := candidate.UniqueKey()
|
|
||||||
if candidate.ID == "" {
|
|
||||||
candidate.ID = stableAdapterProxyID(uniqueKey)
|
|
||||||
}
|
|
||||||
record := proxyRecord{
|
|
||||||
Version: recordVersion, ID: candidate.ID, Scheme: string(candidate.Scheme), Host: candidate.Host,
|
|
||||||
Port: int64(candidate.Port), Username: candidate.Username, Password: password,
|
|
||||||
CredentialVersion: candidate.CredentialVersion, SourceUpstream: upstreamID,
|
|
||||||
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
|
|
||||||
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
|
|
||||||
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
|
|
||||||
Tags: cloneTags(candidate.Tags), IndexKeys: a.availableIndexKeys(candidate),
|
|
||||||
}
|
|
||||||
if candidate.LastCheckedAt != nil {
|
|
||||||
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
|
||||||
}
|
|
||||||
if candidate.LastSuccessAt != nil {
|
|
||||||
record.LastSuccessAtMS = candidate.LastSuccessAt.UnixMilli()
|
|
||||||
}
|
|
||||||
encoded, err := encodeProxyRecord(record)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, ErrInvalidRecord) {
|
|
||||||
return upsertCandidate{}, false, activitypool.ErrInvalidBatch
|
|
||||||
}
|
|
||||||
return upsertCandidate{}, false, err
|
|
||||||
}
|
|
||||||
return upsertCandidate{
|
|
||||||
ProxyID: candidate.ID, UniqueDigest: digestToken(uniqueKey), Upstream: upstreamID, Record: encoded,
|
|
||||||
}, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) upsertChunk(
|
|
||||||
ctx context.Context,
|
|
||||||
observedAt time.Time,
|
|
||||||
maxSize int,
|
|
||||||
candidates []upsertCandidate,
|
|
||||||
) (upsertScriptReply, error) {
|
|
||||||
operationID, err := newOperationID()
|
|
||||||
if err != nil {
|
|
||||||
return upsertScriptReply{}, err
|
|
||||||
}
|
|
||||||
payload, err := json.Marshal(candidates)
|
|
||||||
if err != nil {
|
|
||||||
return upsertScriptReply{}, fmt.Errorf("encode Redis upsert candidates: %w", err)
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, upsertScript, []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
|
||||||
}, observedAt.UnixMilli(), a.options.CleanupLimit, maxSize, operationTTLMillis(a.options.OperationTTL), string(payload))
|
|
||||||
if err != nil {
|
|
||||||
return upsertScriptReply{}, err
|
|
||||||
}
|
|
||||||
var reply upsertScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return upsertScriptReply{}, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
return reply, nil
|
|
||||||
case scriptInvalid:
|
|
||||||
return upsertScriptReply{}, activitypool.ErrInvalidBatch
|
|
||||||
default:
|
|
||||||
return upsertScriptReply{}, invalidScriptReply("unexpected upsert status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) availableIndexKeys(candidate proxyDomain.Proxy) []string {
|
|
||||||
keys := []string{a.keys.protocol(string(candidate.Scheme))}
|
|
||||||
if region := candidate.Tags["region"]; region != "" {
|
|
||||||
keys = append(keys, a.keys.region(region))
|
|
||||||
}
|
|
||||||
if carrier := candidate.Tags["carrier"]; carrier != "" {
|
|
||||||
keys = append(keys, a.keys.carrier(carrier))
|
|
||||||
}
|
|
||||||
return append(keys, a.keys.upstream(candidate.SourceUpstream))
|
|
||||||
}
|
|
||||||
|
|
||||||
func proxyRecordEntry(record proxyRecord) activitypool.Entry {
|
|
||||||
createdAt := time.UnixMilli(record.CreatedAtMS).UTC()
|
|
||||||
expiresAt := time.UnixMilli(record.ExpiresAtMS).UTC()
|
|
||||||
usableUntil := time.UnixMilli(record.UsableUntilMS).UTC()
|
|
||||||
proxy := proxyDomain.Proxy{
|
|
||||||
ID: record.ID, Scheme: proxyDomain.Scheme(record.Scheme), Host: record.Host, Port: uint16(record.Port),
|
|
||||||
Username: record.Username, CredentialVersion: record.CredentialVersion,
|
|
||||||
SourceUpstream: record.SourceUpstream, CreatedAt: createdAt, ExpiresAt: &expiresAt,
|
|
||||||
UsableUntil: &usableUntil, Latency: time.Duration(record.LatencyNS),
|
|
||||||
MaxConcurrency: record.MaxConcurrency, State: proxyDomain.State(record.State), Tags: cloneTags(record.Tags),
|
|
||||||
}
|
|
||||||
if record.LastCheckedAtMS > 0 {
|
|
||||||
value := time.UnixMilli(record.LastCheckedAtMS).UTC()
|
|
||||||
proxy.LastCheckedAt = &value
|
|
||||||
}
|
|
||||||
if record.LastSuccessAtMS > 0 {
|
|
||||||
value := time.UnixMilli(record.LastSuccessAtMS).UTC()
|
|
||||||
proxy.LastSuccessAt = &value
|
|
||||||
}
|
|
||||||
return activitypool.Entry{
|
|
||||||
Proxy: proxy, UsableUntil: usableUntil,
|
|
||||||
OwnerWorkerID: record.OwnerWorkerID, State: proxyDomain.State(record.State),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validCandidateIdentity(candidate proxyDomain.Proxy) bool {
|
|
||||||
if candidate.Host == "" || candidate.Port == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return validScheme(string(candidate.Scheme))
|
|
||||||
}
|
|
||||||
|
|
||||||
func stableAdapterProxyID(uniqueKey string) string {
|
|
||||||
digest := sha256.Sum256([]byte(uniqueKey))
|
|
||||||
return "px_" + hex.EncodeToString(digest[:12])
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneTags(tags map[string]string) map[string]string {
|
|
||||||
if tags == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := make(map[string]string, len(tags))
|
|
||||||
for key, value := range tags {
|
|
||||||
cloned[key] = value
|
|
||||||
}
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func invalidScriptReply(reason string) error {
|
|
||||||
return errors.Join(extractionDomain.ErrStoreUnavailable, errors.New(reason))
|
|
||||||
}
|
|
||||||
@ -1,230 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRedisUpsertEnforcesMaxSizeWithoutLeavingRejectedMappings(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
first := testProxy("proxy-a", "192.0.2.10")
|
|
||||||
second := testProxy("proxy-b", "192.0.2.11")
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second, MaxSize: 1,
|
|
||||||
Proxies: []proxyDomain.Proxy{first, second},
|
|
||||||
})
|
|
||||||
if err != nil || result.Accepted != 2 || result.Inserted != 1 || result.Dropped != 1 {
|
|
||||||
t.Fatalf("UpsertFetched() = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
retry, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now.Add(time.Second), ConfiguredTTL: 30 * time.Second, MaxSize: 2,
|
|
||||||
Proxies: []proxyDomain.Proxy{second},
|
|
||||||
})
|
|
||||||
if err != nil || retry.Inserted != 1 || retry.Refreshed != 0 {
|
|
||||||
t.Fatalf("UpsertFetched(capacity retry) = %+v, %v", retry, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisUpsertChunksLargeProviderResponsesWithGlobalCapacity(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
proxies := make([]proxyDomain.Proxy, 300)
|
|
||||||
for index := range proxies {
|
|
||||||
proxies[index] = testProxy("proxy-"+fmt.Sprint(index), fmt.Sprintf("192.0.2.%d", index%250+1))
|
|
||||||
proxies[index].Port = uint16(10_000 + index)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 275, Proxies: proxies,
|
|
||||||
})
|
|
||||||
if err != nil || result.Accepted != 300 || result.Inserted != 275 || result.Dropped != 25 {
|
|
||||||
t.Fatalf("UpsertFetched(large batch) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisUpsertPreservesIncumbentLifecycleAndRuntimeHealth(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
candidate := testProxy("proxy-a", "192.0.2.10")
|
|
||||||
|
|
||||||
first, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second,
|
|
||||||
AllocationSafetyMargin: 3 * time.Second, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || first.Inserted != 1 {
|
|
||||||
t.Fatalf("first UpsertFetched() = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(checking): %v", err)
|
|
||||||
}
|
|
||||||
healthyAt := now.Add(2 * time.Second)
|
|
||||||
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: healthyAt,
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: 25 * time.Millisecond,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(available): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
refreshed, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now.Add(3 * time.Second), ConfiguredTTL: time.Minute,
|
|
||||||
AllocationSafetyMargin: 5 * time.Second, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || refreshed.Refreshed != 1 || refreshed.Inserted != 0 {
|
|
||||||
t.Fatalf("same-provider refresh = %+v, %v", refreshed, err)
|
|
||||||
}
|
|
||||||
entry, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: healthyAt, NextState: proxyDomain.StateAvailable,
|
|
||||||
})
|
|
||||||
if err != nil || entry.State != proxyDomain.StateAvailable || entry.Proxy.Latency != 25*time.Millisecond ||
|
|
||||||
entry.Proxy.SourceUpstream != "provider-a" || entry.Proxy.ExpiresAt == nil ||
|
|
||||||
!entry.Proxy.ExpiresAt.Equal(now.Add(63*time.Second)) {
|
|
||||||
t.Fatalf("refreshed entry = %+v, %v", entry, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
duplicate, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-b", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now.Add(4 * time.Second), ConfiguredTTL: 5 * time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || duplicate.Refreshed != 1 || duplicate.Inserted != 0 {
|
|
||||||
t.Fatalf("cross-provider duplicate = %+v, %v", duplicate, err)
|
|
||||||
}
|
|
||||||
entry, err = fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: healthyAt, NextState: proxyDomain.StateAvailable,
|
|
||||||
})
|
|
||||||
if err != nil || entry.Proxy.SourceUpstream != "provider-a" ||
|
|
||||||
entry.Proxy.ExpiresAt == nil || !entry.Proxy.ExpiresAt.Equal(now.Add(63*time.Second)) {
|
|
||||||
t.Fatalf("incumbent entry = %+v, %v", entry, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
replacementAt := now.Add(64 * time.Second)
|
|
||||||
replaced, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-b", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: replacementAt, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || replaced.Inserted != 1 || replaced.Refreshed != 0 {
|
|
||||||
t.Fatalf("expired incumbent replacement = %+v, %v", replaced, err)
|
|
||||||
}
|
|
||||||
entry, err = fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: replacementAt.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
})
|
|
||||||
if err != nil || entry.Proxy.SourceUpstream != "provider-b" {
|
|
||||||
t.Fatalf("replacement entry = %+v, %v", entry, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
candidate := testProxy("proxy-a", "192.0.2.10")
|
|
||||||
candidate.Username = "user"
|
|
||||||
candidate.SecretRef = "cred_missing"
|
|
||||||
candidate.CredentialVersion = "v1"
|
|
||||||
|
|
||||||
_, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if !errors.Is(err, credentials.ErrCredentialMissing) {
|
|
||||||
t.Fatalf("UpsertFetched(missing credential) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
candidate.SecretRef = ""
|
|
||||||
candidate.CredentialVersion = ""
|
|
||||||
retry, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || retry.Inserted != 1 || retry.Refreshed != 0 {
|
|
||||||
t.Fatalf("UpsertFetched(after resolution failure) = %+v, %v", retry, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reference, err := fixture.Credentials.Put(context.Background(), "proxy-b", credentials.Value{
|
|
||||||
Username: "user", Password: "password",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Credentials.Put(): %v", err)
|
|
||||||
}
|
|
||||||
withCredential := testProxy("proxy-b", "192.0.2.11")
|
|
||||||
withCredential.Username = "user"
|
|
||||||
withCredential.SecretRef = reference.SecretRef
|
|
||||||
withCredential.CredentialVersion = reference.CredentialVersion
|
|
||||||
stored, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{withCredential},
|
|
||||||
})
|
|
||||||
if err != nil || stored.Inserted != 1 {
|
|
||||||
t.Fatalf("UpsertFetched(resolved credential) = %+v, %v", stored, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(checking): %v", err)
|
|
||||||
}
|
|
||||||
checkedAt := now.Add(2 * time.Second)
|
|
||||||
available, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: checkedAt,
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: 30 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil || available.State != proxyDomain.StateAvailable ||
|
|
||||||
available.Proxy.LastSuccessAt == nil || !available.Proxy.LastSuccessAt.Equal(checkedAt) {
|
|
||||||
t.Fatalf("ApplyHealth(available) = %+v, %v", available, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
replayed, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: checkedAt,
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: time.Second,
|
|
||||||
})
|
|
||||||
if err != nil || replayed.Proxy.Latency != 30*time.Millisecond {
|
|
||||||
t.Fatalf("ApplyHealth(replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
_, err = fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: checkedAt, NextState: proxyDomain.StateSuspect,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, activitypool.ErrStaleHealthUpdate) {
|
|
||||||
t.Fatalf("ApplyHealth(conflicting replay) error = %v", err)
|
|
||||||
}
|
|
||||||
_, err = fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateSuspect,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, activitypool.ErrStaleHealthUpdate) {
|
|
||||||
t.Fatalf("ApplyHealth(stale) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func redisTestNow() time.Time {
|
|
||||||
return time.Now().UTC().Add(time.Hour).Truncate(time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
func testProxy(id, host string) proxyDomain.Proxy {
|
|
||||||
return proxyDomain.Proxy{
|
|
||||||
ID: id, Scheme: proxyDomain.SchemeHTTP, Host: host, Port: 8080,
|
|
||||||
State: proxyDomain.StateFetched, Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -364,11 +364,7 @@ func TestHandlerExtractMapsErrorsToProblemResponsesWithoutSensitiveLeakage(t *te
|
|||||||
name: "extraction unavailable",
|
name: "extraction unavailable",
|
||||||
requestBody: `{"count":1}`,
|
requestBody: `{"count":1}`,
|
||||||
contentType: "application/json",
|
contentType: "application/json",
|
||||||
extractErr: errors.Join(
|
extractErr: controllerExtraction.ErrUnavailable,
|
||||||
controllerExtraction.ErrUnavailable,
|
|
||||||
domainExtraction.ErrStoreUnavailable,
|
|
||||||
errors.New("redis dial failed: TOKEN"),
|
|
||||||
),
|
|
||||||
wantStatus: http.StatusServiceUnavailable,
|
wantStatus: http.StatusServiceUnavailable,
|
||||||
wantCode: "SERVICE_UNAVAILABLE",
|
wantCode: "SERVICE_UNAVAILABLE",
|
||||||
},
|
},
|
||||||
@ -414,8 +410,7 @@ func TestHandlerExtractMapsErrorsToProblemResponsesWithoutSensitiveLeakage(t *te
|
|||||||
if problem.Status != test.wantStatus || problem.Code != test.wantCode {
|
if problem.Status != test.wantStatus || problem.Code != test.wantCode {
|
||||||
t.Fatalf("problem = %+v", problem)
|
t.Fatalf("problem = %+v", problem)
|
||||||
}
|
}
|
||||||
if body := response.Body.String(); strings.Contains(body, `"password"`) ||
|
if body := response.Body.String(); strings.Contains(body, `"password"`) || strings.Contains(body, "secret") {
|
||||||
strings.Contains(body, "secret") || strings.Contains(body, "redis") || strings.Contains(body, "TOKEN") {
|
|
||||||
t.Fatalf("error body leaked sensitive data: %s", body)
|
t.Fatalf("error body leaked sensitive data: %s", body)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@ -140,9 +140,6 @@ func (s *Service) Extract(ctx context.Context, request Request) (Response, error
|
|||||||
Upstreams: append([]string(nil), request.Filters.Upstreams...),
|
Upstreams: append([]string(nil), request.Filters.Upstreams...),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, domain.ErrStoreUnavailable) {
|
|
||||||
return response, errors.Join(ErrUnavailable, err)
|
|
||||||
}
|
|
||||||
return response, err
|
return response, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -145,27 +145,6 @@ func TestServiceUsesSourceIdentityForEphemeralIdempotency(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestServiceClassifiesStoreUnavailableAndPreservesCause(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
storeErr := errors.Join(domain.ErrStoreUnavailable, errors.New("redis dial failed: TOKEN"))
|
|
||||||
service, err := NewService(&recordingStore{err: storeErr}, Policy{
|
|
||||||
MaxCountPerRequest: 1,
|
|
||||||
DefaultFulfillment: domain.Partial,
|
|
||||||
}, allowAllAdmission{}, time.Now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewService(): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = service.Extract(context.Background(), Request{
|
|
||||||
RequestID: "req-1",
|
|
||||||
ClientID: "client-1",
|
|
||||||
Count: 1,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrUnavailable) || !errors.Is(err, domain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("Extract() error = %v, want unavailable classification with store cause", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
|
func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
|
||||||
firstTime := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
firstTime := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||||
secondTime := firstTime.Add(time.Minute)
|
secondTime := firstTime.Add(time.Minute)
|
||||||
@ -174,7 +153,6 @@ func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
|
|||||||
upserted, err := store.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
upserted, err := store.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||||
ObservedAt: firstTime,
|
ObservedAt: firstTime,
|
||||||
ConfiguredTTL: 2 * time.Minute,
|
ConfiguredTTL: 2 * time.Minute,
|
||||||
MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
Proxies: []proxyDomain.Proxy{{
|
||||||
ID: "p1", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
ID: "p1", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
||||||
State: proxyDomain.StateAvailable, LastCheckedAt: &checkedAt,
|
State: proxyDomain.StateAvailable, LastCheckedAt: &checkedAt,
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package pool
|
package pool
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
ownershipDomain "proxy-pool/internal/domain/ownership"
|
ownershipDomain "proxy-pool/internal/domain/ownership"
|
||||||
@ -31,62 +30,44 @@ func NewOwnershipManager(repository ownershipDomain.Repository) (*OwnershipManag
|
|||||||
return &OwnershipManager{repository: repository}, nil
|
return &OwnershipManager{repository: repository}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) Assign(ctx context.Context, now time.Time, proxyID, workerID string, ttl time.Duration) (Assignment, error) {
|
func (m *OwnershipManager) Assign(now time.Time, proxyID, workerID string, ttl time.Duration) (Assignment, error) {
|
||||||
if m == nil || m.repository == nil || ctx == nil {
|
if m == nil || m.repository == nil {
|
||||||
return Assignment{}, ErrInvalidOwnership
|
return Assignment{}, ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.Assign(now, proxyID, workerID, ttl)
|
||||||
return Assignment{}, err
|
|
||||||
}
|
|
||||||
return m.repository.Assign(ctx, now, proxyID, workerID, ttl)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) Renew(ctx context.Context, now time.Time, proxyID, workerID string, epoch uint64, ttl time.Duration) (Assignment, error) {
|
func (m *OwnershipManager) Renew(now time.Time, proxyID, workerID string, epoch uint64, ttl time.Duration) (Assignment, error) {
|
||||||
if m == nil || m.repository == nil || ctx == nil {
|
if m == nil || m.repository == nil {
|
||||||
return Assignment{}, ErrInvalidOwnership
|
return Assignment{}, ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.Renew(now, proxyID, workerID, epoch, ttl)
|
||||||
return Assignment{}, err
|
|
||||||
}
|
|
||||||
return m.repository.Renew(ctx, now, proxyID, workerID, epoch, ttl)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) BeginDrain(ctx context.Context, proxyID, workerID string, epoch uint64) (Assignment, error) {
|
func (m *OwnershipManager) BeginDrain(proxyID, workerID string, epoch uint64) (Assignment, error) {
|
||||||
if m == nil || m.repository == nil || ctx == nil {
|
if m == nil || m.repository == nil {
|
||||||
return Assignment{}, ErrInvalidOwnership
|
return Assignment{}, ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.BeginDrain(proxyID, workerID, epoch)
|
||||||
return Assignment{}, err
|
|
||||||
}
|
|
||||||
return m.repository.BeginDrain(ctx, proxyID, workerID, epoch)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) AcknowledgeDrain(ctx context.Context, proxyID, workerID string, epoch uint64, active, reserved int64) error {
|
func (m *OwnershipManager) AcknowledgeDrain(proxyID, workerID string, epoch uint64, active, reserved int64) error {
|
||||||
if m == nil || m.repository == nil || ctx == nil {
|
if m == nil || m.repository == nil {
|
||||||
return ErrInvalidOwnership
|
return ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.AcknowledgeDrain(proxyID, workerID, epoch, active, reserved)
|
||||||
return err
|
|
||||||
}
|
|
||||||
return m.repository.AcknowledgeDrain(ctx, proxyID, workerID, epoch, active, reserved)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) Get(ctx context.Context, proxyID string) (Assignment, bool, error) {
|
func (m *OwnershipManager) Get(proxyID string) (Assignment, bool) {
|
||||||
if m == nil || m.repository == nil || ctx == nil {
|
if m == nil || m.repository == nil {
|
||||||
return Assignment{}, false, ErrInvalidOwnership
|
return Assignment{}, false
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.Get(proxyID)
|
||||||
return Assignment{}, false, err
|
|
||||||
}
|
|
||||||
return m.repository.Get(ctx, proxyID)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *OwnershipManager) Expire(ctx context.Context, now time.Time, limit int) ([]Assignment, error) {
|
func (m *OwnershipManager) Expire(now time.Time) []Assignment {
|
||||||
if m == nil || m.repository == nil || ctx == nil || limit <= 0 {
|
if m == nil || m.repository == nil {
|
||||||
return nil, ErrInvalidOwnership
|
return nil
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
return m.repository.Expire(now)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return m.repository.Expire(ctx, now, limit)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -11,7 +11,6 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
ownershipDomain "proxy-pool/internal/domain/ownership"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -25,7 +24,7 @@ func TestOwnershipManagerPreventsDualAssignment(t *testing.T) {
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
_, err := manager.Assign(context.Background(), now, "proxy-1", fmt.Sprintf("worker-%d", index), time.Minute)
|
_, err := manager.Assign(now, "proxy-1", fmt.Sprintf("worker-%d", index), time.Minute)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
succeeded.Add(1)
|
succeeded.Add(1)
|
||||||
return
|
return
|
||||||
@ -45,22 +44,21 @@ func TestOwnershipManagerPreventsDualAssignment(t *testing.T) {
|
|||||||
func TestOwnershipManagerRenewsOnlyCurrentAssignment(t *testing.T) {
|
func TestOwnershipManagerRenewsOnlyCurrentAssignment(t *testing.T) {
|
||||||
manager := newTestOwnershipManager(t, "proxy-1")
|
manager := newTestOwnershipManager(t, "proxy-1")
|
||||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||||
ctx := context.Background()
|
assigned, err := manager.Assign(now, "proxy-1", "worker-1", time.Minute)
|
||||||
assigned, err := manager.Assign(ctx, now, "proxy-1", "worker-1", time.Minute)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(): %v", err)
|
t.Fatalf("Assign(): %v", err)
|
||||||
}
|
}
|
||||||
renewed, err := manager.Renew(ctx, now.Add(30*time.Second), "proxy-1", "worker-1", assigned.Epoch, time.Minute)
|
renewed, err := manager.Renew(now.Add(30*time.Second), "proxy-1", "worker-1", assigned.Epoch, time.Minute)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Renew(): %v", err)
|
t.Fatalf("Renew(): %v", err)
|
||||||
}
|
}
|
||||||
if renewed.Version != assigned.Version+1 || !renewed.ExpiresAt.Equal(now.Add(90*time.Second)) {
|
if renewed.Version != assigned.Version+1 || !renewed.ExpiresAt.Equal(now.Add(90*time.Second)) {
|
||||||
t.Fatalf("renewed assignment = %+v", renewed)
|
t.Fatalf("renewed assignment = %+v", renewed)
|
||||||
}
|
}
|
||||||
if _, err := manager.Renew(ctx, now, "proxy-1", "worker-2", assigned.Epoch, time.Minute); !errors.Is(err, ErrStaleAssignment) {
|
if _, err := manager.Renew(now, "proxy-1", "worker-2", assigned.Epoch, time.Minute); !errors.Is(err, ErrStaleAssignment) {
|
||||||
t.Fatalf("Renew(stale) error = %v, want ErrStaleAssignment", err)
|
t.Fatalf("Renew(stale) error = %v, want ErrStaleAssignment", err)
|
||||||
}
|
}
|
||||||
if expired, err := manager.Expire(ctx, now.Add(time.Minute), 32); err != nil || len(expired) != 0 {
|
if expired := manager.Expire(now.Add(time.Minute)); len(expired) != 0 {
|
||||||
t.Fatalf("renewed assignment expired at old deadline: %+v", expired)
|
t.Fatalf("renewed assignment expired at old deadline: %+v", expired)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -79,7 +77,7 @@ func TestSharedRepositoryMakesOwnershipAndExtractionMutuallyExclusive(t *testing
|
|||||||
extracted := make(chan bool, 1)
|
extracted := make(chan bool, 1)
|
||||||
go func() {
|
go func() {
|
||||||
<-start
|
<-start
|
||||||
_, assignErr := manager.Assign(context.Background(), now, "proxy-1", "worker-1", time.Minute)
|
_, assignErr := manager.Assign(now, "proxy-1", "worker-1", time.Minute)
|
||||||
if assignErr != nil && !errors.Is(assignErr, ErrOwnershipUnavailable) {
|
if assignErr != nil && !errors.Is(assignErr, ErrOwnershipUnavailable) {
|
||||||
t.Errorf("iteration %d Assign(): %v", iteration, assignErr)
|
t.Errorf("iteration %d Assign(): %v", iteration, assignErr)
|
||||||
}
|
}
|
||||||
@ -113,12 +111,11 @@ func TestSharedRepositoryMakesOwnershipAndExtractionMutuallyExclusive(t *testing
|
|||||||
func TestOwnershipManagerRequiresDrainAckAtZeroRuntime(t *testing.T) {
|
func TestOwnershipManagerRequiresDrainAckAtZeroRuntime(t *testing.T) {
|
||||||
manager := newTestOwnershipManager(t, "proxy-1")
|
manager := newTestOwnershipManager(t, "proxy-1")
|
||||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||||
ctx := context.Background()
|
assignment, err := manager.Assign(now, "proxy-1", "worker-1", time.Minute)
|
||||||
assignment, err := manager.Assign(ctx, now, "proxy-1", "worker-1", time.Minute)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(): %v", err)
|
t.Fatalf("Assign(): %v", err)
|
||||||
}
|
}
|
||||||
draining, err := manager.BeginDrain(ctx, "proxy-1", "worker-1", assignment.Epoch)
|
draining, err := manager.BeginDrain("proxy-1", "worker-1", assignment.Epoch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("BeginDrain(): %v", err)
|
t.Fatalf("BeginDrain(): %v", err)
|
||||||
}
|
}
|
||||||
@ -126,13 +123,13 @@ func TestOwnershipManagerRequiresDrainAckAtZeroRuntime(t *testing.T) {
|
|||||||
t.Fatalf("draining assignment = %+v", draining)
|
t.Fatalf("draining assignment = %+v", draining)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := manager.AcknowledgeDrain(ctx, "proxy-1", "worker-1", assignment.Epoch, 1, 0); !errors.Is(err, ErrDrainNotReady) {
|
if err := manager.AcknowledgeDrain("proxy-1", "worker-1", assignment.Epoch, 1, 0); !errors.Is(err, ErrDrainNotReady) {
|
||||||
t.Fatalf("AcknowledgeDrain(active) error = %v, want ErrDrainNotReady", err)
|
t.Fatalf("AcknowledgeDrain(active) error = %v, want ErrDrainNotReady", err)
|
||||||
}
|
}
|
||||||
if err := manager.AcknowledgeDrain(ctx, "proxy-1", "worker-1", assignment.Epoch, 0, 0); err != nil {
|
if err := manager.AcknowledgeDrain("proxy-1", "worker-1", assignment.Epoch, 0, 0); err != nil {
|
||||||
t.Fatalf("AcknowledgeDrain(zero): %v", err)
|
t.Fatalf("AcknowledgeDrain(zero): %v", err)
|
||||||
}
|
}
|
||||||
if _, ok, err := manager.Get(ctx, "proxy-1"); err != nil || ok {
|
if _, ok := manager.Get("proxy-1"); ok {
|
||||||
t.Fatal("assignment still exists after drain acknowledgement")
|
t.Fatal("assignment still exists after drain acknowledgement")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -140,19 +137,18 @@ func TestOwnershipManagerRequiresDrainAckAtZeroRuntime(t *testing.T) {
|
|||||||
func TestOwnershipManagerExpiresCrashedWorkerAssignment(t *testing.T) {
|
func TestOwnershipManagerExpiresCrashedWorkerAssignment(t *testing.T) {
|
||||||
manager := newTestOwnershipManager(t, "proxy-1")
|
manager := newTestOwnershipManager(t, "proxy-1")
|
||||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||||
ctx := context.Background()
|
first, err := manager.Assign(now, "proxy-1", "worker-1", time.Minute)
|
||||||
first, err := manager.Assign(ctx, now, "proxy-1", "worker-1", time.Minute)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(first): %v", err)
|
t.Fatalf("Assign(first): %v", err)
|
||||||
}
|
}
|
||||||
if expired, err := manager.Expire(ctx, now.Add(59*time.Second), 32); err != nil || len(expired) != 0 {
|
if expired := manager.Expire(now.Add(59 * time.Second)); len(expired) != 0 {
|
||||||
t.Fatalf("expired early: %+v", expired)
|
t.Fatalf("expired early: %+v", expired)
|
||||||
}
|
}
|
||||||
if expired, err := manager.Expire(ctx, now.Add(time.Minute), 32); err != nil || len(expired) != 1 || expired[0].ProxyID != "proxy-1" {
|
if expired := manager.Expire(now.Add(time.Minute)); len(expired) != 1 || expired[0].ProxyID != "proxy-1" {
|
||||||
t.Fatalf("Expire() = %+v, want proxy-1", expired)
|
t.Fatalf("Expire() = %+v, want proxy-1", expired)
|
||||||
}
|
}
|
||||||
|
|
||||||
second, err := manager.Assign(ctx, now.Add(time.Minute), "proxy-1", "worker-2", time.Minute)
|
second, err := manager.Assign(now.Add(time.Minute), "proxy-1", "worker-2", time.Minute)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(second): %v", err)
|
t.Fatalf("Assign(second): %v", err)
|
||||||
}
|
}
|
||||||
@ -161,125 +157,6 @@ func TestOwnershipManagerExpiresCrashedWorkerAssignment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOwnershipManagerPropagatesCanceledContextWithoutCallingRepository(t *testing.T) {
|
|
||||||
repository := &ownershipRepositoryStub{}
|
|
||||||
manager, err := NewOwnershipManager(repository)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewOwnershipManager(): %v", err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
_, err = manager.Assign(ctx, time.Now(), "proxy-1", "worker-1", time.Minute)
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Assign() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if repository.assignCalled {
|
|
||||||
t.Fatal("repository Assign called with canceled context")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOwnershipManagerPassesContextAndRepositoryError(t *testing.T) {
|
|
||||||
storageErr := errors.New("redis unavailable")
|
|
||||||
ctx := context.WithValue(context.Background(), ownershipContextKey{}, "request-1")
|
|
||||||
repository := &ownershipRepositoryStub{
|
|
||||||
assign: func(got context.Context, _ time.Time, _, _ string, _ time.Duration) (Assignment, error) {
|
|
||||||
if got != ctx {
|
|
||||||
t.Fatal("Assign() did not pass the original context")
|
|
||||||
}
|
|
||||||
return Assignment{}, storageErr
|
|
||||||
},
|
|
||||||
}
|
|
||||||
manager, err := NewOwnershipManager(repository)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewOwnershipManager(): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = manager.Assign(ctx, time.Now(), "proxy-1", "worker-1", time.Minute)
|
|
||||||
if !errors.Is(err, storageErr) {
|
|
||||||
t.Fatalf("Assign() error = %v, want repository error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOwnershipManagerPassesExpireLimit(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
|
||||||
want := []Assignment{{ProxyID: "proxy-1"}}
|
|
||||||
repository := &ownershipRepositoryStub{
|
|
||||||
expire: func(got context.Context, gotNow time.Time, gotLimit int) ([]Assignment, error) {
|
|
||||||
if got != ctx || !gotNow.Equal(now) || gotLimit != 32 {
|
|
||||||
t.Fatalf("Expire() arguments = (%v, %v, %d)", got, gotNow, gotLimit)
|
|
||||||
}
|
|
||||||
return want, nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
manager, err := NewOwnershipManager(repository)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewOwnershipManager(): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := manager.Expire(ctx, now, 32)
|
|
||||||
if err != nil || len(got) != 1 || got[0].ProxyID != want[0].ProxyID {
|
|
||||||
t.Fatalf("Expire() = %+v, %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOwnershipManagerRejectsInvalidContextAndExpireLimit(t *testing.T) {
|
|
||||||
manager, err := NewOwnershipManager(&ownershipRepositoryStub{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewOwnershipManager(): %v", err)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
if _, err := manager.Assign(nil, now, "proxy-1", "worker-1", time.Minute); !errors.Is(err, ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Assign(nil context) error = %v, want ErrInvalidOwnership", err)
|
|
||||||
}
|
|
||||||
if _, err := manager.Expire(context.Background(), now, 0); !errors.Is(err, ErrInvalidOwnership) {
|
|
||||||
t.Fatalf("Expire(zero limit) error = %v, want ErrInvalidOwnership", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ownershipContextKey struct{}
|
|
||||||
|
|
||||||
type ownershipRepositoryStub struct {
|
|
||||||
assign func(context.Context, time.Time, string, string, time.Duration) (Assignment, error)
|
|
||||||
expire func(context.Context, time.Time, int) ([]Assignment, error)
|
|
||||||
assignCalled bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) Assign(ctx context.Context, now time.Time, proxyID, workerID string, ttl time.Duration) (Assignment, error) {
|
|
||||||
r.assignCalled = true
|
|
||||||
if r.assign != nil {
|
|
||||||
return r.assign(ctx, now, proxyID, workerID, ttl)
|
|
||||||
}
|
|
||||||
return Assignment{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) Renew(context.Context, time.Time, string, string, uint64, time.Duration) (Assignment, error) {
|
|
||||||
return Assignment{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) BeginDrain(context.Context, string, string, uint64) (Assignment, error) {
|
|
||||||
return Assignment{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) AcknowledgeDrain(context.Context, string, string, uint64, int64, int64) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) Get(context.Context, string) (Assignment, bool, error) {
|
|
||||||
return Assignment{}, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *ownershipRepositoryStub) Expire(ctx context.Context, now time.Time, limit int) ([]Assignment, error) {
|
|
||||||
if r.expire != nil {
|
|
||||||
return r.expire(ctx, now, limit)
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ ownershipDomain.Repository = (*ownershipRepositoryStub)(nil)
|
|
||||||
|
|
||||||
func newTestOwnershipManager(t *testing.T, proxyIDs ...string) *OwnershipManager {
|
func newTestOwnershipManager(t *testing.T, proxyIDs ...string) *OwnershipManager {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||||
@ -301,7 +178,7 @@ func newTestActivityPool(t *testing.T, now time.Time, proxyIDs ...string) *activ
|
|||||||
}
|
}
|
||||||
store := activitypool.NewMemoryPool()
|
store := activitypool.NewMemoryPool()
|
||||||
result, err := store.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
result, err := store.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||||
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 100, Proxies: proxies,
|
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, Proxies: proxies,
|
||||||
})
|
})
|
||||||
if err != nil || result.Inserted != len(proxyIDs) {
|
if err != nil || result.Inserted != len(proxyIDs) {
|
||||||
t.Fatalf("UpsertFetched() = %+v, %v", result, err)
|
t.Fatalf("UpsertFetched() = %+v, %v", result, err)
|
||||||
|
|||||||
@ -19,7 +19,6 @@ type Config struct {
|
|||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
MaxSize int
|
|
||||||
TTL time.Duration
|
TTL time.Duration
|
||||||
AllocationSafetyMargin time.Duration
|
AllocationSafetyMargin time.Duration
|
||||||
Retry RetryConfig
|
Retry RetryConfig
|
||||||
@ -63,8 +62,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
|
|||||||
if config.UpstreamID == "" {
|
if config.UpstreamID == "" {
|
||||||
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
||||||
}
|
}
|
||||||
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 || config.MaxInFlight <= 0 {
|
||||||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
|
|
||||||
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
||||||
}
|
}
|
||||||
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
||||||
@ -215,7 +213,6 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
ObservedAt: r.runtime.Clock.Now().UTC(),
|
ObservedAt: r.runtime.Clock.Now().UTC(),
|
||||||
ConfiguredTTL: r.config.TTL,
|
ConfiguredTTL: r.config.TTL,
|
||||||
AllocationSafetyMargin: r.config.AllocationSafetyMargin,
|
AllocationSafetyMargin: r.config.AllocationSafetyMargin,
|
||||||
MaxSize: r.config.MaxSize,
|
|
||||||
Proxies: retained,
|
Proxies: retained,
|
||||||
})
|
})
|
||||||
candidateErr = err
|
candidateErr = err
|
||||||
|
|||||||
@ -31,7 +31,6 @@ func TestReconcilerWritesProviderTTLPolicyToEphemeralPool(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 321,
|
|
||||||
TTL: 30 * time.Second,
|
TTL: 30 * time.Second,
|
||||||
AllocationSafetyMargin: 3 * time.Second,
|
AllocationSafetyMargin: 3 * time.Second,
|
||||||
}, ports, Runtime{Clock: clock})
|
}, ports, Runtime{Clock: clock})
|
||||||
@ -45,7 +44,7 @@ func TestReconcilerWritesProviderTTLPolicyToEphemeralPool(t *testing.T) {
|
|||||||
}
|
}
|
||||||
batch := <-batches
|
batch := <-batches
|
||||||
if !batch.ObservedAt.Equal(now) || batch.ConfiguredTTL != 30*time.Second ||
|
if !batch.ObservedAt.Equal(now) || batch.ConfiguredTTL != 30*time.Second ||
|
||||||
batch.AllocationSafetyMargin != 3*time.Second || batch.MaxSize != 321 {
|
batch.AllocationSafetyMargin != 3*time.Second {
|
||||||
t.Fatalf("activity batch = %+v", batch)
|
t.Fatalf("activity batch = %+v", batch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -58,7 +57,6 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, Ports{
|
}, Ports{
|
||||||
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
calls.Add(1)
|
calls.Add(1)
|
||||||
@ -119,7 +117,6 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, successfulPorts(func() { calledAt <- clock.Now() }, results), Runtime{
|
}, successfulPorts(func() { calledAt <- clock.Now() }, results), Runtime{
|
||||||
Clock: clock,
|
Clock: clock,
|
||||||
Sleeper: sleeper,
|
Sleeper: sleeper,
|
||||||
@ -170,7 +167,6 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 3,
|
MaxAttempts: 3,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{
|
Retry: RetryConfig{
|
||||||
Initial: 100 * time.Millisecond,
|
Initial: 100 * time.Millisecond,
|
||||||
Max: time.Second,
|
Max: time.Second,
|
||||||
@ -248,7 +244,6 @@ func TestReconcilerClassifiesFetchResults(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -282,7 +277,6 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 2,
|
MaxAttempts: 2,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: time.Second, Jitter: 20},
|
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: time.Second, Jitter: 20},
|
||||||
}, ports, Runtime{Clock: clock, Sleeper: sleeper, Random: fixedRandom(1)})
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper, Random: fixedRandom(1)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -323,7 +317,6 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 2,
|
MaxAttempts: 2,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: 2 * time.Second},
|
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: 2 * time.Second},
|
||||||
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -378,7 +371,6 @@ func TestReconcilerAppliesAttemptTimeoutToEveryPort(t *testing.T) {
|
|||||||
Timeout: 250 * time.Millisecond,
|
Timeout: 250 * time.Millisecond,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -409,7 +401,6 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -465,7 +456,6 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -511,7 +501,6 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 1,
|
MaxAttempts: 1,
|
||||||
MaxInFlight: 2,
|
MaxInFlight: 2,
|
||||||
MaxSize: 100,
|
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -555,7 +544,6 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 3,
|
MaxAttempts: 3,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
||||||
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -588,7 +576,6 @@ func TestReconcilerDoesNotRetryPermanentAdapterError(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 3,
|
MaxAttempts: 3,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{Initial: time.Second, Max: time.Second},
|
Retry: RetryConfig{Initial: time.Second, Max: time.Second},
|
||||||
}, ports, Runtime{Sleeper: sleeper})
|
}, ports, Runtime{Sleeper: sleeper})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -615,7 +602,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
})
|
})
|
||||||
reconciler, err := NewReconciler(Config{
|
reconciler, err := NewReconciler(Config{
|
||||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1,
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -657,7 +644,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
|||||||
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
|
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
|
||||||
})
|
})
|
||||||
reconciler, err := NewReconciler(Config{
|
reconciler, err := NewReconciler(Config{
|
||||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1,
|
||||||
}, ports)
|
}, ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -690,7 +677,6 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|||||||
Timeout: time.Second,
|
Timeout: time.Second,
|
||||||
MaxAttempts: 2,
|
MaxAttempts: 2,
|
||||||
MaxInFlight: 1,
|
MaxInFlight: 1,
|
||||||
MaxSize: 100,
|
|
||||||
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
||||||
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -724,15 +710,14 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
config Config
|
config Config
|
||||||
}{
|
}{
|
||||||
{name: "missing upstream", config: Config{Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
{name: "missing upstream", config: Config{Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1}},
|
||||||
{name: "negative interval", config: Config{UpstreamID: "a", RequestInterval: -1, Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
{name: "negative interval", config: Config{UpstreamID: "a", RequestInterval: -1, Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1}},
|
||||||
{name: "missing timeout", config: Config{UpstreamID: "a", MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
{name: "missing timeout", config: Config{UpstreamID: "a", MaxAttempts: 1, MaxInFlight: 1}},
|
||||||
{name: "missing attempts", config: Config{UpstreamID: "a", Timeout: time.Second, MaxInFlight: 1, MaxSize: 100}},
|
{name: "missing attempts", config: Config{UpstreamID: "a", Timeout: time.Second, MaxInFlight: 1}},
|
||||||
{name: "missing in flight", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxSize: 100}},
|
{name: "missing in flight", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1}},
|
||||||
{name: "missing max size", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1}},
|
{name: "invalid jitter", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, Retry: RetryConfig{Jitter: 101}}},
|
||||||
{name: "invalid jitter", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Jitter: 101}}},
|
{name: "initial exceeds max", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, Retry: RetryConfig{Initial: time.Second, Max: time.Millisecond}}},
|
||||||
{name: "initial exceeds max", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Initial: time.Second, Max: time.Millisecond}}},
|
{name: "incomplete retry pair", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, Retry: RetryConfig{Max: time.Second}}},
|
||||||
{name: "incomplete retry pair", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Max: time.Second}}},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@ -1,170 +0,0 @@
|
|||||||
package activitypool
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMemoryPoolAppliesHealthTransitionsAndRejectsStaleObservation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
pool := NewMemoryPool()
|
|
||||||
inserted, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
|
||||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
|
||||||
State: proxyDomain.StateFetched,
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
if err != nil || inserted.Inserted != 1 {
|
|
||||||
t.Fatalf("UpsertFetched() = %+v, %v", inserted, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checking, err := pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
})
|
|
||||||
if err != nil || checking.State != proxyDomain.StateChecking {
|
|
||||||
t.Fatalf("ApplyHealth(checking) = %+v, %v", checking, err)
|
|
||||||
}
|
|
||||||
available, err := pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(2 * time.Second),
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: 25 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil || available.State != proxyDomain.StateAvailable ||
|
|
||||||
available.Proxy.LastCheckedAt == nil || !available.Proxy.LastCheckedAt.Equal(now.Add(2*time.Second)) ||
|
|
||||||
available.Proxy.LastSuccessAt == nil || !available.Proxy.LastSuccessAt.Equal(now.Add(2*time.Second)) ||
|
|
||||||
available.Proxy.Latency != 25*time.Millisecond {
|
|
||||||
t.Fatalf("ApplyHealth(available) = %+v, %v", available, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateSuspect,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrStaleHealthUpdate) {
|
|
||||||
t.Fatalf("ApplyHealth(stale) error = %v, want ErrStaleHealthUpdate", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
replayed, err := pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(2 * time.Second),
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: time.Second,
|
|
||||||
})
|
|
||||||
if err != nil || replayed.Proxy.Latency != 25*time.Millisecond {
|
|
||||||
t.Fatalf("ApplyHealth(idempotent replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
_, err = pool.ApplyHealth(context.Background(), HealthUpdate{
|
|
||||||
ProxyID: "proxy-a", CheckedAt: now.Add(2 * time.Second), NextState: proxyDomain.StateSuspect,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrStaleHealthUpdate) {
|
|
||||||
t.Fatalf("ApplyHealth(conflicting replay) error = %v, want ErrStaleHealthUpdate", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolRejectsInvalidHealthUpdates(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
pool := NewMemoryPool()
|
|
||||||
if _, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
|
||||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
|
||||||
State: proxyDomain.StateFetched,
|
|
||||||
}},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
update HealthUpdate
|
|
||||||
want error
|
|
||||||
}{
|
|
||||||
{name: "missing proxy ID", update: HealthUpdate{CheckedAt: now, NextState: proxyDomain.StateChecking}, want: ErrInvalidHealthUpdate},
|
|
||||||
{name: "zero observation time", update: HealthUpdate{ProxyID: "proxy-a", NextState: proxyDomain.StateChecking}, want: ErrInvalidHealthUpdate},
|
|
||||||
{name: "negative latency", update: HealthUpdate{ProxyID: "proxy-a", CheckedAt: now, NextState: proxyDomain.StateChecking, Latency: -1}, want: ErrInvalidHealthUpdate},
|
|
||||||
{name: "missing entry", update: HealthUpdate{ProxyID: "missing", CheckedAt: now, NextState: proxyDomain.StateChecking}, want: ErrActivityNotFound},
|
|
||||||
{name: "invalid transition", update: HealthUpdate{ProxyID: "proxy-a", CheckedAt: now, NextState: proxyDomain.StateAvailable}, want: ErrInvalidHealthUpdate},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
if _, err := pool.ApplyHealth(context.Background(), tt.update); !errors.Is(err, tt.want) {
|
|
||||||
t.Fatalf("ApplyHealth() error = %v, want %v", err, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolRejectsNonPositiveMaxSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
_, err := NewMemoryPool().UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: time.Date(2026, 7, 29, 15, 0, 0, 0, time.UTC),
|
|
||||||
ConfiguredTTL: time.Minute,
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrInvalidBatch) {
|
|
||||||
t.Fatalf("UpsertFetched() error = %v, want ErrInvalidBatch", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolEnforcesMaxSizePerIncumbentUpstream(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
pool := NewMemoryPool()
|
|
||||||
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
|
||||||
{ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080},
|
|
||||||
{ID: "proxy-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8080},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(): %v", err)
|
|
||||||
}
|
|
||||||
if result.Accepted != 2 || result.Inserted != 1 || result.Dropped != 1 {
|
|
||||||
t.Fatalf("UpsertFetched() = %+v", result)
|
|
||||||
}
|
|
||||||
retry, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now.Add(time.Second), ConfiguredTTL: time.Minute, MaxSize: 2,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
|
||||||
{ID: "proxy-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8080},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil || retry.Inserted != 1 || retry.Refreshed != 0 {
|
|
||||||
t.Fatalf("UpsertFetched(capacity retry) = %+v, %v", retry, err)
|
|
||||||
}
|
|
||||||
inventory, err := pool.Inventory(context.Background(), "provider-a", now)
|
|
||||||
if err != nil || inventory.Managed != 2 {
|
|
||||||
t.Fatalf("Inventory() = %+v, %v", inventory, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolInventoryAndSweepExpiredAreBounded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
pool := NewMemoryPool()
|
|
||||||
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Second, MaxSize: 2,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
|
||||||
{ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080},
|
|
||||||
{ID: "proxy-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8080},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil || result.Inserted != 2 {
|
|
||||||
t.Fatalf("UpsertFetched() = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
inventory, err := pool.Inventory(context.Background(), "provider-a", now.Add(2*time.Second))
|
|
||||||
if err != nil || inventory.Managed != 0 {
|
|
||||||
t.Fatalf("Inventory(expired) = %+v, %v", inventory, err)
|
|
||||||
}
|
|
||||||
first, err := pool.SweepExpired(context.Background(), now.Add(2*time.Second), 1)
|
|
||||||
if err != nil || first != 1 {
|
|
||||||
t.Fatalf("SweepExpired(first) = %d, %v", first, err)
|
|
||||||
}
|
|
||||||
second, err := pool.SweepExpired(context.Background(), now.Add(2*time.Second), 1)
|
|
||||||
if err != nil || second != 1 {
|
|
||||||
t.Fatalf("SweepExpired(second) = %d, %v", second, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
package activitypool_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
"proxy-pool/internal/domain/activitypool/contracttest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMemoryPoolContract(t *testing.T) {
|
|
||||||
contracttest.Run(t, func(*testing.T) (contracttest.Store, func()) {
|
|
||||||
return activitypool.NewMemoryPool(), func() {}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@ -1,508 +0,0 @@
|
|||||||
package contracttest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
ownershipDomain "proxy-pool/internal/domain/ownership"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Store interface {
|
|
||||||
activitypool.Upserter
|
|
||||||
activitypool.HealthStore
|
|
||||||
activitypool.InventoryReader
|
|
||||||
activitypool.Maintainer
|
|
||||||
extractionDomain.Store
|
|
||||||
ownershipDomain.Repository
|
|
||||||
}
|
|
||||||
|
|
||||||
type Factory func(*testing.T) (Store, func())
|
|
||||||
|
|
||||||
func Run(t *testing.T, factory Factory) {
|
|
||||||
t.Helper()
|
|
||||||
t.Run("upsert capacity and incumbent lifecycle", func(t *testing.T) {
|
|
||||||
runUpsertContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
t.Run("health and filtered extraction", func(t *testing.T) {
|
|
||||||
runHealthAndExtractionContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
t.Run("fulfillment and gateway reserve", func(t *testing.T) {
|
|
||||||
runFulfillmentContract(t, factory)
|
|
||||||
})
|
|
||||||
t.Run("business idempotency", func(t *testing.T) {
|
|
||||||
runIdempotencyContract(t, factory)
|
|
||||||
})
|
|
||||||
t.Run("idempotency is bounded by proxy expiry", func(t *testing.T) {
|
|
||||||
runIdempotencyExpiryContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
t.Run("ownership lifecycle", func(t *testing.T) {
|
|
||||||
runOwnershipContract(t, factory)
|
|
||||||
})
|
|
||||||
t.Run("inventory and bounded maintenance", func(t *testing.T) {
|
|
||||||
runMaintenanceContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
t.Run("concurrent exclusivity", func(t *testing.T) {
|
|
||||||
runConcurrencyContract(t, factory)
|
|
||||||
})
|
|
||||||
t.Run("canceled contexts", func(t *testing.T) {
|
|
||||||
runCancellationContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func runUpsertContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
firstProxy := contractProxy("proxy-a", "192.0.2.10", proxyDomain.StateFetched)
|
|
||||||
secondProxy := contractProxy("proxy-b", "192.0.2.11", proxyDomain.StateFetched)
|
|
||||||
result, err := store.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second, AllocationSafetyMargin: 3 * time.Second,
|
|
||||||
MaxSize: 1, Proxies: []proxyDomain.Proxy{firstProxy, secondProxy},
|
|
||||||
})
|
|
||||||
if err != nil || result.Accepted != 2 || result.Inserted != 1 || result.Dropped != 1 {
|
|
||||||
t.Fatalf("UpsertFetched(capacity) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now, 1)
|
|
||||||
|
|
||||||
refreshed, err := store.UpsertFetched(context.Background(), "provider-b", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now.Add(time.Second), ConfiguredTTL: 5 * time.Minute,
|
|
||||||
AllocationSafetyMargin: 10 * time.Second, MaxSize: 10, Proxies: []proxyDomain.Proxy{firstProxy},
|
|
||||||
})
|
|
||||||
if err != nil || refreshed.Inserted != 0 || refreshed.Refreshed != 1 {
|
|
||||||
t.Fatalf("UpsertFetched(cross-provider refresh) = %+v, %v", refreshed, err)
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now.Add(time.Second), 1)
|
|
||||||
assertInventory(t, store, "provider-b", now.Add(time.Second), 0)
|
|
||||||
|
|
||||||
replaced, err := store.UpsertFetched(context.Background(), "provider-b", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now.Add(31 * time.Second), ConfiguredTTL: time.Minute,
|
|
||||||
AllocationSafetyMargin: 5 * time.Second, MaxSize: 10, Proxies: []proxyDomain.Proxy{firstProxy},
|
|
||||||
})
|
|
||||||
if err != nil || replaced.Inserted != 1 || replaced.Refreshed != 0 {
|
|
||||||
t.Fatalf("UpsertFetched(after incumbent expiry) = %+v, %v", replaced, err)
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now.Add(31*time.Second), 0)
|
|
||||||
assertInventory(t, store, "provider-b", now.Add(31*time.Second), 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runHealthAndExtractionContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
candidate := contractProxy("proxy-a", "192.0.2.10", proxyDomain.StateFetched)
|
|
||||||
candidate.Tags = map[string]string{"region": "cn", "carrier": "ct"}
|
|
||||||
upsertOne(t, store, "provider-a", now, time.Minute, candidate)
|
|
||||||
checking, err := store.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: candidate.ID, CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
})
|
|
||||||
if err != nil || checking.State != proxyDomain.StateChecking {
|
|
||||||
t.Fatalf("ApplyHealth(checking) = %+v, %v", checking, err)
|
|
||||||
}
|
|
||||||
available, err := store.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: candidate.ID, CheckedAt: now.Add(2 * time.Second),
|
|
||||||
NextState: proxyDomain.StateAvailable, Latency: 25 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil || available.State != proxyDomain.StateAvailable ||
|
|
||||||
available.Proxy.LastCheckedAt == nil || !available.Proxy.LastCheckedAt.Equal(now.Add(2*time.Second)) ||
|
|
||||||
available.Proxy.LastSuccessAt == nil || !available.Proxy.LastSuccessAt.Equal(now.Add(2*time.Second)) {
|
|
||||||
t.Fatalf("ApplyHealth(available) = %+v, %v", available, err)
|
|
||||||
}
|
|
||||||
if _, err := store.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
||||||
ProxyID: candidate.ID, CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateSuspect,
|
|
||||||
}); !errors.Is(err, activitypool.ErrStaleHealthUpdate) {
|
|
||||||
t.Fatalf("ApplyHealth(stale) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := store.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-filter", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second),
|
|
||||||
MinRemainingTTL: 10 * time.Second, MaxHealthCheckAge: 5 * time.Second,
|
|
||||||
Protocols: []string{"http"}, Regions: []string{"cn"}, Carriers: []string{"ct"},
|
|
||||||
Upstreams: []string{"provider-a"},
|
|
||||||
})
|
|
||||||
if err != nil || result.Returned != 1 || result.Items[0].ID != candidate.ID ||
|
|
||||||
result.Items[0].State != extractionDomain.Extracted {
|
|
||||||
t.Fatalf("Extract(filtered) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now.Add(3*time.Second), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runFulfillmentContract(t *testing.T, factory Factory) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
store := newStore(t, factory)
|
|
||||||
seedAvailable(t, store, "provider-a", now, time.Minute,
|
|
||||||
contractProxy("proxy-a", "192.0.2.10", proxyDomain.StateAvailable))
|
|
||||||
if result, err := store.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-all", ClientID: "client-a", Requested: 2,
|
|
||||||
Fulfillment: extractionDomain.AllOrNothing, Now: now.Add(time.Second),
|
|
||||||
}); !errors.Is(err, extractionDomain.ErrInsufficientProxies) || result.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(allOrNothing) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
if result, err := store.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-partial", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
}); err != nil || result.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(partial after insufficient) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reservedStore := newStore(t, factory)
|
|
||||||
seedAvailable(t, reservedStore, "provider-a", now, time.Minute,
|
|
||||||
contractProxy("proxy-reserved", "192.0.2.11", proxyDomain.StateAvailable))
|
|
||||||
reserved, err := reservedStore.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-reserved", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second), ReserveForGateway: 1,
|
|
||||||
})
|
|
||||||
if err != nil || reserved.Returned != 0 || len(reserved.Items) != 0 {
|
|
||||||
t.Fatalf("Extract(reserved) = %+v, %v", reserved, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
marginStore := newStore(t, factory)
|
|
||||||
marginProxy := contractProxy("proxy-margin", "192.0.2.12", proxyDomain.StateAvailable)
|
|
||||||
if result, err := marginStore.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second, AllocationSafetyMargin: 5 * time.Second,
|
|
||||||
MaxSize: 10, Proxies: []proxyDomain.Proxy{marginProxy},
|
|
||||||
}); err != nil || result.Inserted != 1 {
|
|
||||||
t.Fatalf("UpsertFetched(safety margin) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
if result, err := marginStore.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-margin", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(26 * time.Second),
|
|
||||||
}); err != nil || result.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(after usableUntil) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runIdempotencyContract(t *testing.T, factory Factory) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
store := newStore(t, factory)
|
|
||||||
candidate := contractProxy("proxy-a", "192.0.2.10", proxyDomain.StateAvailable)
|
|
||||||
candidate.Tags = map[string]string{"region": "cn", "carrier": "ct"}
|
|
||||||
checkedAt := now
|
|
||||||
candidate.LastCheckedAt = &checkedAt
|
|
||||||
seedAvailable(t, store, "provider-a", now, time.Minute, candidate)
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-idem-first", ClientID: "client-a", SourceIP: "192.0.2.100",
|
|
||||||
IdempotencyKey: "idem-contract", IdempotencyTTL: time.Minute,
|
|
||||||
Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
MinRemainingTTL: time.Second, MaxHealthCheckAge: time.Minute,
|
|
||||||
Protocols: []string{"http", "http"}, Regions: []string{"cn", "cn"},
|
|
||||||
Carriers: []string{"ct", "ct"}, Upstreams: []string{"provider-a", "provider-a"},
|
|
||||||
}
|
|
||||||
first, err := store.Extract(context.Background(), command)
|
|
||||||
if err != nil || first.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(idempotent first) = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-idem-replay"
|
|
||||||
command.SourceIP = "198.51.100.200"
|
|
||||||
command.Now = now.Add(2 * time.Second)
|
|
||||||
command.IdempotencyTTL = 2 * time.Minute
|
|
||||||
command.MinRemainingTTL = 2 * time.Second
|
|
||||||
command.MaxHealthCheckAge = 2 * time.Minute
|
|
||||||
command.Protocols = []string{"http"}
|
|
||||||
command.Regions = []string{"cn"}
|
|
||||||
command.Carriers = []string{"ct"}
|
|
||||||
command.Upstreams = []string{"provider-a"}
|
|
||||||
replayed, err := store.Extract(context.Background(), command)
|
|
||||||
if err != nil || replayed.Returned != 1 || replayed.Items[0].ID != first.Items[0].ID ||
|
|
||||||
!replayed.ExtractedAt.Equal(first.ExtractedAt) {
|
|
||||||
t.Fatalf("Extract(idempotent replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
command.RequestID = "req-idem-conflict"
|
|
||||||
command.Requested = 2
|
|
||||||
if _, err := store.Extract(context.Background(), command); !errors.Is(err, extractionDomain.ErrIdempotencyConflict) {
|
|
||||||
t.Fatalf("Extract(idempotent conflict) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
zeroStore := newStore(t, factory)
|
|
||||||
zero := extractionDomain.Command{
|
|
||||||
RequestID: "req-zero", ClientID: "client-a", IdempotencyKey: "idem-zero",
|
|
||||||
Requested: 0, Fulfillment: extractionDomain.Partial, Now: now,
|
|
||||||
}
|
|
||||||
if result, err := zeroStore.Extract(context.Background(), zero); err != nil || result.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(zero first) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
zero.RequestID = "req-zero-replay"
|
|
||||||
zero.Now = now.Add(time.Second)
|
|
||||||
if result, err := zeroStore.Extract(context.Background(), zero); err != nil || result.Returned != 0 {
|
|
||||||
t.Fatalf("Extract(zero replay) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
zero.RequestID = "req-zero-conflict"
|
|
||||||
zero.Requested = 1
|
|
||||||
if _, err := zeroStore.Extract(context.Background(), zero); !errors.Is(err, extractionDomain.ErrIdempotencyConflict) {
|
|
||||||
t.Fatalf("Extract(zero conflict) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runIdempotencyExpiryContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
||||||
candidate := contractProxy("short-lived", "192.0.2.20", proxyDomain.StateAvailable)
|
|
||||||
upsertOne(t, store, "provider-a", now, 600*time.Millisecond, candidate)
|
|
||||||
command := extractionDomain.Command{
|
|
||||||
RequestID: "req-expiry-first", ClientID: "client-a", IdempotencyKey: "idem-expiry-contract",
|
|
||||||
IdempotencyTTL: time.Minute, Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Millisecond),
|
|
||||||
}
|
|
||||||
first, err := store.Extract(context.Background(), command)
|
|
||||||
if err != nil || first.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(short-lived first) = %+v, %v", first, err)
|
|
||||||
}
|
|
||||||
time.Sleep(750 * time.Millisecond)
|
|
||||||
replacementAt := time.Now().UTC().Truncate(time.Millisecond)
|
|
||||||
upsertOne(t, store, "provider-a", replacementAt, time.Minute, candidate)
|
|
||||||
command.RequestID = "req-expiry-second"
|
|
||||||
command.Now = replacementAt.Add(time.Millisecond)
|
|
||||||
again, err := store.Extract(context.Background(), command)
|
|
||||||
if err != nil || again.Returned != 1 || again.ExtractedAt.Equal(first.ExtractedAt) {
|
|
||||||
t.Fatalf("Extract(after idempotency expiry) = %+v, %v", again, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runOwnershipContract(t *testing.T, factory Factory) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
store := newStore(t, factory)
|
|
||||||
seedAvailable(t, store, "provider-a", now, 2*time.Minute,
|
|
||||||
contractProxy("proxy-a", "192.0.2.10", proxyDomain.StateAvailable))
|
|
||||||
assigned, err := store.Assign(context.Background(), now.Add(time.Second), "proxy-a", "worker-a", 20*time.Second)
|
|
||||||
if err != nil || assigned.Epoch == 0 || assigned.Version != 1 {
|
|
||||||
t.Fatalf("Assign() = %+v, %v", assigned, err)
|
|
||||||
}
|
|
||||||
if _, err := store.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-b", time.Minute); !errors.Is(err, ownershipDomain.ErrAlreadyOwned) {
|
|
||||||
t.Fatalf("Assign(already owned) error = %v", err)
|
|
||||||
}
|
|
||||||
renewed, err := store.Renew(context.Background(), now.Add(10*time.Second), "proxy-a", "worker-a", assigned.Epoch, 5*time.Minute)
|
|
||||||
if err != nil || renewed.Version != 2 || !renewed.ExpiresAt.Equal(now.Add(2*time.Minute)) {
|
|
||||||
t.Fatalf("Renew() = %+v, %v", renewed, err)
|
|
||||||
}
|
|
||||||
draining, err := store.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch)
|
|
||||||
if err != nil || !draining.Draining || draining.Version != 3 {
|
|
||||||
t.Fatalf("BeginDrain() = %+v, %v", draining, err)
|
|
||||||
}
|
|
||||||
if replayed, err := store.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch); err != nil || replayed != draining {
|
|
||||||
t.Fatalf("BeginDrain(replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
if err := store.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 1, 0); !errors.Is(err, ownershipDomain.ErrDrainNotReady) {
|
|
||||||
t.Fatalf("AcknowledgeDrain(active) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := store.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); err != nil {
|
|
||||||
t.Fatalf("AcknowledgeDrain(): %v", err)
|
|
||||||
}
|
|
||||||
if current, ok, err := store.Get(context.Background(), "proxy-a"); err != nil || ok {
|
|
||||||
t.Fatalf("Get(after ACK) = %+v, %t, %v", current, ok, err)
|
|
||||||
}
|
|
||||||
if result, err := store.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: "req-after-ack", ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(20 * time.Second),
|
|
||||||
}); err != nil || result.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(after ACK) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
takeoverStore := newStore(t, factory)
|
|
||||||
seedAvailable(t, takeoverStore, "provider-a", now, 2*time.Minute,
|
|
||||||
contractProxy("takeover", "192.0.2.11", proxyDomain.StateAvailable))
|
|
||||||
old, err := takeoverStore.Assign(context.Background(), now.Add(time.Second), "takeover", "worker-a", time.Second)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Assign(takeover old): %v", err)
|
|
||||||
}
|
|
||||||
newAssignment, err := takeoverStore.Assign(context.Background(), now.Add(3*time.Second), "takeover", "worker-b", time.Minute)
|
|
||||||
if err != nil || newAssignment.Epoch <= old.Epoch || newAssignment.WorkerID != "worker-b" {
|
|
||||||
t.Fatalf("Assign(takeover new) = %+v, %v; old=%+v", newAssignment, err, old)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runMaintenanceContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
for index := range 2 {
|
|
||||||
upsertOne(t, store, "provider-a", now, 5*time.Second,
|
|
||||||
contractProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10), proxyDomain.StateFetched))
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now.Add(time.Second), 2)
|
|
||||||
if removed, err := store.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 1 {
|
|
||||||
t.Fatalf("SweepExpired(first) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
if removed, err := store.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 1 {
|
|
||||||
t.Fatalf("SweepExpired(second) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
if removed, err := store.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 0 {
|
|
||||||
t.Fatalf("SweepExpired(empty) = %d, %v", removed, err)
|
|
||||||
}
|
|
||||||
assertInventory(t, store, "provider-a", now.Add(6*time.Second), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runConcurrencyContract(t *testing.T, factory Factory) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
extractStore := newStore(t, factory)
|
|
||||||
for iteration := range 100 {
|
|
||||||
proxyID := fmt.Sprintf("extract-race-%d", iteration)
|
|
||||||
seedAvailable(t, extractStore, "provider-a", now, 10*time.Minute,
|
|
||||||
contractProxy(proxyID, fmt.Sprintf("198.51.100.%d", iteration+1), proxyDomain.StateAvailable))
|
|
||||||
results := make(chan extractionDomain.Result, 2)
|
|
||||||
errorsCh := make(chan error, 2)
|
|
||||||
var workers sync.WaitGroup
|
|
||||||
for worker := range 2 {
|
|
||||||
workers.Add(1)
|
|
||||||
go func(worker int) {
|
|
||||||
defer workers.Done()
|
|
||||||
result, err := extractStore.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: fmt.Sprintf("extract-race-%d-%d", iteration, worker),
|
|
||||||
ClientID: fmt.Sprintf("client-%d", worker), Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
})
|
|
||||||
results <- result
|
|
||||||
errorsCh <- err
|
|
||||||
}(worker)
|
|
||||||
}
|
|
||||||
workers.Wait()
|
|
||||||
for range 2 {
|
|
||||||
if err := <-errorsCh; err != nil {
|
|
||||||
t.Fatalf("iteration %d Extract(): %v", iteration, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
returned := (<-results).Returned + (<-results).Returned
|
|
||||||
if returned != 1 {
|
|
||||||
t.Fatalf("iteration %d extracted = %d, want 1", iteration, returned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ownershipStore := newStore(t, factory)
|
|
||||||
for iteration := range 100 {
|
|
||||||
proxyID := fmt.Sprintf("ownership-race-%d", iteration)
|
|
||||||
seedAvailable(t, ownershipStore, "provider-a", now, 10*time.Minute,
|
|
||||||
contractProxy(proxyID, fmt.Sprintf("203.0.113.%d", iteration+1), proxyDomain.StateAvailable))
|
|
||||||
var workers sync.WaitGroup
|
|
||||||
assigned := make(chan bool, 1)
|
|
||||||
extracted := make(chan bool, 1)
|
|
||||||
errorsCh := make(chan error, 2)
|
|
||||||
workers.Add(2)
|
|
||||||
go func() {
|
|
||||||
defer workers.Done()
|
|
||||||
_, err := ownershipStore.Assign(context.Background(), now.Add(time.Second), proxyID, "worker-a", time.Minute)
|
|
||||||
if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) {
|
|
||||||
errorsCh <- err
|
|
||||||
}
|
|
||||||
assigned <- err == nil
|
|
||||||
}()
|
|
||||||
go func() {
|
|
||||||
defer workers.Done()
|
|
||||||
result, err := ownershipStore.Extract(context.Background(), extractionDomain.Command{
|
|
||||||
RequestID: fmt.Sprintf("ownership-race-%d", iteration), ClientID: "client-a", Requested: 1,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
errorsCh <- err
|
|
||||||
}
|
|
||||||
extracted <- err == nil && result.Returned == 1
|
|
||||||
}()
|
|
||||||
workers.Wait()
|
|
||||||
close(errorsCh)
|
|
||||||
for err := range errorsCh {
|
|
||||||
t.Fatalf("iteration %d ownership race: %v", iteration, err)
|
|
||||||
}
|
|
||||||
wins := 0
|
|
||||||
if <-assigned {
|
|
||||||
wins++
|
|
||||||
}
|
|
||||||
if <-extracted {
|
|
||||||
wins++
|
|
||||||
}
|
|
||||||
if wins != 1 {
|
|
||||||
t.Fatalf("iteration %d winners = %d, want 1", iteration, wins)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runCancellationContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
checks := []struct {
|
|
||||||
name string
|
|
||||||
call func() error
|
|
||||||
}{
|
|
||||||
{name: "upsert", call: func() error {
|
|
||||||
_, err := store.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1})
|
|
||||||
return err
|
|
||||||
}},
|
|
||||||
{name: "health", call: func() error {
|
|
||||||
_, err := store.ApplyHealth(ctx, activitypool.HealthUpdate{ProxyID: "proxy-a", CheckedAt: now, NextState: proxyDomain.StateChecking})
|
|
||||||
return err
|
|
||||||
}},
|
|
||||||
{name: "inventory", call: func() error { _, err := store.Inventory(ctx, "provider-a", now); return err }},
|
|
||||||
{name: "sweep", call: func() error { _, err := store.SweepExpired(ctx, now, 1); return err }},
|
|
||||||
{name: "extract", call: func() error {
|
|
||||||
_, err := store.Extract(ctx, extractionDomain.Command{Requested: 1, Fulfillment: extractionDomain.Partial, Now: now})
|
|
||||||
return err
|
|
||||||
}},
|
|
||||||
{name: "assign", call: func() error { _, err := store.Assign(ctx, now, "proxy-a", "worker-a", time.Minute); return err }},
|
|
||||||
{name: "renew", call: func() error { _, err := store.Renew(ctx, now, "proxy-a", "worker-a", 1, time.Minute); return err }},
|
|
||||||
{name: "begin drain", call: func() error { _, err := store.BeginDrain(ctx, "proxy-a", "worker-a", 1); return err }},
|
|
||||||
{name: "acknowledge drain", call: func() error { return store.AcknowledgeDrain(ctx, "proxy-a", "worker-a", 1, 0, 0) }},
|
|
||||||
{name: "get", call: func() error { _, _, err := store.Get(ctx, "proxy-a"); return err }},
|
|
||||||
{name: "expire", call: func() error { _, err := store.Expire(ctx, now, 1); return err }},
|
|
||||||
}
|
|
||||||
for _, check := range checks {
|
|
||||||
t.Run(check.name, func(t *testing.T) {
|
|
||||||
if err := check.call(); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newStore(t *testing.T, factory Factory) Store {
|
|
||||||
t.Helper()
|
|
||||||
store, cleanup := factory(t)
|
|
||||||
if store == nil {
|
|
||||||
t.Fatal("contract factory returned nil store")
|
|
||||||
}
|
|
||||||
if cleanup != nil {
|
|
||||||
t.Cleanup(cleanup)
|
|
||||||
}
|
|
||||||
return store
|
|
||||||
}
|
|
||||||
|
|
||||||
func upsertOne(t *testing.T, store Store, upstreamID string, now time.Time, ttl time.Duration, candidate proxyDomain.Proxy) {
|
|
||||||
t.Helper()
|
|
||||||
result, err := store.UpsertFetched(context.Background(), upstreamID, activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: ttl, MaxSize: 500, Proxies: []proxyDomain.Proxy{candidate},
|
|
||||||
})
|
|
||||||
if err != nil || result.Inserted != 1 {
|
|
||||||
t.Fatalf("UpsertFetched(%s) = %+v, %v", candidate.ID, result, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func seedAvailable(t *testing.T, store Store, upstreamID string, now time.Time, ttl time.Duration, candidate proxyDomain.Proxy) {
|
|
||||||
t.Helper()
|
|
||||||
candidate.State = proxyDomain.StateAvailable
|
|
||||||
upsertOne(t, store, upstreamID, now, ttl, candidate)
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertInventory(t *testing.T, store Store, upstreamID string, now time.Time, want int) {
|
|
||||||
t.Helper()
|
|
||||||
inventory, err := store.Inventory(context.Background(), upstreamID, now)
|
|
||||||
if err != nil || inventory.UpstreamID != upstreamID || inventory.Managed != want {
|
|
||||||
t.Fatalf("Inventory(%s) = %+v, %v; want %d", upstreamID, inventory, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func contractProxy(id, host string, state proxyDomain.State) proxyDomain.Proxy {
|
|
||||||
return proxyDomain.Proxy{
|
|
||||||
ID: id, Scheme: proxyDomain.SchemeHTTP, Host: host, Port: 8080, State: state,
|
|
||||||
Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func contractNow() time.Time {
|
|
||||||
return time.Now().UTC().Add(time.Hour).Truncate(time.Millisecond)
|
|
||||||
}
|
|
||||||
@ -16,14 +16,7 @@ import (
|
|||||||
|
|
||||||
const defaultIdempotencyTTL = 5 * time.Minute
|
const defaultIdempotencyTTL = 5 * time.Minute
|
||||||
|
|
||||||
var (
|
var ErrInvalidBatch = errors.New("invalid activity pool batch")
|
||||||
ErrInvalidBatch = errors.New("invalid activity pool batch")
|
|
||||||
ErrInvalidHealthUpdate = errors.New("invalid activity pool health update")
|
|
||||||
ErrActivityNotFound = errors.New("activity pool proxy not found")
|
|
||||||
ErrStaleHealthUpdate = errors.New("stale activity pool health update")
|
|
||||||
ErrInvalidInventory = errors.New("invalid activity pool inventory query")
|
|
||||||
ErrInvalidMaintenance = errors.New("invalid activity pool maintenance request")
|
|
||||||
)
|
|
||||||
|
|
||||||
// FetchedBatch describes one ephemeral provider response. Proxies without a
|
// FetchedBatch describes one ephemeral provider response. Proxies without a
|
||||||
// usable expiry are dropped because this pool is intentionally rebuildable.
|
// usable expiry are dropped because this pool is intentionally rebuildable.
|
||||||
@ -31,7 +24,6 @@ type FetchedBatch struct {
|
|||||||
ObservedAt time.Time
|
ObservedAt time.Time
|
||||||
ConfiguredTTL time.Duration
|
ConfiguredTTL time.Duration
|
||||||
AllocationSafetyMargin time.Duration
|
AllocationSafetyMargin time.Duration
|
||||||
MaxSize int
|
|
||||||
Proxies []proxyDomain.Proxy
|
Proxies []proxyDomain.Proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -48,30 +40,6 @@ type Upserter interface {
|
|||||||
UpsertFetched(context.Context, string, FetchedBatch) (UpsertResult, error)
|
UpsertFetched(context.Context, string, FetchedBatch) (UpsertResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type HealthUpdate struct {
|
|
||||||
ProxyID string
|
|
||||||
CheckedAt time.Time
|
|
||||||
NextState proxyDomain.State
|
|
||||||
Latency time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type Inventory struct {
|
|
||||||
UpstreamID string
|
|
||||||
Managed int
|
|
||||||
}
|
|
||||||
|
|
||||||
type HealthStore interface {
|
|
||||||
ApplyHealth(context.Context, HealthUpdate) (Entry, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type InventoryReader interface {
|
|
||||||
Inventory(context.Context, string, time.Time) (Inventory, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Maintainer interface {
|
|
||||||
SweepExpired(context.Context, time.Time, int) (int, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Entry struct {
|
type Entry struct {
|
||||||
Proxy proxyDomain.Proxy
|
Proxy proxyDomain.Proxy
|
||||||
UsableUntil time.Time
|
UsableUntil time.Time
|
||||||
@ -97,9 +65,6 @@ type idempotencyEntry struct {
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
_ Upserter = (*MemoryPool)(nil)
|
_ Upserter = (*MemoryPool)(nil)
|
||||||
_ HealthStore = (*MemoryPool)(nil)
|
|
||||||
_ InventoryReader = (*MemoryPool)(nil)
|
|
||||||
_ Maintainer = (*MemoryPool)(nil)
|
|
||||||
_ extractionDomain.Store = (*MemoryPool)(nil)
|
_ extractionDomain.Store = (*MemoryPool)(nil)
|
||||||
_ ownershipDomain.Repository = (*MemoryPool)(nil)
|
_ ownershipDomain.Repository = (*MemoryPool)(nil)
|
||||||
)
|
)
|
||||||
@ -118,7 +83,7 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if p == nil || upstreamID == "" || batch.ObservedAt.IsZero() || batch.ConfiguredTTL < 0 || batch.MaxSize <= 0 ||
|
if p == nil || upstreamID == "" || batch.ObservedAt.IsZero() || batch.ConfiguredTTL < 0 ||
|
||||||
batch.AllocationSafetyMargin < 0 ||
|
batch.AllocationSafetyMargin < 0 ||
|
||||||
(batch.ConfiguredTTL > 0 && batch.AllocationSafetyMargin >= batch.ConfiguredTTL) {
|
(batch.ConfiguredTTL > 0 && batch.AllocationSafetyMargin >= batch.ConfiguredTTL) {
|
||||||
return result, ErrInvalidBatch
|
return result, ErrInvalidBatch
|
||||||
@ -135,12 +100,6 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
p.purgeExpiredLocked(batch.ObservedAt)
|
p.purgeExpiredLocked(batch.ObservedAt)
|
||||||
managedByUpstream := make(map[string]int)
|
|
||||||
for _, entry := range p.entries {
|
|
||||||
if managedActivityState(entry.State) {
|
|
||||||
managedByUpstream[entry.Proxy.SourceUpstream]++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seenIDs := make(map[string]string, len(batch.Proxies))
|
seenIDs := make(map[string]string, len(batch.Proxies))
|
||||||
for _, candidate := range batch.Proxies {
|
for _, candidate := range batch.Proxies {
|
||||||
if !validProxyIdentity(candidate) {
|
if !validProxyIdentity(candidate) {
|
||||||
@ -206,10 +165,6 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if managedByUpstream[upstreamID] >= batch.MaxSize {
|
|
||||||
result.Dropped++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if candidate.ID == "" {
|
if candidate.ID == "" {
|
||||||
candidate.ID = stableProxyID(key)
|
candidate.ID = stableProxyID(key)
|
||||||
@ -220,119 +175,11 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
|
|||||||
State: candidate.State,
|
State: candidate.State,
|
||||||
}
|
}
|
||||||
p.keyByID[candidate.ID] = key
|
p.keyByID[candidate.ID] = key
|
||||||
if managedActivityState(candidate.State) {
|
|
||||||
managedByUpstream[upstreamID]++
|
|
||||||
}
|
|
||||||
result.Inserted++
|
result.Inserted++
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) ApplyHealth(ctx context.Context, update HealthUpdate) (Entry, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return Entry{}, ErrInvalidHealthUpdate
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Entry{}, err
|
|
||||||
}
|
|
||||||
if p == nil || update.ProxyID == "" || update.CheckedAt.IsZero() || update.NextState == "" || update.Latency < 0 {
|
|
||||||
return Entry{}, ErrInvalidHealthUpdate
|
|
||||||
}
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return Entry{}, err
|
|
||||||
}
|
|
||||||
p.purgeExpiredLocked(update.CheckedAt)
|
|
||||||
entry, ok := p.entryByIDLocked(update.ProxyID)
|
|
||||||
if !ok {
|
|
||||||
return Entry{}, ErrActivityNotFound
|
|
||||||
}
|
|
||||||
if entry.Proxy.LastCheckedAt != nil {
|
|
||||||
if update.CheckedAt.Before(*entry.Proxy.LastCheckedAt) {
|
|
||||||
return Entry{}, ErrStaleHealthUpdate
|
|
||||||
}
|
|
||||||
if update.CheckedAt.Equal(*entry.Proxy.LastCheckedAt) {
|
|
||||||
if entry.State != update.NextState {
|
|
||||||
return Entry{}, ErrStaleHealthUpdate
|
|
||||||
}
|
|
||||||
entry.Proxy = cloneProxy(entry.Proxy)
|
|
||||||
return entry, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if entry.State != update.NextState && !proxyDomain.CanTransition(entry.State, update.NextState) {
|
|
||||||
return Entry{}, ErrInvalidHealthUpdate
|
|
||||||
}
|
|
||||||
checkedAt := update.CheckedAt.UTC()
|
|
||||||
entry.State = update.NextState
|
|
||||||
entry.Proxy.State = update.NextState
|
|
||||||
entry.Proxy.LastCheckedAt = &checkedAt
|
|
||||||
entry.Proxy.Latency = update.Latency
|
|
||||||
if update.NextState == proxyDomain.StateAvailable {
|
|
||||||
lastSuccessAt := checkedAt
|
|
||||||
entry.Proxy.LastSuccessAt = &lastSuccessAt
|
|
||||||
}
|
|
||||||
p.setEntryByIDLocked(update.ProxyID, entry)
|
|
||||||
entry.Proxy = cloneProxy(entry.Proxy)
|
|
||||||
return entry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) Inventory(ctx context.Context, upstreamID string, now time.Time) (Inventory, error) {
|
|
||||||
result := Inventory{UpstreamID: upstreamID}
|
|
||||||
if ctx == nil {
|
|
||||||
return result, ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if p == nil || upstreamID == "" || now.IsZero() {
|
|
||||||
return result, ErrInvalidInventory
|
|
||||||
}
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
for _, entry := range p.entries {
|
|
||||||
if entry.Proxy.SourceUpstream == upstreamID && managedActivityState(entry.State) &&
|
|
||||||
entry.Proxy.ExpiresAt != nil && entry.Proxy.ExpiresAt.After(now) {
|
|
||||||
result.Managed++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return 0, ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if p == nil || now.IsZero() || limit <= 0 {
|
|
||||||
return 0, ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
expiredIDs := make([]string, 0)
|
|
||||||
for _, entry := range p.entries {
|
|
||||||
if entry.Proxy.ExpiresAt != nil && !entry.Proxy.ExpiresAt.After(now) {
|
|
||||||
expiredIDs = append(expiredIDs, entry.Proxy.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(expiredIDs)
|
|
||||||
if len(expiredIDs) > limit {
|
|
||||||
expiredIDs = expiredIDs[:limit]
|
|
||||||
}
|
|
||||||
for _, proxyID := range expiredIDs {
|
|
||||||
p.removeEntryByIDLocked(proxyID)
|
|
||||||
}
|
|
||||||
return len(expiredIDs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) Snapshot(now time.Time) []Entry {
|
func (p *MemoryPool) Snapshot(now time.Time) []Entry {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return nil
|
return nil
|
||||||
@ -387,7 +234,6 @@ func (p *MemoryPool) Extract(ctx context.Context, command extractionDomain.Comma
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if command.Requested == 0 {
|
if command.Requested == 0 {
|
||||||
p.rememberExtractionLocked(idempotencyKey, command, result)
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -422,18 +268,7 @@ func (p *MemoryPool) Extract(ctx context.Context, command extractionDomain.Comma
|
|||||||
if result.Returned > 0 {
|
if result.Returned > 0 {
|
||||||
result.ExtractedAt = command.Now.UTC()
|
result.ExtractedAt = command.Now.UTC()
|
||||||
}
|
}
|
||||||
p.rememberExtractionLocked(idempotencyKey, command, result)
|
if command.IdempotencyKey != "" {
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) rememberExtractionLocked(
|
|
||||||
idempotencyKey string,
|
|
||||||
command extractionDomain.Command,
|
|
||||||
result extractionDomain.Result,
|
|
||||||
) {
|
|
||||||
if command.IdempotencyKey == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
expiresAt := command.Now.Add(idempotencyTTL(command.IdempotencyTTL))
|
expiresAt := command.Now.Add(idempotencyTTL(command.IdempotencyTTL))
|
||||||
for _, item := range result.Items {
|
for _, item := range result.Items {
|
||||||
if !item.ExpiresAt.IsZero() && item.ExpiresAt.Before(expiresAt) {
|
if !item.ExpiresAt.IsZero() && item.ExpiresAt.Before(expiresAt) {
|
||||||
@ -446,19 +281,15 @@ func (p *MemoryPool) rememberExtractionLocked(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return result, nil
|
||||||
func (p *MemoryPool) Assign(ctx context.Context, now time.Time, proxyID, workerID string, ttl time.Duration) (ownershipDomain.Assignment, error) {
|
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *MemoryPool) Assign(now time.Time, proxyID, workerID string, ttl time.Duration) (ownershipDomain.Assignment, error) {
|
||||||
if p == nil || now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
if p == nil || now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
p.purgeExpiredLocked(now)
|
p.purgeExpiredLocked(now)
|
||||||
if current, exists := p.ownership[proxyID]; exists {
|
if current, exists := p.ownership[proxyID]; exists {
|
||||||
if current.ExpiresAt.After(now) {
|
if current.ExpiresAt.After(now) {
|
||||||
@ -485,18 +316,12 @@ func (p *MemoryPool) Assign(ctx context.Context, now time.Time, proxyID, workerI
|
|||||||
return assignment, nil
|
return assignment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) Renew(ctx context.Context, now time.Time, proxyID, workerID string, epoch uint64, ttl time.Duration) (ownershipDomain.Assignment, error) {
|
func (p *MemoryPool) Renew(now time.Time, proxyID, workerID string, epoch uint64, ttl time.Duration) (ownershipDomain.Assignment, error) {
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
if p == nil || now.IsZero() || ttl <= 0 {
|
if p == nil || now.IsZero() || ttl <= 0 {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
p.purgeExpiredLocked(now)
|
p.purgeExpiredLocked(now)
|
||||||
assignment, ok := p.ownership[proxyID]
|
assignment, ok := p.ownership[proxyID]
|
||||||
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch || !assignment.ExpiresAt.After(now) {
|
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch || !assignment.ExpiresAt.After(now) {
|
||||||
@ -512,18 +337,12 @@ func (p *MemoryPool) Renew(ctx context.Context, now time.Time, proxyID, workerID
|
|||||||
return assignment, nil
|
return assignment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) BeginDrain(ctx context.Context, proxyID, workerID string, epoch uint64) (ownershipDomain.Assignment, error) {
|
func (p *MemoryPool) BeginDrain(proxyID, workerID string, epoch uint64) (ownershipDomain.Assignment, error) {
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, err
|
|
||||||
}
|
|
||||||
assignment, ok := p.ownership[proxyID]
|
assignment, ok := p.ownership[proxyID]
|
||||||
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch {
|
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrStaleAssignment
|
||||||
@ -536,18 +355,12 @@ func (p *MemoryPool) BeginDrain(ctx context.Context, proxyID, workerID string, e
|
|||||||
return assignment, nil
|
return assignment, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) AcknowledgeDrain(ctx context.Context, proxyID, workerID string, epoch uint64, active, reserved int64) error {
|
func (p *MemoryPool) AcknowledgeDrain(proxyID, workerID string, epoch uint64, active, reserved int64) error {
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if p == nil || active < 0 || reserved < 0 {
|
if p == nil || active < 0 || reserved < 0 {
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
assignment, ok := p.ownership[proxyID]
|
assignment, ok := p.ownership[proxyID]
|
||||||
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch {
|
if !ok || assignment.WorkerID != workerID || assignment.Epoch != epoch {
|
||||||
return ownershipDomain.ErrStaleAssignment
|
return ownershipDomain.ErrStaleAssignment
|
||||||
@ -566,53 +379,28 @@ func (p *MemoryPool) AcknowledgeDrain(ctx context.Context, proxyID, workerID str
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) Get(ctx context.Context, proxyID string) (ownershipDomain.Assignment, bool, error) {
|
func (p *MemoryPool) Get(proxyID string) (ownershipDomain.Assignment, bool) {
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, false, err
|
|
||||||
}
|
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return ownershipDomain.Assignment{}, false, nil
|
return ownershipDomain.Assignment{}, false
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return ownershipDomain.Assignment{}, false, err
|
|
||||||
}
|
|
||||||
assignment, ok := p.ownership[proxyID]
|
assignment, ok := p.ownership[proxyID]
|
||||||
return assignment, ok, nil
|
return assignment, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) Expire(ctx context.Context, now time.Time, limit int) ([]ownershipDomain.Assignment, error) {
|
func (p *MemoryPool) Expire(now time.Time) []ownershipDomain.Assignment {
|
||||||
if err := ownershipContextError(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if limit <= 0 {
|
|
||||||
return nil, ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return nil, nil
|
return nil
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
expired := make([]ownershipDomain.Assignment, 0)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
eligibleIDs := make([]string, 0)
|
|
||||||
for proxyID, assignment := range p.ownership {
|
for proxyID, assignment := range p.ownership {
|
||||||
_, exists := p.entryByIDLocked(proxyID)
|
entry, exists := p.entryByIDLocked(proxyID)
|
||||||
if exists && assignment.ExpiresAt.After(now) {
|
if exists && assignment.ExpiresAt.After(now) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
eligibleIDs = append(eligibleIDs, proxyID)
|
|
||||||
}
|
|
||||||
sort.Strings(eligibleIDs)
|
|
||||||
if len(eligibleIDs) > limit {
|
|
||||||
eligibleIDs = eligibleIDs[:limit]
|
|
||||||
}
|
|
||||||
expired := make([]ownershipDomain.Assignment, 0, len(eligibleIDs))
|
|
||||||
for _, proxyID := range eligibleIDs {
|
|
||||||
assignment := p.ownership[proxyID]
|
|
||||||
entry, exists := p.entryByIDLocked(proxyID)
|
|
||||||
if exists && entry.OwnerWorkerID == assignment.WorkerID {
|
if exists && entry.OwnerWorkerID == assignment.WorkerID {
|
||||||
entry.OwnerWorkerID = ""
|
entry.OwnerWorkerID = ""
|
||||||
p.setEntryByIDLocked(proxyID, entry)
|
p.setEntryByIDLocked(proxyID, entry)
|
||||||
@ -620,14 +408,9 @@ func (p *MemoryPool) Expire(ctx context.Context, now time.Time, limit int) ([]ow
|
|||||||
expired = append(expired, assignment)
|
expired = append(expired, assignment)
|
||||||
delete(p.ownership, proxyID)
|
delete(p.ownership, proxyID)
|
||||||
}
|
}
|
||||||
return expired, nil
|
p.purgeExpiredLocked(now)
|
||||||
}
|
sort.Slice(expired, func(i, j int) bool { return expired[i].ProxyID < expired[j].ProxyID })
|
||||||
|
return expired
|
||||||
func ownershipContextError(ctx context.Context) error {
|
|
||||||
if ctx == nil {
|
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
|
||||||
}
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) purgeExpiredLocked(now time.Time) int {
|
func (p *MemoryPool) purgeExpiredLocked(now time.Time) int {
|
||||||
@ -636,7 +419,9 @@ func (p *MemoryPool) purgeExpiredLocked(now time.Time) int {
|
|||||||
if entry.Proxy.ExpiresAt == nil || entry.Proxy.ExpiresAt.After(now) {
|
if entry.Proxy.ExpiresAt == nil || entry.Proxy.ExpiresAt.After(now) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
p.removeEntryLocked(key, entry)
|
delete(p.ownership, entry.Proxy.ID)
|
||||||
|
delete(p.keyByID, entry.Proxy.ID)
|
||||||
|
delete(p.entries, key)
|
||||||
removed++
|
removed++
|
||||||
}
|
}
|
||||||
for key, entry := range p.idempotent {
|
for key, entry := range p.idempotent {
|
||||||
@ -647,25 +432,6 @@ func (p *MemoryPool) purgeExpiredLocked(now time.Time) int {
|
|||||||
return removed
|
return removed
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) removeEntryByIDLocked(proxyID string) {
|
|
||||||
key, ok := p.keyByID[proxyID]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
entry, ok := p.entries[key]
|
|
||||||
if !ok {
|
|
||||||
delete(p.keyByID, proxyID)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p.removeEntryLocked(key, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) removeEntryLocked(key string, entry Entry) {
|
|
||||||
delete(p.ownership, entry.Proxy.ID)
|
|
||||||
delete(p.keyByID, entry.Proxy.ID)
|
|
||||||
delete(p.entries, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) entryByIDLocked(proxyID string) (Entry, bool) {
|
func (p *MemoryPool) entryByIDLocked(proxyID string) (Entry, bool) {
|
||||||
key, ok := p.keyByID[proxyID]
|
key, ok := p.keyByID[proxyID]
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -712,7 +478,7 @@ func extractionCandidate(entry Entry) extractionDomain.Candidate {
|
|||||||
ID: entry.Proxy.ID, Protocol: string(entry.Proxy.Scheme), Host: entry.Proxy.Host,
|
ID: entry.Proxy.ID, Protocol: string(entry.Proxy.Scheme), Host: entry.Proxy.Host,
|
||||||
Port: entry.Proxy.Port, Username: entry.Proxy.Username, Region: entry.Proxy.Tags["region"],
|
Port: entry.Proxy.Port, Username: entry.Proxy.Username, Region: entry.Proxy.Tags["region"],
|
||||||
Carrier: entry.Proxy.Tags["carrier"], Upstream: entry.Proxy.SourceUpstream,
|
Carrier: entry.Proxy.Tags["carrier"], Upstream: entry.Proxy.SourceUpstream,
|
||||||
OwnerWorkerID: entry.OwnerWorkerID, State: extractionDomain.Extracted,
|
OwnerWorkerID: entry.OwnerWorkerID, State: extractionDomain.Available,
|
||||||
ExpiresAt: expiresAt, LastCheckedAt: checkedAt,
|
ExpiresAt: expiresAt, LastCheckedAt: checkedAt,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -734,16 +500,6 @@ func validProxyIdentity(candidate proxyDomain.Proxy) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func managedActivityState(state proxyDomain.State) bool {
|
|
||||||
switch state {
|
|
||||||
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable,
|
|
||||||
proxyDomain.StateSuspect, proxyDomain.StateDraining:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy {
|
func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy {
|
||||||
if candidate.ExpiresAt != nil {
|
if candidate.ExpiresAt != nil {
|
||||||
value := *candidate.ExpiresAt
|
value := *candidate.ExpiresAt
|
||||||
@ -791,19 +547,16 @@ func sameIdempotentRequest(left, right extractionDomain.Command) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func equalSet(left, right []string) bool {
|
func equalSet(left, right []string) bool {
|
||||||
leftSet := make(map[string]struct{}, len(left))
|
if len(left) != len(right) {
|
||||||
for _, value := range left {
|
|
||||||
leftSet[value] = struct{}{}
|
|
||||||
}
|
|
||||||
rightSet := make(map[string]struct{}, len(right))
|
|
||||||
for _, value := range right {
|
|
||||||
rightSet[value] = struct{}{}
|
|
||||||
}
|
|
||||||
if len(leftSet) != len(rightSet) {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for value := range leftSet {
|
counts := make(map[string]int, len(left))
|
||||||
if _, ok := rightSet[value]; !ok {
|
for _, value := range left {
|
||||||
|
counts[value]++
|
||||||
|
}
|
||||||
|
for _, value := range right {
|
||||||
|
counts[value]--
|
||||||
|
if counts[value] < 0 {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,7 +19,6 @@ func TestMemoryPoolUpsertAppliesProviderTTLAndRefreshesWithoutGrowth(t *testing.
|
|||||||
ObservedAt: now,
|
ObservedAt: now,
|
||||||
ConfiguredTTL: 30 * time.Second,
|
ConfiguredTTL: 30 * time.Second,
|
||||||
AllocationSafetyMargin: 3 * time.Second,
|
AllocationSafetyMargin: 3 * time.Second,
|
||||||
MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
Proxies: []proxyDomain.Proxy{{
|
||||||
Scheme: proxyDomain.SchemeHTTP,
|
Scheme: proxyDomain.SchemeHTTP,
|
||||||
Host: "192.0.2.10",
|
Host: "192.0.2.10",
|
||||||
@ -71,7 +70,7 @@ func TestMemoryPoolCrossProviderDuplicateDoesNotReplaceSourceLifecycle(t *testin
|
|||||||
State: proxyDomain.StateAvailable,
|
State: proxyDomain.StateAvailable,
|
||||||
}
|
}
|
||||||
first, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
first, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second, MaxSize: 100,
|
ObservedAt: now, ConfiguredTTL: 30 * time.Second,
|
||||||
AllocationSafetyMargin: 3 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
AllocationSafetyMargin: 3 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
||||||
})
|
})
|
||||||
if err != nil || first.Inserted != 1 {
|
if err != nil || first.Inserted != 1 {
|
||||||
@ -79,7 +78,7 @@ func TestMemoryPoolCrossProviderDuplicateDoesNotReplaceSourceLifecycle(t *testin
|
|||||||
}
|
}
|
||||||
|
|
||||||
duplicate, err := pool.UpsertFetched(context.Background(), "provider-b", FetchedBatch{
|
duplicate, err := pool.UpsertFetched(context.Background(), "provider-b", FetchedBatch{
|
||||||
ObservedAt: now.Add(time.Second), ConfiguredTTL: 5 * time.Minute, MaxSize: 100,
|
ObservedAt: now.Add(time.Second), ConfiguredTTL: 5 * time.Minute,
|
||||||
AllocationSafetyMargin: 10 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
AllocationSafetyMargin: 10 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
||||||
})
|
})
|
||||||
if err != nil || duplicate.Inserted != 0 || duplicate.Refreshed != 1 {
|
if err != nil || duplicate.Inserted != 0 || duplicate.Refreshed != 1 {
|
||||||
@ -100,7 +99,7 @@ func TestMemoryPoolCrossProviderDuplicateDoesNotReplaceSourceLifecycle(t *testin
|
|||||||
}
|
}
|
||||||
|
|
||||||
afterExpiry, err := pool.UpsertFetched(context.Background(), "provider-b", FetchedBatch{
|
afterExpiry, err := pool.UpsertFetched(context.Background(), "provider-b", FetchedBatch{
|
||||||
ObservedAt: now.Add(31 * time.Second), ConfiguredTTL: 5 * time.Minute, MaxSize: 100,
|
ObservedAt: now.Add(31 * time.Second), ConfiguredTTL: 5 * time.Minute,
|
||||||
AllocationSafetyMargin: 10 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
AllocationSafetyMargin: 10 * time.Second, Proxies: []proxyDomain.Proxy{proxy},
|
||||||
})
|
})
|
||||||
if err != nil || afterExpiry.Inserted != 1 {
|
if err != nil || afterExpiry.Inserted != 1 {
|
||||||
@ -117,7 +116,7 @@ func TestMemoryPoolRefreshPreservesRuntimeStateAndHealth(t *testing.T) {
|
|||||||
checkedAt := now.Add(-time.Second)
|
checkedAt := now.Add(-time.Second)
|
||||||
pool := NewMemoryPool()
|
pool := NewMemoryPool()
|
||||||
first, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
first, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
||||||
ObservedAt: now, ConfiguredTTL: 30 * time.Second, MaxSize: 100,
|
ObservedAt: now, ConfiguredTTL: 30 * time.Second,
|
||||||
Proxies: []proxyDomain.Proxy{{
|
Proxies: []proxyDomain.Proxy{{
|
||||||
Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
||||||
State: proxyDomain.StateAvailable, LastCheckedAt: &checkedAt,
|
State: proxyDomain.StateAvailable, LastCheckedAt: &checkedAt,
|
||||||
@ -129,7 +128,7 @@ func TestMemoryPoolRefreshPreservesRuntimeStateAndHealth(t *testing.T) {
|
|||||||
initial := pool.Snapshot(now)[0]
|
initial := pool.Snapshot(now)[0]
|
||||||
|
|
||||||
second, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
second, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
||||||
ObservedAt: now.Add(10 * time.Second), ConfiguredTTL: 30 * time.Second, MaxSize: 100,
|
ObservedAt: now.Add(10 * time.Second), ConfiguredTTL: 30 * time.Second,
|
||||||
Proxies: []proxyDomain.Proxy{{
|
Proxies: []proxyDomain.Proxy{{
|
||||||
Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
|
||||||
State: proxyDomain.StateFetched,
|
State: proxyDomain.StateFetched,
|
||||||
@ -155,7 +154,6 @@ func TestMemoryPoolDropsCandidatesWithoutUsableTTL(t *testing.T) {
|
|||||||
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
||||||
ObservedAt: now,
|
ObservedAt: now,
|
||||||
AllocationSafetyMargin: 3 * time.Second,
|
AllocationSafetyMargin: 3 * time.Second,
|
||||||
MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
Proxies: []proxyDomain.Proxy{
|
||||||
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8001},
|
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8001},
|
||||||
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8002, ExpiresAt: &expired},
|
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8002, ExpiresAt: &expired},
|
||||||
@ -176,7 +174,6 @@ func TestMemoryPoolRejectsInvalidBatchAtomically(t *testing.T) {
|
|||||||
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
result, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
||||||
ObservedAt: now,
|
ObservedAt: now,
|
||||||
ConfiguredTTL: time.Minute,
|
ConfiguredTTL: time.Minute,
|
||||||
MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
Proxies: []proxyDomain.Proxy{
|
||||||
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8001},
|
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8001},
|
||||||
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8002, SourceUpstream: "provider-b"},
|
{Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8002, SourceUpstream: "provider-b"},
|
||||||
@ -286,11 +283,11 @@ func TestMemoryPoolAssignRecoversExpiredLeaseWithoutSeparateSweep(t *testing.T)
|
|||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
||||||
pool := poolWithOneProxy(t, now)
|
pool := poolWithOneProxy(t, now)
|
||||||
proxyID := pool.Snapshot(now)[0].Proxy.ID
|
proxyID := pool.Snapshot(now)[0].Proxy.ID
|
||||||
first, err := pool.Assign(context.Background(), now, proxyID, "worker-a", 5*time.Second)
|
first, err := pool.Assign(now, proxyID, "worker-a", 5*time.Second)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(first): %v", err)
|
t.Fatalf("Assign(first): %v", err)
|
||||||
}
|
}
|
||||||
second, err := pool.Assign(context.Background(), now.Add(5*time.Second), proxyID, "worker-b", 5*time.Second)
|
second, err := pool.Assign(now.Add(5*time.Second), proxyID, "worker-b", 5*time.Second)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Assign(after lease expiry): %v", err)
|
t.Fatalf("Assign(after lease expiry): %v", err)
|
||||||
}
|
}
|
||||||
@ -308,7 +305,7 @@ func TestMemoryPoolMakesOwnershipAndExtractionMutuallyExclusive(t *testing.T) {
|
|||||||
extracted := make(chan bool, 1)
|
extracted := make(chan bool, 1)
|
||||||
go func() {
|
go func() {
|
||||||
<-start
|
<-start
|
||||||
_, err := pool.Assign(context.Background(), now, pool.Snapshot(now)[0].Proxy.ID, "worker-a", time.Minute)
|
_, err := pool.Assign(now, pool.Snapshot(now)[0].Proxy.ID, "worker-a", time.Minute)
|
||||||
if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) {
|
if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) {
|
||||||
t.Errorf("Assign(): %v", err)
|
t.Errorf("Assign(): %v", err)
|
||||||
}
|
}
|
||||||
@ -339,101 +336,6 @@ func TestMemoryPoolMakesOwnershipAndExtractionMutuallyExclusive(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryPoolOwnershipMethodsPropagateCanceledContext(t *testing.T) {
|
|
||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
|
||||||
pool := poolWithOneProxy(t, now)
|
|
||||||
proxyID := pool.Snapshot(now)[0].Proxy.ID
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
if _, err := pool.Assign(ctx, now, proxyID, "worker-a", time.Minute); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Assign() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if _, err := pool.Renew(ctx, now, proxyID, "worker-a", 1, time.Minute); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Renew() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if _, err := pool.BeginDrain(ctx, proxyID, "worker-a", 1); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("BeginDrain() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if err := pool.AcknowledgeDrain(ctx, proxyID, "worker-a", 1, 0, 0); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("AcknowledgeDrain() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if _, _, err := pool.Get(ctx, proxyID); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Get() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if _, err := pool.Expire(ctx, now, 1); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Expire() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolOwnershipMethodRechecksContextAfterLock(t *testing.T) {
|
|
||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
|
||||||
pool := poolWithOneProxy(t, now)
|
|
||||||
proxyID := pool.Snapshot(now)[0].Proxy.ID
|
|
||||||
base, cancel := context.WithCancel(context.Background())
|
|
||||||
ctx := &firstCheckContext{Context: base, checked: make(chan struct{})}
|
|
||||||
result := make(chan error, 1)
|
|
||||||
|
|
||||||
pool.mu.Lock()
|
|
||||||
go func() {
|
|
||||||
_, err := pool.Assign(ctx, now, proxyID, "worker-a", time.Minute)
|
|
||||||
result <- err
|
|
||||||
}()
|
|
||||||
<-ctx.checked
|
|
||||||
cancel()
|
|
||||||
pool.mu.Unlock()
|
|
||||||
|
|
||||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Assign() error = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
if _, ok, err := pool.Get(context.Background(), proxyID); err != nil || ok {
|
|
||||||
t.Fatal("canceled Assign() mutated ownership")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMemoryPoolExpireUsesStableProxyIDOrderAndLimit(t *testing.T) {
|
|
||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
|
||||||
pool := NewMemoryPool()
|
|
||||||
proxies := []proxyDomain.Proxy{
|
|
||||||
{ID: "proxy-c", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.3", Port: 8003, State: proxyDomain.StateAvailable},
|
|
||||||
{ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.1", Port: 8001, State: proxyDomain.StateAvailable},
|
|
||||||
{ID: "proxy-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.2", Port: 8002, State: proxyDomain.StateAvailable},
|
|
||||||
}
|
|
||||||
if _, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 100, Proxies: proxies,
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched(): %v", err)
|
|
||||||
}
|
|
||||||
for _, proxyID := range []string{"proxy-c", "proxy-a", "proxy-b"} {
|
|
||||||
if _, err := pool.Assign(context.Background(), now, proxyID, "worker-a", time.Minute); err != nil {
|
|
||||||
t.Fatalf("Assign(%s): %v", proxyID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expired, err := pool.Expire(context.Background(), now.Add(time.Minute), 2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Expire(): %v", err)
|
|
||||||
}
|
|
||||||
if len(expired) != 2 || expired[0].ProxyID != "proxy-a" || expired[1].ProxyID != "proxy-b" {
|
|
||||||
t.Fatalf("Expire() = %+v, want proxy-a then proxy-b", expired)
|
|
||||||
}
|
|
||||||
if _, ok, err := pool.Get(context.Background(), "proxy-c"); err != nil || !ok {
|
|
||||||
t.Fatalf("Get(proxy-c) = ok %v, error %v; want remaining assignment", ok, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type firstCheckContext struct {
|
|
||||||
context.Context
|
|
||||||
checked chan struct{}
|
|
||||||
once sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *firstCheckContext) Err() error {
|
|
||||||
err := c.Context.Err()
|
|
||||||
c.once.Do(func() { close(c.checked) })
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func poolWithOneProxy(t *testing.T, now time.Time) *MemoryPool {
|
func poolWithOneProxy(t *testing.T, now time.Time) *MemoryPool {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
pool := NewMemoryPool()
|
pool := NewMemoryPool()
|
||||||
@ -442,7 +344,6 @@ func poolWithOneProxy(t *testing.T, now time.Time) *MemoryPool {
|
|||||||
ObservedAt: now,
|
ObservedAt: now,
|
||||||
ConfiguredTTL: 30 * time.Second,
|
ConfiguredTTL: 30 * time.Second,
|
||||||
AllocationSafetyMargin: 3 * time.Second,
|
AllocationSafetyMargin: 3 * time.Second,
|
||||||
MaxSize: 100,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
Proxies: []proxyDomain.Proxy{{
|
||||||
Scheme: proxyDomain.SchemeHTTP,
|
Scheme: proxyDomain.SchemeHTTP,
|
||||||
Host: "192.0.2.10",
|
Host: "192.0.2.10",
|
||||||
|
|||||||
@ -24,7 +24,6 @@ var (
|
|||||||
ErrInsufficientProxies = errors.New("insufficient proxies")
|
ErrInsufficientProxies = errors.New("insufficient proxies")
|
||||||
ErrIdempotencyConflict = errors.New("idempotency key was reused with a different extraction request")
|
ErrIdempotencyConflict = errors.New("idempotency key was reused with a different extraction request")
|
||||||
ErrInvalidCommand = errors.New("invalid extraction command")
|
ErrInvalidCommand = errors.New("invalid extraction command")
|
||||||
ErrStoreUnavailable = errors.New("extraction store unavailable")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Candidate struct {
|
type Candidate struct {
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package ownership
|
package ownership
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@ -27,10 +26,10 @@ type Assignment struct {
|
|||||||
// Repository is the shared authority for ownership changes. Implementations
|
// Repository is the shared authority for ownership changes. Implementations
|
||||||
// that also support extraction must serialize both operations transactionally.
|
// that also support extraction must serialize both operations transactionally.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
Assign(context.Context, time.Time, string, string, time.Duration) (Assignment, error)
|
Assign(now time.Time, proxyID, workerID string, ttl time.Duration) (Assignment, error)
|
||||||
Renew(context.Context, time.Time, string, string, uint64, time.Duration) (Assignment, error)
|
Renew(now time.Time, proxyID, workerID string, epoch uint64, ttl time.Duration) (Assignment, error)
|
||||||
BeginDrain(context.Context, string, string, uint64) (Assignment, error)
|
BeginDrain(proxyID, workerID string, epoch uint64) (Assignment, error)
|
||||||
AcknowledgeDrain(context.Context, string, string, uint64, int64, int64) error
|
AcknowledgeDrain(proxyID, workerID string, epoch uint64, active, reserved int64) error
|
||||||
Get(context.Context, string) (Assignment, bool, error)
|
Get(proxyID string) (Assignment, bool)
|
||||||
Expire(context.Context, time.Time, int) ([]Assignment, error)
|
Expire(now time.Time) []Assignment
|
||||||
}
|
}
|
||||||
|
|||||||
14
progress.md
14
progress.md
@ -10,19 +10,12 @@
|
|||||||
`LoadResolved` 回归测试。
|
`LoadResolved` 回归测试。
|
||||||
- 已新增公用 `activitypool` 契约和并发安全内存参考实现；Provider 按各供应商
|
- 已新增公用 `activitypool` 契约和并发安全内存参考实现；Provider 按各供应商
|
||||||
TTL 与安全余量写入，独占提取、短期幂等和 Worker 所有权在同一原子边界内。
|
TTL 与安全余量写入，独占提取、短期幂等和 Worker 所有权在同一原子边界内。
|
||||||
- 已实现生产 Redis Activity Adapter：Upsert、健康更新、原子提取、所有权、
|
|
||||||
Drain/ACK、库存读取与有界过期清理均封装为窄领域端口和 Lua 原子操作。
|
|
||||||
- MemoryPool 与 Redis Adapter 运行同一套公用契约；真实 Redis 8.2 fixture 已覆盖
|
|
||||||
100 轮并发提取和 100 轮所有权竞争，本地 Compose 不持久化短效代理数据。
|
|
||||||
- Proxy 明细不写 PostgreSQL；PostgreSQL 仅保存配置版本、管理状态、Admin
|
- Proxy 明细不写 PostgreSQL；PostgreSQL 仅保存配置版本、管理状态、Admin
|
||||||
审计、管理 outbox 和可选聚合指标。
|
审计、管理 outbox 和可选聚合指标。
|
||||||
- `usableUntil` 已进入 Worker Snapshot 契约，Gateway 在供应商硬过期前按安全
|
- `usableUntil` 已进入 Worker Snapshot 契约，Gateway 在供应商硬过期前按安全
|
||||||
余量停止新分配。
|
余量停止新分配。
|
||||||
- PostgreSQL 管理面 Adapter、Provider Leader/分布式限流与心跳装配、生产命令
|
- 生产 Redis Adapter、PostgreSQL 管理面 Adapter、生产命令入口与代表性
|
||||||
入口、Redis 故障转移验证和代表性 100,000 QPS 集群压测仍待实现。
|
100,000 QPS 集群压测仍待实现。
|
||||||
- 本轮 `.\scripts\verify.ps1`、`.\scripts\test-redis.ps1`、Compose 静态展开和
|
|
||||||
Compose 非持久化策略测试通过；Windows `CGO_ENABLED=0`，race 继续由 Linux
|
|
||||||
CI 执行。
|
|
||||||
|
|
||||||
## 2026-07-28
|
## 2026-07-28
|
||||||
|
|
||||||
@ -43,7 +36,8 @@
|
|||||||
- Windows 当前 `CGO_ENABLED=0` 且无 C 编译器，race 测试由 Linux CI 承担。
|
- Windows 当前 `CGO_ENABLED=0` 且无 C 编译器，race 测试由 Linux CI 承担。
|
||||||
- 100,000 QPS 仍是未验证设计目标；运行进程、存储适配器、完整网络转发与
|
- 100,000 QPS 仍是未验证设计目标；运行进程、存储适配器、完整网络转发与
|
||||||
代表性集群压测尚未实施，已在完成审计中明确列出。
|
代表性集群压测尚未实施，已在完成审计中明确列出。
|
||||||
- 历史文档压缩包已由用户删除，当前交付以仓库内可追踪文档为准。
|
- 已生成 `proxy-pool-docs-v1.0.zip`，包含 50 个条目，SHA-256 为
|
||||||
|
`A6882B71196210CE3594A10992C2A0A73EA3A1CF31D8A570709CA999133CE104`。
|
||||||
- 已实现 random、roundRobin、weighted、leastConnections，并将 Sequential
|
- 已实现 random、roundRobin、weighted、leastConnections，并将 Sequential
|
||||||
拆成共享 Upstream 空结果状态与每 Routing 版本化游标。
|
拆成共享 Upstream 空结果状态与每 Routing 版本化游标。
|
||||||
- 已实现 Provider 合并通知、完整 attempt 超时、永久错误契约、指数退避、
|
- 已实现 Provider 合并通知、完整 attempt 超时、永久错误契约、指数退避、
|
||||||
|
|||||||
@ -1,33 +0,0 @@
|
|||||||
$ErrorActionPreference = "Stop"
|
|
||||||
|
|
||||||
$repositoryRoot = Split-Path -Parent $PSScriptRoot
|
|
||||||
$composeFile = Join-Path $repositoryRoot "deploy/docker-compose.test.yml"
|
|
||||||
$previousRedisURL = [Environment]::GetEnvironmentVariable("PROXY_POOL_TEST_REDIS_URL", "Process")
|
|
||||||
|
|
||||||
try {
|
|
||||||
docker compose -f $composeFile up -d --wait
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "starting Redis test fixture failed with exit code $LASTEXITCODE"
|
|
||||||
}
|
|
||||||
|
|
||||||
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
|
|
||||||
Push-Location $repositoryRoot
|
|
||||||
try {
|
|
||||||
go test -count=1 -tags=integration -timeout 60s ./internal/adapters/redisactivity/...
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "Redis integration tests failed with exit code $LASTEXITCODE"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
Pop-Location
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
if ($null -eq $previousRedisURL) {
|
|
||||||
Remove-Item Env:PROXY_POOL_TEST_REDIS_URL -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$env:PROXY_POOL_TEST_REDIS_URL = $previousRedisURL
|
|
||||||
}
|
|
||||||
docker compose -f $composeFile down --remove-orphans
|
|
||||||
}
|
|
||||||
@ -26,8 +26,6 @@
|
|||||||
8. [已完成] 按需求矩阵逐项审计并生成版本化文档包
|
8. [已完成] 按需求矩阵逐项审计并生成版本化文档包
|
||||||
9. [已完成] 将 Proxy 明细、独占提取、短期幂等和 Worker 所有权统一到
|
9. [已完成] 将 Proxy 明细、独占提取、短期幂等和 Worker 所有权统一到
|
||||||
TTL 活动池契约；PostgreSQL 退出代理数据路径
|
TTL 活动池契约；PostgreSQL 退出代理数据路径
|
||||||
10. [已完成] 实现生产 Redis Activity Adapter、原子 Lua、公用行为契约和
|
|
||||||
Redis 8.2 集成 fixture；本地 Redis 禁止短效代理数据持久化
|
|
||||||
|
|
||||||
## 串并行关系
|
## 串并行关系
|
||||||
|
|
||||||
@ -48,5 +46,5 @@
|
|||||||
|
|
||||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证；未启动
|
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证；未启动
|
||||||
目标运行拓扑。
|
目标运行拓扑。
|
||||||
- `cmd/proxy-*`、PostgreSQL 管理面 Adapter、Provider Leader/分布式限流、
|
- `cmd/proxy-*`、生产 Redis 活动池 Adapter、PostgreSQL 管理面 Adapter 与
|
||||||
Checker 运行时、Redis 故障转移验证与代表性集群压测属于后续实施范围。
|
Checker 运行时属于后续实施范围，见完成审计。
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user