Compare commits

..

7 Commits

Author SHA1 Message Date
youfak
e9945d933f docs: record provider coordination delivery
Some checks are pending
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run
2026-07-30 14:16:02 +08:00
youfak
5e430923e5 feat: add Redis provider coordination 2026-07-30 14:15:48 +08:00
youfak
f45def4191 docs: record controller metrics delivery 2026-07-30 12:31:04 +08:00
youfak
d4e77a64a9 feat: add controller metrics probes 2026-07-30 12:30:42 +08:00
youfak
78fba16115 docs: record controller bootstrap delivery 2026-07-30 11:38:57 +08:00
youfak
0e1aed7f29 feat: add controller startup bootstrap 2026-07-30 11:38:33 +08:00
youfak
6a3660d639 feat: add redis activity state inventory 2026-07-30 11:38:07 +08:00
92 changed files with 4015 additions and 137 deletions

View File

@ -48,3 +48,6 @@ jobs:
- name: PostgreSQL admin-state contract - name: PostgreSQL admin-state contract
shell: pwsh shell: pwsh
run: ./scripts/test-postgres.ps1 run: ./scripts/test-postgres.ps1
- name: Controller dual-store bootstrap
shell: pwsh
run: ./scripts/test-controller.ps1

1
.gitignore vendored
View File

@ -9,6 +9,7 @@ coverage/
*.out *.out
*.test *.test
*.prof *.prof
*.exe
.tmp-proto/ .tmp-proto/
# Local configuration and secrets # Local configuration and secrets

View File

@ -0,0 +1,63 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/bootstrap"
)
const configEnvironment = "PROXY_POOL_CONFIG"
type environmentLookup func(string) string
type controllerRun func(context.Context, bootstrap.Options) error
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
os.Exit(execute(ctx, os.Args[1:], os.Getenv, bootstrap.Run, os.Stderr))
}
func execute(
ctx context.Context,
args []string,
getenv environmentLookup,
run controllerRun,
stderr io.Writer,
) int {
flags := flag.NewFlagSet("proxy-controller", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", "", "configuration file path")
if err := flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if flags.NArg() != 0 {
_, _ = fmt.Fprintln(stderr, "proxy-controller: unexpected positional arguments")
return 2
}
if *configPath == "" && getenv != nil {
*configPath = getenv(configEnvironment)
}
if strings.TrimSpace(*configPath) != *configPath || *configPath == "" || ctx == nil || run == nil {
_, _ = fmt.Fprintf(stderr, "proxy-controller: -config or %s is required\n", configEnvironment)
return 2
}
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}
_, _ = fmt.Fprintf(stderr, "proxy-controller: %v\n", err)
return 1
}

View File

@ -0,0 +1,86 @@
package main
import (
"bytes"
"context"
"errors"
"testing"
"proxy-pool/internal/controller/bootstrap"
)
func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
t.Parallel()
var received bootstrap.Options
code := execute(context.Background(), []string{"-config", "flag.yaml"}, func(name string) string {
if name == configEnvironment {
return "environment.yaml"
}
return ""
}, func(_ context.Context, options bootstrap.Options) error {
received = options
return nil
}, &bytes.Buffer{})
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
t.Fatalf("execute() = %d, options = %+v", code, received)
}
}
func TestExecuteFallsBackToEnvironment(t *testing.T) {
t.Parallel()
var received bootstrap.Options
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
_ context.Context,
options bootstrap.Options,
) error {
received = options
return nil
}, &bytes.Buffer{})
if code != 0 || received.ConfigPath != "environment.yaml" {
t.Fatalf("execute() = %d, config = %q", code, received.ConfigPath)
}
}
func TestExecuteReturnsUsageCodeWithoutConfiguration(t *testing.T) {
t.Parallel()
called := false
var stderr bytes.Buffer
code := execute(context.Background(), nil, func(string) string { return "" }, func(
context.Context,
bootstrap.Options,
) error {
called = true
return nil
}, &stderr)
if code != 2 || called || stderr.Len() == 0 {
t.Fatalf("execute() = %d, called = %t, stderr = %q", code, called, stderr.String())
}
}
func TestExecuteMapsStartupFailureAndSignalCancellation(t *testing.T) {
t.Parallel()
var stderr bytes.Buffer
want := errors.New("startup failed")
code := execute(context.Background(), []string{"-config", "config.yaml"}, func(string) string { return "" }, func(
context.Context,
bootstrap.Options,
) error {
return want
}, &stderr)
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
stderr.Reset()
code = execute(ctx, []string{"-config", "config.yaml"}, func(string) string { return "" }, func(
context.Context,
bootstrap.Options,
) error {
return context.Canceled
}, &stderr)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("execute(canceled) = %d, stderr = %q", code, stderr.String())
}
}

View File

@ -5,6 +5,7 @@ security:
defaults: defaults:
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 5s timeout: 5s
maxAttempts: 3 maxAttempts: 3
@ -139,10 +140,15 @@ upstreams:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 20 maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 5m ttl: 5m
allocationSafetyMargin: 30s allocationSafetyMargin: 30s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 5s timeout: 5s
maxAttempts: 3 maxAttempts: 3
@ -192,10 +198,15 @@ upstreams:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 10 maxConcurrencyPerProxy: 10
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 2m ttl: 2m
allocationSafetyMargin: 20s allocationSafetyMargin: 20s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 2s requestInterval: 2s
timeout: 5s timeout: 5s
maxAttempts: 3 maxAttempts: 3

View File

@ -76,6 +76,26 @@ func TestPostgresFixtureScriptUsesDedicatedComposeProject(t *testing.T) {
} }
} }
func TestControllerFixtureScriptStartsBothStoresInDedicatedProject(t *testing.T) {
payload, err := os.ReadFile("../scripts/test-controller.ps1")
if err != nil {
t.Fatalf("read test-controller.ps1: %v", err)
}
script := string(payload)
for _, required := range []string{
`-p $composeProject`,
`up -d --wait --wait-timeout 60 postgres redis`,
`PROXY_POOL_TEST_POSTGRES_URL`,
`PROXY_POOL_TEST_REDIS_URL`,
`go test -count=1 -tags=integration -timeout 60s ./internal/controller/bootstrap`,
`down --volumes --remove-orphans`,
} {
if !strings.Contains(script, required) {
t.Errorf("test-controller.ps1 missing %q", required)
}
}
}
func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) { func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) {
document := loadComposeDocument(t) document := loadComposeDocument(t)
redis, ok := document.Services["redis"] redis, ok := document.Services["redis"]

View File

@ -111,10 +111,15 @@ upstreams:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 20 maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 5m ttl: 5m
allocationSafetyMargin: 20s allocationSafetyMargin: 20s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3
@ -156,10 +161,15 @@ upstreams:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 20 maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 5m ttl: 5m
allocationSafetyMargin: 20s allocationSafetyMargin: 20s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3

View File

@ -110,10 +110,15 @@ data:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 20 maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 5m ttl: 5m
allocationSafetyMargin: 20s allocationSafetyMargin: 20s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3
@ -152,10 +157,15 @@ data:
shrinkDelay: 30s shrinkDelay: 30s
capacity: capacity:
maxConcurrencyPerProxy: 20 maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle: lifecycle:
ttl: 5m ttl: 5m
allocationSafetyMargin: 20s allocationSafetyMargin: 20s
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3

View File

@ -10,8 +10,16 @@
go run ./deploy/tools/configcheck deploy/config/local.yaml go run ./deploy/tools/configcheck deploy/config/local.yaml
``` ```
规划中的生产入口为 `proxy-controller -config CONFIG_FILE`;该命令完成实现和 Controller 入口已实现,源码运行方式为:
进程级测试前,不作为当前可执行能力。
```powershell
go run ./cmd/proxy-controller -config CONFIG_FILE
```
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
所有时间值使用 Go duration例如 `500ms`、`30s`、`5m`。示例中的 所有时间值使用 Go duration例如 `500ms`、`30s`、`5m`。示例中的
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量 `${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
@ -28,6 +36,10 @@ go run ./deploy/tools/configcheck deploy/config/local.yaml
新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新 新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新
分配,已有连接进入 Drain不强制中断。 分配,已有连接进入 Drain不强制中断。
当前 Controller 启动时使用同一份不可变快照完成存储、入口和提取策略装配。
Admin 重载会原子提交新配置、审计并发布到配置 Store监听地址、存储连接和
已构造的安全/提取策略尚未自动重建,这些字段变更后需要重启 Controller。
## 2. 根结构 ## 2. 根结构
```yaml ```yaml
@ -237,8 +249,13 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000, shrinkDelay: 30s} pool: {maxSize: 1000, shrinkDelay: 30s}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill:
reconcileInterval: 1s
minimumAvailableSlots: 2000
targetAvailableSlots: 5000
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: fetch:
estimatedIPsPerCall: 100
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3
@ -303,6 +320,8 @@ proxyAuth:
### 7.3 Fetch 限制 ### 7.3 Fetch 限制
- `estimatedIPsPerCall`:冷启动时每次 Provider 调用预计返回的合法 Proxy 数,
同时用于 `pool.maxSize` 的 pending 预占;不得从任意 Query 或 Body 字段推断。
- `requestInterval`:同一 Provider 请求间隔。 - `requestInterval`:同一 Provider 请求间隔。
- `timeout`:单次调用超时。 - `timeout`:单次调用超时。
- `maxAttempts`:单次补池动作最大尝试次数。 - `maxAttempts`:单次补池动作最大尝试次数。
@ -314,7 +333,18 @@ proxyAuth:
大量缺池信号必须合并成 singleflight 或容量为 1 的通知,不能按 Gateway 请求 大量缺池信号必须合并成 singleflight 或容量为 1 的通知,不能按 Gateway 请求
数量线性触发 Provider API。 数量线性触发 Provider API。
### 7.4 生命周期与健康 ### 7.4 Refill 水位
- `reconcileInterval`:无事件时重新读取库存的兜底周期,启动时仍立即检查一次。
- `minimumAvailableSlots`:可用并发槽位低于该值时进入补池。
- `targetAvailableSlots`:进入补池后持续补到该目标,再退出补池状态。
三个字段均为必填正值,且 `targetAvailableSlots > minimumAvailableSlots`
目标不得超过 `pool.maxSize * capacity.maxConcurrencyPerProxy` 的理论上限。
`requestInterval` 只限制外部 API 调用不能兼任库存复核周期。pending Proxy
`estimatedIPsPerCall * maxConcurrencyPerProxy` 折算槽位,避免并发补池超量。
### 7.5 生命周期与健康
- 明确绝对过期时间优先于响应 TTL响应 TTL 优先于配置 `lifecycle.ttl` - 明确绝对过期时间优先于响应 TTL响应 TTL 优先于配置 `lifecycle.ttl`
- 距离过期不足 `allocationSafetyMargin` 时停止新分配。 - 距离过期不足 `allocationSafetyMargin` 时停止新分配。
@ -346,6 +376,13 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签 PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。 禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`
`/readyz``/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标Provider、提取和容量等
业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
服务流量门槛PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
变更需要重启 Controller。
## 9. 启动前校验清单 ## 9. 启动前校验清单
1. `version` 必须为 `1`,未知字段拒绝。 1. `version` 必须为 `1`,未知字段拒绝。
@ -353,8 +390,9 @@ PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Me
3. 非回环监听器满足认证或来源 CIDR 保护。 3. 非回环监听器满足认证或来源 CIDR 保护。
4. Routing 名称唯一,正则可编译,引用的 Upstream 存在。 4. Routing 名称唯一,正则可编译,引用的 Upstream 存在。
5. Sequential 至少引用两个 Upstream、阈值大于零`onUnavailable.action` 明确。 5. Sequential 至少引用两个 Upstream、阈值大于零`onUnavailable.action` 明确。
6. 启用的 Upstream 有正数 `pool.maxSize`、并发和 Fetch 限制。 6. 启用的 Upstream 有正数 `pool.maxSize`、并发、Refill 水位和 Fetch 估值/限制。
7. `allocationSafetyMargin < ttl` 7. `allocationSafetyMargin < ttl`
8. `fetch.maxTotal == 0``fetch.maxTotal >= pool.maxSize` 8. `fetch.maxTotal == 0``fetch.maxTotal >= pool.maxSize`
9. Distribution 的 fulfillment 合法,单次数量大于零。 9. `minimumAvailableSlots < targetAvailableSlots`,且目标不超过理论并发容量。
10. Secret 未写入日志可见配置转储。 10. Distribution 的 fulfillment 合法,单次数量大于零。
11. Secret 未写入日志可见配置转储。

View File

@ -226,7 +226,7 @@ flowchart TD
SF -->|yes| W[Coalesce signal] SF -->|yes| W[Coalesce signal]
SF -->|no| R[Read current demand] SF -->|no| R[Read current demand]
R --> L[Acquire Provider leader] R --> L[Acquire Provider leader]
L --> C{Under maxSize/maxTotal?} L --> C{Below refill target and under maxSize/maxTotal?}
C -->|no| X[Stop] C -->|no| X[Stop]
C -->|yes| I[Wait requestInterval] C -->|yes| I[Wait requestInterval]
I --> M[Acquire maxInFlight] I --> M[Acquire maxInFlight]
@ -238,6 +238,12 @@ flowchart TD
DD --> HC[Create FETCHED and schedule check] DD --> HC[Create FETCHED and schedule check]
``` ```
每个 Upstream 的 Leader、最小请求间隔和在途 Permit 由同一个 Redis 原子协调
模块维护。Leader 租约使用 generation + 单调 epoch fenceRedis 状态整体丢失后
生成新 generation 并重新竞选。任何续租不确定、记录损坏或 Redis 断连均
fail-closed不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
复核期间若仍有 pending Fetch则等待下一轮再同步 Managed避免重复计数。
### 8.1 Empty、Duplicate 与 Error ### 8.1 Empty、Duplicate 与 Error
- **Empty**HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。 - **Empty**HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。

View File

@ -13,7 +13,7 @@ proxy-pool/
│ ├── config/ # 严格配置解析和校验 │ ├── config/ # 严格配置解析和校验
│ ├── domain/ # 无传输、无存储依赖的领域模型 │ ├── domain/ # 无传输、无存储依赖的领域模型
│ ├── gateway/ # snapshot、dispatch、server、transport │ ├── gateway/ # snapshot、dispatch、server、transport
│ ├── controller/ # provider、pool、routing、extraction、health、runtime │ ├── controller/ # provider、pool、extraction、operations、runtime、bootstrap
│ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配 │ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配
│ └── platform/ # HTTP、安全、日志、指标、停机和进程装配 │ └── platform/ # HTTP、安全、日志、指标、停机和进程装配
├── api/ # OpenAPI 与 Protobuf 契约 ├── api/ # OpenAPI 与 Protobuf 契约

View File

@ -37,7 +37,7 @@ internal/
domain/extraction/{extraction.go,store.go} domain/extraction/{extraction.go,store.go}
domain/client/client.go domain/client/client.go
gateway/{server,dispatch,snapshot,transport}/ gateway/{server,dispatch,snapshot,transport}/
controller/{provider,pool,routing,extraction,health,distribution}/ controller/{provider,pool,routing,extraction,health,distribution,operations,runtime,bootstrap}/
adapters/{memory,postgres,redis,providerapi}/ adapters/{memory,postgres,redis,providerapi}/
platform/{logging,metrics,shutdown}/ platform/{logging,metrics,shutdown}/
api/{openapi,proto}/ api/{openapi,proto}/
@ -196,14 +196,31 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。 Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态 Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
聚合与配置重载接到同一公用 seam严格文件加载、脱敏管理摘要及原子配置发布 聚合与配置重载接到同一公用 seam严格文件加载、脱敏管理摘要及原子配置发布
已通过失败路径和并发测试。生产命令入口及其连接池/迁移启动装配仍待实现。 已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
Distribution/Admin 服务构造、错误合并和资源关闭Provider 调度及完整 HTTP
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture覆盖
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider 已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖 Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
`usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、 `usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、
重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis 重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis
Adapter 已通过真实 Redis 8.2 运行同一套公用契约;原子 Lua 覆盖提取、所有权和 Adapter 已通过真实 Redis 8.2 运行同一套公用契约;原子 Lua 覆盖提取、所有权和
有界清理。Redis Sentinel/故障转移验证与代表性多节点压测仍待实施。 有界清理。新增低基数 StateInventory Hash五类写脚本在同一原子边界维护状态
计数,读取不扫描 Proxy 明细;过期清理积压或负计数时 fail-closed。Redis
Sentinel/故障转移验证与代表性多节点压测仍待实施。
Provider 分布式协调已新增公用 `Coordinator.RunLeader` / `LeaderSession` seam 与
独立 `redisprovider` Adapter。真实 Redis 8.2 已验证同 Upstream 双实例互斥、
generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、TTL
回收及 Redis 状态丢失后的新 generation 自动重建Redis 异常期间不发放请求。
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`Pool Reconciler
已实现迟滞与 pending 槽位折算FetchBudget 仅在无 pending 时同步 Redis 权威
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成,
因此本轮不勾选 Task 10 的组合验收项。
## Task 11: Checker and Health Reducer ## Task 11: Checker and Health Reducer

View File

@ -18,9 +18,12 @@
## 2. 本地拓扑模板 ## 2. 本地拓扑模板
当前仓库交付设计、契约、部署拓扑和关键领域实现;`cmd/proxy-*` 的完整运行时 `cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
装配属于 `implementation-plan.md` 后续任务。此处 Compose/Kubernetes 资产用于 Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
评审网络、资源、探针和依赖关系,当前只执行静态渲染,不把模板写成可运行服务。 指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
后续任务。
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
视为完整可运行拓扑。
### 2.1 前置条件 ### 2.1 前置条件
@ -60,11 +63,17 @@ tmpfs。`ApplyMigrations` 在同一物理连接上执行仓库内嵌的幂等前
每个契约创建唯一 Schema结束时只删除该 Schema 和临时 Compose 项目。该脚本 每个契约创建唯一 Schema结束时只删除该 Schema 和临时 Compose 项目。该脚本
禁止指向开发或生产数据库。 禁止指向开发或生产数据库。
`.\scripts\test-controller.ps1` 同时启动两个隔离 fixture验证 Controller
bootstrap 的迁移、启动配置提交、Redis Readiness、Admin Status、`/readyz` 与
Prometheus 输出。脚本不启动
部署模板中的 Controller 容器,也不连接开发或生产存储。
目标拓扑入口: 目标拓扑入口:
- Gateway`127.0.0.1:8080` - Gateway`127.0.0.1:8080`
- Distribution`http://127.0.0.1:8081` - Distribution`http://127.0.0.1:8081`
- Admin`http://127.0.0.1:8082` - Admin`http://127.0.0.1:8082`
- Controller Metrics`http://127.0.0.1:9090`
- HAProxy 状态:`http://127.0.0.1:8404/stats` - HAProxy 状态:`http://127.0.0.1:8404/stats`
- Prometheus`http://127.0.0.1:9091` - Prometheus`http://127.0.0.1:9091`
- Grafana`http://127.0.0.1:3000` - Grafana`http://127.0.0.1:3000`
@ -105,6 +114,9 @@ kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=10m
的完整 Snapshot 且仍有准入能力时才 Ready。 的完整 Snapshot 且仍有准入能力时才 Ready。
- Controller 按能力判定就绪Distribution/Fetch 依赖 Redis 活动池Admin 持久化 - Controller 按能力判定就绪Distribution/Fetch 依赖 Redis 活动池Admin 持久化
写依赖 PostgreSQL 与兼容迁移。PostgreSQL 故障不得单独使 Extract 返回 503。 写依赖 PostgreSQL 与兼容迁移。PostgreSQL 故障不得单独使 Extract 返回 503。
- 同一 Controller 同时启用 Distribution 与 Admin 时Pod `/readyz` 以 Redis
活动池为服务流量门槛PostgreSQL 故障由 Admin 接口独立返回不可用,不把仍可
完成的 Extract 从 Service 摘除。Admin-only 进程才同时检查 PostgreSQL 与 Redis。
- Checker 在任务消费与结果上报通道可用时 Ready。 - Checker 在任务消费与结果上报通道可用时 Ready。
- `/metrics`独立于业务入口NetworkPolicy 仅允许监控命名空间访问。 - `/metrics`独立于业务入口NetworkPolicy 仅允许监控命名空间访问。

View File

@ -76,12 +76,14 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt
以下已有设计、接口或部署位置,但尚无端到端生产实现: 以下已有设计、接口或部署位置,但尚无端到端生产实现:
1. `cmd/proxy-gateway/controller/checker/loadgen` 进程装配。 1. `cmd/proxy-gateway/checker/loadgen` 进程装配;`proxy-controller` 已完成
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配,但 Provider 和业务
指标链未闭环。
2. Gateway 进程装配、生产连接池调优与代表性流量压测。 2. Gateway 进程装配、生产连接池调优与代表性流量压测。
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。 3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
4. PostgreSQL 连接池、迁移和 pgx Adapter 的生产命令启动装配,以及可选聚合指标 4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成
Schema、领域 seam、Memory/pgx Adapter、真实 PostgreSQL 18 契约和 Admin 公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成Controller
应用层接线已经完成 三监听器与探针集成已完成;可选聚合指标和完整容器进程部署验证仍待实现
5. Redis Provider Leader、分布式速率与 Client 限制、Worker 心跳和自动重建; 5. Redis Provider Leader、分布式速率与 Client 限制、Worker 心跳和自动重建;
TTL 活动池、原子提取和 Worker ownership 已完成。 TTL 活动池、原子提取和 Worker ownership 已完成。
6. Worker 网络快照流Redis ownership drain/ACK/过期回收已完成。 6. Worker 网络快照流Redis ownership drain/ACK/过期回收已完成。

View File

@ -7,9 +7,9 @@
| ID | 最终需求 | 来源 | 验证证据 | | ID | 最终需求 | 来源 | 验证证据 |
|---|---|---|---| |---|---|---|---|
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;四个 `cmd/proxy-*` 构建产物待实现 | | ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 命令已实现Gateway/Checker/Loadgen 构建产物待实现 |
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 | | ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Distribution/Admin 独立监听已测试Gateway/Metrics 生产入口待装配 | | ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机Gateway 生产入口待装配 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成分布式 Leader 与运行装配待完成 | | ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成分布式 Leader 与运行装配待完成 |
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 | | ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
@ -32,7 +32,7 @@
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` | | FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 | | FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
| FETCH-003 | 错误使用指数退避和抖动429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go``providerapi/http_adapter_test.go` | | FETCH-003 | 错误使用指数退避和抖动429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go``providerapi/http_adapter_test.go` |
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | 单进程 Reconciler 已完成Redis Leader 租约及多实例互斥测试待完成 | | FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试Provider Fleet/bootstrap 接线待完成 |
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 | | FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
| FETCH-006 | 重复候选不当作 Empty记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 | | FETCH-006 | 重复候选不当作 Empty记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 | | FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
@ -46,7 +46,7 @@
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 | | PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 | | PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 | | CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
| CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | TTL/状态/Active/Reserved 的 `AvailableSlots` 与 Reconciler 已测试ownership、目标健康及 Gateway reserve 聚合待完成 | | CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 | | CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 | | CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 | | CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
@ -86,5 +86,5 @@
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 | | OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
| OPS-002 | 优雅停机停止新请求/Fetch等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 | | OPS-002 | 优雅停机停止新请求/Fetch等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 | | OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | 文档和配置已约束Prometheus 指标模块及描述符测试待实现 | | OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现且当前只暴露无业务标签的 Go/进程指标;低基数业务 Collector 与描述符测试待实现 |
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 | | TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |

View File

@ -50,6 +50,12 @@ go build ./...
该 fixture 使用唯一命名空间,不执行 `FLUSHDB`,并关闭 AOF、RDB 与数据卷; 该 fixture 使用唯一命名空间,不执行 `FLUSHDB`,并关闭 AOF、RDB 与数据卷;
测试结束后按命名空间清理活动池、所有权和幂等键。 测试结束后按命名空间清理活动池、所有权和幂等键。
Controller 的 PostgreSQL + Redis 启动组合测试使用:
```powershell
.\scripts\test-controller.ps1
```
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock集成和 单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock集成和
soak 测试单独标记,不混入快速单测。 soak 测试单独标记,不混入快速单测。

View File

@ -108,6 +108,16 @@ Routing CAS、Repeatable Read 快照、审计分页、Routing no-op、`SKIP LOCK
数据卷。静态与 `information_schema` 双重检查证明只存在六张管理表,且没有 数据卷。静态与 `information_schema` 双重检查证明只存在六张管理表,且没有
Proxy、凭据、逐次提取、Worker ownership 或幂等明细列。 Proxy、凭据、逐次提取、Worker ownership 或幂等明细列。
Controller bootstrap 的双存储组合验证命令是:
```powershell
.\scripts\test-controller.ps1
```
该 fixture 同时启动 PostgreSQL 18 与 Redis 8.2,验证迁移、启动配置提交、
Redis Readiness 和 Admin StatusHTTP Runner 使用测试 Adapter避免占用业务
监听端口。测试数据仅存在于隔离 Compose 项目和 PostgreSQL tmpfs。
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、 Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、 权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态 持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态

View File

@ -48,6 +48,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 100} pool: {maxSize: 100}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s} lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -28,6 +28,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 2000} pool: {maxSize: 2000}
capacity: {maxConcurrencyPerProxy: 20} capacity: {maxConcurrencyPerProxy: 20}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -30,6 +30,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s} lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -32,6 +32,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 3000} pool: {maxSize: 3000}
capacity: {maxConcurrencyPerProxy: 20} capacity: {maxConcurrencyPerProxy: 20}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -33,6 +33,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 3m, allocationSafetyMargin: 30s} lifecycle: {ttl: 3m, allocationSafetyMargin: 30s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -40,6 +40,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -32,6 +32,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -22,8 +22,9 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
provider-b: provider-b:
enabled: true enabled: true
@ -33,6 +34,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 3m, allocationSafetyMargin: 15s} lifecycle: {ttl: 3m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -21,8 +21,9 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
provider-b: provider-b:
enabled: true enabled: true
@ -32,6 +33,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -19,8 +19,9 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
provider-b: provider-b:
enabled: true enabled: true
exposure: [gateway] exposure: [gateway]
@ -29,5 +30,6 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}

View File

@ -19,8 +19,9 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
provider-b: provider-b:
enabled: true enabled: true
exposure: [gateway] exposure: [gateway]
@ -29,5 +30,6 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}

View File

@ -23,6 +23,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 10000} pool: {maxSize: 10000}
capacity: {maxConcurrencyPerProxy: 50} capacity: {maxConcurrencyPerProxy: 50}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -29,6 +29,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000}
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -37,6 +37,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 2000} pool: {maxSize: 2000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s} lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -29,6 +29,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 5m, allocationSafetyMargin: 60s} lifecycle: {ttl: 5m, allocationSafetyMargin: 60s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -26,6 +26,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -27,6 +27,7 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 500} pool: {maxSize: 500}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -27,8 +27,10 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 10} capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s} lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
fetch: fetch:
estimatedIPsPerCall: 10
requestInterval: 2s requestInterval: 2s
timeout: 5s timeout: 5s
maxAttempts: 3 maxAttempts: 3

View File

@ -22,6 +22,7 @@ upstreams:
password: "${SOCKS_PASSWORD}" password: "${SOCKS_PASSWORD}"
pool: {maxSize: 1000} pool: {maxSize: 1000}
capacity: {maxConcurrencyPerProxy: 20} capacity: {maxConcurrencyPerProxy: 20}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 10m, allocationSafetyMargin: 60s} lifecycle: {ttl: 10m, allocationSafetyMargin: 60s}
fetch: {requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 10, requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3}

View File

@ -24,8 +24,10 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 100} pool: {maxSize: 100}
capacity: {maxConcurrencyPerProxy: 1} capacity: {maxConcurrencyPerProxy: 1}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
fetch: fetch:
estimatedIPsPerCall: 10
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 3 maxAttempts: 3

12
go.mod
View File

@ -4,17 +4,25 @@ go 1.26.0
require ( require (
github.com/jackc/pgx/v5 v5.6.0 github.com/jackc/pgx/v5 v5.6.0
github.com/prometheus/client_golang v1.24.1
github.com/redis/go-redis/v9 v9.19.0 github.com/redis/go-redis/v9 v9.19.0
go.yaml.in/yaml/v4 v4.0.0-rc.3 go.yaml.in/yaml/v4 v4.0.0-rc.3
) )
require ( require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
go.uber.org/atomic v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect
golang.org/x/crypto v0.17.0 // indirect golang.org/x/crypto v0.17.0 // indirect
golang.org/x/sync v0.1.0 // indirect golang.org/x/sync v0.22.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
) )

40
go.sum
View File

@ -1,3 +1,5 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@ -7,6 +9,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
@ -15,31 +19,51 @@ github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@ -10,6 +10,7 @@ import (
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction" extractionDomain "proxy-pool/internal/domain/extraction"
proxyDomain "proxy-pool/internal/domain/proxy" proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/platform/credentials" "proxy-pool/internal/platform/credentials"
@ -92,6 +93,7 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys, adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners, adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory, adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
adapter.keys.stateInventory,
} }
for _, key := range staticKeys { for _, key := range staticKeys {
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 { if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
@ -118,6 +120,38 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
} }
} }
func TestStateInventoryFieldsAreCollisionFreeAndStatusDoesNotScanRecords(t *testing.T) {
t.Parallel()
first := stateInventoryField("provider:a", "FETCHED")
second := stateInventoryField("provider", "a:FETCHED")
if first == second || first == "" || second == "" {
t.Fatalf("state inventory fields collide: %q and %q", first, second)
}
upper := strings.ToUpper(statusSource)
if strings.Contains(upper, "HGETALL") || strings.Contains(upper, "HSCAN") {
t.Fatal("status script scans Redis hashes")
}
}
func TestReadStateInventoryRejectsInvalidCalls(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
var adapter *Adapter
if _, err := adapter.ReadStateInventory(context.Background(), []string{"provider-a"}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
t.Fatalf("nil adapter error = %v", err)
}
if _, err := adapter.ReadStateInventory(nil, []string{"provider-a"}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
t.Fatalf("nil context error = %v", err)
}
adapter = &Adapter{}
if _, err := adapter.ReadStateInventory(context.Background(), []string{""}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
t.Fatalf("empty upstream error = %v", err)
}
if _, err := adapter.ReadStateInventory(context.Background(), []string{"provider-a"}, time.Time{}); !errors.Is(err, activitypool.ErrInvalidInventory) {
t.Fatalf("zero time error = %v", err)
}
}
func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) { func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
t.Parallel() t.Parallel()
record := proxyRecord{ record := proxyRecord{

View File

@ -86,7 +86,8 @@ func (a *Adapter) Extract(ctx context.Context, command extractionDomain.Command)
} }
keys := []string{ keys := []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available, a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, operationKey, idempotencyKey, a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
operationKey, idempotencyKey,
} }
keys = append(keys, a.extractionDriverKeys(digestInput)...) keys = append(keys, a.extractionDriverKeys(digestInput)...)
idempotencyTTL := command.IdempotencyTTL idempotencyTTL := command.IdempotencyTTL

View File

@ -25,7 +25,8 @@ func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpd
} }
result, err := runScript(ctx, a.client, healthScript, []string{ result, err := runScript(ctx, a.client, healthScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available, a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID), a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.operation(operationID),
}, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency), }, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency),
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID) a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID)
if err != nil { if err != nil {

View File

@ -19,6 +19,7 @@ type keyspace struct {
ownerExpiry string ownerExpiry string
epoch string epoch string
inventory string inventory string
stateInventory string
} }
func newKeyspace(namespace string) keyspace { func newKeyspace(namespace string) keyspace {
@ -34,9 +35,14 @@ func newKeyspace(namespace string) keyspace {
ownerExpiry: prefix + ":owner-expiry", ownerExpiry: prefix + ":owner-expiry",
epoch: prefix + ":epoch", epoch: prefix + ":epoch",
inventory: prefix + ":inventory", inventory: prefix + ":inventory",
stateInventory: prefix + ":state-inventory",
} }
} }
func stateInventoryField(upstreamID, state string) string {
return strconv.Itoa(len(upstreamID)) + ":" + upstreamID + ":" + state
}
func (keys keyspace) idempotency(clientID, idempotencyKey string) string { func (keys keyspace) idempotency(clientID, idempotencyKey string) string {
return keys.prefix + ":idem:" + digestParts(clientID, idempotencyKey) return keys.prefix + ":idem:" + digestParts(clientID, idempotencyKey)
} }

View File

@ -78,7 +78,8 @@ func (a *Adapter) runMaintenance(
} }
result, err := runScript(ctx, a.client, sweepScript, []string{ result, err := runScript(ctx, a.client, sweepScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available, a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID), a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.operation(operationID),
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL)) }, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL))
if err != nil { if err != nil {
return maintenanceScriptReply{}, err return maintenanceScriptReply{}, err

View File

@ -223,7 +223,8 @@ func (a *Adapter) runOwnership(
} }
result, err := runScript(ctx, a.client, ownershipScript, []string{ result, err := runScript(ctx, a.client, ownershipScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available, a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.epoch, operationKey, a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.epoch, operationKey,
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit, }, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
nowMS, proxyID, workerID, epoch, value, active, reserved) nowMS, proxyID, workerID, epoch, value, active, reserved)
if err != nil { if err != nil {

View File

@ -58,6 +58,22 @@ type maintenanceScriptReply struct {
Count int `json:"count"` Count int `json:"count"`
} }
type statusScriptReply struct {
Status scriptStatus `json:"status"`
Inventories []statusScriptInventory `json:"inventories"`
}
type statusScriptInventory struct {
UpstreamID string `json:"upstreamId"`
Fetched int64 `json:"fetched"`
Checking int64 `json:"checking"`
Available int64 `json:"available"`
Suspect int64 `json:"suspect"`
Draining int64 `json:"draining"`
Unhealthy int64 `json:"unhealthy"`
Extracted int64 `json:"extracted"`
}
//go:embed scripts/upsert.lua //go:embed scripts/upsert.lua
var upsertSource string var upsertSource string
@ -73,12 +89,16 @@ var ownershipSource string
//go:embed scripts/sweep.lua //go:embed scripts/sweep.lua
var sweepSource string var sweepSource string
//go:embed scripts/status.lua
var statusSource string
var ( var (
upsertScript = redis.NewScript(upsertSource) upsertScript = redis.NewScript(upsertSource)
healthScript = redis.NewScript(healthSource) healthScript = redis.NewScript(healthSource)
extractScript = redis.NewScript(extractSource) extractScript = redis.NewScript(extractSource)
ownershipScript = redis.NewScript(ownershipSource) ownershipScript = redis.NewScript(ownershipSource)
sweepScript = redis.NewScript(sweepSource) sweepScript = redis.NewScript(sweepSource)
statusScript = redis.NewScript(statusSource)
) )
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) { func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {

View File

@ -4,10 +4,11 @@ local idkeys_key = KEYS[3]
local expiry_key = KEYS[4] local expiry_key = KEYS[4]
local available_key = KEYS[5] local available_key = KEYS[5]
local inventory_key = KEYS[6] local inventory_key = KEYS[6]
local owners_key = KEYS[7] local state_inventory_key = KEYS[7]
local owner_expiry_key = KEYS[8] local owners_key = KEYS[8]
local operation_key = KEYS[9] local owner_expiry_key = KEYS[9]
local idempotency_key = KEYS[10] local operation_key = KEYS[10]
local idempotency_key = KEYS[11]
local now_ms = tonumber(ARGV[1]) local now_ms = tonumber(ARGV[1])
local requested = tonumber(ARGV[2]) local requested = tonumber(ARGV[2])
@ -73,6 +74,33 @@ local function decrement_inventory(upstream)
end end
end end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function increment_state(upstream, state)
if not is_counted(state) then
return
end
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
if value <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function remove_available(proxy_id, record) local function remove_available(proxy_id, record)
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
local index_keys = record and record.indexKeys local index_keys = record and record.indexKeys
@ -95,6 +123,9 @@ local function remove_proxy(proxy_id)
if decoded and type(record) == 'table' and is_managed(record.state) then if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
if decoded and type(record) == 'table' then
decrement_state(record.sourceUpstream, record.state)
end
else else
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
end end
@ -205,7 +236,7 @@ end
local driver_key = available_key local driver_key = available_key
local driver_size = redis.call('ZCARD', available_key) local driver_size = redis.call('ZCARD', available_key)
for index = 11, #KEYS do for index = 12, #KEYS do
local size = redis.call('ZCARD', KEYS[index]) local size = redis.call('ZCARD', KEYS[index])
if size < driver_size then if size < driver_size then
driver_key = KEYS[index] driver_key = KEYS[index]
@ -281,7 +312,9 @@ for index = 1, selected_count do
if is_managed(record.state) then if is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
decrement_state(record.sourceUpstream, record.state)
record.state = 'EXTRACTED' record.state = 'EXTRACTED'
increment_state(record.sourceUpstream, record.state)
local encoded = cjson.encode(record) local encoded = cjson.encode(record)
redis.call('HSET', records_key, selected.id, encoded) redis.call('HSET', records_key, selected.id, encoded)

View File

@ -4,9 +4,10 @@ local idkeys_key = KEYS[3]
local expiry_key = KEYS[4] local expiry_key = KEYS[4]
local available_key = KEYS[5] local available_key = KEYS[5]
local inventory_key = KEYS[6] local inventory_key = KEYS[6]
local owners_key = KEYS[7] local state_inventory_key = KEYS[7]
local owner_expiry_key = KEYS[8] local owners_key = KEYS[8]
local operation_key = KEYS[9] local owner_expiry_key = KEYS[9]
local operation_key = KEYS[10]
local checked_at_ms = tonumber(ARGV[1]) local checked_at_ms = tonumber(ARGV[1])
local next_state = ARGV[2] local next_state = ARGV[2]
@ -35,6 +36,33 @@ local function decrement_inventory(upstream)
end end
end end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function increment_state(upstream, state)
if not is_counted(state) then
return
end
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
if value <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function remove_available(id, record) local function remove_available(id, record)
redis.call('ZREM', available_key, id) redis.call('ZREM', available_key, id)
for _, index_key in ipairs(record and record.indexKeys or {}) do for _, index_key in ipairs(record and record.indexKeys or {}) do
@ -51,6 +79,7 @@ local function remove_proxy(id)
if is_managed(record.state) then if is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
decrement_state(record.sourceUpstream, record.state)
else else
redis.call('ZREM', available_key, id) redis.call('ZREM', available_key, id)
end end
@ -132,7 +161,8 @@ if record.state ~= next_state and not (transitions[record.state] and transitions
return finish({status = 'invalid'}) return finish({status = 'invalid'})
end end
local was_managed = is_managed(record.state) local previous_state = record.state
local was_managed = is_managed(previous_state)
local will_be_managed = is_managed(next_state) local will_be_managed = is_managed(next_state)
remove_available(proxy_id, record) remove_available(proxy_id, record)
record.state = next_state record.state = next_state
@ -146,6 +176,10 @@ if was_managed and not will_be_managed then
elseif not was_managed and will_be_managed then elseif not was_managed and will_be_managed then
redis.call('HINCRBY', inventory_key, record.sourceUpstream, 1) redis.call('HINCRBY', inventory_key, record.sourceUpstream, 1)
end end
if previous_state ~= next_state then
decrement_state(record.sourceUpstream, previous_state)
increment_state(record.sourceUpstream, next_state)
end
local encoded = cjson.encode(record) local encoded = cjson.encode(record)
redis.call('HSET', records_key, proxy_id, encoded) redis.call('HSET', records_key, proxy_id, encoded)
@ -163,6 +197,7 @@ touch(idkeys_key, tonumber(record.expiresAtMs))
touch(expiry_key, tonumber(record.expiresAtMs)) touch(expiry_key, tonumber(record.expiresAtMs))
touch(available_key, tonumber(record.expiresAtMs)) touch(available_key, tonumber(record.expiresAtMs))
touch(inventory_key, tonumber(record.expiresAtMs)) touch(inventory_key, tonumber(record.expiresAtMs))
touch(state_inventory_key, tonumber(record.expiresAtMs))
touch(owners_key, tonumber(record.expiresAtMs)) touch(owners_key, tonumber(record.expiresAtMs))
touch(owner_expiry_key, tonumber(record.expiresAtMs)) touch(owner_expiry_key, tonumber(record.expiresAtMs))

View File

@ -4,10 +4,11 @@ local idkeys_key = KEYS[3]
local expiry_key = KEYS[4] local expiry_key = KEYS[4]
local available_key = KEYS[5] local available_key = KEYS[5]
local inventory_key = KEYS[6] local inventory_key = KEYS[6]
local owners_key = KEYS[7] local state_inventory_key = KEYS[7]
local owner_expiry_key = KEYS[8] local owners_key = KEYS[8]
local epoch_key = KEYS[9] local owner_expiry_key = KEYS[9]
local operation_key = KEYS[10] local epoch_key = KEYS[10]
local operation_key = KEYS[11]
local operation = ARGV[1] local operation = ARGV[1]
local operation_ttl_ms = tonumber(ARGV[2]) local operation_ttl_ms = tonumber(ARGV[2])
@ -51,6 +52,26 @@ local function decrement_inventory(upstream)
end end
end end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
if count <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function touch(key, expires_at_ms) local function touch(key, expires_at_ms)
if redis.call('EXISTS', key) == 0 then if redis.call('EXISTS', key) == 0 then
return return
@ -100,6 +121,9 @@ local function remove_proxy(id)
if decoded and type(record) == 'table' and is_managed(record.state) then if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
if decoded and type(record) == 'table' then
decrement_state(record.sourceUpstream, record.state)
end
else else
redis.call('ZREM', available_key, id) redis.call('ZREM', available_key, id)
end end

View File

@ -0,0 +1,117 @@
local records_key = KEYS[1]
local unique_key = KEYS[2]
local idkeys_key = KEYS[3]
local expiry_key = KEYS[4]
local available_key = KEYS[5]
local inventory_key = KEYS[6]
local state_inventory_key = KEYS[7]
local owners_key = KEYS[8]
local owner_expiry_key = KEYS[9]
local now_ms = tonumber(ARGV[1])
local cleanup_limit = tonumber(ARGV[2])
local upstream_ids = cjson.decode(ARGV[3])
local function is_managed(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING'
end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function decrement_inventory(upstream)
if type(upstream) ~= 'string' or upstream == '' then
return
end
local count = redis.call('HINCRBY', inventory_key, upstream, -1)
if count < 0 then
redis.call('HSET', inventory_key, upstream, 0)
end
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
if count <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function remove_available(proxy_id, record)
redis.call('ZREM', available_key, proxy_id)
local index_keys = record and record.indexKeys
if type(index_keys) == 'table' then
for _, index_key in ipairs(index_keys) do
if type(index_key) == 'string' and index_key ~= '' then
redis.call('ZREM', index_key, proxy_id)
end
end
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
if raw then
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(proxy_id, decoded and record or nil)
if decoded and type(record) == 'table' then
if is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
decrement_state(record.sourceUpstream, record.state)
end
else
redis.call('ZREM', available_key, proxy_id)
end
local digest = redis.call('HGET', idkeys_key, proxy_id)
if digest and redis.call('HGET', unique_key, digest) == proxy_id then
redis.call('HDEL', unique_key, digest)
end
redis.call('HDEL', idkeys_key, proxy_id)
redis.call('HDEL', records_key, proxy_id)
redis.call('ZREM', expiry_key, proxy_id)
redis.call('HDEL', owners_key, proxy_id)
redis.call('ZREM', owner_expiry_key, proxy_id)
end
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', now_ms, 'LIMIT', 0, cleanup_limit)
for _, proxy_id in ipairs(expired) do
remove_proxy(proxy_id)
end
local oldest = redis.call('ZRANGE', expiry_key, 0, 0, 'WITHSCORES')
if #oldest == 2 and tonumber(oldest[2]) <= now_ms then
return cjson.encode({status = 'unavailable', inventories = cjson.decode('[]')})
end
local inventories = cjson.decode('[]')
local states = {'FETCHED', 'CHECKING', 'AVAILABLE', 'SUSPECT', 'DRAINING', 'UNHEALTHY', 'EXTRACTED'}
for _, upstream_id in ipairs(upstream_ids) do
if type(upstream_id) ~= 'string' or upstream_id == '' then
return cjson.encode({status = 'invalid', inventories = cjson.decode('[]')})
end
local counts = {}
for _, state in ipairs(states) do
local count = tonumber(redis.call('HGET', state_inventory_key, state_field(upstream_id, state)) or '0')
if count < 0 then
return cjson.encode({status = 'unavailable', inventories = cjson.decode('[]')})
end
counts[string.lower(state)] = count
end
counts.upstreamId = upstream_id
inventories[#inventories + 1] = counts
end
return cjson.encode({status = 'ok', inventories = inventories})

View File

@ -4,9 +4,10 @@ local idkeys_key = KEYS[3]
local expiry_key = KEYS[4] local expiry_key = KEYS[4]
local available_key = KEYS[5] local available_key = KEYS[5]
local inventory_key = KEYS[6] local inventory_key = KEYS[6]
local owners_key = KEYS[7] local state_inventory_key = KEYS[7]
local owner_expiry_key = KEYS[8] local owners_key = KEYS[8]
local operation_key = KEYS[9] local owner_expiry_key = KEYS[9]
local operation_key = KEYS[10]
local operation = ARGV[1] local operation = ARGV[1]
local now_ms = tonumber(ARGV[2]) local now_ms = tonumber(ARGV[2])
@ -40,6 +41,26 @@ local function decrement_inventory(upstream)
end end
end end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
if count <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function remove_available(proxy_id, record) local function remove_available(proxy_id, record)
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
local index_keys = record and record.indexKeys local index_keys = record and record.indexKeys
@ -62,6 +83,9 @@ local function remove_proxy(proxy_id)
if decoded and type(record) == 'table' and is_managed(record.state) then if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
if decoded and type(record) == 'table' then
decrement_state(record.sourceUpstream, record.state)
end
else else
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
end end

View File

@ -4,9 +4,10 @@ local idkeys_key = KEYS[3]
local expiry_key = KEYS[4] local expiry_key = KEYS[4]
local available_key = KEYS[5] local available_key = KEYS[5]
local inventory_key = KEYS[6] local inventory_key = KEYS[6]
local owners_key = KEYS[7] local state_inventory_key = KEYS[7]
local owner_expiry_key = KEYS[8] local owners_key = KEYS[8]
local operation_key = KEYS[9] local owner_expiry_key = KEYS[9]
local operation_key = KEYS[10]
local now_ms = tonumber(ARGV[1]) local now_ms = tonumber(ARGV[1])
local cleanup_limit = tonumber(ARGV[2]) local cleanup_limit = tonumber(ARGV[2])
@ -34,6 +35,33 @@ local function decrement_inventory(upstream)
end end
end end
local function state_field(upstream, state)
return string.len(upstream) .. ':' .. upstream .. ':' .. state
end
local function is_counted(state)
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
end
local function increment_state(upstream, state)
if not is_counted(state) then
return
end
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
end
local function decrement_state(upstream, state)
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
return
end
local field = state_field(upstream, state)
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
if value <= 0 then
redis.call('HDEL', state_inventory_key, field)
end
end
local function remove_available(proxy_id, record) local function remove_available(proxy_id, record)
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
local indexes = record and record.indexKeys or {} local indexes = record and record.indexKeys or {}
@ -51,6 +79,7 @@ local function remove_proxy(proxy_id)
if is_managed(record.state) then if is_managed(record.state) then
decrement_inventory(record.sourceUpstream) decrement_inventory(record.sourceUpstream)
end end
decrement_state(record.sourceUpstream, record.state)
else else
redis.call('ZREM', available_key, proxy_id) redis.call('ZREM', available_key, proxy_id)
end end
@ -166,6 +195,7 @@ for _, candidate in ipairs(candidates) do
if is_managed(incoming.state) then if is_managed(incoming.state) then
redis.call('HINCRBY', inventory_key, candidate.upstream, 1) redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
end end
increment_state(candidate.upstream, incoming.state)
add_available(candidate.proxyId, incoming) add_available(candidate.proxyId, incoming)
if tonumber(incoming.expiresAtMs) > max_expiry_ms then if tonumber(incoming.expiresAtMs) > max_expiry_ms then
max_expiry_ms = tonumber(incoming.expiresAtMs) max_expiry_ms = tonumber(incoming.expiresAtMs)
@ -182,6 +212,7 @@ if max_expiry_ms > 0 then
touch(expiry_key, max_expiry_ms) touch(expiry_key, max_expiry_ms)
touch(available_key, max_expiry_ms) touch(available_key, max_expiry_ms)
touch(inventory_key, max_expiry_ms) touch(inventory_key, max_expiry_ms)
touch(state_inventory_key, max_expiry_ms)
touch(owners_key, max_expiry_ms) touch(owners_key, max_expiry_ms)
touch(owner_expiry_key, max_expiry_ms) touch(owner_expiry_key, max_expiry_ms)
end end

View File

@ -0,0 +1,72 @@
package redisactivity
import (
"context"
"encoding/json"
"time"
"proxy-pool/internal/domain/activitypool"
)
var _ activitypool.StateInventoryReader = (*Adapter)(nil)
func (a *Adapter) ReadStateInventory(
ctx context.Context,
upstreamIDs []string,
now time.Time,
) ([]activitypool.StateInventory, error) {
if ctx == nil {
return nil, activitypool.ErrInvalidInventory
}
if err := ctx.Err(); err != nil {
return nil, err
}
if a == nil || now.IsZero() {
return nil, activitypool.ErrInvalidInventory
}
for _, upstreamID := range upstreamIDs {
if upstreamID == "" {
return nil, activitypool.ErrInvalidInventory
}
}
if len(upstreamIDs) == 0 {
return []activitypool.StateInventory{}, nil
}
payload, err := json.Marshal(upstreamIDs)
if err != nil {
return nil, err
}
result, err := runScript(ctx, a.client, statusScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
}, now.UnixMilli(), a.options.CleanupLimit, string(payload))
if err != nil {
return nil, err
}
var reply statusScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return nil, err
}
if reply.Status == scriptInvalid {
return nil, activitypool.ErrInvalidInventory
}
if reply.Status == scriptUnavailable {
return nil, invalidScriptReply("expired cleanup is backlogged")
}
if reply.Status != scriptOK || len(reply.Inventories) != len(upstreamIDs) {
return nil, invalidScriptReply("unexpected state inventory reply")
}
inventories := make([]activitypool.StateInventory, len(reply.Inventories))
for index, item := range reply.Inventories {
if item.UpstreamID != upstreamIDs[index] || item.Fetched < 0 || item.Checking < 0 ||
item.Available < 0 || item.Suspect < 0 || item.Draining < 0 || item.Unhealthy < 0 || item.Extracted < 0 {
return nil, invalidScriptReply("invalid state inventory counters")
}
inventories[index] = activitypool.StateInventory{
UpstreamID: item.UpstreamID, Fetched: item.Fetched, Checking: item.Checking,
Available: item.Available, Suspect: item.Suspect, Draining: item.Draining,
Unhealthy: item.Unhealthy, Extracted: item.Extracted,
}
}
return inventories, nil
}

View File

@ -0,0 +1,61 @@
//go:build integration
package redisactivity
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestReadStateInventoryFailsClosedWhileExpiredCleanupIsBacklogged(t *testing.T) {
fixture := newRedisTestFixture(t)
bounded, err := New(fixture.Client, Options{
Namespace: fixture.Namespace, Credentials: fixture.Credentials,
OperationTTL: time.Minute, MaxCandidateScan: 32, CleanupLimit: 1,
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC)
_, err = bounded.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: now, ConfiguredTTL: time.Second, MaxSize: 10,
Proxies: []proxyDomain.Proxy{
{ID: "expired-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080, State: proxyDomain.StateFetched},
{ID: "expired-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8080, State: proxyDomain.StateFetched},
},
})
if err != nil {
t.Fatalf("UpsertFetched() error = %v", err)
}
if _, err = bounded.ReadStateInventory(context.Background(), []string{"provider-a"}, now.Add(2*time.Second)); !errors.Is(err, extractionDomain.ErrStoreUnavailable) {
t.Fatalf("ReadStateInventory(backlog) error = %v", err)
}
inventories, err := bounded.ReadStateInventory(context.Background(), []string{"provider-a"}, now.Add(2*time.Second))
if err != nil || len(inventories) != 1 || inventories[0] != (activitypool.StateInventory{UpstreamID: "provider-a"}) {
t.Fatalf("ReadStateInventory(after cleanup) = %+v, %v", inventories, err)
}
}
func TestReadStateInventoryFailsClosedOnNegativeCounters(t *testing.T) {
fixture := newRedisTestFixture(t)
if err := fixture.Client.HSet(
context.Background(),
fixture.Adapter.keys.stateInventory,
stateInventoryField("provider-a", string(proxyDomain.StateAvailable)),
-1,
).Err(); err != nil {
t.Fatalf("seed invalid state counter: %v", err)
}
if _, err := fixture.Adapter.ReadStateInventory(
context.Background(), []string{"provider-a"}, time.Now().UTC(),
); !errors.Is(err, extractionDomain.ErrStoreUnavailable) {
t.Fatalf("ReadStateInventory(negative counter) error = %v", err)
}
}

View File

@ -175,7 +175,8 @@ func (a *Adapter) upsertChunk(
} }
result, err := runScript(ctx, a.client, upsertScript, []string{ result, err := runScript(ctx, a.client, upsertScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available, a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID), a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.operation(operationID),
}, observedAt.UnixMilli(), a.options.CleanupLimit, maxSize, operationTTLMillis(a.options.OperationTTL), string(payload)) }, observedAt.UnixMilli(), a.options.CleanupLimit, maxSize, operationTTLMillis(a.options.OperationTTL), string(payload))
if err != nil { if err != nil {
return upsertScriptReply{}, err return upsertScriptReply{}, err

View File

@ -0,0 +1,355 @@
package redisprovider
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"math"
"reflect"
"regexp"
"strings"
"sync"
"time"
"github.com/redis/go-redis/v9"
controllerProvider "proxy-pool/internal/controller/provider"
)
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
type Options struct {
Namespace string
HolderID string
LeaseTTL time.Duration
RenewEvery time.Duration
RetryInterval time.Duration
PermitGrace time.Duration
}
type Adapter struct {
client redis.Scripter
options Options
keys keyBuilder
}
var _ controllerProvider.Coordinator = (*Adapter)(nil)
func New(client redis.Scripter, options Options) (*Adapter, error) {
options.Namespace = strings.TrimSpace(options.Namespace)
options.HolderID = strings.TrimSpace(options.HolderID)
if nilInterface(client) || !namespacePattern.MatchString(options.Namespace) || options.HolderID == "" ||
options.LeaseTTL <= 0 || options.RenewEvery <= 0 || options.RenewEvery > options.LeaseTTL/3 ||
options.RetryInterval <= 0 || options.PermitGrace < 0 {
return nil, controllerProvider.ErrInvalidCoordination
}
return &Adapter{client: client, options: options, keys: keyBuilder{namespace: options.Namespace}}, nil
}
func (adapter *Adapter) RunLeader(
ctx context.Context,
upstreamID string,
limits controllerProvider.CoordinationLimits,
work func(context.Context, controllerProvider.LeaderSession) error,
) error {
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
return controllerProvider.ErrInvalidCoordination
}
if err := ctx.Err(); err != nil {
return err
}
keys, err := adapter.keys.forUpstream(upstreamID)
if err != nil {
return err
}
token, err := randomToken()
if err != nil {
return errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
}
for ctx.Err() == nil {
generationCandidate, tokenErr := randomToken()
if tokenErr != nil {
return errors.Join(controllerProvider.ErrCoordinationUnavailable, tokenErr)
}
reply, acquireErr := runScript(ctx, adapter.client, keys,
"acquire_leader", generationCandidate, adapter.options.HolderID, token,
durationMillis(adapter.options.LeaseTTL),
)
if acquireErr != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
return err
}
continue
}
switch reply.Status {
case "busy":
delay := adapter.options.RetryInterval
if reply.WaitMS > 0 && time.Duration(reply.WaitMS)*time.Millisecond < delay {
delay = time.Duration(reply.WaitMS) * time.Millisecond
}
if err := wait(ctx, delay); err != nil {
return err
}
continue
case "ok":
if reply.Generation == "" || reply.Epoch == 0 {
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
return err
}
continue
}
default:
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
return err
}
continue
}
session := &leaderSession{
adapter: adapter, keys: keys, upstreamID: upstreamID, limits: limits,
generation: reply.Generation, holderID: adapter.options.HolderID,
token: token, epoch: reply.Epoch,
}
lost, runErr := adapter.runLeaderTerm(ctx, session, work)
if runErr != nil {
return runErr
}
if !lost {
return nil
}
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
return err
}
}
return ctx.Err()
}
func (adapter *Adapter) runLeaderTerm(
ctx context.Context,
session *leaderSession,
work func(context.Context, controllerProvider.LeaderSession) error,
) (bool, error) {
leaderCtx, cancel := context.WithCancel(ctx)
defer cancel()
session.ctx = leaderCtx
workDone := make(chan error, 1)
go func() { workDone <- work(leaderCtx, session) }()
ticker := time.NewTicker(adapter.options.RenewEvery)
defer ticker.Stop()
deadline := time.NewTimer(adapter.options.LeaseTTL - adapter.options.RenewEvery)
defer deadline.Stop()
for {
select {
case <-ctx.Done():
cancel()
<-workDone
adapter.releaseLeader(session)
return false, ctx.Err()
case workErr := <-workDone:
cancel()
adapter.releaseLeader(session)
return false, leaderWorkResult(ctx, workErr)
case <-deadline.C:
cancel()
<-workDone
return true, nil
case <-ticker.C:
renewCtx, renewCancel := context.WithTimeout(leaderCtx, adapter.options.RenewEvery)
reply, err := runScript(renewCtx, adapter.client, session.keys,
"renew_leader", session.generation, session.holderID, session.token,
session.epoch, durationMillis(adapter.options.LeaseTTL),
)
renewCancel()
if err != nil || reply.Status != "ok" {
cancel()
<-workDone
return true, nil
}
resetTimer(deadline, adapter.options.LeaseTTL-adapter.options.RenewEvery)
}
}
}
func leaderWorkResult(ctx context.Context, workErr error) error {
if err := ctx.Err(); err != nil {
return err
}
if workErr != nil {
return workErr
}
return controllerProvider.ErrLeaderWorkStopped
}
func (adapter *Adapter) releaseLeader(session *leaderSession) {
ctx, cancel := context.WithTimeout(context.Background(), adapter.options.RenewEvery)
defer cancel()
_, _ = runScript(ctx, adapter.client, session.keys,
"release_leader", session.generation, session.holderID, session.token, session.epoch,
)
}
type leaderSession struct {
adapter *Adapter
keys upstreamKeys
upstreamID string
limits controllerProvider.CoordinationLimits
ctx context.Context
generation string
holderID string
token string
epoch uint64
}
var _ controllerProvider.LeaderSession = (*leaderSession)(nil)
func (session *leaderSession) Fence() controllerProvider.Fence {
if session == nil {
return controllerProvider.Fence{}
}
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
}
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
return nil, controllerProvider.ErrInvalidCoordination
}
operationCtx, cancel := context.WithCancel(ctx)
stop := context.AfterFunc(session.ctx, cancel)
defer func() {
stop()
cancel()
}()
permitToken, err := randomToken()
if err != nil {
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
}
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
for operationCtx.Err() == nil {
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
durationMillis(permitTTL),
)
if scriptErr != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
}
if operationCtx.Err() != nil {
return nil, operationCtx.Err()
}
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
return nil, err
}
continue
}
switch reply.Status {
case "ok":
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
case "stale":
return nil, controllerProvider.ErrLeadershipLost
case "rate_limited", "at_capacity":
delay := time.Duration(reply.WaitMS) * time.Millisecond
if delay <= 0 {
delay = session.adapter.options.RetryInterval
}
if err := wait(operationCtx, delay); err != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
}
return nil, err
}
default:
return nil, controllerProvider.ErrCoordinationUnavailable
}
}
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
}
return nil, operationCtx.Err()
}
type requestPermit struct {
adapter *Adapter
keys upstreamKeys
token string
mu sync.Mutex
done bool
}
func (permit *requestPermit) Release(ctx context.Context) error {
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
return controllerProvider.ErrInvalidCoordination
}
permit.mu.Lock()
defer permit.mu.Unlock()
if permit.done {
return nil
}
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
if err != nil {
return err
}
if reply.Status != "ok" {
return controllerProvider.ErrCoordinationUnavailable
}
permit.done = true
return nil
}
func randomToken() (string, error) {
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
return "", err
}
return hex.EncodeToString(token[:]), nil
}
func durationMillis(value time.Duration) int64 {
milliseconds := value / time.Millisecond
if value%time.Millisecond != 0 {
milliseconds++
}
return int64(milliseconds)
}
func wait(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-timer.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func resetTimer(timer *time.Timer, duration time.Duration) {
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timer.Reset(duration)
}
func nilInterface(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

View File

@ -0,0 +1,129 @@
package redisprovider
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/redis/go-redis/v9"
controllerProvider "proxy-pool/internal/controller/provider"
)
func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
t.Parallel()
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
t.Cleanup(func() { _ = client.Close() })
valid := Options{
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond, PermitGrace: time.Second,
}
var typedNil *redis.Client
tests := []struct {
name string
client redis.Scripter
options Options
}{
{name: "nil client", options: valid},
{name: "typed nil client", client: typedNil, options: valid},
{name: "empty namespace", client: client, options: withNamespace(valid, "")},
{name: "unsafe namespace", client: client, options: withNamespace(valid, "bad:value")},
{name: "empty holder", client: client, options: withHolder(valid, "")},
{name: "zero lease", client: client, options: withLeaseTTL(valid, 0)},
{name: "renew exceeds third", client: client, options: withRenewEvery(valid, 2*time.Second)},
{name: "zero retry", client: client, options: withRetryInterval(valid, 0)},
{name: "negative grace", client: client, options: withPermitGrace(valid, -1)},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
adapter, err := New(test.client, test.options)
if err == nil || adapter != nil {
t.Fatalf("New() = (%v, %v), want nil adapter and error", adapter, err)
}
})
}
}
func TestNewBuildsPerUpstreamClusterSafeKeys(t *testing.T) {
t.Parallel()
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
t.Cleanup(func() { _ = client.Close() })
adapter, err := New(client, Options{
Namespace: " controller ", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond, PermitGrace: time.Second,
})
if err != nil {
t.Fatalf("New(): %v", err)
}
keys, err := adapter.keys.forUpstream("provider:{unsafe}")
if err != nil {
t.Fatalf("forUpstream(): %v", err)
}
all := keys.all()
for _, key := range all {
if strings.Contains(key, "provider:{unsafe}") || strings.Count(key, "{") != 1 ||
strings.Count(key, "}") != 1 || !strings.Contains(key, "{provider:") {
t.Fatalf("unsafe provider coordination key %q", key)
}
}
if strings.Split(all[0], "}")[0] != strings.Split(all[len(all)-1], "}")[0] {
t.Fatalf("keys do not share one upstream hash tag: %v", all)
}
if _, err := adapter.keys.forUpstream(""); !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
t.Fatalf("forUpstream(empty) error = %v", err)
}
}
func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
t.Parallel()
var adapter *Adapter
limits := controllerProvider.CoordinationLimits{
RequestInterval: time.Second, MaxInFlight: 1, MaxAttemptDuration: time.Second,
}
work := func(context.Context, controllerProvider.LeaderSession) error { return nil }
if err := adapter.RunLeader(context.Background(), "provider-a", limits, work); !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
t.Fatalf("nil adapter error = %v", err)
}
}
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := leaderWorkResult(ctx, nil); !errors.Is(err, context.Canceled) {
t.Fatalf("leaderWorkResult() error = %v, want context cancellation", err)
}
}
func withNamespace(options Options, value string) Options {
options.Namespace = value
return options
}
func withHolder(options Options, value string) Options {
options.HolderID = value
return options
}
func withLeaseTTL(options Options, value time.Duration) Options {
options.LeaseTTL = value
return options
}
func withRenewEvery(options Options, value time.Duration) Options {
options.RenewEvery = value
return options
}
func withRetryInterval(options Options, value time.Duration) Options {
options.RetryInterval = value
return options
}
func withPermitGrace(options Options, value time.Duration) Options {
options.PermitGrace = value
return options
}

View File

@ -0,0 +1,297 @@
//go:build integration
package redisprovider
import (
"context"
"fmt"
"os"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
controllerProvider "proxy-pool/internal/controller/provider"
)
var integrationNamespaceSequence atomic.Uint64
func TestRedisCoordinatorElectsOneLeaderAndFencesFailover(t *testing.T) {
fixture := newRedisFixture(t)
first := fixture.coordinator(t, "controller-a")
second := fixture.coordinator(t, "controller-b")
limits := controllerProvider.CoordinationLimits{
RequestInterval: 50 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: 300 * time.Millisecond,
}
ctxA, cancelA := context.WithCancel(context.Background())
ctxB, cancelB := context.WithCancel(context.Background())
defer cancelA()
defer cancelB()
started := make(chan leadershipFixture, 4)
var active atomic.Int64
var maximum atomic.Int64
work := func(holder string) func(context.Context, controllerProvider.LeaderSession) error {
return func(ctx context.Context, session controllerProvider.LeaderSession) error {
current := active.Add(1)
for {
observed := maximum.Load()
if current <= observed || maximum.CompareAndSwap(observed, current) {
break
}
}
started <- leadershipFixture{holder: holder, fence: session.Fence()}
<-ctx.Done()
active.Add(-1)
return nil
}
}
doneA := make(chan error, 1)
doneB := make(chan error, 1)
go func() { doneA <- first.RunLeader(ctxA, "provider-a", limits, work("controller-a")) }()
go func() { doneB <- second.RunLeader(ctxB, "provider-a", limits, work("controller-b")) }()
initial := receiveLeadership(t, started)
time.Sleep(200 * time.Millisecond)
select {
case duplicate := <-started:
t.Fatalf("simultaneous leaders started: first=%+v duplicate=%+v", initial, duplicate)
default:
}
if got := maximum.Load(); got != 1 {
t.Fatalf("maximum simultaneous leaders = %d, want 1", got)
}
if initial.holder == "controller-a" {
cancelA()
} else {
cancelB()
}
replacement := receiveLeadership(t, started)
if replacement.holder == initial.holder {
t.Fatalf("replacement holder = %q, want the other controller", replacement.holder)
}
if replacement.fence.Generation != initial.fence.Generation || replacement.fence.Epoch <= initial.fence.Epoch {
t.Fatalf("replacement fence = %+v, initial = %+v", replacement.fence, initial.fence)
}
if got := maximum.Load(); got != 1 {
t.Fatalf("maximum simultaneous leaders after failover = %d, want 1", got)
}
cancelA()
cancelB()
waitRunner(t, doneA)
waitRunner(t, doneB)
}
func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T) {
fixture := newRedisFixture(t)
coordinator := fixture.coordinator(t, "controller-a")
limits := controllerProvider.CoordinationLimits{
RequestInterval: 250 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: time.Second,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessions := make(chan controllerProvider.LeaderSession, 1)
done := make(chan error, 1)
go func() {
done <- coordinator.RunLeader(ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
sessions <- session
<-workCtx.Done()
return nil
})
}()
session := receiveSession(t, sessions)
first, err := session.AcquireFetch(context.Background())
if err != nil {
t.Fatalf("first AcquireFetch(): %v", err)
}
startedAt := time.Now()
secondResult := make(chan permitResultFixture, 1)
go func() {
permit, acquireErr := session.AcquireFetch(context.Background())
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
}()
select {
case result := <-secondResult:
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
case <-time.After(100 * time.Millisecond):
}
if err := first.Release(context.Background()); err != nil {
t.Fatalf("first Release(): %v", err)
}
result := receivePermit(t, secondResult)
if result.err != nil || result.permit == nil {
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
}
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("second Release(): %v", err)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("idempotent second Release(): %v", err)
}
cancel()
waitRunner(t, done)
}
func TestRedisCoordinatorRebuildsWithNewGenerationAfterStateLoss(t *testing.T) {
fixture := newRedisFixture(t)
coordinator := fixture.coordinator(t, "controller-a")
limits := controllerProvider.CoordinationLimits{
RequestInterval: 50 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: time.Second,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
started := make(chan controllerProvider.Fence, 4)
done := make(chan error, 1)
go func() {
done <- coordinator.RunLeader(ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
started <- session.Fence()
<-workCtx.Done()
return nil
})
}()
first := receiveFence(t, started)
fixture.deleteKeys(t)
second := receiveFence(t, started)
if second.Generation == first.Generation {
t.Fatalf("generation after Redis state loss = %q, want a new generation", second.Generation)
}
cancel()
waitRunner(t, done)
}
type redisFixture struct {
client *redis.Client
namespace string
}
func newRedisFixture(t *testing.T) redisFixture {
t.Helper()
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
if redisURL == "" {
t.Skip("PROXY_POOL_TEST_REDIS_URL is not set")
}
options, err := redis.ParseURL(redisURL)
if err != nil {
t.Fatalf("parse PROXY_POOL_TEST_REDIS_URL: %v", err)
}
client := redis.NewClient(options)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
_ = client.Close()
t.Fatalf("ping Redis: %v", err)
}
namespace := fmt.Sprintf("provider-it-%d-%d-%d", os.Getpid(), time.Now().UnixNano(), integrationNamespaceSequence.Add(1))
t.Cleanup(func() {
redisFixture{client: client, namespace: namespace}.deleteKeys(t)
_ = client.Close()
})
return redisFixture{client: client, namespace: namespace}
}
func (fixture redisFixture) deleteKeys(t *testing.T) {
t.Helper()
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cleanupCancel()
var cursor uint64
for {
keys, next, err := fixture.client.Scan(cleanupCtx, cursor, "pp:"+fixture.namespace+":*", 128).Result()
if err != nil {
t.Fatalf("scan Redis provider keys: %v", err)
}
if len(keys) > 0 {
if err := fixture.client.Unlink(cleanupCtx, keys...).Err(); err != nil {
t.Fatalf("remove Redis provider keys: %v", err)
}
}
cursor = next
if cursor == 0 {
return
}
}
}
func (fixture redisFixture) coordinator(t *testing.T, holder string) *Adapter {
t.Helper()
adapter, err := New(fixture.client, Options{
Namespace: fixture.namespace, HolderID: holder, LeaseTTL: 600 * time.Millisecond,
RenewEvery: 150 * time.Millisecond, RetryInterval: 20 * time.Millisecond, PermitGrace: 100 * time.Millisecond,
})
if err != nil {
t.Fatalf("New(%s): %v", holder, err)
}
return adapter
}
func receiveLeadership(t *testing.T, values <-chan leadershipFixture) leadershipFixture {
t.Helper()
select {
case value := <-values:
return value
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for leadership")
return leadershipFixture{}
}
}
type leadershipFixture struct {
holder string
fence controllerProvider.Fence
}
func receiveSession(t *testing.T, values <-chan controllerProvider.LeaderSession) controllerProvider.LeaderSession {
t.Helper()
select {
case value := <-values:
return value
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for leader session")
return nil
}
}
func receiveFence(t *testing.T, values <-chan controllerProvider.Fence) controllerProvider.Fence {
t.Helper()
select {
case value := <-values:
return value
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for provider fence")
return controllerProvider.Fence{}
}
}
func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResultFixture {
t.Helper()
select {
case value := <-values:
return value
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for request permit")
return permitResultFixture{}
}
}
type permitResultFixture struct {
permit controllerProvider.RequestPermit
err error
}
func waitRunner(t *testing.T, done <-chan error) {
t.Helper()
select {
case err := <-done:
if err != nil && err != context.Canceled {
t.Fatalf("RunLeader() error = %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for RunLeader shutdown")
}
}

View File

@ -0,0 +1,51 @@
package redisprovider
import (
"crypto/sha256"
"encoding/hex"
"strconv"
"strings"
controllerProvider "proxy-pool/internal/controller/provider"
)
type keyBuilder struct {
namespace string
}
type upstreamKeys struct {
generation string
epoch string
leader string
next string
inflight string
}
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" {
return upstreamKeys{}, controllerProvider.ErrInvalidCoordination
}
digest := digestParts(upstreamID)
prefix := "pp:" + builder.namespace + ":{provider:" + digest + "}"
return upstreamKeys{
generation: prefix + ":generation",
epoch: prefix + ":epoch",
leader: prefix + ":leader",
next: prefix + ":next-request",
inflight: prefix + ":inflight",
}, nil
}
func (keys upstreamKeys) all() []string {
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
}
func digestParts(values ...string) string {
digest := sha256.New()
for _, value := range values {
_, _ = digest.Write([]byte(strconv.Itoa(len(value))))
_, _ = digest.Write([]byte{':'})
_, _ = digest.Write([]byte(value))
}
return hex.EncodeToString(digest.Sum(nil))
}

View File

@ -0,0 +1,61 @@
package redisprovider
import (
"bytes"
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"github.com/redis/go-redis/v9"
controllerProvider "proxy-pool/internal/controller/provider"
)
type scriptReply struct {
Status string `json:"status"`
Generation string `json:"generation,omitempty"`
Epoch uint64 `json:"epoch,string,omitempty"`
WaitMS int64 `json:"waitMs,omitempty"`
}
//go:embed scripts/provider.lua
var providerSource string
var providerScript = redis.NewScript(providerSource)
func runScript(ctx context.Context, client redis.Scripter, keys upstreamKeys, args ...any) (scriptReply, error) {
var reply scriptReply
result, err := providerScript.Run(ctx, client, keys.all(), args...).Result()
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return reply, err
}
return reply, errors.Join(controllerProvider.ErrCoordinationUnavailable,
fmt.Errorf("run Redis provider script: %w", err))
}
var payload []byte
switch value := result.(type) {
case string:
payload = []byte(value)
case []byte:
payload = value
default:
return reply, errors.Join(controllerProvider.ErrCoordinationUnavailable,
fmt.Errorf("decode Redis provider script: unexpected reply type %T", result))
}
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&reply); err != nil || reply.Status == "" {
return scriptReply{}, errors.Join(controllerProvider.ErrCoordinationUnavailable,
fmt.Errorf("decode Redis provider script reply: %w", err))
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return scriptReply{}, errors.Join(controllerProvider.ErrCoordinationUnavailable,
errors.New("decode Redis provider script reply: trailing value"))
}
return reply, nil
}

View File

@ -0,0 +1,158 @@
local operation = ARGV[1]
local function now_ms()
local value = redis.call('TIME')
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
end
local function reply(status, generation, epoch, wait_ms)
return cjson.encode({
status = status,
generation = generation,
epoch = tostring(epoch or '0'),
waitMs = wait_ms
})
end
local function read_leader()
local encoded = redis.call('GET', KEYS[3])
if not encoded then
return nil, nil
end
local ok, value = pcall(cjson.decode, encoded)
if not ok or type(value) ~= 'table' or type(value.generation) ~= 'string' or
type(value.holderId) ~= 'string' or type(value.token) ~= 'string' or
type(value.epoch) ~= 'string' then
return nil, 'invalid'
end
return value, nil
end
local function same_leader(value, generation, holder_id, token, epoch)
return value and value.generation == generation and value.holderId == holder_id and
value.token == token and value.epoch == tostring(epoch)
end
if operation == 'acquire_leader' then
local generation_candidate = ARGV[2]
local holder_id = ARGV[3]
local token = ARGV[4]
local lease_ttl = tonumber(ARGV[5])
if not lease_ttl or lease_ttl <= 0 then
return reply('invalid', '', 0, 0)
end
redis.call('SET', KEYS[1], generation_candidate, 'NX')
local generation = redis.call('GET', KEYS[1])
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, 0, 0)
end
if current then
if current.generation == generation and current.holderId == holder_id and current.token == token then
current.expiresAtMs = now_ms() + lease_ttl
redis.call('SET', KEYS[3], cjson.encode(current), 'PX', lease_ttl)
return reply('ok', generation, current.epoch, 0)
end
local remaining = redis.call('PTTL', KEYS[3])
return reply('busy', generation, 0, math.max(remaining, 1))
end
redis.call('INCR', KEYS[2])
local epoch = redis.call('GET', KEYS[2])
local leader = {
version = 1,
generation = generation,
holderId = holder_id,
token = token,
epoch = epoch,
expiresAtMs = now_ms() + lease_ttl
}
redis.call('SET', KEYS[3], cjson.encode(leader), 'PX', lease_ttl)
return reply('ok', generation, epoch, 0)
end
if operation == 'renew_leader' then
local generation = ARGV[2]
local holder_id = ARGV[3]
local token = ARGV[4]
local epoch = ARGV[5]
local lease_ttl = tonumber(ARGV[6])
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, epoch or 0, 0)
end
if not same_leader(current, generation, holder_id, token, epoch) then
return reply('stale', generation, epoch or 0, 0)
end
current.expiresAtMs = now_ms() + lease_ttl
redis.call('SET', KEYS[3], cjson.encode(current), 'PX', lease_ttl)
return reply('ok', generation, epoch, 0)
end
if operation == 'release_leader' then
local generation = ARGV[2]
local holder_id = ARGV[3]
local token = ARGV[4]
local epoch = ARGV[5]
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, epoch or 0, 0)
end
if same_leader(current, generation, holder_id, token, epoch) then
redis.call('DEL', KEYS[3])
end
return reply('ok', generation, epoch or 0, 0)
end
if operation == 'acquire_fetch' then
local generation = ARGV[2]
local holder_id = ARGV[3]
local leader_token = ARGV[4]
local epoch = ARGV[5]
local permit_token = ARGV[6]
local request_interval = tonumber(ARGV[7])
local max_in_flight = tonumber(ARGV[8])
local permit_ttl = tonumber(ARGV[9])
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, epoch or 0, 0)
end
if not same_leader(current, generation, holder_id, leader_token, epoch) then
return reply('stale', generation, epoch or 0, 0)
end
local now = now_ms()
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
if existing then
return reply('ok', generation, epoch, 0)
end
local next_request = redis.call('GET', KEYS[4])
if next_request and not tonumber(next_request) then
return reply('unavailable', generation, epoch, 0)
end
if next_request and tonumber(next_request) > now then
return reply('rate_limited', generation, epoch, tonumber(next_request) - now)
end
if redis.call('ZCARD', KEYS[5]) >= max_in_flight then
local earliest = redis.call('ZRANGE', KEYS[5], 0, 0, 'WITHSCORES')
local wait_ms = 1
if earliest[2] then
wait_ms = math.max(tonumber(earliest[2]) - now, 1)
end
return reply('at_capacity', generation, epoch, wait_ms)
end
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
if request_interval > 0 then
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
else
redis.call('DEL', KEYS[4])
end
return reply('ok', generation, epoch, 0)
end
if operation == 'release_fetch' then
redis.call('ZREM', KEYS[5], ARGV[2])
return reply('ok', '', 0, 0)
end
return reply('invalid', '', 0, 0)

View File

@ -163,6 +163,7 @@ type Upstream struct {
ProxyAuth ProxyAuth `yaml:"proxyAuth"` ProxyAuth ProxyAuth `yaml:"proxyAuth"`
Pool Pool `yaml:"pool"` Pool Pool `yaml:"pool"`
Capacity Capacity `yaml:"capacity"` Capacity Capacity `yaml:"capacity"`
Refill Refill `yaml:"refill"`
Lifecycle Lifecycle `yaml:"lifecycle"` Lifecycle Lifecycle `yaml:"lifecycle"`
Fetch Fetch `yaml:"fetch"` Fetch Fetch `yaml:"fetch"`
Check Check `yaml:"check"` Check Check `yaml:"check"`
@ -217,12 +218,19 @@ type Capacity struct {
MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"` MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"`
} }
type Refill struct {
ReconcileInterval Duration `yaml:"reconcileInterval"`
MinimumAvailableSlots int64 `yaml:"minimumAvailableSlots"`
TargetAvailableSlots int64 `yaml:"targetAvailableSlots"`
}
type Lifecycle struct { type Lifecycle struct {
TTL Duration `yaml:"ttl"` TTL Duration `yaml:"ttl"`
AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"` AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"`
} }
type Fetch struct { type Fetch struct {
EstimatedIPsPerCall int `yaml:"estimatedIPsPerCall"`
RequestInterval Duration `yaml:"requestInterval"` RequestInterval Duration `yaml:"requestInterval"`
Timeout Duration `yaml:"timeout"` Timeout Duration `yaml:"timeout"`
MaxAttempts int `yaml:"maxAttempts"` MaxAttempts int `yaml:"maxAttempts"`

View File

@ -61,10 +61,15 @@ upstreams:
maxSize: 100 maxSize: 100
capacity: capacity:
maxConcurrencyPerProxy: 10 maxConcurrencyPerProxy: 10
refill:
reconcileInterval: 1s
minimumAvailableSlots: 200
targetAvailableSlots: 500
lifecycle: lifecycle:
ttl: 120s ttl: 120s
allocationSafetyMargin: 10s allocationSafetyMargin: 10s
fetch: fetch:
estimatedIPsPerCall: 20
requestInterval: 1s requestInterval: 1s
timeout: 3s timeout: 3s
maxAttempts: 5 maxAttempts: 5
@ -86,7 +91,10 @@ func TestLoadStrictValidConfiguration(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Load(): %v", err) t.Fatalf("Load(): %v", err)
} }
if cfg.Version != 1 || cfg.Upstreams["provider-a"].Pool.MaxSize != 100 { provider := cfg.Upstreams["provider-a"]
if cfg.Version != 1 || provider.Pool.MaxSize != 100 || provider.Fetch.EstimatedIPsPerCall != 20 ||
provider.Refill.ReconcileInterval.Value() != time.Second || provider.Refill.MinimumAvailableSlots != 200 ||
provider.Refill.TargetAvailableSlots != 500 {
t.Fatalf("unexpected config: %+v", cfg) t.Fatalf("unexpected config: %+v", cfg)
} }
} }
@ -318,6 +326,31 @@ func TestValidateAcceptsBearerListenerAuthentication(t *testing.T) {
} }
} }
func TestValidateMetricsListener(t *testing.T) {
t.Parallel()
cfg := mustLoadValidConfig(t)
cfg.Metrics = Metrics{Enabled: true}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
t.Fatalf("Validate(metrics without listen) error = %v", err)
}
cfg.Metrics.Listen = "not-an-address"
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
t.Fatalf("Validate(invalid metrics listen) error = %v", err)
}
cfg.Metrics.Listen = "127.0.0.1:70000"
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
t.Fatalf("Validate(out-of-range metrics port) error = %v", err)
}
cfg.Metrics.Listen = "0.0.0.0:9090"
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(public metrics listener) error = %v", err)
}
cfg.Metrics = Metrics{Enabled: false, Listen: "not-an-address"}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(disabled metrics listener) error = %v", err)
}
}
func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) { func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@ -504,6 +537,47 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
}, },
want: "fetch.maxInFlight", want: "fetch.maxInFlight",
}, },
{
name: "zero estimated IPs per call",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) { upstream.Fetch.EstimatedIPsPerCall = 0 })
},
want: "estimatedIPsPerCall",
},
{
name: "estimated IPs exceed pool size",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) {
upstream.Fetch.EstimatedIPsPerCall = upstream.Pool.MaxSize + 1
})
},
want: "estimatedIPsPerCall",
},
{
name: "zero refill interval",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) { upstream.Refill.ReconcileInterval = 0 })
},
want: "refill.reconcileInterval",
},
{
name: "refill target does not exceed minimum",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) {
upstream.Refill.TargetAvailableSlots = upstream.Refill.MinimumAvailableSlots
})
},
want: "targetAvailableSlots",
},
{
name: "refill target exceeds theoretical capacity",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) {
upstream.Refill.TargetAvailableSlots = int64(upstream.Pool.MaxSize)*int64(upstream.Capacity.MaxConcurrencyPerProxy) + 1
})
},
want: "targetAvailableSlots",
},
{ {
name: "negative fetch response limit", name: "negative fetch response limit",
mutate: func(cfg *Config) { mutate: func(cfg *Config) {

View File

@ -4,6 +4,7 @@ import (
"errors" "errors"
"sync" "sync"
"testing" "testing"
"time"
) )
func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) { func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
@ -77,8 +78,13 @@ func storeTestConfig(upstreamName string) *Config {
API: ProviderAPI{Auth: ProviderAuth{Type: "none"}}, API: ProviderAPI{Auth: ProviderAuth{Type: "none"}},
ProxyAuth: ProxyAuth{Type: "response"}, ProxyAuth: ProxyAuth{Type: "response"},
Pool: Pool{MaxSize: 10}, Capacity: Capacity{MaxConcurrencyPerProxy: 1}, Pool: Pool{MaxSize: 10}, Capacity: Capacity{MaxConcurrencyPerProxy: 1},
Refill: Refill{
ReconcileInterval: Duration(time.Second), MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
},
Lifecycle: Lifecycle{TTL: Duration(60_000_000_000), AllocationSafetyMargin: Duration(10_000_000_000)}, Lifecycle: Lifecycle{TTL: Duration(60_000_000_000), AllocationSafetyMargin: Duration(10_000_000_000)},
Fetch: Fetch{Timeout: Duration(1_000_000_000), MaxAttempts: 1, MaxInFlight: 1}, Fetch: Fetch{
EstimatedIPsPerCall: 1, Timeout: Duration(time.Second), MaxAttempts: 1, MaxInFlight: 1,
},
}, },
}, },
Routing: []Routing{{ Routing: []Routing{{

View File

@ -2,9 +2,11 @@ package config
import ( import (
"fmt" "fmt"
"math"
"net" "net"
"net/url" "net/url"
"regexp" "regexp"
"strconv"
"strings" "strings"
) )
@ -28,6 +30,14 @@ func Validate(cfg *Config) error {
return err return err
} }
} }
if cfg.Metrics.Enabled {
if cfg.Metrics.Listen == "" {
return fmt.Errorf("validate metrics listen: address is required")
}
if _, err := validateListenAddress("metrics", cfg.Metrics.Listen); err != nil {
return err
}
}
if fetchConfigured(cfg.Defaults.Fetch) { if fetchConfigured(cfg.Defaults.Fetch) {
if err := validateFetch("defaults.fetch", cfg.Defaults.Fetch); err != nil { if err := validateFetch("defaults.fetch", cfg.Defaults.Fetch); err != nil {
return err return err
@ -110,9 +120,9 @@ func validateListener(name string, listener Listener, security Security) error {
return fmt.Errorf("validate %s limits.%s: must be non-negative", name, limit.name) return fmt.Errorf("validate %s limits.%s: must be non-negative", name, limit.name)
} }
} }
host, _, err := net.SplitHostPort(listener.Listen) host, err := validateListenAddress(name, listener.Listen)
if err != nil { if err != nil {
return fmt.Errorf("validate %s listen: %w", name, err) return err
} }
if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 { if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 {
return fmt.Errorf("validate %s: unprotected public listener is forbidden", name) return fmt.Errorf("validate %s: unprotected public listener is forbidden", name)
@ -139,6 +149,17 @@ func validateListener(name string, listener Listener, security Security) error {
return nil return nil
} }
func validateListenAddress(name, address string) (string, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return "", fmt.Errorf("validate %s listen: %w", name, err)
}
if _, err = strconv.ParseUint(port, 10, 16); err != nil {
return "", fmt.Errorf("validate %s listen: invalid port", name)
}
return host, nil
}
func validateRouting(index int, route Routing, upstreams map[string]Upstream, seen map[string]struct{}) error { func validateRouting(index int, route Routing, upstreams map[string]Upstream, seen map[string]struct{}) error {
if route.Name == "" { if route.Name == "" {
return fmt.Errorf("validate routing[%d]: name is required", index) return fmt.Errorf("validate routing[%d]: name is required", index)
@ -318,6 +339,22 @@ func validateUpstream(name string, upstream Upstream) error {
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil { if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
return err return err
} }
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
return err
}
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
return err
}
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
}
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
}
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
}
if err := requirePositive(scope+" lifecycle.ttl", upstream.Lifecycle.TTL); err != nil { if err := requirePositive(scope+" lifecycle.ttl", upstream.Lifecycle.TTL); err != nil {
return err return err
} }
@ -330,6 +367,12 @@ func validateUpstream(name string, upstream Upstream) error {
if err := validateFetch(scope+" fetch", upstream.Fetch); err != nil { if err := validateFetch(scope+" fetch", upstream.Fetch); err != nil {
return err return err
} }
if upstream.Fetch.EstimatedIPsPerCall > upstream.Pool.MaxSize {
return fmt.Errorf("validate %s fetch.estimatedIPsPerCall: cannot exceed pool.maxSize", scope)
}
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.EstimatedIPsPerCall > upstream.Fetch.MaxTotal {
return fmt.Errorf("validate %s fetch.estimatedIPsPerCall: cannot exceed fetch.maxTotal", scope)
}
if err := validateCheck(scope+" check", upstream.Check); err != nil { if err := validateCheck(scope+" check", upstream.Check); err != nil {
return err return err
} }
@ -357,6 +400,9 @@ func validateUpstream(name string, upstream Upstream) error {
} }
func validateFetch(scope string, fetch Fetch) error { func validateFetch(scope string, fetch Fetch) error {
if err := requirePositive(scope+".estimatedIPsPerCall", fetch.EstimatedIPsPerCall); err != nil {
return err
}
if err := requireNonNegative(scope+".requestInterval", fetch.RequestInterval); err != nil { if err := requireNonNegative(scope+".requestInterval", fetch.RequestInterval); err != nil {
return err return err
} }
@ -422,7 +468,7 @@ func validateCheck(scope string, check Check) error {
} }
func fetchConfigured(fetch Fetch) bool { func fetchConfigured(fetch Fetch) bool {
return fetch.RequestInterval != 0 || fetch.Timeout != 0 || fetch.MaxAttempts != 0 || return fetch.EstimatedIPsPerCall != 0 || fetch.RequestInterval != 0 || fetch.Timeout != 0 || fetch.MaxAttempts != 0 ||
fetch.MaxInFlight != 0 || fetch.MaxTotal != 0 || fetch.MaxResponseBytes != 0 || fetch.MaxInFlight != 0 || fetch.MaxTotal != 0 || fetch.MaxResponseBytes != 0 ||
fetch.TemplateTimeout != 0 || fetch.Retry.Initial != 0 || fetch.Retry.Max != 0 || fetch.TemplateTimeout != 0 || fetch.Retry.Initial != 0 || fetch.Retry.Max != 0 ||
fetch.Retry.Jitter != 0 fetch.Retry.Jitter != 0

View File

@ -150,7 +150,8 @@ upstreams:
proxyAuth: {type: response} proxyAuth: {type: response}
pool: {maxSize: 2000} pool: {maxSize: 2000}
capacity: {maxConcurrencyPerProxy: 20} capacity: {maxConcurrencyPerProxy: 20}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 100, targetAvailableSlots: 200}
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} fetch: {estimatedIPsPerCall: 100, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
` `

View File

@ -178,6 +178,23 @@ func (service *ApplicationService) ReloadConfiguration(ctx context.Context, comm
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err) return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
} }
} }
return service.ApplyConfiguration(ctx, command, loaded)
}
// ApplyConfiguration persists and publishes one already loaded configuration snapshot.
// Startup and runtime reload paths share this method so storage connections and the
// committed management view cannot be built from different reads of the source file.
func (service *ApplicationService) ApplyConfiguration(
ctx context.Context,
command ReloadCommand,
loaded LoadedConfiguration,
) (MutationResult, error) {
if ctx == nil {
return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration
}
if err := ctx.Err(); err != nil {
return MutationResult{RequestID: command.RequestID}, err
}
if loaded.Value == nil || strings.TrimSpace(loaded.Source) != loaded.Source || loaded.Source == "" || if loaded.Value == nil || strings.TrimSpace(loaded.Source) != loaded.Source || loaded.Source == "" ||
len(loaded.Source) > adminstate.MaxSourceBytes { len(loaded.Source) > adminstate.MaxSourceBytes {
return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration

View File

@ -293,6 +293,42 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
} }
} }
func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloading(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 30, 9, 0, 0, 0, time.UTC)
configuration := validReloadConfiguration()
publisher := &recordingConfigurationPublisher{}
state := &recordingAdminState{
mutation: adminstate.MutationResult{RequestID: "controller-startup", Changed: true, Revision: 1},
}
service, err := NewApplicationService(ApplicationDependencies{
State: state,
Operations: staticOperationalStatusReader{},
Configuration: forbiddenConfigurationLoader{},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
result, err := service.ApplyConfiguration(context.Background(), ReloadCommand{
RequestID: "controller-startup", ActorID: "proxy-controller",
}, LoadedConfiguration{Value: configuration, Source: "configs/controller.yaml"})
if err != nil {
t.Fatalf("ApplyConfiguration() error = %v", err)
}
if result.Version != 1 || !result.Changed {
t.Fatalf("ApplyConfiguration() result = %+v", result)
}
if len(publisher.published) != 1 || publisher.published[0] != configuration {
t.Fatalf("published configurations = %+v", publisher.published)
}
if state.lastConfig.Source != "configs/controller.yaml" || state.lastConfig.Actor.ID != "proxy-controller" ||
!state.lastConfig.OccurredAt.Equal(now) {
t.Fatalf("CommitConfig() metadata = %+v", state.lastConfig)
}
}
func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration(t *testing.T) { func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration(t *testing.T) {
t.Parallel() t.Parallel()
tests := []struct { tests := []struct {
@ -437,8 +473,13 @@ func validReloadUpstream(secret string) config.Upstream {
API: config.ProviderAPI{Auth: config.ProviderAuth{Type: "none"}}, API: config.ProviderAPI{Auth: config.ProviderAuth{Type: "none"}},
ProxyAuth: config.ProxyAuth{Type: "static", Username: "user", Password: secret}, ProxyAuth: config.ProxyAuth{Type: "static", Username: "user", Password: secret},
Pool: config.Pool{MaxSize: 10}, Capacity: config.Capacity{MaxConcurrencyPerProxy: 2}, Pool: config.Pool{MaxSize: 10}, Capacity: config.Capacity{MaxConcurrencyPerProxy: 2},
Refill: config.Refill{
ReconcileInterval: config.Duration(time.Second), MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
},
Lifecycle: config.Lifecycle{TTL: config.Duration(time.Minute), AllocationSafetyMargin: config.Duration(10 * time.Second)}, Lifecycle: config.Lifecycle{TTL: config.Duration(time.Minute), AllocationSafetyMargin: config.Duration(10 * time.Second)},
Fetch: config.Fetch{Timeout: config.Duration(time.Second), MaxAttempts: 2, MaxInFlight: 1}, Fetch: config.Fetch{
EstimatedIPsPerCall: 1, Timeout: config.Duration(time.Second), MaxAttempts: 2, MaxInFlight: 1,
},
} }
} }
@ -500,6 +541,12 @@ type staticConfigurationLoader struct {
err error err error
} }
type forbiddenConfigurationLoader struct{}
func (forbiddenConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) {
panic("ApplyConfiguration must not reload the source")
}
func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) { func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) {
return loader.loaded, loader.err return loader.loaded, loader.err
} }

View File

@ -0,0 +1,192 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/distribution"
"proxy-pool/internal/controller/extraction"
"proxy-pool/internal/controller/operations"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/admission"
"proxy-pool/internal/platform/httpserver"
platformMetrics "proxy-pool/internal/platform/metrics"
)
var (
ErrInvalidOptions = errors.New("invalid controller bootstrap options")
ErrStartup = errors.New("controller startup failed")
)
type Options struct {
ConfigPath string
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
}
type activityStore interface {
extractionDomain.Store
activitypool.StateInventoryReader
}
type ports struct {
state admin.StateRepository
activity activityStore
readiness distribution.ReadinessChecker
metricsReadiness platformMetrics.ReadinessChecker
close func() error
}
type infrastructure interface {
Open(context.Context, *config.Config) (ports, error)
}
type controllerRunner interface {
Run(context.Context) error
}
type runtimeFactory interface {
New(*config.Config, controllerRuntime.Dependencies, controllerRuntime.Options) (controllerRunner, error)
}
func Run(ctx context.Context, options Options) error {
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
}
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
nilInterface(options.Resolver) || nilInterface(infrastructure) || nilInterface(factory) {
return ErrInvalidOptions
}
if err := ctx.Err(); err != nil {
return err
}
if options.Now == nil {
options.Now = time.Now
}
loader, err := admin.NewFileConfigurationLoader(options.ConfigPath, options.Resolver)
if err != nil {
return errors.Join(ErrInvalidOptions, err)
}
loaded, err := loader.LoadConfiguration(ctx)
if err != nil {
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
}
configurationStore, err := config.NewStore(loaded.Value)
if err != nil {
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
}
opened, err := infrastructure.Open(ctx, loaded.Value)
if err != nil {
return fmt.Errorf("%w: open infrastructure: %w", ErrStartup, err)
}
if opened.close == nil {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
defer func() {
resultErr = errors.Join(resultErr, opened.close())
}()
dependencies := controllerRuntime.Dependencies{}
if loaded.Value.Distribution.Enabled {
if nilInterface(opened.activity) || nilInterface(opened.readiness) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
service, serviceErr := extraction.NewService(opened.activity, extractionPolicy(loaded.Value), admission.AllowAll{}, options.Now)
if serviceErr != nil {
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
}
dependencies.Extractor = service
dependencies.Readiness = opened.readiness
}
if loaded.Value.Admin.Enabled {
if nilInterface(opened.state) || nilInterface(opened.activity) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
if statusErr != nil {
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
}
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
}, admin.ApplicationOptions{Now: options.Now})
if serviceErr != nil {
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
}
if _, applyErr := service.ApplyConfiguration(ctx, admin.ReloadCommand{
RequestID: "controller-startup", ActorID: "proxy-controller",
}, loaded); applyErr != nil {
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
}
dependencies.AdminService = service
}
if loaded.Value.Metrics.Enabled {
if nilInterface(opened.metricsReadiness) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
})
if handlerErr != nil {
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
}
dependencies.MetricsHandler = handler
}
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
}
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
return runner.Run(ctx)
}
func extractionPolicy(configuration *config.Config) extraction.Policy {
configured := configuration.Distribution.Extraction
return extraction.Policy{
MaxCountPerRequest: configured.MaxCountPerRequest,
DefaultFulfillment: extractionDomain.Fulfillment(configured.Fulfillment),
MinRemainingTTL: configured.MinRemainingTTL.Value(),
MaxHealthCheckAge: configured.MaxHealthCheckAge.Value(),
ReserveForGateway: configured.ReserveForGateway,
IdempotencyTTL: configured.IdempotencyTTL.Value(),
}
}
func nilInterface(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}
type productionRuntimeFactory struct{}
func (productionRuntimeFactory) New(
configuration *config.Config,
dependencies controllerRuntime.Dependencies,
options controllerRuntime.Options,
) (controllerRunner, error) {
return controllerRuntime.New(configuration, dependencies, options)
}

View File

@ -0,0 +1,84 @@
//go:build integration
package bootstrap
import (
"context"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
controllerRuntime "proxy-pool/internal/controller/runtime"
)
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
postgresURL := os.Getenv("PROXY_POOL_TEST_POSTGRES_URL")
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
if postgresURL == "" || redisURL == "" {
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
}
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
factory := &integrationRuntimeFactory{}
err := run(context.Background(), Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
},
}, &productionInfrastructure{}, factory)
if err != nil {
t.Fatalf("run() error = %v", err)
}
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
t.Fatalf("Admin Status = %+v", factory.status)
}
if factory.status.Upstreams[0].Name != "provider-a" || factory.status.Upstreams[0].Available != 0 ||
factory.status.Upstreams[1].Name != "provider-b" {
t.Fatalf("Admin Status upstreams = %+v", factory.status.Upstreams)
}
if factory.readyStatus != http.StatusOK || factory.metricsStatus != http.StatusOK ||
!strings.Contains(factory.metricsBody, "go_") {
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
}
}
type integrationRuntimeFactory struct {
status admin.Status
readyStatus int
metricsStatus int
metricsBody string
}
func (factory *integrationRuntimeFactory) New(
_ *config.Config,
dependencies controllerRuntime.Dependencies,
_ controllerRuntime.Options,
) (controllerRunner, error) {
return integrationRunner{run: func(ctx context.Context) error {
if err := dependencies.Readiness.Ready(ctx); err != nil {
return err
}
ready := httptest.NewRecorder()
dependencies.MetricsHandler.ServeHTTP(ready, httptest.NewRequest(http.MethodGet, "/readyz", nil))
factory.readyStatus = ready.Code
metrics := httptest.NewRecorder()
dependencies.MetricsHandler.ServeHTTP(metrics, httptest.NewRequest(http.MethodGet, "/metrics", nil))
factory.metricsStatus = metrics.Code
factory.metricsBody = metrics.Body.String()
status, err := dependencies.AdminService.Status(ctx)
factory.status = status
return err
}}, nil
}
type integrationRunner struct {
run func(context.Context) error
}
func (runner integrationRunner) Run(ctx context.Context) error { return runner.run(ctx) }

View File

@ -0,0 +1,211 @@
package bootstrap
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/config"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
extractionDomain "proxy-pool/internal/domain/extraction"
)
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
t.Parallel()
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
state := adminstate.NewMemoryStore()
activity := &stubActivityStore{}
closeErr := errors.New("close failed")
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
close: func() error { return closeErr },
}}
runErr := errors.New("runtime failed")
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
err := run(context.Background(), Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
}, infrastructure, factory)
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
t.Fatalf("run() error = %v, want runtime and close errors", err)
}
if resolver.reads != 1 {
t.Fatalf("configuration reads = %d, want 1", resolver.reads)
}
if infrastructure.opens != 1 || infrastructure.configuration == nil {
t.Fatalf("infrastructure opens = %d, config = %p", infrastructure.opens, infrastructure.configuration)
}
snapshot, snapshotErr := state.Snapshot(context.Background())
if snapshotErr != nil || snapshot.Config == nil || snapshot.Config.Source != "controller.yaml" || snapshot.Revision != 1 {
t.Fatalf("management snapshot = %+v, %v", snapshot, snapshotErr)
}
if factory.configuration == nil || factory.dependencies.Extractor == nil ||
factory.dependencies.Readiness == nil || factory.dependencies.AdminService == nil ||
factory.dependencies.MetricsHandler == nil {
t.Fatalf("runtime assembly = config:%p dependencies:%+v", factory.configuration, factory.dependencies)
}
}
func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
t.Parallel()
valid := Options{ConfigPath: "controller.yaml", Resolver: &memoryResolver{}, Now: time.Now}
tests := []struct {
name string
ctx context.Context
options Options
}{
{name: "nil context", options: valid},
{name: "missing path", ctx: context.Background(), options: Options{Resolver: valid.Resolver, Now: time.Now}},
{name: "unclean path", ctx: context.Background(), options: Options{ConfigPath: " controller.yaml", Resolver: valid.Resolver, Now: time.Now}},
{name: "missing resolver", ctx: context.Background(), options: Options{ConfigPath: "controller.yaml", Now: time.Now}},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
if err := run(test.ctx, test.options, &stubInfrastructure{}, &recordingRuntimeFactory{}); !errors.Is(err, ErrInvalidOptions) {
t.Fatalf("run() error = %v", err)
}
})
}
}
type memoryResolver struct {
files map[string][]byte
reads int
}
func (*memoryResolver) LookupEnv(string) (string, bool) { return "", false }
func (resolver *memoryResolver) ReadFile(path string) ([]byte, error) {
resolver.reads++
content, ok := resolver.files[path]
if !ok {
return nil, errors.New("file missing")
}
return append([]byte(nil), content...), nil
}
type stubInfrastructure struct {
ports ports
err error
opens int
configuration *config.Config
}
func (infrastructure *stubInfrastructure) Open(
_ context.Context,
configuration *config.Config,
) (ports, error) {
infrastructure.opens++
infrastructure.configuration = configuration
return infrastructure.ports, infrastructure.err
}
type recordingRuntimeFactory struct {
configuration *config.Config
dependencies controllerRuntime.Dependencies
runner controllerRunner
err error
}
func (factory *recordingRuntimeFactory) New(
configuration *config.Config,
dependencies controllerRuntime.Dependencies,
options controllerRuntime.Options,
) (controllerRunner, error) {
factory.configuration = configuration
factory.dependencies = dependencies
return factory.runner, factory.err
}
type runnerStub struct{ err error }
func (runner runnerStub) Run(context.Context) error { return runner.err }
type readyStub struct{}
func (readyStub) Ready(context.Context) error { return nil }
type stubActivityStore struct{}
func (*stubActivityStore) Extract(_ context.Context, command extractionDomain.Command) (extractionDomain.Result, error) {
return extractionDomain.Result{Requested: command.Requested}, nil
}
func (*stubActivityStore) ReadStateInventory(
_ context.Context,
upstreamIDs []string,
_ time.Time,
) ([]activitypool.StateInventory, error) {
result := make([]activitypool.StateInventory, len(upstreamIDs))
for index, upstreamID := range upstreamIDs {
result[index].UpstreamID = upstreamID
}
return result, nil
}
const bootstrapTestConfig = `
version: 1
security:
requireProtectionOnPublicListen: true
gateway:
enabled: false
distribution:
enabled: true
listen: 127.0.0.1:0
auth: {mode: none}
extraction:
fulfillment: partial
maxCountPerRequest: 20
minRemainingTTL: 5s
maxHealthCheckAge: 15s
reserveForGateway: 5
idempotencyTTL: 5m
admin:
enabled: true
listen: 127.0.0.1:0
auth: {mode: none}
metrics:
enabled: true
listen: 127.0.0.1:0
storage:
postgresURL: postgres://fixture
redisURL: redis://fixture
routing:
- name: extract
enabled: true
purpose: extract
upstreams: [provider-a, provider-b]
strategy: {type: sequential, switchAfterEmptyFetch: 5, endBehavior: stayLast}
onUnavailable: {action: reject}
upstreams:
provider-a: &upstream
enabled: true
exposure: [extract]
provider: {billingMode: fetch, protocols: [http]}
api:
url: https://provider.invalid/proxies
method: GET
template: '{{.}}'
auth: {type: none}
proxyAuth: {type: response}
pool: {maxSize: 100}
capacity: {maxConcurrencyPerProxy: 10}
refill: {reconcileInterval: 1s, minimumAvailableSlots: 100, targetAvailableSlots: 200}
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
check:
interval: 30s
jitter: 20
maxInFlight: 100
timeout: 2s
maxAttempts: 2
maxConsecutiveFailures: 3
urls: [https://example.invalid/health]
provider-b: *upstream
`

View File

@ -0,0 +1,218 @@
package bootstrap
import (
"context"
"errors"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
"proxy-pool/internal/adapters/postgresadmin"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/config"
"proxy-pool/internal/platform/credentials"
platformMetrics "proxy-pool/internal/platform/metrics"
)
const (
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisCleanupLimit = 1_024
)
var (
ErrPostgresConfiguration = errors.New("invalid PostgreSQL configuration")
ErrPostgresUnavailable = errors.New("PostgreSQL unavailable")
ErrRedisConfiguration = errors.New("invalid Redis configuration")
ErrRedisUnavailable = errors.New("Redis unavailable")
)
type productionInfrastructure struct{}
func (*productionInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (_ ports, resultErr error) {
if ctx == nil || configuration == nil {
return ports{}, ErrInvalidOptions
}
var postgresPool *pgxpool.Pool
var redisClient *redis.Client
closeResources := func() error {
var closeErr error
if redisClient != nil {
closeErr = redisClient.Close()
}
if postgresPool != nil {
postgresPool.Close()
}
return closeErr
}
defer func() {
if resultErr != nil {
_ = closeResources()
}
}()
opened := ports{close: closeResources}
if configuration.Admin.Enabled {
if strings.TrimSpace(configuration.Storage.PostgresURL) == "" {
return ports{}, ErrPostgresConfiguration
}
poolConfig, err := pgxpool.ParseConfig(configuration.Storage.PostgresURL)
if err != nil {
return ports{}, ErrPostgresConfiguration
}
poolConfig.ConnConfig.RuntimeParams["application_name"] = "proxy-controller"
postgresPool, err = pgxpool.NewWithConfig(ctx, poolConfig)
if err != nil {
return ports{}, ErrPostgresUnavailable
}
if err = postgresPool.Ping(ctx); err != nil {
return ports{}, contextOr(ctx, ErrPostgresUnavailable)
}
if err = postgresadmin.ApplyMigrations(ctx, postgresPool); err != nil {
return ports{}, err
}
opened.state, err = postgresadmin.New(postgresPool)
if err != nil {
return ports{}, err
}
}
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
return ports{}, ErrRedisConfiguration
}
redisOptions, err := redis.ParseURL(configuration.Storage.RedisURL)
if err != nil {
return ports{}, ErrRedisConfiguration
}
redisClient = redis.NewClient(redisOptions)
if err = redisClient.Ping(ctx).Err(); err != nil {
return ports{}, contextOr(ctx, ErrRedisUnavailable)
}
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
if err != nil {
return ports{}, err
}
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
Namespace: redisNamespace,
Credentials: credentialStore,
OperationTTL: redisOperationTTL,
MaxCandidateScan: candidateScan(configuration),
CleanupLimit: redisCleanupLimit,
})
if err != nil {
return ports{}, err
}
opened.activity = adapter
opened.readiness = redisReadiness{client: redisClient}
}
if configuration.Metrics.Enabled {
opened.metricsReadiness = selectMetricsReadiness(
configuration,
storeReadiness{postgres: postgresPool, redis: redisClient},
redisReadiness{client: redisClient},
)
}
return opened, nil
}
func selectMetricsReadiness(
configuration *config.Config,
admin, activity platformMetrics.ReadinessChecker,
) platformMetrics.ReadinessChecker {
if configuration.Distribution.Enabled {
return activity
}
if configuration.Admin.Enabled {
return admin
}
return alwaysReady{}
}
type alwaysReady struct{}
func (alwaysReady) Ready(ctx context.Context) error {
if ctx == nil {
return ErrInvalidOptions
}
return ctx.Err()
}
type storeReadiness struct {
postgres *pgxpool.Pool
redis *redis.Client
}
func (readiness storeReadiness) Ready(ctx context.Context) error {
if ctx == nil {
return ErrInvalidOptions
}
if readiness.postgres != nil {
if err := readiness.postgres.Ping(ctx); err != nil {
return contextOr(ctx, ErrPostgresUnavailable)
}
}
if readiness.redis != nil {
if err := readiness.redis.Ping(ctx).Err(); err != nil {
return contextOr(ctx, ErrRedisUnavailable)
}
}
return nil
}
type redisReadiness struct {
client *redis.Client
}
func (readiness redisReadiness) Ready(ctx context.Context) error {
if ctx == nil || readiness.client == nil {
return ErrRedisUnavailable
}
if err := readiness.client.Ping(ctx).Err(); err != nil {
return contextOr(ctx, ErrRedisUnavailable)
}
return nil
}
func credentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
continue
}
if capacity > maximum-upstream.Pool.MaxSize {
return maximum
}
capacity += upstream.Pool.MaxSize
}
if capacity == 0 {
return 1
}
return capacity
}
func candidateScan(configuration *config.Config) int {
configured := configuration.Distribution.Extraction
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
return int(^uint(0) >> 1)
}
value := configured.MaxCountPerRequest + configured.ReserveForGateway
if value < redisMinimumScan {
return redisMinimumScan
}
return value
}
func contextOr(ctx context.Context, fallback error) error {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
return fallback
}

View File

@ -0,0 +1,96 @@
package bootstrap
import (
"context"
"errors"
"strings"
"sync/atomic"
"testing"
"proxy-pool/internal/config"
)
func TestProductionInfrastructureRejectsInvalidStorageWithoutLeakingURLs(t *testing.T) {
t.Parallel()
postgresSecret := "postgres-secret"
_, err := (&productionInfrastructure{}).Open(context.Background(), &config.Config{
Admin: config.Listener{Enabled: true},
Storage: config.Storage{PostgresURL: "postgres://user:" + postgresSecret + "@%zz"},
})
if !errors.Is(err, ErrPostgresConfiguration) || strings.Contains(err.Error(), postgresSecret) {
t.Fatalf("Open(invalid PostgreSQL) error = %v", err)
}
redisSecret := "redis-secret"
_, err = (&productionInfrastructure{}).Open(context.Background(), &config.Config{
Distribution: config.Distribution{Listener: config.Listener{Enabled: true}},
Storage: config.Storage{RedisURL: "redis://user:" + redisSecret + "@%zz"},
})
if !errors.Is(err, ErrRedisConfiguration) || strings.Contains(err.Error(), redisSecret) {
t.Fatalf("Open(invalid Redis) error = %v", err)
}
}
func TestSelectMetricsReadinessPreservesDistributionWhenAdminStoreFails(t *testing.T) {
t.Parallel()
adminCalls := &atomic.Int64{}
activityCalls := &atomic.Int64{}
adminReady := readinessFunc(func(context.Context) error {
adminCalls.Add(1)
return ErrPostgresUnavailable
})
activityReady := readinessFunc(func(context.Context) error {
activityCalls.Add(1)
return nil
})
selected := selectMetricsReadiness(&config.Config{
Admin: config.Listener{Enabled: true},
Distribution: config.Distribution{Listener: config.Listener{Enabled: true}},
}, adminReady, activityReady)
if err := selected.Ready(context.Background()); err != nil {
t.Fatalf("Ready() error = %v", err)
}
if adminCalls.Load() != 0 || activityCalls.Load() != 1 {
t.Fatalf("readiness calls = admin:%d activity:%d", adminCalls.Load(), activityCalls.Load())
}
}
func TestSelectMetricsReadinessUsesAdminStoresWithoutDistribution(t *testing.T) {
t.Parallel()
wantErr := errors.New("admin unavailable")
selected := selectMetricsReadiness(
&config.Config{Admin: config.Listener{Enabled: true}},
readinessFunc(func(context.Context) error { return wantErr }),
readinessFunc(func(context.Context) error { return nil }),
)
if err := selected.Ready(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Ready() error = %v, want %v", err, wantErr)
}
}
type readinessFunc func(context.Context) error
func (function readinessFunc) Ready(ctx context.Context) error { return function(ctx) }
func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
t.Parallel()
configuration := &config.Config{
Distribution: config.Distribution{Extraction: config.Extraction{
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
},
}
if got := credentialCapacity(configuration); got != 5_000 {
t.Fatalf("credentialCapacity() = %d, want 5000", got)
}
if got := candidateScan(configuration); got != 5_100 {
t.Fatalf("candidateScan() = %d, want 5100", got)
}
configuration.Distribution.Extraction = config.Extraction{MaxCountPerRequest: 1}
if got := candidateScan(configuration); got != redisMinimumScan {
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
}
}

View File

@ -0,0 +1,104 @@
package operations
import (
"context"
"errors"
"reflect"
"sort"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/activitypool"
)
var (
ErrInvalidReader = errors.New("invalid operational status reader")
ErrUnavailable = errors.New("operational status unavailable")
)
type ConfigurationReader interface {
Current() *config.Config
}
type Reader struct {
configuration ConfigurationReader
inventory activitypool.StateInventoryReader
now func() time.Time
}
var _ admin.OperationalStatusReader = (*Reader)(nil)
func NewReader(
configuration ConfigurationReader,
inventory activitypool.StateInventoryReader,
now func() time.Time,
) (*Reader, error) {
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
return nil, ErrInvalidReader
}
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
}
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
if ctx == nil || reader == nil {
return admin.OperationalStatus{}, ErrInvalidReader
}
if err := ctx.Err(); err != nil {
return admin.OperationalStatus{}, err
}
configuration := reader.configuration.Current()
if configuration == nil {
return admin.OperationalStatus{}, ErrUnavailable
}
upstreamIDs := make([]string, 0, len(configuration.Upstreams))
for upstreamID := range configuration.Upstreams {
upstreamIDs = append(upstreamIDs, upstreamID)
}
sort.Strings(upstreamIDs)
inventories, err := reader.inventory.ReadStateInventory(ctx, upstreamIDs, reader.now().UTC())
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return admin.OperationalStatus{}, err
}
return admin.OperationalStatus{}, errors.Join(ErrUnavailable, err)
}
if len(inventories) != len(upstreamIDs) {
return admin.OperationalStatus{}, ErrUnavailable
}
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
for index, inventory := range inventories {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
return admin.OperationalStatus{}, ErrUnavailable
}
status.Upstreams[index] = admin.UpstreamActivity{
Name: inventory.UpstreamID,
Available: inventory.Available,
Checking: inventory.Checking,
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
}
}
return status, nil
}
func invalidInventory(inventory activitypool.StateInventory) bool {
return inventory.Fetched < 0 || inventory.Checking < 0 || inventory.Available < 0 ||
inventory.Suspect < 0 || inventory.Draining < 0 || inventory.Unhealthy < 0 || inventory.Extracted < 0
}
func nilInterface(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

View File

@ -0,0 +1,118 @@
package operations
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/domain/activitypool"
)
func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
inventory := &recordingStateInventoryReader{result: []activitypool.StateInventory{
{UpstreamID: "provider-a", Fetched: 3, Checking: 2, Available: 11, Suspect: 1, Draining: 4, Extracted: 8},
{UpstreamID: "provider-b", Available: 7},
}}
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
}}, inventory, func() time.Time { return now })
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
status, err := reader.ReadOperationalStatus(context.Background())
if err != nil {
t.Fatalf("ReadOperationalStatus() error = %v", err)
}
if len(inventory.upstreamIDs) != 2 || inventory.upstreamIDs[0] != "provider-a" || inventory.upstreamIDs[1] != "provider-b" ||
!inventory.now.Equal(now) {
t.Fatalf("ReadStateInventory() input = %+v at %v", inventory.upstreamIDs, inventory.now)
}
if status.SnapshotVersion != 0 || len(status.Workers) != 0 || len(status.Upstreams) != 2 {
t.Fatalf("operational status shape = %+v", status)
}
first := status.Upstreams[0]
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
first.Draining != 4 || first.Extracted != 8 {
t.Fatalf("first upstream = %+v", first)
}
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
t.Fatalf("second upstream = %+v", status.Upstreams[1])
}
}
func TestReaderRejectsMissingOrMalformedDependencies(t *testing.T) {
t.Parallel()
now := func() time.Time { return time.Now().UTC() }
validConfig := staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-a": {}},
}}
validInventory := &recordingStateInventoryReader{result: []activitypool.StateInventory{{UpstreamID: "provider-a"}}}
if _, err := NewReader(nil, validInventory, now); !errors.Is(err, ErrInvalidReader) {
t.Fatalf("NewReader(nil config) error = %v", err)
}
if _, err := NewReader(validConfig, nil, now); !errors.Is(err, ErrInvalidReader) {
t.Fatalf("NewReader(nil inventory) error = %v", err)
}
if _, err := NewReader(validConfig, validInventory, nil); !errors.Is(err, ErrInvalidReader) {
t.Fatalf("NewReader(nil clock) error = %v", err)
}
malformed := &recordingStateInventoryReader{result: []activitypool.StateInventory{{UpstreamID: "wrong"}}}
reader, err := NewReader(validConfig, malformed, now)
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
if _, err := reader.ReadOperationalStatus(context.Background()); !errors.Is(err, ErrUnavailable) {
t.Fatalf("ReadOperationalStatus(malformed) error = %v", err)
}
}
func TestReaderPreservesCancellationAndClassifiesDependencyFailures(t *testing.T) {
t.Parallel()
dependencyError := errors.New("redis down")
inventory := &recordingStateInventoryReader{err: dependencyError}
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-a": {}},
}}, inventory, time.Now)
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
if _, err := reader.ReadOperationalStatus(context.Background()); !errors.Is(err, ErrUnavailable) || !errors.Is(err, dependencyError) {
t.Fatalf("ReadOperationalStatus(dependency) error = %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := reader.ReadOperationalStatus(ctx); !errors.Is(err, context.Canceled) || errors.Is(err, ErrUnavailable) {
t.Fatalf("ReadOperationalStatus(canceled) error = %v", err)
}
}
type staticConfigurationReader struct {
configuration *config.Config
}
func (reader staticConfigurationReader) Current() *config.Config { return reader.configuration }
type recordingStateInventoryReader struct {
result []activitypool.StateInventory
err error
upstreamIDs []string
now time.Time
}
func (reader *recordingStateInventoryReader) ReadStateInventory(
_ context.Context,
upstreamIDs []string,
now time.Time,
) ([]activitypool.StateInventory, error) {
reader.upstreamIDs = append([]string(nil), upstreamIDs...)
reader.now = now
return append([]activitypool.StateInventory(nil), reader.result...), reader.err
}

View File

@ -13,6 +13,8 @@ var (
ErrInvalidFetchCompletion = errors.New("invalid fetch completion") ErrInvalidFetchCompletion = errors.New("invalid fetch completion")
ErrFetchPermitFinished = errors.New("fetch permit is already finished") ErrFetchPermitFinished = errors.New("fetch permit is already finished")
ErrInvalidManagedRelease = errors.New("invalid managed proxy release") ErrInvalidManagedRelease = errors.New("invalid managed proxy release")
ErrInvalidManagedSynchronization = errors.New("invalid managed proxy synchronization")
ErrManagedSynchronizationInFlight = errors.New("managed proxy synchronization has pending fetches")
) )
type FetchBudgetConfig struct { type FetchBudgetConfig struct {
@ -112,6 +114,22 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
return b.usage return b.usage
} }
// SynchronizeManaged replaces the local current-inventory count with the
// authoritative activity-store observation. Pending requests and cumulative
// fetch usage remain owned by this budget.
func (b *FetchBudget) SynchronizeManaged(managed int) error {
if b == nil || managed < 0 {
return ErrInvalidManagedSynchronization
}
b.mu.Lock()
defer b.mu.Unlock()
if b.usage.PendingExpected > 0 {
return ErrManagedSynchronizationInFlight
}
b.usage.Managed = managed
return nil
}
// ReleaseManaged returns current-inventory capacity after extraction, expiry, // ReleaseManaged returns current-inventory capacity after extraction, expiry,
// or removal. It deliberately does not restore the cumulative fetch quota. // or removal. It deliberately does not restore the cumulative fetch quota.
func (b *FetchBudget) ReleaseManaged(count int) error { func (b *FetchBudget) ReleaseManaged(count int) error {

View File

@ -61,6 +61,48 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
} }
} }
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
permit, ok, err := budget.ReserveFetch("a")
if err != nil || !ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
}
if err := budget.SynchronizeManaged(1); !errors.Is(err, ErrManagedSynchronizationInFlight) {
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
}
usage := budget.Snapshot()
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
}
if err := permit.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
if err := budget.SynchronizeManaged(1); err != nil {
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
}
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
}
}
func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 1,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
if err := budget.SynchronizeManaged(-1); !errors.Is(err, ErrInvalidManagedSynchronization) {
t.Fatalf("SynchronizeManaged() error = %v, want ErrInvalidManagedSynchronization", err)
}
}
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) { func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@ -2,6 +2,8 @@ package pool
import ( import (
"errors" "errors"
"math"
"sync"
"time" "time"
"proxy-pool/internal/domain/upstream" "proxy-pool/internal/domain/upstream"
@ -11,7 +13,9 @@ var ErrInvalidReconcilePolicy = errors.New("invalid pool reconcile policy")
type ReconcilePolicy struct { type ReconcilePolicy struct {
MinimumAvailableSlots int64 MinimumAvailableSlots int64
TargetAvailableSlots int64
ExpectedPerFetch int ExpectedPerFetch int
ExpectedSlotsPerFetch int64
SafetyMargin time.Duration SafetyMargin time.Duration
} }
@ -24,6 +28,7 @@ type ReconcileDecision struct {
PendingExpected int PendingExpected int
FetchedTotal int64 FetchedTotal int64
FetchAllowance int FetchAllowance int
EffectiveSlots int64
Triggered bool Triggered bool
} }
@ -31,34 +36,75 @@ type Reconciler struct {
policy ReconcilePolicy policy ReconcilePolicy
budget *FetchBudget budget *FetchBudget
notifier FetchNotifier notifier FetchNotifier
mu sync.Mutex
refilling bool
slotsPerProxy int64
} }
func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNotifier) (*Reconciler, error) { func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNotifier) (*Reconciler, error) {
if policy.MinimumAvailableSlots <= 0 || policy.ExpectedPerFetch <= 0 || if policy.MinimumAvailableSlots <= 0 || policy.TargetAvailableSlots <= policy.MinimumAvailableSlots ||
policy.ExpectedPerFetch <= 0 || policy.ExpectedSlotsPerFetch <= 0 ||
policy.SafetyMargin < 0 || budget == nil || notifier == nil { policy.SafetyMargin < 0 || budget == nil || notifier == nil {
return nil, ErrInvalidReconcilePolicy return nil, ErrInvalidReconcilePolicy
} }
if budget.expected != policy.ExpectedPerFetch { if budget.expected != policy.ExpectedPerFetch {
return nil, ErrInvalidReconcilePolicy return nil, ErrInvalidReconcilePolicy
} }
return &Reconciler{policy: policy, budget: budget, notifier: notifier}, nil if policy.ExpectedSlotsPerFetch%int64(policy.ExpectedPerFetch) != 0 {
return nil, ErrInvalidReconcilePolicy
}
return &Reconciler{
policy: policy, budget: budget, notifier: notifier,
slotsPerProxy: policy.ExpectedSlotsPerFetch / int64(policy.ExpectedPerFetch),
}, nil
} }
// Reconcile centralizes the cold-path decision. The notifier may coalesce many // Reconcile centralizes the cold-path decision. The notifier may coalesce many
// calls; Provider Reconciler atomically reserves the budget before doing I/O. // calls; Provider Reconciler atomically reserves the budget before doing I/O.
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision { func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
usage := r.budget.Snapshot() usage := r.budget.Snapshot()
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
decision := ReconcileDecision{ decision := ReconcileDecision{
AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin), AvailableSlots: availableSlots,
PendingExpected: usage.PendingExpected, PendingExpected: usage.PendingExpected,
FetchedTotal: usage.FetchedTotal, FetchedTotal: usage.FetchedTotal,
FetchAllowance: r.budget.FetchAllowance(), FetchAllowance: r.budget.FetchAllowance(),
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
} }
if decision.AvailableSlots >= r.policy.MinimumAvailableSlots || r.mu.Lock()
decision.FetchAllowance < r.policy.ExpectedPerFetch { if r.refilling {
if usage.PendingExpected == 0 && decision.AvailableSlots >= r.policy.TargetAvailableSlots {
r.refilling = false
}
} else if decision.AvailableSlots < r.policy.MinimumAvailableSlots {
r.refilling = true
}
trigger := r.refilling &&
decision.EffectiveSlots < r.policy.TargetAvailableSlots &&
decision.FetchAllowance >= r.policy.ExpectedPerFetch
r.mu.Unlock()
if !trigger {
return decision return decision
} }
r.notifier.Notify() r.notifier.Notify()
decision.Triggered = true decision.Triggered = true
return decision return decision
} }
func saturatingMultiply(left, right int64) int64 {
if left <= 0 || right <= 0 {
return 0
}
if left > math.MaxInt64/right {
return math.MaxInt64
}
return left * right
}
func saturatingAdd(left, right int64) int64 {
if left >= math.MaxInt64-right {
return math.MaxInt64
}
return left + right
}

View File

@ -18,7 +18,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
notifier := &recordingFetchNotifier{} notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{ reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 5, MinimumAvailableSlots: 5,
TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedPerFetch: 2,
ExpectedSlotsPerFetch: 2,
SafetyMargin: 10 * time.Second, SafetyMargin: 10 * time.Second,
}, budget, notifier) }, budget, notifier)
if err != nil { if err != nil {
@ -56,7 +58,8 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
defer permit.Cancel() defer permit.Cancel()
notifier := &recordingFetchNotifier{} notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{ reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 1, ExpectedPerFetch: 2, MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
}, budget, notifier) }, budget, notifier)
if err != nil { if err != nil {
t.Fatalf("NewReconciler(): %v", err) t.Fatalf("NewReconciler(): %v", err)
@ -71,6 +74,79 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
} }
} }
func TestPoolReconcilerUsesTargetWatermarkUntilRefillCompletes(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3,
TargetAvailableSlots: 8,
ExpectedPerFetch: 2,
ExpectedSlotsPerFetch: 2,
}, budget, notifier)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
belowMinimum := reconciler.Reconcile(now, inventoryWithSlots(now, 2))
betweenWatermarks := reconciler.Reconcile(now, inventoryWithSlots(now, 5))
atTarget := reconciler.Reconcile(now, inventoryWithSlots(now, 8))
aboveMinimumAfterCompletion := reconciler.Reconcile(now, inventoryWithSlots(now, 5))
if !belowMinimum.Triggered || !betweenWatermarks.Triggered || atTarget.Triggered || aboveMinimumAfterCompletion.Triggered {
t.Fatalf("triggered states = [%v %v %v %v], want [true true false false]",
belowMinimum.Triggered, betweenWatermarks.Triggered, atTarget.Triggered, aboveMinimumAfterCompletion.Triggered)
}
if notifier.calls != 2 {
t.Fatalf("Notify() calls = %d, want 2", notifier.calls)
}
}
func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 6,
}, budget, notifier)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); !decision.Triggered {
t.Fatalf("initial Reconcile() = %+v, want trigger", decision)
}
permit, ok, err := budget.ReserveFetch("provider-a")
if err != nil || !ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
}
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
}
if err := permit.Complete(2, 1); err != nil {
t.Fatalf("Complete(): %v", err)
}
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
t.Fatalf("post-fetch Reconcile() = %+v, want refill episode to continue", decision)
}
}
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,
}}}
}
type recordingFetchNotifier struct{ calls int } type recordingFetchNotifier struct{ calls int }
func (n *recordingFetchNotifier) Notify() { n.calls++ } func (n *recordingFetchNotifier) Notify() { n.calls++ }

View File

@ -0,0 +1,41 @@
package provider
import (
"context"
"errors"
"time"
)
var (
ErrInvalidCoordination = errors.New("invalid provider coordination")
ErrCoordinationUnavailable = errors.New("provider coordination unavailable")
ErrLeadershipLost = errors.New("provider leadership lost")
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
)
type CoordinationLimits struct {
RequestInterval time.Duration
MaxInFlight int
MaxAttemptDuration time.Duration
}
type Fence struct {
Generation string
Epoch uint64
}
type Coordinator interface {
// RunLeader waits for leadership and runs work only while its fencing lease
// is valid. The work context is canceled before a known lease expiry.
RunLeader(context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error) error
}
type LeaderSession interface {
Fence() Fence
AcquireFetch(context.Context) (RequestPermit, error)
}
type RequestPermit interface {
// Release is idempotent. A failed release expires automatically in storage.
Release(context.Context) error
}

View File

@ -26,6 +26,7 @@ type Dependencies struct {
Extractor distribution.Extractor Extractor distribution.Extractor
Readiness distribution.ReadinessChecker Readiness distribution.ReadinessChecker
AdminService admin.Service AdminService admin.Service
MetricsHandler http.Handler
} }
type Options struct { type Options struct {
@ -37,6 +38,7 @@ type Options struct {
type Listeners struct { type Listeners struct {
Distribution net.Listener Distribution net.Listener
Admin net.Listener Admin net.Listener
Metrics net.Listener
} }
type Runtime struct { type Runtime struct {
@ -46,11 +48,14 @@ type Runtime struct {
adminEnabled bool adminEnabled bool
adminAddress string adminAddress string
adminHandler http.Handler adminHandler http.Handler
metricsEnabled bool
metricsAddress string
metricsHandler http.Handler
httpOptions httpserver.Options httpOptions httpserver.Options
} }
func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runtime, error) { func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runtime, error) {
if cfg == nil || (!cfg.Distribution.Enabled && !cfg.Admin.Enabled) { if cfg == nil || (!cfg.Distribution.Enabled && !cfg.Admin.Enabled && !cfg.Metrics.Enabled) {
return nil, ErrInvalidRuntime return nil, ErrInvalidRuntime
} }
distributionBodyLimit, adminBodyLimit, err := resolveBodyLimits(options) distributionBodyLimit, adminBodyLimit, err := resolveBodyLimits(options)
@ -104,6 +109,15 @@ func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runti
result.adminAddress = cfg.Admin.Listen result.adminAddress = cfg.Admin.Listen
result.adminHandler = handler result.adminHandler = handler
} }
if cfg.Metrics.Enabled {
if strings.TrimSpace(cfg.Metrics.Listen) == "" || dependencies.MetricsHandler == nil {
return nil, ErrInvalidRuntime
}
result.metricsEnabled = true
result.metricsAddress = cfg.Metrics.Listen
result.metricsHandler = dependencies.MetricsHandler
}
return result, nil return result, nil
} }
@ -111,7 +125,7 @@ func (runtime *Runtime) Run(ctx context.Context) error {
if runtime == nil || ctx == nil { if runtime == nil || ctx == nil {
return ErrInvalidRuntime return ErrInvalidRuntime
} }
bindings := make([]httpserver.Binding, 0, 2) bindings := make([]httpserver.Binding, 0, 3)
if runtime.distributionEnabled { if runtime.distributionEnabled {
bindings = append(bindings, httpserver.Binding{ bindings = append(bindings, httpserver.Binding{
Name: "distribution", Address: runtime.distributionAddress, Handler: runtime.distributionHandler, Name: "distribution", Address: runtime.distributionAddress, Handler: runtime.distributionHandler,
@ -122,6 +136,11 @@ func (runtime *Runtime) Run(ctx context.Context) error {
Name: "admin", Address: runtime.adminAddress, Handler: runtime.adminHandler, Name: "admin", Address: runtime.adminAddress, Handler: runtime.adminHandler,
}) })
} }
if runtime.metricsEnabled {
bindings = append(bindings, httpserver.Binding{
Name: "metrics", Address: runtime.metricsAddress, Handler: runtime.metricsHandler,
})
}
if err := httpserver.ListenAndServe(ctx, runtime.httpOptions, bindings...); err != nil { if err := httpserver.ListenAndServe(ctx, runtime.httpOptions, bindings...); err != nil {
return fmt.Errorf("run controller HTTP runtime: %w", err) return fmt.Errorf("run controller HTTP runtime: %w", err)
} }
@ -131,10 +150,11 @@ func (runtime *Runtime) Run(ctx context.Context) error {
func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error { func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error {
if runtime == nil || ctx == nil || if runtime == nil || ctx == nil ||
runtime.distributionEnabled != (listeners.Distribution != nil) || runtime.distributionEnabled != (listeners.Distribution != nil) ||
runtime.adminEnabled != (listeners.Admin != nil) { runtime.adminEnabled != (listeners.Admin != nil) ||
runtime.metricsEnabled != (listeners.Metrics != nil) {
return ErrInvalidRuntime return ErrInvalidRuntime
} }
endpoints := make([]httpserver.Endpoint, 0, 2) endpoints := make([]httpserver.Endpoint, 0, 3)
if runtime.distributionEnabled { if runtime.distributionEnabled {
endpoints = append(endpoints, httpserver.Endpoint{ endpoints = append(endpoints, httpserver.Endpoint{
Name: "distribution", Listener: listeners.Distribution, Handler: runtime.distributionHandler, Name: "distribution", Listener: listeners.Distribution, Handler: runtime.distributionHandler,
@ -145,6 +165,11 @@ func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error {
Name: "admin", Listener: listeners.Admin, Handler: runtime.adminHandler, Name: "admin", Listener: listeners.Admin, Handler: runtime.adminHandler,
}) })
} }
if runtime.metricsEnabled {
endpoints = append(endpoints, httpserver.Endpoint{
Name: "metrics", Listener: listeners.Metrics, Handler: runtime.metricsHandler,
})
}
if err := httpserver.Serve(ctx, runtime.httpOptions, endpoints...); err != nil { if err := httpserver.Serve(ctx, runtime.httpOptions, endpoints...); err != nil {
return fmt.Errorf("serve controller HTTP runtime: %w", err) return fmt.Errorf("serve controller HTTP runtime: %w", err)
} }

View File

@ -19,34 +19,47 @@ import (
func TestRuntimeServesDistributionAndAdminOnIndependentListeners(t *testing.T) { func TestRuntimeServesDistributionAndAdminOnIndependentListeners(t *testing.T) {
t.Parallel() t.Parallel()
cfg := runtimeConfig() cfg := runtimeConfig()
cfg.Metrics = config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}
adminService := &stubAdminService{status: admin.Status{ConfigVersion: "cfg-7", SnapshotVersion: 11}} adminService := &stubAdminService{status: admin.Status{ConfigVersion: "cfg-7", SnapshotVersion: 11}}
runtime, err := New(cfg, Dependencies{ runtime, err := New(cfg, Dependencies{
Extractor: stubExtractor{}, Extractor: stubExtractor{},
Readiness: stubReadiness{}, Readiness: stubReadiness{},
AdminService: adminService, AdminService: adminService,
MetricsHandler: http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/readyz" {
http.NotFound(response, request)
return
}
response.WriteHeader(http.StatusOK)
}),
}, Options{HTTP: testHTTPOptions()}) }, Options{HTTP: testHTTPOptions()})
if err != nil { if err != nil {
t.Fatalf("New() error = %v", err) t.Fatalf("New() error = %v", err)
} }
distributionListener := mustListen(t) distributionListener := mustListen(t)
adminListener := mustListen(t) adminListener := mustListen(t)
metricsListener := mustListen(t)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 1) result := make(chan error, 1)
go func() { go func() {
result <- runtime.Serve(ctx, Listeners{ result <- runtime.Serve(ctx, Listeners{
Distribution: distributionListener, Distribution: distributionListener,
Admin: adminListener, Admin: adminListener,
Metrics: metricsListener,
}) })
}() }()
distributionURL := "http://" + distributionListener.Addr().String() distributionURL := "http://" + distributionListener.Addr().String()
adminURL := "http://" + adminListener.Addr().String() adminURL := "http://" + adminListener.Addr().String()
metricsURL := "http://" + metricsListener.Addr().String()
assertStatus(t, http.MethodGet, distributionURL+"/health/live", nil, http.StatusOK) assertStatus(t, http.MethodGet, distributionURL+"/health/live", nil, http.StatusOK)
assertStatus(t, http.MethodGet, distributionURL+"/api/v1/status", nil, http.StatusNotFound) assertStatus(t, http.MethodGet, distributionURL+"/api/v1/status", nil, http.StatusNotFound)
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", nil, http.StatusUnauthorized) assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", nil, http.StatusUnauthorized)
adminHeaders := http.Header{"Authorization": []string{"Bearer admin-token"}} adminHeaders := http.Header{"Authorization": []string{"Bearer admin-token"}}
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", adminHeaders, http.StatusOK) assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", adminHeaders, http.StatusOK)
assertStatus(t, http.MethodGet, adminURL+"/health/live", adminHeaders, http.StatusNotFound) assertStatus(t, http.MethodGet, adminURL+"/health/live", adminHeaders, http.StatusNotFound)
assertStatus(t, http.MethodGet, metricsURL+"/readyz", nil, http.StatusOK)
assertStatus(t, http.MethodGet, metricsURL+"/api/v1/status", nil, http.StatusNotFound)
if adminService.statusCalls.Load() != 1 { if adminService.statusCalls.Load() != 1 {
t.Fatalf("admin status calls = %d, want 1", adminService.statusCalls.Load()) t.Fatalf("admin status calls = %d, want 1", adminService.statusCalls.Load())
} }
@ -107,6 +120,44 @@ func TestServeRequiresExactlyTheEnabledListeners(t *testing.T) {
} }
} }
func TestRuntimeServesMetricsOnIndependentListener(t *testing.T) {
t.Parallel()
cfg := &config.Config{Metrics: config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}}
metricsHandler := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/readyz" {
http.NotFound(response, request)
return
}
response.WriteHeader(http.StatusOK)
})
runtime, err := New(cfg, Dependencies{MetricsHandler: metricsHandler}, Options{HTTP: testHTTPOptions()})
if err != nil {
t.Fatalf("New(metrics-only) error = %v", err)
}
listener := mustListen(t)
ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 1)
go func() { result <- runtime.Serve(ctx, Listeners{Metrics: listener}) }()
assertStatus(t, http.MethodGet, "http://"+listener.Addr().String()+"/readyz", nil, http.StatusOK)
cancel()
select {
case err := <-result:
if err != nil {
t.Fatalf("Serve(metrics-only) error = %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Serve(metrics-only) did not stop")
}
}
func TestNewRequiresMetricsHandlerWhenEnabled(t *testing.T) {
t.Parallel()
cfg := &config.Config{Metrics: config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}}
if _, err := New(cfg, Dependencies{}, Options{}); !errors.Is(err, ErrInvalidRuntime) {
t.Fatalf("New(metrics without handler) error = %v", err)
}
}
func runtimeConfig() *config.Config { func runtimeConfig() *config.Config {
return &config.Config{ return &config.Config{
Distribution: config.Distribution{ Distribution: config.Distribution{

View File

@ -18,6 +18,7 @@ type Store interface {
activitypool.Upserter activitypool.Upserter
activitypool.HealthStore activitypool.HealthStore
activitypool.InventoryReader activitypool.InventoryReader
activitypool.StateInventoryReader
activitypool.Maintainer activitypool.Maintainer
extractionDomain.Store extractionDomain.Store
ownershipDomain.Repository ownershipDomain.Repository
@ -48,6 +49,9 @@ func Run(t *testing.T, factory Factory) {
t.Run("inventory and bounded maintenance", func(t *testing.T) { t.Run("inventory and bounded maintenance", func(t *testing.T) {
runMaintenanceContract(t, newStore(t, factory)) runMaintenanceContract(t, newStore(t, factory))
}) })
t.Run("state inventory lifecycle", func(t *testing.T) {
runStateInventoryContract(t, newStore(t, factory))
})
t.Run("concurrent exclusivity", func(t *testing.T) { t.Run("concurrent exclusivity", func(t *testing.T) {
runConcurrencyContract(t, factory) runConcurrencyContract(t, factory)
}) })
@ -338,6 +342,88 @@ func runMaintenanceContract(t *testing.T, store Store) {
assertInventory(t, store, "provider-a", now.Add(6*time.Second), 0) assertInventory(t, store, "provider-a", now.Add(6*time.Second), 0)
} }
func runStateInventoryContract(t *testing.T, store Store) {
t.Helper()
now := contractNow()
upstreamA := "provider:a:FETCHED"
upstreamB := "provider:a"
states := []proxyDomain.State{
proxyDomain.StateFetched,
proxyDomain.StateChecking,
proxyDomain.StateAvailable,
proxyDomain.StateSuspect,
proxyDomain.StateDraining,
proxyDomain.StateUnhealthy,
proxyDomain.StateExtracted,
}
for index, state := range states {
upsertOne(t, store, upstreamA, now, time.Minute,
contractProxy(fmt.Sprintf("state-%d", index), fmt.Sprintf("192.0.2.%d", index+30), state))
}
upsertOne(t, store, upstreamB, now, time.Minute,
contractProxy("collision-control", "198.51.100.30", proxyDomain.StateFetched))
inventories, err := store.ReadStateInventory(context.Background(), []string{upstreamB, upstreamA}, now)
if err != nil || len(inventories) != 2 {
t.Fatalf("ReadStateInventory() = %+v, %v", inventories, err)
}
if inventories[0] != (activitypool.StateInventory{UpstreamID: upstreamB, Fetched: 1}) {
t.Fatalf("ReadStateInventory(collision control) = %+v", inventories[0])
}
wantAll := activitypool.StateInventory{
UpstreamID: upstreamA, Fetched: 1, Checking: 1, Available: 1, Suspect: 1,
Draining: 1, Unhealthy: 1, Extracted: 1,
}
if inventories[1] != wantAll {
t.Fatalf("ReadStateInventory(all states) = %+v, want %+v", inventories[1], wantAll)
}
transition := activitypool.HealthUpdate{
ProxyID: "state-0", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
}
if _, err := store.ApplyHealth(context.Background(), transition); err != nil {
t.Fatalf("ApplyHealth(state inventory transition): %v", err)
}
if _, err := store.ApplyHealth(context.Background(), transition); err != nil {
t.Fatalf("ApplyHealth(state inventory replay): %v", err)
}
afterTransition, err := store.ReadStateInventory(context.Background(), []string{upstreamA}, now.Add(time.Second))
if err != nil || len(afterTransition) != 1 || afterTransition[0].Fetched != 0 || afterTransition[0].Checking != 2 {
t.Fatalf("ReadStateInventory(after transition) = %+v, %v", afterTransition, err)
}
extractCommand := extractionDomain.Command{
RequestID: "state-inventory-extract", ClientID: "client-a", Requested: 1,
IdempotencyKey: "state-inventory-extract", IdempotencyTTL: time.Minute,
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
Upstreams: []string{upstreamA},
}
result, err := store.Extract(context.Background(), extractCommand)
if err != nil || result.Returned != 1 {
t.Fatalf("Extract(state inventory) = %+v, %v", result, err)
}
extractCommand.RequestID = "state-inventory-extract-replay"
if replayed, err := store.Extract(context.Background(), extractCommand); err != nil || replayed.Returned != 1 ||
replayed.Items[0].ID != result.Items[0].ID {
t.Fatalf("Extract(state inventory replay) = %+v, %v", replayed, err)
}
afterExtract, err := store.ReadStateInventory(context.Background(), []string{upstreamA}, now.Add(2*time.Second))
if err != nil || len(afterExtract) != 1 || afterExtract[0].Available != 0 || afterExtract[0].Extracted != 2 {
t.Fatalf("ReadStateInventory(after extract) = %+v, %v", afterExtract, err)
}
afterExpiry, err := store.ReadStateInventory(context.Background(), []string{upstreamA, upstreamB}, now.Add(2*time.Minute))
if err != nil || len(afterExpiry) != 2 || afterExpiry[0] != (activitypool.StateInventory{UpstreamID: upstreamA}) ||
afterExpiry[1] != (activitypool.StateInventory{UpstreamID: upstreamB}) {
t.Fatalf("ReadStateInventory(after expiry) = %+v, %v", afterExpiry, err)
}
empty, err := store.ReadStateInventory(context.Background(), nil, now)
if err != nil || len(empty) != 0 {
t.Fatalf("ReadStateInventory(empty) = %+v, %v", empty, err)
}
}
func runConcurrencyContract(t *testing.T, factory Factory) { func runConcurrencyContract(t *testing.T, factory Factory) {
t.Helper() t.Helper()
now := contractNow() now := contractNow()
@ -439,6 +525,10 @@ func runCancellationContract(t *testing.T, store Store) {
return err return err
}}, }},
{name: "inventory", call: func() error { _, err := store.Inventory(ctx, "provider-a", now); return err }}, {name: "inventory", call: func() error { _, err := store.Inventory(ctx, "provider-a", now); return err }},
{name: "state inventory", call: func() error {
_, err := store.ReadStateInventory(ctx, []string{"provider-a"}, now)
return err
}},
{name: "sweep", call: func() error { _, err := store.SweepExpired(ctx, now, 1); return err }}, {name: "sweep", call: func() error { _, err := store.SweepExpired(ctx, now, 1); return err }},
{name: "extract", call: func() error { {name: "extract", call: func() error {
_, err := store.Extract(ctx, extractionDomain.Command{Requested: 1, Fulfillment: extractionDomain.Partial, Now: now}) _, err := store.Extract(ctx, extractionDomain.Command{Requested: 1, Fulfillment: extractionDomain.Partial, Now: now})

View File

@ -60,6 +60,19 @@ type Inventory struct {
Managed int Managed int
} }
// StateInventory is a low-cardinality operational view of one upstream.
// Expired and removed entries are intentionally excluded.
type StateInventory struct {
UpstreamID string
Fetched int64
Checking int64
Available int64
Suspect int64
Draining int64
Unhealthy int64
Extracted int64
}
type HealthStore interface { type HealthStore interface {
ApplyHealth(context.Context, HealthUpdate) (Entry, error) ApplyHealth(context.Context, HealthUpdate) (Entry, error)
} }
@ -68,6 +81,10 @@ type InventoryReader interface {
Inventory(context.Context, string, time.Time) (Inventory, error) Inventory(context.Context, string, time.Time) (Inventory, error)
} }
type StateInventoryReader interface {
ReadStateInventory(context.Context, []string, time.Time) ([]StateInventory, error)
}
type Maintainer interface { type Maintainer interface {
SweepExpired(context.Context, time.Time, int) (int, error) SweepExpired(context.Context, time.Time, int) (int, error)
} }
@ -99,6 +116,7 @@ var (
_ Upserter = (*MemoryPool)(nil) _ Upserter = (*MemoryPool)(nil)
_ HealthStore = (*MemoryPool)(nil) _ HealthStore = (*MemoryPool)(nil)
_ InventoryReader = (*MemoryPool)(nil) _ InventoryReader = (*MemoryPool)(nil)
_ StateInventoryReader = (*MemoryPool)(nil)
_ Maintainer = (*MemoryPool)(nil) _ Maintainer = (*MemoryPool)(nil)
_ extractionDomain.Store = (*MemoryPool)(nil) _ extractionDomain.Store = (*MemoryPool)(nil)
_ ownershipDomain.Repository = (*MemoryPool)(nil) _ ownershipDomain.Repository = (*MemoryPool)(nil)
@ -302,6 +320,50 @@ func (p *MemoryPool) Inventory(ctx context.Context, upstreamID string, now time.
return result, nil return result, nil
} }
func (p *MemoryPool) ReadStateInventory(
ctx context.Context,
upstreamIDs []string,
now time.Time,
) ([]StateInventory, error) {
if ctx == nil {
return nil, ErrInvalidInventory
}
if err := ctx.Err(); err != nil {
return nil, err
}
if p == nil || now.IsZero() {
return nil, ErrInvalidInventory
}
result := make([]StateInventory, len(upstreamIDs))
positions := make(map[string][]int, len(upstreamIDs))
for index, upstreamID := range upstreamIDs {
if upstreamID == "" {
return nil, ErrInvalidInventory
}
result[index].UpstreamID = upstreamID
positions[upstreamID] = append(positions[upstreamID], index)
}
if len(result) == 0 {
return result, nil
}
p.mu.Lock()
defer p.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
for _, entry := range p.entries {
indexes := positions[entry.Proxy.SourceUpstream]
if len(indexes) == 0 || entry.Proxy.ExpiresAt == nil || !entry.Proxy.ExpiresAt.After(now) {
continue
}
for _, index := range indexes {
incrementStateInventory(&result[index], entry.State)
}
}
return result, nil
}
func (p *MemoryPool) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) { func (p *MemoryPool) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) {
if ctx == nil { if ctx == nil {
return 0, ErrInvalidMaintenance return 0, ErrInvalidMaintenance
@ -744,6 +806,28 @@ func managedActivityState(state proxyDomain.State) bool {
} }
} }
func incrementStateInventory(inventory *StateInventory, state proxyDomain.State) {
if inventory == nil {
return
}
switch state {
case proxyDomain.StateFetched:
inventory.Fetched++
case proxyDomain.StateChecking:
inventory.Checking++
case proxyDomain.StateAvailable:
inventory.Available++
case proxyDomain.StateSuspect:
inventory.Suspect++
case proxyDomain.StateDraining:
inventory.Draining++
case proxyDomain.StateUnhealthy:
inventory.Unhealthy++
case proxyDomain.StateExtracted:
inventory.Extracted++
}
}
func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy { func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy {
if candidate.ExpiresAt != nil { if candidate.ExpiresAt != nil {
value := *candidate.ExpiresAt value := *candidate.ExpiresAt

View File

@ -0,0 +1,14 @@
package admission
import "context"
// AllowAll validates the common admission contract without applying another
// quota. It is used when an outer transport boundary already owns rate limits.
type AllowAll struct{}
func (AllowAll) Admit(ctx context.Context, key string) error {
if ctx == nil || key == "" {
return ErrInvalidIdentity
}
return ctx.Err()
}

View File

@ -0,0 +1,23 @@
package admission
import (
"context"
"errors"
"testing"
)
func TestAllowAllPreservesContextAndIdentityValidation(t *testing.T) {
t.Parallel()
var admission AllowAll
if err := admission.Admit(context.Background(), "client-a"); err != nil {
t.Fatalf("Admit(valid) error = %v", err)
}
if err := admission.Admit(context.Background(), ""); !errors.Is(err, ErrInvalidIdentity) {
t.Fatalf("Admit(empty identity) error = %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := admission.Admit(ctx, "client-a"); !errors.Is(err, context.Canceled) {
t.Fatalf("Admit(canceled) error = %v", err)
}
}

View File

@ -0,0 +1,70 @@
package metrics
import (
"context"
"errors"
"net/http"
"reflect"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var ErrInvalidDependencies = errors.New("invalid metrics dependencies")
type ReadinessChecker interface {
Ready(context.Context) error
}
type Dependencies struct {
Gatherer prometheus.Gatherer
Readiness ReadinessChecker
}
func NewHandler(dependencies Dependencies) (http.Handler, error) {
if nilInterface(dependencies.Gatherer) || nilInterface(dependencies.Readiness) {
return nil, ErrInvalidDependencies
}
metricsHandler := promhttp.HandlerFor(dependencies.Gatherer, promhttp.HandlerOpts{})
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodGet {
response.Header().Set("Allow", http.MethodGet)
http.Error(response, "method not allowed", http.StatusMethodNotAllowed)
return
}
switch request.URL.Path {
case "/livez":
writeStatus(response, http.StatusOK, "live")
case "/readyz":
if err := dependencies.Readiness.Ready(request.Context()); err != nil {
writeStatus(response, http.StatusServiceUnavailable, "unavailable")
return
}
writeStatus(response, http.StatusOK, "ready")
case "/metrics":
metricsHandler.ServeHTTP(response, request)
default:
http.NotFound(response, request)
}
}), nil
}
func writeStatus(response http.ResponseWriter, status int, value string) {
response.Header().Set("Content-Type", "text/plain; charset=utf-8")
response.Header().Set("X-Content-Type-Options", "nosniff")
response.WriteHeader(status)
_, _ = response.Write([]byte(value + "\n"))
}
func nilInterface(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

View File

@ -0,0 +1,121 @@
package metrics
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/prometheus/client_golang/prometheus"
)
func TestHandlerServesMetricsAndIndependentProbes(t *testing.T) {
t.Parallel()
registry := prometheus.NewRegistry()
gauge := prometheus.NewGauge(prometheus.GaugeOpts{Name: "proxy_pool_test_inventory"})
gauge.Set(7)
registry.MustRegister(gauge)
readiness := &recordingReadiness{}
handler, err := NewHandler(Dependencies{Gatherer: registry, Readiness: readiness})
if err != nil {
t.Fatalf("NewHandler() error = %v", err)
}
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
assertProbe(t, http.MethodGet, server.URL+"/livez", http.StatusOK, "live")
if readiness.calls.Load() != 0 {
t.Fatalf("livez readiness calls = %d, want 0", readiness.calls.Load())
}
assertProbe(t, http.MethodGet, server.URL+"/readyz", http.StatusOK, "ready")
if readiness.calls.Load() != 1 {
t.Fatalf("readyz readiness calls = %d, want 1", readiness.calls.Load())
}
response, err := http.Get(server.URL + "/metrics")
if err != nil {
t.Fatalf("GET /metrics: %v", err)
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("ReadAll(/metrics): %v", err)
}
if response.StatusCode != http.StatusOK || !strings.Contains(string(body), "proxy_pool_test_inventory 7") {
t.Fatalf("GET /metrics = %d %q", response.StatusCode, body)
}
}
func TestHandlerFailsReadyWithoutLeakingDependencyError(t *testing.T) {
t.Parallel()
const secret = "redis://user:secret@redis:6379"
handler, err := NewHandler(Dependencies{
Gatherer: prometheus.NewRegistry(),
Readiness: &recordingReadiness{err: errors.New(secret)},
})
if err != nil {
t.Fatalf("NewHandler() error = %v", err)
}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
if recorder.Code != http.StatusServiceUnavailable || strings.Contains(recorder.Body.String(), secret) {
t.Fatalf("GET /readyz = %d %q", recorder.Code, recorder.Body.String())
}
}
func TestHandlerRejectsInvalidDependenciesMethodsAndPaths(t *testing.T) {
t.Parallel()
if _, err := NewHandler(Dependencies{}); !errors.Is(err, ErrInvalidDependencies) {
t.Fatalf("NewHandler(empty) error = %v", err)
}
handler, err := NewHandler(Dependencies{
Gatherer: prometheus.NewRegistry(), Readiness: &recordingReadiness{},
})
if err != nil {
t.Fatalf("NewHandler() error = %v", err)
}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/livez", nil))
if recorder.Code != http.StatusMethodNotAllowed || recorder.Header().Get("Allow") != http.MethodGet {
t.Fatalf("POST /livez = %d Allow=%q", recorder.Code, recorder.Header().Get("Allow"))
}
recorder = httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/missing", nil))
if recorder.Code != http.StatusNotFound {
t.Fatalf("GET /missing = %d", recorder.Code)
}
}
type recordingReadiness struct {
calls atomic.Int64
err error
}
func (readiness *recordingReadiness) Ready(context.Context) error {
readiness.calls.Add(1)
return readiness.err
}
func assertProbe(t *testing.T, method, target string, wantStatus int, wantBody string) {
t.Helper()
request, err := http.NewRequest(method, target, nil)
if err != nil {
t.Fatalf("NewRequest(): %v", err)
}
response, err := http.DefaultClient.Do(request)
if err != nil {
t.Fatalf("Do(%s): %v", target, err)
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("ReadAll(%s): %v", target, err)
}
if response.StatusCode != wantStatus || strings.TrimSpace(string(body)) != wantBody {
t.Fatalf("%s = %d %q", target, response.StatusCode, body)
}
}

View File

@ -2,6 +2,49 @@
## 2026-07-30 ## 2026-07-30
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
`redisprovider` AdapterRedis Lua 原子维护 generation、epoch、Leader 租约、
全局 requestInterval 与带 TTL 的 maxInFlight Permit异常时 fail-closed。
- 真实 Redis 8.2 已验证两个 Controller 同 Upstream 只有一个 Leader、取消后
epoch 单调接管、跨实例请求限速/在途上限、Permit 幂等释放,以及 Redis 协调
状态丢失后使用新 generation 自动重建。
- 新增显式 `refill.reconcileInterval/minimumAvailableSlots/targetAvailableSlots`
`fetch.estimatedIPsPerCall`20 份示例、主配置、Compose/Kubernetes 配置均通过
严格解析和理论容量边界校验。
- Pool Reconciler 已按并发槽位实现 minimum/target 迟滞并计入 pending 预估;
FetchBudget 只在没有 pending 请求时接受 Redis Managed 同步,消除 Upsert 与
Permit Complete 短窗口中的重复计数风险。
- Provider Fleet、Worker Active/Reserved 容量汇总和 Controller bootstrap 接线
尚未完成,本轮不增加 51/73 的总验收计数。
- 新增公用 `platform/metrics.NewHandler`,固定提供 `/livez`、`/readyz` 与
`/metrics`Readiness 失败只返回脱敏状态,不泄露底层存储错误。
- Metrics 地址进入严格配置校验Controller Runtime 支持 Metrics-only 和
Distribution/Admin/Metrics 三监听器隔离、首错联动及统一优雅停机。
- Controller bootstrap 使用同一根生命周期装配 Prometheus 默认 Gatherer
Metrics Readiness 按启用能力低成本检查 PostgreSQL/Redis不触发 Provider 或
扫描 Proxy 明细。
- `go test` 定向包和 `.\scripts\test-controller.ps1` 通过;真实 PostgreSQL 18 +
Redis 8.2 fixture 已验证 `/readyz` 与 Prometheus 输出,临时容器和网络已清理。
- 新增 `cmd/proxy-controller` 与公用 `controller/bootstrap.Run`;配置只加载一次,
同一快照用于存储连接、PostgreSQL 管理态提交和 HTTP Runtime 构造,避免启动
期间二次读取产生配置撕裂。
- Bootstrap 已封装 PostgreSQL Ping/迁移/pgx Adapter、Redis Ping/活动池、
Distribution/Admin 服务构造、运行错误与关闭错误合并;`main` 仅处理
`-config`、`PROXY_POOL_CONFIG`、信号上下文和退出码。
- 活动池新增公用 `StateInventoryReader`Memory/Redis 使用同一契约Redis
通过低基数 Hash 和五个原子 Lua 维护七类状态计数,不扫描 Proxy 明细。
- 新增 `controller/operations.Reader`,把当前配置中的 Upstream 与活动状态映射到
Admin StatusWorker/Snapshot 与 Provider 统计尚无来源时保持空/零,不用
`Managed` 冒充 `Available`
- Redis 状态读取在有界过期清理仍有积压或检测到负计数时 fail-closed避免
Admin 返回包含失效代理或损坏计数的成功响应。
- 本轮全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、
`go build ./cmd/proxy-controller``git diff --check` 通过;新加的 Redis
backlog/负计数场景也已通过真实 Redis 8.2 fixture。
- 新增 `test-controller.ps1` 双存储 fixture真实 PostgreSQL 18 + Redis 8.2 已
通过迁移、启动配置提交、Redis Readiness 与 Admin Status 组合验证。
- 固定 `github.com/jackc/pgx/v5 v5.6.0`,实现封装在 `adminstate.Store` 后的 - 固定 `github.com/jackc/pgx/v5 v5.6.0`,实现封装在 `adminstate.Store` 后的
PostgreSQL 深适配器配置、Upstream、Routing mutation 在同一事务中提交 PostgreSQL 深适配器配置、Upstream、Routing mutation 在同一事务中提交
revision、管理状态、审计与 Outbox数据库错误不泄漏 DSN、SQL 或参数。 revision、管理状态、审计与 Outbox数据库错误不泄漏 DSN、SQL 或参数。

View File

@ -0,0 +1,42 @@
$ErrorActionPreference = "Stop"
$repositoryRoot = Split-Path -Parent $PSScriptRoot
$composeFile = Join-Path $repositoryRoot "deploy/docker-compose.test.yml"
$composeProject = "proxy-pool-controller-test"
$previousPostgresURL = [Environment]::GetEnvironmentVariable("PROXY_POOL_TEST_POSTGRES_URL", "Process")
$previousRedisURL = [Environment]::GetEnvironmentVariable("PROXY_POOL_TEST_REDIS_URL", "Process")
try {
docker compose -p $composeProject -f $composeFile up -d --wait --wait-timeout 60 postgres redis
if ($LASTEXITCODE -ne 0) {
throw "starting Controller test fixtures failed with exit code $LASTEXITCODE"
}
$env:PROXY_POOL_TEST_POSTGRES_URL = "postgres://proxy_pool_test:proxy-pool-test@127.0.0.1:15432/proxy_pool_test?sslmode=disable"
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
Push-Location $repositoryRoot
try {
go test -count=1 -tags=integration -timeout 60s ./internal/controller/bootstrap
if ($LASTEXITCODE -ne 0) {
throw "Controller integration tests failed with exit code $LASTEXITCODE"
}
}
finally {
Pop-Location
}
}
finally {
if ($null -eq $previousPostgresURL) {
Remove-Item Env:PROXY_POOL_TEST_POSTGRES_URL -ErrorAction SilentlyContinue
}
else {
$env:PROXY_POOL_TEST_POSTGRES_URL = $previousPostgresURL
}
if ($null -eq $previousRedisURL) {
Remove-Item Env:PROXY_POOL_TEST_REDIS_URL -ErrorAction SilentlyContinue
}
else {
$env:PROXY_POOL_TEST_REDIS_URL = $previousRedisURL
}
docker compose -p $composeProject -f $composeFile down --volumes --remove-orphans
}

View File

@ -14,7 +14,7 @@ try {
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15" $env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
Push-Location $repositoryRoot Push-Location $repositoryRoot
try { try {
go test -count=1 -tags=integration -timeout 60s ./internal/adapters/redisactivity/... go test -count=1 -tags=integration -timeout 60s ./internal/adapters/redisactivity/... ./internal/adapters/redisprovider/...
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
throw "Redis integration tests failed with exit code $LASTEXITCODE" throw "Redis integration tests failed with exit code $LASTEXITCODE"
} }

View File

@ -33,6 +33,9 @@
和真实 PostgreSQL 18 集成测试已完成 和真实 PostgreSQL 18 集成测试已完成
12. [进行中] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界; 12. [进行中] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界;
机器契约和文档类滞后勾选已按仓库证据校正 机器契约和文档类滞后勾选已按仓库证据校正
13. [进行中] 落地 `proxy-controller` 进程装配配置单次加载、PostgreSQL 迁移、
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
双存储 bootstrap 和探针集成已通过Provider、业务指标与完整容器进程链仍待实现
## 串并行关系 ## 串并行关系
@ -53,7 +56,8 @@
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证Redis 8.2 - Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证Redis 8.2
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。 与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
- `cmd/proxy-*`、PostgreSQL 管理面 Adapter、Provider Leader/分布式限流、 - `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
Checker 运行时、Redis 故障转移验证与代表性集群压测属于后续实施范围。 Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
故障转移验证与代表性集群压测属于后续实施范围。
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成, - `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
验收项完成率约 69.9%,不等同于生产就绪度。 验收项完成率约 69.9%,不等同于生产就绪度。