Compare commits
No commits in common. "e9945d933f6c064dd74c47feae048a49a1e7df51" and "96701e4241f6b3149a2897f7e8499f4069ee63db" have entirely different histories.
e9945d933f
...
96701e4241
3
.github/workflows/ci.yml
vendored
3
.github/workflows/ci.yml
vendored
@ -48,6 +48,3 @@ jobs:
|
|||||||
- name: PostgreSQL admin-state contract
|
- name: PostgreSQL admin-state contract
|
||||||
shell: pwsh
|
shell: pwsh
|
||||||
run: ./scripts/test-postgres.ps1
|
run: ./scripts/test-postgres.ps1
|
||||||
- name: Controller dual-store bootstrap
|
|
||||||
shell: pwsh
|
|
||||||
run: ./scripts/test-controller.ps1
|
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -9,7 +9,6 @@ coverage/
|
|||||||
*.out
|
*.out
|
||||||
*.test
|
*.test
|
||||||
*.prof
|
*.prof
|
||||||
*.exe
|
|
||||||
.tmp-proto/
|
.tmp-proto/
|
||||||
|
|
||||||
# Local configuration and secrets
|
# Local configuration and secrets
|
||||||
|
|||||||
@ -1,63 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/controller/bootstrap"
|
|
||||||
)
|
|
||||||
|
|
||||||
const configEnvironment = "PROXY_POOL_CONFIG"
|
|
||||||
|
|
||||||
type environmentLookup func(string) string
|
|
||||||
type controllerRun func(context.Context, bootstrap.Options) error
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
||||||
defer stop()
|
|
||||||
os.Exit(execute(ctx, os.Args[1:], os.Getenv, bootstrap.Run, os.Stderr))
|
|
||||||
}
|
|
||||||
|
|
||||||
func execute(
|
|
||||||
ctx context.Context,
|
|
||||||
args []string,
|
|
||||||
getenv environmentLookup,
|
|
||||||
run controllerRun,
|
|
||||||
stderr io.Writer,
|
|
||||||
) int {
|
|
||||||
flags := flag.NewFlagSet("proxy-controller", flag.ContinueOnError)
|
|
||||||
flags.SetOutput(stderr)
|
|
||||||
configPath := flags.String("config", "", "configuration file path")
|
|
||||||
if err := flags.Parse(args); err != nil {
|
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if flags.NArg() != 0 {
|
|
||||||
_, _ = fmt.Fprintln(stderr, "proxy-controller: unexpected positional arguments")
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if *configPath == "" && getenv != nil {
|
|
||||||
*configPath = getenv(configEnvironment)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(*configPath) != *configPath || *configPath == "" || ctx == nil || run == nil {
|
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-controller: -config or %s is required\n", configEnvironment)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
|
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-controller: %v\n", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
@ -1,86 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"proxy-pool/internal/controller/bootstrap"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
var received bootstrap.Options
|
|
||||||
code := execute(context.Background(), []string{"-config", "flag.yaml"}, func(name string) string {
|
|
||||||
if name == configEnvironment {
|
|
||||||
return "environment.yaml"
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}, func(_ context.Context, options bootstrap.Options) error {
|
|
||||||
received = options
|
|
||||||
return nil
|
|
||||||
}, &bytes.Buffer{})
|
|
||||||
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
|
|
||||||
t.Fatalf("execute() = %d, options = %+v", code, received)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
var received bootstrap.Options
|
|
||||||
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
|
|
||||||
_ context.Context,
|
|
||||||
options bootstrap.Options,
|
|
||||||
) error {
|
|
||||||
received = options
|
|
||||||
return nil
|
|
||||||
}, &bytes.Buffer{})
|
|
||||||
if code != 0 || received.ConfigPath != "environment.yaml" {
|
|
||||||
t.Fatalf("execute() = %d, config = %q", code, received.ConfigPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteReturnsUsageCodeWithoutConfiguration(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
called := false
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
code := execute(context.Background(), nil, func(string) string { return "" }, func(
|
|
||||||
context.Context,
|
|
||||||
bootstrap.Options,
|
|
||||||
) error {
|
|
||||||
called = true
|
|
||||||
return nil
|
|
||||||
}, &stderr)
|
|
||||||
if code != 2 || called || stderr.Len() == 0 {
|
|
||||||
t.Fatalf("execute() = %d, called = %t, stderr = %q", code, called, stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteMapsStartupFailureAndSignalCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
want := errors.New("startup failed")
|
|
||||||
code := execute(context.Background(), []string{"-config", "config.yaml"}, func(string) string { return "" }, func(
|
|
||||||
context.Context,
|
|
||||||
bootstrap.Options,
|
|
||||||
) error {
|
|
||||||
return want
|
|
||||||
}, &stderr)
|
|
||||||
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
|
|
||||||
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
stderr.Reset()
|
|
||||||
code = execute(ctx, []string{"-config", "config.yaml"}, func(string) string { return "" }, func(
|
|
||||||
context.Context,
|
|
||||||
bootstrap.Options,
|
|
||||||
) error {
|
|
||||||
return context.Canceled
|
|
||||||
}, &stderr)
|
|
||||||
if code != 0 || stderr.Len() != 0 {
|
|
||||||
t.Fatalf("execute(canceled) = %d, stderr = %q", code, stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -5,7 +5,6 @@ security:
|
|||||||
|
|
||||||
defaults:
|
defaults:
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
@ -140,15 +139,10 @@ upstreams:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 20
|
maxConcurrencyPerProxy: 20
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 5m
|
ttl: 5m
|
||||||
allocationSafetyMargin: 30s
|
allocationSafetyMargin: 30s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
@ -198,15 +192,10 @@ upstreams:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 10
|
maxConcurrencyPerProxy: 10
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 2m
|
ttl: 2m
|
||||||
allocationSafetyMargin: 20s
|
allocationSafetyMargin: 20s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 2s
|
requestInterval: 2s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
|
|||||||
@ -76,26 +76,6 @@ func TestPostgresFixtureScriptUsesDedicatedComposeProject(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestControllerFixtureScriptStartsBothStoresInDedicatedProject(t *testing.T) {
|
|
||||||
payload, err := os.ReadFile("../scripts/test-controller.ps1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read test-controller.ps1: %v", err)
|
|
||||||
}
|
|
||||||
script := string(payload)
|
|
||||||
for _, required := range []string{
|
|
||||||
`-p $composeProject`,
|
|
||||||
`up -d --wait --wait-timeout 60 postgres redis`,
|
|
||||||
`PROXY_POOL_TEST_POSTGRES_URL`,
|
|
||||||
`PROXY_POOL_TEST_REDIS_URL`,
|
|
||||||
`go test -count=1 -tags=integration -timeout 60s ./internal/controller/bootstrap`,
|
|
||||||
`down --volumes --remove-orphans`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(script, required) {
|
|
||||||
t.Errorf("test-controller.ps1 missing %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) {
|
func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) {
|
||||||
document := loadComposeDocument(t)
|
document := loadComposeDocument(t)
|
||||||
redis, ok := document.Services["redis"]
|
redis, ok := document.Services["redis"]
|
||||||
|
|||||||
@ -111,15 +111,10 @@ upstreams:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 20
|
maxConcurrencyPerProxy: 20
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 5m
|
ttl: 5m
|
||||||
allocationSafetyMargin: 20s
|
allocationSafetyMargin: 20s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
@ -161,15 +156,10 @@ upstreams:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 20
|
maxConcurrencyPerProxy: 20
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 5m
|
ttl: 5m
|
||||||
allocationSafetyMargin: 20s
|
allocationSafetyMargin: 20s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
|
|||||||
@ -110,15 +110,10 @@ data:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 20
|
maxConcurrencyPerProxy: 20
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 5m
|
ttl: 5m
|
||||||
allocationSafetyMargin: 20s
|
allocationSafetyMargin: 20s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
@ -157,15 +152,10 @@ data:
|
|||||||
shrinkDelay: 30s
|
shrinkDelay: 30s
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 20
|
maxConcurrencyPerProxy: 20
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 5m
|
ttl: 5m
|
||||||
allocationSafetyMargin: 20s
|
allocationSafetyMargin: 20s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
|
|||||||
@ -10,16 +10,8 @@
|
|||||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
Controller 入口已实现,源码运行方式为:
|
规划中的生产入口为 `proxy-controller -config CONFIG_FILE`;该命令完成实现和
|
||||||
|
进程级测试前,不作为当前可执行能力。
|
||||||
```powershell
|
|
||||||
go run ./cmd/proxy-controller -config CONFIG_FILE
|
|
||||||
```
|
|
||||||
|
|
||||||
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
|
|
||||||
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
|
|
||||||
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
|
|
||||||
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
|
|
||||||
|
|
||||||
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
||||||
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
||||||
@ -36,10 +28,6 @@ Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范
|
|||||||
新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新
|
新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新
|
||||||
分配,已有连接进入 Drain,不强制中断。
|
分配,已有连接进入 Drain,不强制中断。
|
||||||
|
|
||||||
当前 Controller 启动时使用同一份不可变快照完成存储、入口和提取策略装配。
|
|
||||||
Admin 重载会原子提交新配置、审计并发布到配置 Store;监听地址、存储连接和
|
|
||||||
已构造的安全/提取策略尚未自动重建,这些字段变更后需要重启 Controller。
|
|
||||||
|
|
||||||
## 2. 根结构
|
## 2. 根结构
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@ -249,13 +237,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000, shrinkDelay: 30s}
|
pool: {maxSize: 1000, shrinkDelay: 30s}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 2000
|
|
||||||
targetAvailableSlots: 5000
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
@ -320,8 +303,6 @@ proxyAuth:
|
|||||||
|
|
||||||
### 7.3 Fetch 限制
|
### 7.3 Fetch 限制
|
||||||
|
|
||||||
- `estimatedIPsPerCall`:冷启动时每次 Provider 调用预计返回的合法 Proxy 数,
|
|
||||||
同时用于 `pool.maxSize` 的 pending 预占;不得从任意 Query 或 Body 字段推断。
|
|
||||||
- `requestInterval`:同一 Provider 请求间隔。
|
- `requestInterval`:同一 Provider 请求间隔。
|
||||||
- `timeout`:单次调用超时。
|
- `timeout`:单次调用超时。
|
||||||
- `maxAttempts`:单次补池动作最大尝试次数。
|
- `maxAttempts`:单次补池动作最大尝试次数。
|
||||||
@ -333,18 +314,7 @@ proxyAuth:
|
|||||||
大量缺池信号必须合并成 singleflight 或容量为 1 的通知,不能按 Gateway 请求
|
大量缺池信号必须合并成 singleflight 或容量为 1 的通知,不能按 Gateway 请求
|
||||||
数量线性触发 Provider API。
|
数量线性触发 Provider API。
|
||||||
|
|
||||||
### 7.4 Refill 水位
|
### 7.4 生命周期与健康
|
||||||
|
|
||||||
- `reconcileInterval`:无事件时重新读取库存的兜底周期,启动时仍立即检查一次。
|
|
||||||
- `minimumAvailableSlots`:可用并发槽位低于该值时进入补池。
|
|
||||||
- `targetAvailableSlots`:进入补池后持续补到该目标,再退出补池状态。
|
|
||||||
|
|
||||||
三个字段均为必填正值,且 `targetAvailableSlots > minimumAvailableSlots`。
|
|
||||||
目标不得超过 `pool.maxSize * capacity.maxConcurrencyPerProxy` 的理论上限。
|
|
||||||
`requestInterval` 只限制外部 API 调用,不能兼任库存复核周期。pending Proxy
|
|
||||||
按 `estimatedIPsPerCall * maxConcurrencyPerProxy` 折算槽位,避免并发补池超量。
|
|
||||||
|
|
||||||
### 7.5 生命周期与健康
|
|
||||||
|
|
||||||
- 明确绝对过期时间优先于响应 TTL,响应 TTL 优先于配置 `lifecycle.ttl`。
|
- 明确绝对过期时间优先于响应 TTL,响应 TTL 优先于配置 `lifecycle.ttl`。
|
||||||
- 距离过期不足 `allocationSafetyMargin` 时停止新分配。
|
- 距离过期不足 `allocationSafetyMargin` 时停止新分配。
|
||||||
@ -376,13 +346,6 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
|
|||||||
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
|
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
|
||||||
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
||||||
|
|
||||||
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`、
|
|
||||||
`/readyz` 和 `/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
|
|
||||||
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标,Provider、提取和容量等
|
|
||||||
业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
|
|
||||||
服务流量门槛,PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
|
|
||||||
变更需要重启 Controller。
|
|
||||||
|
|
||||||
## 9. 启动前校验清单
|
## 9. 启动前校验清单
|
||||||
|
|
||||||
1. `version` 必须为 `1`,未知字段拒绝。
|
1. `version` 必须为 `1`,未知字段拒绝。
|
||||||
@ -390,9 +353,8 @@ Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `
|
|||||||
3. 非回环监听器满足认证或来源 CIDR 保护。
|
3. 非回环监听器满足认证或来源 CIDR 保护。
|
||||||
4. Routing 名称唯一,正则可编译,引用的 Upstream 存在。
|
4. Routing 名称唯一,正则可编译,引用的 Upstream 存在。
|
||||||
5. Sequential 至少引用两个 Upstream、阈值大于零,`onUnavailable.action` 明确。
|
5. Sequential 至少引用两个 Upstream、阈值大于零,`onUnavailable.action` 明确。
|
||||||
6. 启用的 Upstream 有正数 `pool.maxSize`、并发、Refill 水位和 Fetch 估值/限制。
|
6. 启用的 Upstream 有正数 `pool.maxSize`、并发和 Fetch 限制。
|
||||||
7. `allocationSafetyMargin < ttl`。
|
7. `allocationSafetyMargin < ttl`。
|
||||||
8. `fetch.maxTotal == 0` 或 `fetch.maxTotal >= pool.maxSize`。
|
8. `fetch.maxTotal == 0` 或 `fetch.maxTotal >= pool.maxSize`。
|
||||||
9. `minimumAvailableSlots < targetAvailableSlots`,且目标不超过理论并发容量。
|
9. Distribution 的 fulfillment 合法,单次数量大于零。
|
||||||
10. Distribution 的 fulfillment 合法,单次数量大于零。
|
10. Secret 未写入日志可见配置转储。
|
||||||
11. Secret 未写入日志可见配置转储。
|
|
||||||
|
|||||||
@ -226,7 +226,7 @@ flowchart TD
|
|||||||
SF -->|yes| W[Coalesce signal]
|
SF -->|yes| W[Coalesce signal]
|
||||||
SF -->|no| R[Read current demand]
|
SF -->|no| R[Read current demand]
|
||||||
R --> L[Acquire Provider leader]
|
R --> L[Acquire Provider leader]
|
||||||
L --> C{Below refill target and under maxSize/maxTotal?}
|
L --> C{Under maxSize/maxTotal?}
|
||||||
C -->|no| X[Stop]
|
C -->|no| X[Stop]
|
||||||
C -->|yes| I[Wait requestInterval]
|
C -->|yes| I[Wait requestInterval]
|
||||||
I --> M[Acquire maxInFlight]
|
I --> M[Acquire maxInFlight]
|
||||||
@ -238,12 +238,6 @@ flowchart TD
|
|||||||
DD --> HC[Create FETCHED and schedule check]
|
DD --> HC[Create FETCHED and schedule check]
|
||||||
```
|
```
|
||||||
|
|
||||||
每个 Upstream 的 Leader、最小请求间隔和在途 Permit 由同一个 Redis 原子协调
|
|
||||||
模块维护。Leader 租约使用 generation + 单调 epoch fence;Redis 状态整体丢失后
|
|
||||||
生成新 generation 并重新竞选。任何续租不确定、记录损坏或 Redis 断连均
|
|
||||||
fail-closed,不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
|
|
||||||
复核期间若仍有 pending Fetch,则等待下一轮再同步 Managed,避免重复计数。
|
|
||||||
|
|
||||||
### 8.1 Empty、Duplicate 与 Error
|
### 8.1 Empty、Duplicate 与 Error
|
||||||
|
|
||||||
- **Empty**:HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
|
- **Empty**:HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
|
||||||
|
|||||||
@ -13,7 +13,7 @@ proxy-pool/
|
|||||||
│ ├── config/ # 严格配置解析和校验
|
│ ├── config/ # 严格配置解析和校验
|
||||||
│ ├── domain/ # 无传输、无存储依赖的领域模型
|
│ ├── domain/ # 无传输、无存储依赖的领域模型
|
||||||
│ ├── gateway/ # snapshot、dispatch、server、transport
|
│ ├── gateway/ # snapshot、dispatch、server、transport
|
||||||
│ ├── controller/ # provider、pool、extraction、operations、runtime、bootstrap
|
│ ├── controller/ # provider、pool、routing、extraction、health、runtime
|
||||||
│ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配
|
│ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配
|
||||||
│ └── platform/ # HTTP、安全、日志、指标、停机和进程装配
|
│ └── platform/ # HTTP、安全、日志、指标、停机和进程装配
|
||||||
├── api/ # OpenAPI 与 Protobuf 契约
|
├── api/ # OpenAPI 与 Protobuf 契约
|
||||||
|
|||||||
@ -37,7 +37,7 @@ internal/
|
|||||||
domain/extraction/{extraction.go,store.go}
|
domain/extraction/{extraction.go,store.go}
|
||||||
domain/client/client.go
|
domain/client/client.go
|
||||||
gateway/{server,dispatch,snapshot,transport}/
|
gateway/{server,dispatch,snapshot,transport}/
|
||||||
controller/{provider,pool,routing,extraction,health,distribution,operations,runtime,bootstrap}/
|
controller/{provider,pool,routing,extraction,health,distribution}/
|
||||||
adapters/{memory,postgres,redis,providerapi}/
|
adapters/{memory,postgres,redis,providerapi}/
|
||||||
platform/{logging,metrics,shutdown}/
|
platform/{logging,metrics,shutdown}/
|
||||||
api/{openapi,proto}/
|
api/{openapi,proto}/
|
||||||
@ -196,31 +196,14 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
|
|||||||
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
|
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
|
||||||
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
|
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
|
||||||
聚合与配置重载接到同一公用 seam;严格文件加载、脱敏管理摘要及原子配置发布
|
聚合与配置重载接到同一公用 seam;严格文件加载、脱敏管理摘要及原子配置发布
|
||||||
已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
|
已通过失败路径和并发测试。生产命令入口及其连接池/迁移启动装配仍待实现。
|
||||||
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
|
|
||||||
Distribution/Admin 服务构造、错误合并和资源关闭;Provider 调度及完整 HTTP
|
|
||||||
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`、
|
|
||||||
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
|
||||||
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
|
||||||
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
|
||||||
|
|
||||||
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
||||||
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
||||||
`usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、
|
`usableUntil` 向 Worker Snapshot 的传播与 Gateway 本地截止过滤、
|
||||||
重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis
|
重复刷新、过期淘汰、独占提取、短期幂等及 Worker ownership 互斥。生产 Redis
|
||||||
Adapter 已通过真实 Redis 8.2 运行同一套公用契约;原子 Lua 覆盖提取、所有权和
|
Adapter 已通过真实 Redis 8.2 运行同一套公用契约;原子 Lua 覆盖提取、所有权和
|
||||||
有界清理。新增低基数 StateInventory Hash,五类写脚本在同一原子边界维护状态
|
有界清理。Redis Sentinel/故障转移验证与代表性多节点压测仍待实施。
|
||||||
计数,读取不扫描 Proxy 明细;过期清理积压或负计数时 fail-closed。Redis
|
|
||||||
Sentinel/故障转移验证与代表性多节点压测仍待实施。
|
|
||||||
|
|
||||||
Provider 分布式协调已新增公用 `Coordinator.RunLeader` / `LeaderSession` seam 与
|
|
||||||
独立 `redisprovider` Adapter。真实 Redis 8.2 已验证同 Upstream 双实例互斥、
|
|
||||||
generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、TTL
|
|
||||||
回收及 Redis 状态丢失后的新 generation 自动重建;Redis 异常期间不发放请求。
|
|
||||||
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`,Pool Reconciler
|
|
||||||
已实现迟滞与 pending 槽位折算,FetchBudget 仅在无 pending 时同步 Redis 权威
|
|
||||||
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成,
|
|
||||||
因此本轮不勾选 Task 10 的组合验收项。
|
|
||||||
|
|
||||||
## Task 11: Checker and Health Reducer
|
## Task 11: Checker and Health Reducer
|
||||||
|
|
||||||
|
|||||||
@ -18,12 +18,9 @@
|
|||||||
|
|
||||||
## 2. 本地拓扑模板
|
## 2. 本地拓扑模板
|
||||||
|
|
||||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
当前仓库交付设计、契约、部署拓扑和关键领域实现;`cmd/proxy-*` 的完整运行时
|
||||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
|
装配属于 `implementation-plan.md` 后续任务。此处 Compose/Kubernetes 资产用于
|
||||||
指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
|
评审网络、资源、探针和依赖关系,当前只执行静态渲染,不把模板写成可运行服务。
|
||||||
后续任务。
|
|
||||||
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
|
||||||
视为完整可运行拓扑。
|
|
||||||
|
|
||||||
### 2.1 前置条件
|
### 2.1 前置条件
|
||||||
|
|
||||||
@ -63,17 +60,11 @@ tmpfs。`ApplyMigrations` 在同一物理连接上执行仓库内嵌的幂等前
|
|||||||
每个契约创建唯一 Schema,结束时只删除该 Schema 和临时 Compose 项目。该脚本
|
每个契约创建唯一 Schema,结束时只删除该 Schema 和临时 Compose 项目。该脚本
|
||||||
禁止指向开发或生产数据库。
|
禁止指向开发或生产数据库。
|
||||||
|
|
||||||
`.\scripts\test-controller.ps1` 同时启动两个隔离 fixture,验证 Controller
|
|
||||||
bootstrap 的迁移、启动配置提交、Redis Readiness、Admin Status、`/readyz` 与
|
|
||||||
Prometheus 输出。脚本不启动
|
|
||||||
部署模板中的 Controller 容器,也不连接开发或生产存储。
|
|
||||||
|
|
||||||
目标拓扑入口:
|
目标拓扑入口:
|
||||||
|
|
||||||
- Gateway:`127.0.0.1:8080`
|
- Gateway:`127.0.0.1:8080`
|
||||||
- Distribution:`http://127.0.0.1:8081`
|
- Distribution:`http://127.0.0.1:8081`
|
||||||
- Admin:`http://127.0.0.1:8082`
|
- Admin:`http://127.0.0.1:8082`
|
||||||
- Controller Metrics:`http://127.0.0.1:9090`
|
|
||||||
- HAProxy 状态:`http://127.0.0.1:8404/stats`
|
- HAProxy 状态:`http://127.0.0.1:8404/stats`
|
||||||
- Prometheus:`http://127.0.0.1:9091`
|
- Prometheus:`http://127.0.0.1:9091`
|
||||||
- Grafana:`http://127.0.0.1:3000`
|
- Grafana:`http://127.0.0.1:3000`
|
||||||
@ -114,9 +105,6 @@ kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=10m
|
|||||||
的完整 Snapshot 且仍有准入能力时才 Ready。
|
的完整 Snapshot 且仍有准入能力时才 Ready。
|
||||||
- Controller 按能力判定就绪:Distribution/Fetch 依赖 Redis 活动池,Admin 持久化
|
- Controller 按能力判定就绪:Distribution/Fetch 依赖 Redis 活动池,Admin 持久化
|
||||||
写依赖 PostgreSQL 与兼容迁移。PostgreSQL 故障不得单独使 Extract 返回 503。
|
写依赖 PostgreSQL 与兼容迁移。PostgreSQL 故障不得单独使 Extract 返回 503。
|
||||||
- 同一 Controller 同时启用 Distribution 与 Admin 时,Pod `/readyz` 以 Redis
|
|
||||||
活动池为服务流量门槛;PostgreSQL 故障由 Admin 接口独立返回不可用,不把仍可
|
|
||||||
完成的 Extract 从 Service 摘除。Admin-only 进程才同时检查 PostgreSQL 与 Redis。
|
|
||||||
- Checker 在任务消费与结果上报通道可用时 Ready。
|
- Checker 在任务消费与结果上报通道可用时 Ready。
|
||||||
- `/metrics`:独立于业务入口,NetworkPolicy 仅允许监控命名空间访问。
|
- `/metrics`:独立于业务入口,NetworkPolicy 仅允许监控命名空间访问。
|
||||||
|
|
||||||
|
|||||||
@ -76,14 +76,12 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt
|
|||||||
|
|
||||||
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
||||||
|
|
||||||
1. `cmd/proxy-gateway/checker/loadgen` 进程装配;`proxy-controller` 已完成
|
1. `cmd/proxy-gateway/controller/checker/loadgen` 进程装配。
|
||||||
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配,但 Provider 和业务
|
|
||||||
指标链未闭环。
|
|
||||||
2. Gateway 进程装配、生产连接池调优与代表性流量压测。
|
2. Gateway 进程装配、生产连接池调优与代表性流量压测。
|
||||||
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
|
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
|
||||||
4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成;
|
4. PostgreSQL 连接池、迁移和 pgx Adapter 的生产命令启动装配,以及可选聚合指标;
|
||||||
公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成,Controller
|
Schema、领域 seam、Memory/pgx Adapter、真实 PostgreSQL 18 契约和 Admin
|
||||||
三监听器与探针集成已完成;可选聚合指标和完整容器进程部署验证仍待实现。
|
应用层接线已经完成。
|
||||||
5. Redis Provider Leader、分布式速率与 Client 限制、Worker 心跳和自动重建;
|
5. Redis Provider Leader、分布式速率与 Client 限制、Worker 心跳和自动重建;
|
||||||
TTL 活动池、原子提取和 Worker ownership 已完成。
|
TTL 活动池、原子提取和 Worker ownership 已完成。
|
||||||
6. Worker 网络快照流;Redis ownership drain/ACK/过期回收已完成。
|
6. Worker 网络快照流;Redis ownership drain/ACK/过期回收已完成。
|
||||||
|
|||||||
@ -7,9 +7,9 @@
|
|||||||
|
|
||||||
| ID | 最终需求 | 来源 | 验证证据 |
|
| ID | 最终需求 | 来源 | 验证证据 |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 命令已实现,Gateway/Checker/Loadgen 构建产物待实现 |
|
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;四个 `cmd/proxy-*` 构建产物待实现 |
|
||||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
|
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
|
||||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 生产入口待装配 |
|
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Distribution/Admin 独立监听已测试;Gateway/Metrics 生产入口待装配 |
|
||||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成;分布式 Leader 与运行装配待完成 |
|
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成;分布式 Leader 与运行装配待完成 |
|
||||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
||||||
|
|
||||||
@ -32,7 +32,7 @@
|
|||||||
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
|
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
|
||||||
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
|
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
|
||||||
| FETCH-003 | 错误使用指数退避和抖动,429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
|
| FETCH-003 | 错误使用指数退避和抖动,429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
|
||||||
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;Provider Fleet/bootstrap 接线待完成 |
|
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | 单进程 Reconciler 已完成;Redis Leader 租约及多实例互斥测试待完成 |
|
||||||
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
|
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
|
||||||
| FETCH-006 | 重复候选不当作 Empty,记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
|
| FETCH-006 | 重复候选不当作 Empty,记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
|
||||||
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
|
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
|
||||||
@ -46,7 +46,7 @@
|
|||||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
|
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
|
||||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
|
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | TTL/状态/Active/Reserved 的 `AvailableSlots` 与 Reconciler 已测试;ownership、目标健康及 Gateway reserve 聚合待完成 |
|
||||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||||
@ -86,5 +86,5 @@
|
|||||||
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
||||||
| OPS-002 | 优雅停机停止新请求/Fetch,等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
| OPS-002 | 优雅停机停止新请求/Fetch,等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
||||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
|
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
|
||||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现且当前只暴露无业务标签的 Go/进程指标;低基数业务 Collector 与描述符测试待实现 |
|
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | 文档和配置已约束;Prometheus 指标模块及描述符测试待实现 |
|
||||||
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单;Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |
|
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单;Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |
|
||||||
|
|||||||
@ -50,12 +50,6 @@ go build ./...
|
|||||||
该 fixture 使用唯一命名空间,不执行 `FLUSHDB`,并关闭 AOF、RDB 与数据卷;
|
该 fixture 使用唯一命名空间,不执行 `FLUSHDB`,并关闭 AOF、RDB 与数据卷;
|
||||||
测试结束后按命名空间清理活动池、所有权和幂等键。
|
测试结束后按命名空间清理活动池、所有权和幂等键。
|
||||||
|
|
||||||
Controller 的 PostgreSQL + Redis 启动组合测试使用:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\test-controller.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock;集成和
|
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock;集成和
|
||||||
soak 测试单独标记,不混入快速单测。
|
soak 测试单独标记,不混入快速单测。
|
||||||
|
|
||||||
|
|||||||
@ -108,16 +108,6 @@ Routing CAS、Repeatable Read 快照、审计分页、Routing no-op、`SKIP LOCK
|
|||||||
数据卷。静态与 `information_schema` 双重检查证明只存在六张管理表,且没有
|
数据卷。静态与 `information_schema` 双重检查证明只存在六张管理表,且没有
|
||||||
Proxy、凭据、逐次提取、Worker ownership 或幂等明细列。
|
Proxy、凭据、逐次提取、Worker ownership 或幂等明细列。
|
||||||
|
|
||||||
Controller bootstrap 的双存储组合验证命令是:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\test-controller.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
该 fixture 同时启动 PostgreSQL 18 与 Redis 8.2,验证迁移、启动配置提交、
|
|
||||||
Redis Readiness 和 Admin Status;HTTP Runner 使用测试 Adapter,避免占用业务
|
|
||||||
监听端口。测试数据仅存在于隔离 Compose 项目和 PostgreSQL tmpfs。
|
|
||||||
|
|
||||||
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
|
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
|
||||||
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
|
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
|
||||||
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
|
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
|
||||||
|
|||||||
@ -48,7 +48,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 100}
|
pool: {maxSize: 100}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -28,7 +28,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 2000}
|
pool: {maxSize: 2000}
|
||||||
capacity: {maxConcurrencyPerProxy: 20}
|
capacity: {maxConcurrencyPerProxy: 20}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -30,7 +30,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -32,7 +32,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 3000}
|
pool: {maxSize: 3000}
|
||||||
capacity: {maxConcurrencyPerProxy: 20}
|
capacity: {maxConcurrencyPerProxy: 20}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -33,7 +33,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 30s}
|
lifecycle: {ttl: 3m, allocationSafetyMargin: 30s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
||||||
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -40,7 +40,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -32,7 +32,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -22,9 +22,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
provider-b:
|
provider-b:
|
||||||
enabled: true
|
enabled: true
|
||||||
@ -34,7 +33,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 3m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -21,9 +21,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
provider-b:
|
provider-b:
|
||||||
enabled: true
|
enabled: true
|
||||||
@ -33,7 +32,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -19,9 +19,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
provider-b:
|
provider-b:
|
||||||
enabled: true
|
enabled: true
|
||||||
exposure: [gateway]
|
exposure: [gateway]
|
||||||
@ -30,6 +29,5 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
|
|||||||
@ -19,9 +19,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
provider-b:
|
provider-b:
|
||||||
enabled: true
|
enabled: true
|
||||||
exposure: [gateway]
|
exposure: [gateway]
|
||||||
@ -30,6 +29,5 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
|
|||||||
@ -23,7 +23,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 10000}
|
pool: {maxSize: 10000}
|
||||||
capacity: {maxConcurrencyPerProxy: 50}
|
capacity: {maxConcurrencyPerProxy: 50}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -29,7 +29,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000}
|
||||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -37,7 +37,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 2000}
|
pool: {maxSize: 2000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -29,7 +29,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 60s}
|
lifecycle: {ttl: 5m, allocationSafetyMargin: 60s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
||||||
check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -26,7 +26,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -27,7 +27,6 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 500}
|
pool: {maxSize: 500}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -27,10 +27,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
capacity: {maxConcurrencyPerProxy: 10}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 10
|
|
||||||
requestInterval: 2s
|
requestInterval: 2s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
|
|||||||
@ -22,7 +22,6 @@ upstreams:
|
|||||||
password: "${SOCKS_PASSWORD}"
|
password: "${SOCKS_PASSWORD}"
|
||||||
pool: {maxSize: 1000}
|
pool: {maxSize: 1000}
|
||||||
capacity: {maxConcurrencyPerProxy: 20}
|
capacity: {maxConcurrencyPerProxy: 20}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 10m, allocationSafetyMargin: 60s}
|
lifecycle: {ttl: 10m, allocationSafetyMargin: 60s}
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
|
|||||||
@ -24,10 +24,8 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 100}
|
pool: {maxSize: 100}
|
||||||
capacity: {maxConcurrencyPerProxy: 1}
|
capacity: {maxConcurrencyPerProxy: 1}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 25, targetAvailableSlots: 50}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 10
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 3
|
maxAttempts: 3
|
||||||
|
|||||||
12
go.mod
12
go.mod
@ -4,25 +4,17 @@ go 1.26.0
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/jackc/pgx/v5 v5.6.0
|
github.com/jackc/pgx/v5 v5.6.0
|
||||||
github.com/prometheus/client_golang v1.24.1
|
|
||||||
github.com/redis/go-redis/v9 v9.19.0
|
github.com/redis/go-redis/v9 v9.19.0
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
|
||||||
github.com/prometheus/client_model v0.6.2 // indirect
|
|
||||||
github.com/prometheus/common v0.70.1 // indirect
|
|
||||||
github.com/prometheus/procfs v0.21.1 // indirect
|
|
||||||
go.uber.org/atomic v1.11.0 // indirect
|
go.uber.org/atomic v1.11.0 // indirect
|
||||||
golang.org/x/crypto v0.17.0 // indirect
|
golang.org/x/crypto v0.17.0 // indirect
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
golang.org/x/sync v0.1.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
golang.org/x/text v0.40.0 // indirect
|
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
40
go.sum
40
go.sum
@ -1,5 +1,3 @@
|
|||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||||
@ -9,8 +7,6 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
|||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
|
||||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||||
@ -19,51 +15,31 @@ github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
|
|||||||
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
|
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
|
||||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
|
||||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
|
||||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
|
||||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
|
||||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
|
||||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
|
||||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
|
||||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
|
||||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
|
||||||
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
|
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
|
||||||
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
|
||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
|
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
|
||||||
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@ -10,7 +10,6 @@ import (
|
|||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
"proxy-pool/internal/platform/credentials"
|
"proxy-pool/internal/platform/credentials"
|
||||||
@ -93,7 +92,6 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
|||||||
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
||||||
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
||||||
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
||||||
adapter.keys.stateInventory,
|
|
||||||
}
|
}
|
||||||
for _, key := range staticKeys {
|
for _, key := range staticKeys {
|
||||||
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
||||||
@ -120,38 +118,6 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStateInventoryFieldsAreCollisionFreeAndStatusDoesNotScanRecords(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
first := stateInventoryField("provider:a", "FETCHED")
|
|
||||||
second := stateInventoryField("provider", "a:FETCHED")
|
|
||||||
if first == second || first == "" || second == "" {
|
|
||||||
t.Fatalf("state inventory fields collide: %q and %q", first, second)
|
|
||||||
}
|
|
||||||
upper := strings.ToUpper(statusSource)
|
|
||||||
if strings.Contains(upper, "HGETALL") || strings.Contains(upper, "HSCAN") {
|
|
||||||
t.Fatal("status script scans Redis hashes")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadStateInventoryRejectsInvalidCalls(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
||||||
var adapter *Adapter
|
|
||||||
if _, err := adapter.ReadStateInventory(context.Background(), []string{"provider-a"}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
||||||
t.Fatalf("nil adapter error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := adapter.ReadStateInventory(nil, []string{"provider-a"}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
||||||
t.Fatalf("nil context error = %v", err)
|
|
||||||
}
|
|
||||||
adapter = &Adapter{}
|
|
||||||
if _, err := adapter.ReadStateInventory(context.Background(), []string{""}, now); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
||||||
t.Fatalf("empty upstream error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := adapter.ReadStateInventory(context.Background(), []string{"provider-a"}, time.Time{}); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
||||||
t.Fatalf("zero time error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
|
func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
record := proxyRecord{
|
record := proxyRecord{
|
||||||
|
|||||||
@ -86,8 +86,7 @@ func (a *Adapter) Extract(ctx context.Context, command extractionDomain.Command)
|
|||||||
}
|
}
|
||||||
keys := []string{
|
keys := []string{
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, operationKey, idempotencyKey,
|
||||||
operationKey, idempotencyKey,
|
|
||||||
}
|
}
|
||||||
keys = append(keys, a.extractionDriverKeys(digestInput)...)
|
keys = append(keys, a.extractionDriverKeys(digestInput)...)
|
||||||
idempotencyTTL := command.IdempotencyTTL
|
idempotencyTTL := command.IdempotencyTTL
|
||||||
|
|||||||
@ -25,8 +25,7 @@ func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpd
|
|||||||
}
|
}
|
||||||
result, err := runScript(ctx, a.client, healthScript, []string{
|
result, err := runScript(ctx, a.client, healthScript, []string{
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
||||||
a.keys.operation(operationID),
|
|
||||||
}, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency),
|
}, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency),
|
||||||
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID)
|
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -19,7 +19,6 @@ type keyspace struct {
|
|||||||
ownerExpiry string
|
ownerExpiry string
|
||||||
epoch string
|
epoch string
|
||||||
inventory string
|
inventory string
|
||||||
stateInventory string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newKeyspace(namespace string) keyspace {
|
func newKeyspace(namespace string) keyspace {
|
||||||
@ -35,14 +34,9 @@ func newKeyspace(namespace string) keyspace {
|
|||||||
ownerExpiry: prefix + ":owner-expiry",
|
ownerExpiry: prefix + ":owner-expiry",
|
||||||
epoch: prefix + ":epoch",
|
epoch: prefix + ":epoch",
|
||||||
inventory: prefix + ":inventory",
|
inventory: prefix + ":inventory",
|
||||||
stateInventory: prefix + ":state-inventory",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func stateInventoryField(upstreamID, state string) string {
|
|
||||||
return strconv.Itoa(len(upstreamID)) + ":" + upstreamID + ":" + state
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys keyspace) idempotency(clientID, idempotencyKey string) string {
|
func (keys keyspace) idempotency(clientID, idempotencyKey string) string {
|
||||||
return keys.prefix + ":idem:" + digestParts(clientID, idempotencyKey)
|
return keys.prefix + ":idem:" + digestParts(clientID, idempotencyKey)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -78,8 +78,7 @@ func (a *Adapter) runMaintenance(
|
|||||||
}
|
}
|
||||||
result, err := runScript(ctx, a.client, sweepScript, []string{
|
result, err := runScript(ctx, a.client, sweepScript, []string{
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
||||||
a.keys.operation(operationID),
|
|
||||||
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL))
|
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return maintenanceScriptReply{}, err
|
return maintenanceScriptReply{}, err
|
||||||
|
|||||||
@ -223,8 +223,7 @@ func (a *Adapter) runOwnership(
|
|||||||
}
|
}
|
||||||
result, err := runScript(ctx, a.client, ownershipScript, []string{
|
result, err := runScript(ctx, a.client, ownershipScript, []string{
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.epoch, operationKey,
|
||||||
a.keys.epoch, operationKey,
|
|
||||||
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
||||||
nowMS, proxyID, workerID, epoch, value, active, reserved)
|
nowMS, proxyID, workerID, epoch, value, active, reserved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -58,22 +58,6 @@ type maintenanceScriptReply struct {
|
|||||||
Count int `json:"count"`
|
Count int `json:"count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type statusScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
Inventories []statusScriptInventory `json:"inventories"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type statusScriptInventory struct {
|
|
||||||
UpstreamID string `json:"upstreamId"`
|
|
||||||
Fetched int64 `json:"fetched"`
|
|
||||||
Checking int64 `json:"checking"`
|
|
||||||
Available int64 `json:"available"`
|
|
||||||
Suspect int64 `json:"suspect"`
|
|
||||||
Draining int64 `json:"draining"`
|
|
||||||
Unhealthy int64 `json:"unhealthy"`
|
|
||||||
Extracted int64 `json:"extracted"`
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:embed scripts/upsert.lua
|
//go:embed scripts/upsert.lua
|
||||||
var upsertSource string
|
var upsertSource string
|
||||||
|
|
||||||
@ -89,16 +73,12 @@ var ownershipSource string
|
|||||||
//go:embed scripts/sweep.lua
|
//go:embed scripts/sweep.lua
|
||||||
var sweepSource string
|
var sweepSource string
|
||||||
|
|
||||||
//go:embed scripts/status.lua
|
|
||||||
var statusSource string
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
upsertScript = redis.NewScript(upsertSource)
|
upsertScript = redis.NewScript(upsertSource)
|
||||||
healthScript = redis.NewScript(healthSource)
|
healthScript = redis.NewScript(healthSource)
|
||||||
extractScript = redis.NewScript(extractSource)
|
extractScript = redis.NewScript(extractSource)
|
||||||
ownershipScript = redis.NewScript(ownershipSource)
|
ownershipScript = redis.NewScript(ownershipSource)
|
||||||
sweepScript = redis.NewScript(sweepSource)
|
sweepScript = redis.NewScript(sweepSource)
|
||||||
statusScript = redis.NewScript(statusSource)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
||||||
|
|||||||
@ -4,11 +4,10 @@ local idkeys_key = KEYS[3]
|
|||||||
local expiry_key = KEYS[4]
|
local expiry_key = KEYS[4]
|
||||||
local available_key = KEYS[5]
|
local available_key = KEYS[5]
|
||||||
local inventory_key = KEYS[6]
|
local inventory_key = KEYS[6]
|
||||||
local state_inventory_key = KEYS[7]
|
local owners_key = KEYS[7]
|
||||||
local owners_key = KEYS[8]
|
local owner_expiry_key = KEYS[8]
|
||||||
local owner_expiry_key = KEYS[9]
|
local operation_key = KEYS[9]
|
||||||
local operation_key = KEYS[10]
|
local idempotency_key = KEYS[10]
|
||||||
local idempotency_key = KEYS[11]
|
|
||||||
|
|
||||||
local now_ms = tonumber(ARGV[1])
|
local now_ms = tonumber(ARGV[1])
|
||||||
local requested = tonumber(ARGV[2])
|
local requested = tonumber(ARGV[2])
|
||||||
@ -74,33 +73,6 @@ local function decrement_inventory(upstream)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function increment_state(upstream, state)
|
|
||||||
if not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if value <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
local function remove_available(proxy_id, record)
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
local index_keys = record and record.indexKeys
|
local index_keys = record and record.indexKeys
|
||||||
@ -123,9 +95,6 @@ local function remove_proxy(proxy_id)
|
|||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
if decoded and type(record) == 'table' then
|
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
end
|
|
||||||
else
|
else
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
end
|
end
|
||||||
@ -236,7 +205,7 @@ end
|
|||||||
|
|
||||||
local driver_key = available_key
|
local driver_key = available_key
|
||||||
local driver_size = redis.call('ZCARD', available_key)
|
local driver_size = redis.call('ZCARD', available_key)
|
||||||
for index = 12, #KEYS do
|
for index = 11, #KEYS do
|
||||||
local size = redis.call('ZCARD', KEYS[index])
|
local size = redis.call('ZCARD', KEYS[index])
|
||||||
if size < driver_size then
|
if size < driver_size then
|
||||||
driver_key = KEYS[index]
|
driver_key = KEYS[index]
|
||||||
@ -312,9 +281,7 @@ for index = 1, selected_count do
|
|||||||
if is_managed(record.state) then
|
if is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
record.state = 'EXTRACTED'
|
record.state = 'EXTRACTED'
|
||||||
increment_state(record.sourceUpstream, record.state)
|
|
||||||
local encoded = cjson.encode(record)
|
local encoded = cjson.encode(record)
|
||||||
redis.call('HSET', records_key, selected.id, encoded)
|
redis.call('HSET', records_key, selected.id, encoded)
|
||||||
|
|
||||||
|
|||||||
@ -4,10 +4,9 @@ local idkeys_key = KEYS[3]
|
|||||||
local expiry_key = KEYS[4]
|
local expiry_key = KEYS[4]
|
||||||
local available_key = KEYS[5]
|
local available_key = KEYS[5]
|
||||||
local inventory_key = KEYS[6]
|
local inventory_key = KEYS[6]
|
||||||
local state_inventory_key = KEYS[7]
|
local owners_key = KEYS[7]
|
||||||
local owners_key = KEYS[8]
|
local owner_expiry_key = KEYS[8]
|
||||||
local owner_expiry_key = KEYS[9]
|
local operation_key = KEYS[9]
|
||||||
local operation_key = KEYS[10]
|
|
||||||
|
|
||||||
local checked_at_ms = tonumber(ARGV[1])
|
local checked_at_ms = tonumber(ARGV[1])
|
||||||
local next_state = ARGV[2]
|
local next_state = ARGV[2]
|
||||||
@ -36,33 +35,6 @@ local function decrement_inventory(upstream)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function increment_state(upstream, state)
|
|
||||||
if not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if value <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(id, record)
|
local function remove_available(id, record)
|
||||||
redis.call('ZREM', available_key, id)
|
redis.call('ZREM', available_key, id)
|
||||||
for _, index_key in ipairs(record and record.indexKeys or {}) do
|
for _, index_key in ipairs(record and record.indexKeys or {}) do
|
||||||
@ -79,7 +51,6 @@ local function remove_proxy(id)
|
|||||||
if is_managed(record.state) then
|
if is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
else
|
else
|
||||||
redis.call('ZREM', available_key, id)
|
redis.call('ZREM', available_key, id)
|
||||||
end
|
end
|
||||||
@ -161,8 +132,7 @@ if record.state ~= next_state and not (transitions[record.state] and transitions
|
|||||||
return finish({status = 'invalid'})
|
return finish({status = 'invalid'})
|
||||||
end
|
end
|
||||||
|
|
||||||
local previous_state = record.state
|
local was_managed = is_managed(record.state)
|
||||||
local was_managed = is_managed(previous_state)
|
|
||||||
local will_be_managed = is_managed(next_state)
|
local will_be_managed = is_managed(next_state)
|
||||||
remove_available(proxy_id, record)
|
remove_available(proxy_id, record)
|
||||||
record.state = next_state
|
record.state = next_state
|
||||||
@ -176,10 +146,6 @@ if was_managed and not will_be_managed then
|
|||||||
elseif not was_managed and will_be_managed then
|
elseif not was_managed and will_be_managed then
|
||||||
redis.call('HINCRBY', inventory_key, record.sourceUpstream, 1)
|
redis.call('HINCRBY', inventory_key, record.sourceUpstream, 1)
|
||||||
end
|
end
|
||||||
if previous_state ~= next_state then
|
|
||||||
decrement_state(record.sourceUpstream, previous_state)
|
|
||||||
increment_state(record.sourceUpstream, next_state)
|
|
||||||
end
|
|
||||||
|
|
||||||
local encoded = cjson.encode(record)
|
local encoded = cjson.encode(record)
|
||||||
redis.call('HSET', records_key, proxy_id, encoded)
|
redis.call('HSET', records_key, proxy_id, encoded)
|
||||||
@ -197,7 +163,6 @@ touch(idkeys_key, tonumber(record.expiresAtMs))
|
|||||||
touch(expiry_key, tonumber(record.expiresAtMs))
|
touch(expiry_key, tonumber(record.expiresAtMs))
|
||||||
touch(available_key, tonumber(record.expiresAtMs))
|
touch(available_key, tonumber(record.expiresAtMs))
|
||||||
touch(inventory_key, tonumber(record.expiresAtMs))
|
touch(inventory_key, tonumber(record.expiresAtMs))
|
||||||
touch(state_inventory_key, tonumber(record.expiresAtMs))
|
|
||||||
touch(owners_key, tonumber(record.expiresAtMs))
|
touch(owners_key, tonumber(record.expiresAtMs))
|
||||||
touch(owner_expiry_key, tonumber(record.expiresAtMs))
|
touch(owner_expiry_key, tonumber(record.expiresAtMs))
|
||||||
|
|
||||||
|
|||||||
@ -4,11 +4,10 @@ local idkeys_key = KEYS[3]
|
|||||||
local expiry_key = KEYS[4]
|
local expiry_key = KEYS[4]
|
||||||
local available_key = KEYS[5]
|
local available_key = KEYS[5]
|
||||||
local inventory_key = KEYS[6]
|
local inventory_key = KEYS[6]
|
||||||
local state_inventory_key = KEYS[7]
|
local owners_key = KEYS[7]
|
||||||
local owners_key = KEYS[8]
|
local owner_expiry_key = KEYS[8]
|
||||||
local owner_expiry_key = KEYS[9]
|
local epoch_key = KEYS[9]
|
||||||
local epoch_key = KEYS[10]
|
local operation_key = KEYS[10]
|
||||||
local operation_key = KEYS[11]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
local operation_ttl_ms = tonumber(ARGV[2])
|
local operation_ttl_ms = tonumber(ARGV[2])
|
||||||
@ -52,26 +51,6 @@ local function decrement_inventory(upstream)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if count <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function touch(key, expires_at_ms)
|
local function touch(key, expires_at_ms)
|
||||||
if redis.call('EXISTS', key) == 0 then
|
if redis.call('EXISTS', key) == 0 then
|
||||||
return
|
return
|
||||||
@ -121,9 +100,6 @@ local function remove_proxy(id)
|
|||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
if decoded and type(record) == 'table' then
|
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
end
|
|
||||||
else
|
else
|
||||||
redis.call('ZREM', available_key, id)
|
redis.call('ZREM', available_key, id)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -1,117 +0,0 @@
|
|||||||
local records_key = KEYS[1]
|
|
||||||
local unique_key = KEYS[2]
|
|
||||||
local idkeys_key = KEYS[3]
|
|
||||||
local expiry_key = KEYS[4]
|
|
||||||
local available_key = KEYS[5]
|
|
||||||
local inventory_key = KEYS[6]
|
|
||||||
local state_inventory_key = KEYS[7]
|
|
||||||
local owners_key = KEYS[8]
|
|
||||||
local owner_expiry_key = KEYS[9]
|
|
||||||
|
|
||||||
local now_ms = tonumber(ARGV[1])
|
|
||||||
local cleanup_limit = tonumber(ARGV[2])
|
|
||||||
local upstream_ids = cjson.decode(ARGV[3])
|
|
||||||
|
|
||||||
local function is_managed(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_inventory(upstream)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local count = redis.call('HINCRBY', inventory_key, upstream, -1)
|
|
||||||
if count < 0 then
|
|
||||||
redis.call('HSET', inventory_key, upstream, 0)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if count <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
local index_keys = record and record.indexKeys
|
|
||||||
if type(index_keys) == 'table' then
|
|
||||||
for _, index_key in ipairs(index_keys) do
|
|
||||||
if type(index_key) == 'string' and index_key ~= '' then
|
|
||||||
redis.call('ZREM', index_key, proxy_id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_proxy(proxy_id)
|
|
||||||
local raw = redis.call('HGET', records_key, proxy_id)
|
|
||||||
local record = nil
|
|
||||||
if raw then
|
|
||||||
local decoded
|
|
||||||
decoded, record = pcall(cjson.decode, raw)
|
|
||||||
remove_available(proxy_id, decoded and record or nil)
|
|
||||||
if decoded and type(record) == 'table' then
|
|
||||||
if is_managed(record.state) then
|
|
||||||
decrement_inventory(record.sourceUpstream)
|
|
||||||
end
|
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
end
|
|
||||||
else
|
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
|
||||||
end
|
|
||||||
local digest = redis.call('HGET', idkeys_key, proxy_id)
|
|
||||||
if digest and redis.call('HGET', unique_key, digest) == proxy_id then
|
|
||||||
redis.call('HDEL', unique_key, digest)
|
|
||||||
end
|
|
||||||
redis.call('HDEL', idkeys_key, proxy_id)
|
|
||||||
redis.call('HDEL', records_key, proxy_id)
|
|
||||||
redis.call('ZREM', expiry_key, proxy_id)
|
|
||||||
redis.call('HDEL', owners_key, proxy_id)
|
|
||||||
redis.call('ZREM', owner_expiry_key, proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local expired = redis.call('ZRANGEBYSCORE', expiry_key, '-inf', now_ms, 'LIMIT', 0, cleanup_limit)
|
|
||||||
for _, proxy_id in ipairs(expired) do
|
|
||||||
remove_proxy(proxy_id)
|
|
||||||
end
|
|
||||||
|
|
||||||
local oldest = redis.call('ZRANGE', expiry_key, 0, 0, 'WITHSCORES')
|
|
||||||
if #oldest == 2 and tonumber(oldest[2]) <= now_ms then
|
|
||||||
return cjson.encode({status = 'unavailable', inventories = cjson.decode('[]')})
|
|
||||||
end
|
|
||||||
|
|
||||||
local inventories = cjson.decode('[]')
|
|
||||||
local states = {'FETCHED', 'CHECKING', 'AVAILABLE', 'SUSPECT', 'DRAINING', 'UNHEALTHY', 'EXTRACTED'}
|
|
||||||
for _, upstream_id in ipairs(upstream_ids) do
|
|
||||||
if type(upstream_id) ~= 'string' or upstream_id == '' then
|
|
||||||
return cjson.encode({status = 'invalid', inventories = cjson.decode('[]')})
|
|
||||||
end
|
|
||||||
local counts = {}
|
|
||||||
for _, state in ipairs(states) do
|
|
||||||
local count = tonumber(redis.call('HGET', state_inventory_key, state_field(upstream_id, state)) or '0')
|
|
||||||
if count < 0 then
|
|
||||||
return cjson.encode({status = 'unavailable', inventories = cjson.decode('[]')})
|
|
||||||
end
|
|
||||||
counts[string.lower(state)] = count
|
|
||||||
end
|
|
||||||
counts.upstreamId = upstream_id
|
|
||||||
inventories[#inventories + 1] = counts
|
|
||||||
end
|
|
||||||
|
|
||||||
return cjson.encode({status = 'ok', inventories = inventories})
|
|
||||||
@ -4,10 +4,9 @@ local idkeys_key = KEYS[3]
|
|||||||
local expiry_key = KEYS[4]
|
local expiry_key = KEYS[4]
|
||||||
local available_key = KEYS[5]
|
local available_key = KEYS[5]
|
||||||
local inventory_key = KEYS[6]
|
local inventory_key = KEYS[6]
|
||||||
local state_inventory_key = KEYS[7]
|
local owners_key = KEYS[7]
|
||||||
local owners_key = KEYS[8]
|
local owner_expiry_key = KEYS[8]
|
||||||
local owner_expiry_key = KEYS[9]
|
local operation_key = KEYS[9]
|
||||||
local operation_key = KEYS[10]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
local now_ms = tonumber(ARGV[2])
|
local now_ms = tonumber(ARGV[2])
|
||||||
@ -41,26 +40,6 @@ local function decrement_inventory(upstream)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local count = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if count <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
local function remove_available(proxy_id, record)
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
local index_keys = record and record.indexKeys
|
local index_keys = record and record.indexKeys
|
||||||
@ -83,9 +62,6 @@ local function remove_proxy(proxy_id)
|
|||||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
if decoded and type(record) == 'table' then
|
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
end
|
|
||||||
else
|
else
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -4,10 +4,9 @@ local idkeys_key = KEYS[3]
|
|||||||
local expiry_key = KEYS[4]
|
local expiry_key = KEYS[4]
|
||||||
local available_key = KEYS[5]
|
local available_key = KEYS[5]
|
||||||
local inventory_key = KEYS[6]
|
local inventory_key = KEYS[6]
|
||||||
local state_inventory_key = KEYS[7]
|
local owners_key = KEYS[7]
|
||||||
local owners_key = KEYS[8]
|
local owner_expiry_key = KEYS[8]
|
||||||
local owner_expiry_key = KEYS[9]
|
local operation_key = KEYS[9]
|
||||||
local operation_key = KEYS[10]
|
|
||||||
|
|
||||||
local now_ms = tonumber(ARGV[1])
|
local now_ms = tonumber(ARGV[1])
|
||||||
local cleanup_limit = tonumber(ARGV[2])
|
local cleanup_limit = tonumber(ARGV[2])
|
||||||
@ -35,33 +34,6 @@ local function decrement_inventory(upstream)
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
local function state_field(upstream, state)
|
|
||||||
return string.len(upstream) .. ':' .. upstream .. ':' .. state
|
|
||||||
end
|
|
||||||
|
|
||||||
local function is_counted(state)
|
|
||||||
return state == 'FETCHED' or state == 'CHECKING' or state == 'AVAILABLE' or
|
|
||||||
state == 'SUSPECT' or state == 'DRAINING' or state == 'UNHEALTHY' or state == 'EXTRACTED'
|
|
||||||
end
|
|
||||||
|
|
||||||
local function increment_state(upstream, state)
|
|
||||||
if not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
redis.call('HINCRBY', state_inventory_key, state_field(upstream, state), 1)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function decrement_state(upstream, state)
|
|
||||||
if type(upstream) ~= 'string' or upstream == '' or not is_counted(state) then
|
|
||||||
return
|
|
||||||
end
|
|
||||||
local field = state_field(upstream, state)
|
|
||||||
local value = redis.call('HINCRBY', state_inventory_key, field, -1)
|
|
||||||
if value <= 0 then
|
|
||||||
redis.call('HDEL', state_inventory_key, field)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
local function remove_available(proxy_id, record)
|
local function remove_available(proxy_id, record)
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
local indexes = record and record.indexKeys or {}
|
local indexes = record and record.indexKeys or {}
|
||||||
@ -79,7 +51,6 @@ local function remove_proxy(proxy_id)
|
|||||||
if is_managed(record.state) then
|
if is_managed(record.state) then
|
||||||
decrement_inventory(record.sourceUpstream)
|
decrement_inventory(record.sourceUpstream)
|
||||||
end
|
end
|
||||||
decrement_state(record.sourceUpstream, record.state)
|
|
||||||
else
|
else
|
||||||
redis.call('ZREM', available_key, proxy_id)
|
redis.call('ZREM', available_key, proxy_id)
|
||||||
end
|
end
|
||||||
@ -195,7 +166,6 @@ for _, candidate in ipairs(candidates) do
|
|||||||
if is_managed(incoming.state) then
|
if is_managed(incoming.state) then
|
||||||
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
||||||
end
|
end
|
||||||
increment_state(candidate.upstream, incoming.state)
|
|
||||||
add_available(candidate.proxyId, incoming)
|
add_available(candidate.proxyId, incoming)
|
||||||
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
||||||
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
||||||
@ -212,7 +182,6 @@ if max_expiry_ms > 0 then
|
|||||||
touch(expiry_key, max_expiry_ms)
|
touch(expiry_key, max_expiry_ms)
|
||||||
touch(available_key, max_expiry_ms)
|
touch(available_key, max_expiry_ms)
|
||||||
touch(inventory_key, max_expiry_ms)
|
touch(inventory_key, max_expiry_ms)
|
||||||
touch(state_inventory_key, max_expiry_ms)
|
|
||||||
touch(owners_key, max_expiry_ms)
|
touch(owners_key, max_expiry_ms)
|
||||||
touch(owner_expiry_key, max_expiry_ms)
|
touch(owner_expiry_key, max_expiry_ms)
|
||||||
end
|
end
|
||||||
|
|||||||
@ -1,72 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ activitypool.StateInventoryReader = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) ReadStateInventory(
|
|
||||||
ctx context.Context,
|
|
||||||
upstreamIDs []string,
|
|
||||||
now time.Time,
|
|
||||||
) ([]activitypool.StateInventory, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return nil, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if a == nil || now.IsZero() {
|
|
||||||
return nil, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
for _, upstreamID := range upstreamIDs {
|
|
||||||
if upstreamID == "" {
|
|
||||||
return nil, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(upstreamIDs) == 0 {
|
|
||||||
return []activitypool.StateInventory{}, nil
|
|
||||||
}
|
|
||||||
payload, err := json.Marshal(upstreamIDs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, statusScript, []string{
|
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
|
||||||
}, now.UnixMilli(), a.options.CleanupLimit, string(payload))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var reply statusScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return nil, activitypool.ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if reply.Status == scriptUnavailable {
|
|
||||||
return nil, invalidScriptReply("expired cleanup is backlogged")
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK || len(reply.Inventories) != len(upstreamIDs) {
|
|
||||||
return nil, invalidScriptReply("unexpected state inventory reply")
|
|
||||||
}
|
|
||||||
inventories := make([]activitypool.StateInventory, len(reply.Inventories))
|
|
||||||
for index, item := range reply.Inventories {
|
|
||||||
if item.UpstreamID != upstreamIDs[index] || item.Fetched < 0 || item.Checking < 0 ||
|
|
||||||
item.Available < 0 || item.Suspect < 0 || item.Draining < 0 || item.Unhealthy < 0 || item.Extracted < 0 {
|
|
||||||
return nil, invalidScriptReply("invalid state inventory counters")
|
|
||||||
}
|
|
||||||
inventories[index] = activitypool.StateInventory{
|
|
||||||
UpstreamID: item.UpstreamID, Fetched: item.Fetched, Checking: item.Checking,
|
|
||||||
Available: item.Available, Suspect: item.Suspect, Draining: item.Draining,
|
|
||||||
Unhealthy: item.Unhealthy, Extracted: item.Extracted,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return inventories, nil
|
|
||||||
}
|
|
||||||
@ -1,61 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestReadStateInventoryFailsClosedWhileExpiredCleanupIsBacklogged(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
bounded, err := New(fixture.Client, Options{
|
|
||||||
Namespace: fixture.Namespace, Credentials: fixture.Credentials,
|
|
||||||
OperationTTL: time.Minute, MaxCandidateScan: 32, CleanupLimit: 1,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() error = %v", err)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC)
|
|
||||||
_, err = bounded.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Second, MaxSize: 10,
|
|
||||||
Proxies: []proxyDomain.Proxy{
|
|
||||||
{ID: "expired-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080, State: proxyDomain.StateFetched},
|
|
||||||
{ID: "expired-b", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.11", Port: 8080, State: proxyDomain.StateFetched},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertFetched() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err = bounded.ReadStateInventory(context.Background(), []string{"provider-a"}, now.Add(2*time.Second)); !errors.Is(err, extractionDomain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("ReadStateInventory(backlog) error = %v", err)
|
|
||||||
}
|
|
||||||
inventories, err := bounded.ReadStateInventory(context.Background(), []string{"provider-a"}, now.Add(2*time.Second))
|
|
||||||
if err != nil || len(inventories) != 1 || inventories[0] != (activitypool.StateInventory{UpstreamID: "provider-a"}) {
|
|
||||||
t.Fatalf("ReadStateInventory(after cleanup) = %+v, %v", inventories, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadStateInventoryFailsClosedOnNegativeCounters(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
if err := fixture.Client.HSet(
|
|
||||||
context.Background(),
|
|
||||||
fixture.Adapter.keys.stateInventory,
|
|
||||||
stateInventoryField("provider-a", string(proxyDomain.StateAvailable)),
|
|
||||||
-1,
|
|
||||||
).Err(); err != nil {
|
|
||||||
t.Fatalf("seed invalid state counter: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := fixture.Adapter.ReadStateInventory(
|
|
||||||
context.Background(), []string{"provider-a"}, time.Now().UTC(),
|
|
||||||
); !errors.Is(err, extractionDomain.ErrStoreUnavailable) {
|
|
||||||
t.Fatalf("ReadStateInventory(negative counter) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -175,8 +175,7 @@ func (a *Adapter) upsertChunk(
|
|||||||
}
|
}
|
||||||
result, err := runScript(ctx, a.client, upsertScript, []string{
|
result, err := runScript(ctx, a.client, upsertScript, []string{
|
||||||
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.owners, a.keys.ownerExpiry, a.keys.operation(operationID),
|
||||||
a.keys.operation(operationID),
|
|
||||||
}, observedAt.UnixMilli(), a.options.CleanupLimit, maxSize, operationTTLMillis(a.options.OperationTTL), string(payload))
|
}, observedAt.UnixMilli(), a.options.CleanupLimit, maxSize, operationTTLMillis(a.options.OperationTTL), string(payload))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return upsertScriptReply{}, err
|
return upsertScriptReply{}, err
|
||||||
|
|||||||
@ -1,355 +0,0 @@
|
|||||||
package redisprovider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"math"
|
|
||||||
"reflect"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
controllerProvider "proxy-pool/internal/controller/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
|
||||||
|
|
||||||
type Options struct {
|
|
||||||
Namespace string
|
|
||||||
HolderID string
|
|
||||||
LeaseTTL time.Duration
|
|
||||||
RenewEvery time.Duration
|
|
||||||
RetryInterval time.Duration
|
|
||||||
PermitGrace time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type Adapter struct {
|
|
||||||
client redis.Scripter
|
|
||||||
options Options
|
|
||||||
keys keyBuilder
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ controllerProvider.Coordinator = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func New(client redis.Scripter, options Options) (*Adapter, error) {
|
|
||||||
options.Namespace = strings.TrimSpace(options.Namespace)
|
|
||||||
options.HolderID = strings.TrimSpace(options.HolderID)
|
|
||||||
if nilInterface(client) || !namespacePattern.MatchString(options.Namespace) || options.HolderID == "" ||
|
|
||||||
options.LeaseTTL <= 0 || options.RenewEvery <= 0 || options.RenewEvery > options.LeaseTTL/3 ||
|
|
||||||
options.RetryInterval <= 0 || options.PermitGrace < 0 {
|
|
||||||
return nil, controllerProvider.ErrInvalidCoordination
|
|
||||||
}
|
|
||||||
return &Adapter{client: client, options: options, keys: keyBuilder{namespace: options.Namespace}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (adapter *Adapter) RunLeader(
|
|
||||||
ctx context.Context,
|
|
||||||
upstreamID string,
|
|
||||||
limits controllerProvider.CoordinationLimits,
|
|
||||||
work func(context.Context, controllerProvider.LeaderSession) error,
|
|
||||||
) error {
|
|
||||||
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
|
||||||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
|
||||||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
|
||||||
return controllerProvider.ErrInvalidCoordination
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
keys, err := adapter.keys.forUpstream(upstreamID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
token, err := randomToken()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for ctx.Err() == nil {
|
|
||||||
generationCandidate, tokenErr := randomToken()
|
|
||||||
if tokenErr != nil {
|
|
||||||
return errors.Join(controllerProvider.ErrCoordinationUnavailable, tokenErr)
|
|
||||||
}
|
|
||||||
reply, acquireErr := runScript(ctx, adapter.client, keys,
|
|
||||||
"acquire_leader", generationCandidate, adapter.options.HolderID, token,
|
|
||||||
durationMillis(adapter.options.LeaseTTL),
|
|
||||||
)
|
|
||||||
if acquireErr != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case "busy":
|
|
||||||
delay := adapter.options.RetryInterval
|
|
||||||
if reply.WaitMS > 0 && time.Duration(reply.WaitMS)*time.Millisecond < delay {
|
|
||||||
delay = time.Duration(reply.WaitMS) * time.Millisecond
|
|
||||||
}
|
|
||||||
if err := wait(ctx, delay); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
case "ok":
|
|
||||||
if reply.Generation == "" || reply.Epoch == 0 {
|
|
||||||
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
session := &leaderSession{
|
|
||||||
adapter: adapter, keys: keys, upstreamID: upstreamID, limits: limits,
|
|
||||||
generation: reply.Generation, holderID: adapter.options.HolderID,
|
|
||||||
token: token, epoch: reply.Epoch,
|
|
||||||
}
|
|
||||||
lost, runErr := adapter.runLeaderTerm(ctx, session, work)
|
|
||||||
if runErr != nil {
|
|
||||||
return runErr
|
|
||||||
}
|
|
||||||
if !lost {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := wait(ctx, adapter.options.RetryInterval); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (adapter *Adapter) runLeaderTerm(
|
|
||||||
ctx context.Context,
|
|
||||||
session *leaderSession,
|
|
||||||
work func(context.Context, controllerProvider.LeaderSession) error,
|
|
||||||
) (bool, error) {
|
|
||||||
leaderCtx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
session.ctx = leaderCtx
|
|
||||||
workDone := make(chan error, 1)
|
|
||||||
go func() { workDone <- work(leaderCtx, session) }()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(adapter.options.RenewEvery)
|
|
||||||
defer ticker.Stop()
|
|
||||||
deadline := time.NewTimer(adapter.options.LeaseTTL - adapter.options.RenewEvery)
|
|
||||||
defer deadline.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
cancel()
|
|
||||||
<-workDone
|
|
||||||
adapter.releaseLeader(session)
|
|
||||||
return false, ctx.Err()
|
|
||||||
case workErr := <-workDone:
|
|
||||||
cancel()
|
|
||||||
adapter.releaseLeader(session)
|
|
||||||
return false, leaderWorkResult(ctx, workErr)
|
|
||||||
case <-deadline.C:
|
|
||||||
cancel()
|
|
||||||
<-workDone
|
|
||||||
return true, nil
|
|
||||||
case <-ticker.C:
|
|
||||||
renewCtx, renewCancel := context.WithTimeout(leaderCtx, adapter.options.RenewEvery)
|
|
||||||
reply, err := runScript(renewCtx, adapter.client, session.keys,
|
|
||||||
"renew_leader", session.generation, session.holderID, session.token,
|
|
||||||
session.epoch, durationMillis(adapter.options.LeaseTTL),
|
|
||||||
)
|
|
||||||
renewCancel()
|
|
||||||
if err != nil || reply.Status != "ok" {
|
|
||||||
cancel()
|
|
||||||
<-workDone
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
resetTimer(deadline, adapter.options.LeaseTTL-adapter.options.RenewEvery)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func leaderWorkResult(ctx context.Context, workErr error) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if workErr != nil {
|
|
||||||
return workErr
|
|
||||||
}
|
|
||||||
return controllerProvider.ErrLeaderWorkStopped
|
|
||||||
}
|
|
||||||
|
|
||||||
func (adapter *Adapter) releaseLeader(session *leaderSession) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), adapter.options.RenewEvery)
|
|
||||||
defer cancel()
|
|
||||||
_, _ = runScript(ctx, adapter.client, session.keys,
|
|
||||||
"release_leader", session.generation, session.holderID, session.token, session.epoch,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
type leaderSession struct {
|
|
||||||
adapter *Adapter
|
|
||||||
keys upstreamKeys
|
|
||||||
upstreamID string
|
|
||||||
limits controllerProvider.CoordinationLimits
|
|
||||||
ctx context.Context
|
|
||||||
generation string
|
|
||||||
holderID string
|
|
||||||
token string
|
|
||||||
epoch uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ controllerProvider.LeaderSession = (*leaderSession)(nil)
|
|
||||||
|
|
||||||
func (session *leaderSession) Fence() controllerProvider.Fence {
|
|
||||||
if session == nil {
|
|
||||||
return controllerProvider.Fence{}
|
|
||||||
}
|
|
||||||
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
|
|
||||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
|
|
||||||
return nil, controllerProvider.ErrInvalidCoordination
|
|
||||||
}
|
|
||||||
operationCtx, cancel := context.WithCancel(ctx)
|
|
||||||
stop := context.AfterFunc(session.ctx, cancel)
|
|
||||||
defer func() {
|
|
||||||
stop()
|
|
||||||
cancel()
|
|
||||||
}()
|
|
||||||
permitToken, err := randomToken()
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
|
||||||
}
|
|
||||||
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
|
||||||
for operationCtx.Err() == nil {
|
|
||||||
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
|
||||||
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
|
||||||
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
|
||||||
durationMillis(permitTTL),
|
|
||||||
)
|
|
||||||
if scriptErr != nil {
|
|
||||||
if session.ctx.Err() != nil {
|
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
|
||||||
}
|
|
||||||
if operationCtx.Err() != nil {
|
|
||||||
return nil, operationCtx.Err()
|
|
||||||
}
|
|
||||||
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case "ok":
|
|
||||||
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
|
|
||||||
case "stale":
|
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
|
||||||
case "rate_limited", "at_capacity":
|
|
||||||
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
|
||||||
if delay <= 0 {
|
|
||||||
delay = session.adapter.options.RetryInterval
|
|
||||||
}
|
|
||||||
if err := wait(operationCtx, delay); err != nil {
|
|
||||||
if session.ctx.Err() != nil {
|
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, controllerProvider.ErrCoordinationUnavailable
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if session.ctx.Err() != nil {
|
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
|
||||||
}
|
|
||||||
return nil, operationCtx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
type requestPermit struct {
|
|
||||||
adapter *Adapter
|
|
||||||
keys upstreamKeys
|
|
||||||
token string
|
|
||||||
mu sync.Mutex
|
|
||||||
done bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (permit *requestPermit) Release(ctx context.Context) error {
|
|
||||||
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
|
|
||||||
return controllerProvider.ErrInvalidCoordination
|
|
||||||
}
|
|
||||||
permit.mu.Lock()
|
|
||||||
defer permit.mu.Unlock()
|
|
||||||
if permit.done {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if reply.Status != "ok" {
|
|
||||||
return controllerProvider.ErrCoordinationUnavailable
|
|
||||||
}
|
|
||||||
permit.done = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func randomToken() (string, error) {
|
|
||||||
var token [16]byte
|
|
||||||
if _, err := rand.Read(token[:]); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(token[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func durationMillis(value time.Duration) int64 {
|
|
||||||
milliseconds := value / time.Millisecond
|
|
||||||
if value%time.Millisecond != 0 {
|
|
||||||
milliseconds++
|
|
||||||
}
|
|
||||||
return int64(milliseconds)
|
|
||||||
}
|
|
||||||
|
|
||||||
func wait(ctx context.Context, duration time.Duration) error {
|
|
||||||
timer := time.NewTimer(duration)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-timer.C:
|
|
||||||
return nil
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func resetTimer(timer *time.Timer, duration time.Duration) {
|
|
||||||
if !timer.Stop() {
|
|
||||||
select {
|
|
||||||
case <-timer.C:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
timer.Reset(duration)
|
|
||||||
}
|
|
||||||
|
|
||||||
func nilInterface(value any) bool {
|
|
||||||
if value == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
reflected := reflect.ValueOf(value)
|
|
||||||
switch reflected.Kind() {
|
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
|
||||||
return reflected.IsNil()
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,129 +0,0 @@
|
|||||||
package redisprovider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
controllerProvider "proxy-pool/internal/controller/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
||||||
t.Cleanup(func() { _ = client.Close() })
|
|
||||||
valid := Options{
|
|
||||||
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
|
|
||||||
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond, PermitGrace: time.Second,
|
|
||||||
}
|
|
||||||
var typedNil *redis.Client
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
client redis.Scripter
|
|
||||||
options Options
|
|
||||||
}{
|
|
||||||
{name: "nil client", options: valid},
|
|
||||||
{name: "typed nil client", client: typedNil, options: valid},
|
|
||||||
{name: "empty namespace", client: client, options: withNamespace(valid, "")},
|
|
||||||
{name: "unsafe namespace", client: client, options: withNamespace(valid, "bad:value")},
|
|
||||||
{name: "empty holder", client: client, options: withHolder(valid, "")},
|
|
||||||
{name: "zero lease", client: client, options: withLeaseTTL(valid, 0)},
|
|
||||||
{name: "renew exceeds third", client: client, options: withRenewEvery(valid, 2*time.Second)},
|
|
||||||
{name: "zero retry", client: client, options: withRetryInterval(valid, 0)},
|
|
||||||
{name: "negative grace", client: client, options: withPermitGrace(valid, -1)},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
adapter, err := New(test.client, test.options)
|
|
||||||
if err == nil || adapter != nil {
|
|
||||||
t.Fatalf("New() = (%v, %v), want nil adapter and error", adapter, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewBuildsPerUpstreamClusterSafeKeys(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
|
||||||
t.Cleanup(func() { _ = client.Close() })
|
|
||||||
adapter, err := New(client, Options{
|
|
||||||
Namespace: " controller ", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
|
|
||||||
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond, PermitGrace: time.Second,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(): %v", err)
|
|
||||||
}
|
|
||||||
keys, err := adapter.keys.forUpstream("provider:{unsafe}")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("forUpstream(): %v", err)
|
|
||||||
}
|
|
||||||
all := keys.all()
|
|
||||||
for _, key := range all {
|
|
||||||
if strings.Contains(key, "provider:{unsafe}") || strings.Count(key, "{") != 1 ||
|
|
||||||
strings.Count(key, "}") != 1 || !strings.Contains(key, "{provider:") {
|
|
||||||
t.Fatalf("unsafe provider coordination key %q", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.Split(all[0], "}")[0] != strings.Split(all[len(all)-1], "}")[0] {
|
|
||||||
t.Fatalf("keys do not share one upstream hash tag: %v", all)
|
|
||||||
}
|
|
||||||
if _, err := adapter.keys.forUpstream(""); !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
|
|
||||||
t.Fatalf("forUpstream(empty) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
var adapter *Adapter
|
|
||||||
limits := controllerProvider.CoordinationLimits{
|
|
||||||
RequestInterval: time.Second, MaxInFlight: 1, MaxAttemptDuration: time.Second,
|
|
||||||
}
|
|
||||||
work := func(context.Context, controllerProvider.LeaderSession) error { return nil }
|
|
||||||
if err := adapter.RunLeader(context.Background(), "provider-a", limits, work); !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
|
|
||||||
t.Fatalf("nil adapter error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if err := leaderWorkResult(ctx, nil); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("leaderWorkResult() error = %v, want context cancellation", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func withNamespace(options Options, value string) Options {
|
|
||||||
options.Namespace = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withHolder(options Options, value string) Options {
|
|
||||||
options.HolderID = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withLeaseTTL(options Options, value time.Duration) Options {
|
|
||||||
options.LeaseTTL = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withRenewEvery(options Options, value time.Duration) Options {
|
|
||||||
options.RenewEvery = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withRetryInterval(options Options, value time.Duration) Options {
|
|
||||||
options.RetryInterval = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
func withPermitGrace(options Options, value time.Duration) Options {
|
|
||||||
options.PermitGrace = value
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
@ -1,297 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package redisprovider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
controllerProvider "proxy-pool/internal/controller/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
var integrationNamespaceSequence atomic.Uint64
|
|
||||||
|
|
||||||
func TestRedisCoordinatorElectsOneLeaderAndFencesFailover(t *testing.T) {
|
|
||||||
fixture := newRedisFixture(t)
|
|
||||||
first := fixture.coordinator(t, "controller-a")
|
|
||||||
second := fixture.coordinator(t, "controller-b")
|
|
||||||
limits := controllerProvider.CoordinationLimits{
|
|
||||||
RequestInterval: 50 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: 300 * time.Millisecond,
|
|
||||||
}
|
|
||||||
|
|
||||||
ctxA, cancelA := context.WithCancel(context.Background())
|
|
||||||
ctxB, cancelB := context.WithCancel(context.Background())
|
|
||||||
defer cancelA()
|
|
||||||
defer cancelB()
|
|
||||||
started := make(chan leadershipFixture, 4)
|
|
||||||
var active atomic.Int64
|
|
||||||
var maximum atomic.Int64
|
|
||||||
work := func(holder string) func(context.Context, controllerProvider.LeaderSession) error {
|
|
||||||
return func(ctx context.Context, session controllerProvider.LeaderSession) error {
|
|
||||||
current := active.Add(1)
|
|
||||||
for {
|
|
||||||
observed := maximum.Load()
|
|
||||||
if current <= observed || maximum.CompareAndSwap(observed, current) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
started <- leadershipFixture{holder: holder, fence: session.Fence()}
|
|
||||||
<-ctx.Done()
|
|
||||||
active.Add(-1)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
doneA := make(chan error, 1)
|
|
||||||
doneB := make(chan error, 1)
|
|
||||||
go func() { doneA <- first.RunLeader(ctxA, "provider-a", limits, work("controller-a")) }()
|
|
||||||
go func() { doneB <- second.RunLeader(ctxB, "provider-a", limits, work("controller-b")) }()
|
|
||||||
|
|
||||||
initial := receiveLeadership(t, started)
|
|
||||||
time.Sleep(200 * time.Millisecond)
|
|
||||||
select {
|
|
||||||
case duplicate := <-started:
|
|
||||||
t.Fatalf("simultaneous leaders started: first=%+v duplicate=%+v", initial, duplicate)
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if got := maximum.Load(); got != 1 {
|
|
||||||
t.Fatalf("maximum simultaneous leaders = %d, want 1", got)
|
|
||||||
}
|
|
||||||
if initial.holder == "controller-a" {
|
|
||||||
cancelA()
|
|
||||||
} else {
|
|
||||||
cancelB()
|
|
||||||
}
|
|
||||||
replacement := receiveLeadership(t, started)
|
|
||||||
if replacement.holder == initial.holder {
|
|
||||||
t.Fatalf("replacement holder = %q, want the other controller", replacement.holder)
|
|
||||||
}
|
|
||||||
if replacement.fence.Generation != initial.fence.Generation || replacement.fence.Epoch <= initial.fence.Epoch {
|
|
||||||
t.Fatalf("replacement fence = %+v, initial = %+v", replacement.fence, initial.fence)
|
|
||||||
}
|
|
||||||
if got := maximum.Load(); got != 1 {
|
|
||||||
t.Fatalf("maximum simultaneous leaders after failover = %d, want 1", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
cancelA()
|
|
||||||
cancelB()
|
|
||||||
waitRunner(t, doneA)
|
|
||||||
waitRunner(t, doneB)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T) {
|
|
||||||
fixture := newRedisFixture(t)
|
|
||||||
coordinator := fixture.coordinator(t, "controller-a")
|
|
||||||
limits := controllerProvider.CoordinationLimits{
|
|
||||||
RequestInterval: 250 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: time.Second,
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
sessions := make(chan controllerProvider.LeaderSession, 1)
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- coordinator.RunLeader(ctx, "provider-a", limits,
|
|
||||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
|
||||||
sessions <- session
|
|
||||||
<-workCtx.Done()
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
session := receiveSession(t, sessions)
|
|
||||||
first, err := session.AcquireFetch(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first AcquireFetch(): %v", err)
|
|
||||||
}
|
|
||||||
startedAt := time.Now()
|
|
||||||
secondResult := make(chan permitResultFixture, 1)
|
|
||||||
go func() {
|
|
||||||
permit, acquireErr := session.AcquireFetch(context.Background())
|
|
||||||
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case result := <-secondResult:
|
|
||||||
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
}
|
|
||||||
if err := first.Release(context.Background()); err != nil {
|
|
||||||
t.Fatalf("first Release(): %v", err)
|
|
||||||
}
|
|
||||||
result := receivePermit(t, secondResult)
|
|
||||||
if result.err != nil || result.permit == nil {
|
|
||||||
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
|
|
||||||
}
|
|
||||||
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
|
||||||
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
|
||||||
}
|
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
|
||||||
t.Fatalf("second Release(): %v", err)
|
|
||||||
}
|
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
|
||||||
t.Fatalf("idempotent second Release(): %v", err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
waitRunner(t, done)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisCoordinatorRebuildsWithNewGenerationAfterStateLoss(t *testing.T) {
|
|
||||||
fixture := newRedisFixture(t)
|
|
||||||
coordinator := fixture.coordinator(t, "controller-a")
|
|
||||||
limits := controllerProvider.CoordinationLimits{
|
|
||||||
RequestInterval: 50 * time.Millisecond, MaxInFlight: 1, MaxAttemptDuration: time.Second,
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
started := make(chan controllerProvider.Fence, 4)
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- coordinator.RunLeader(ctx, "provider-a", limits,
|
|
||||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
|
||||||
started <- session.Fence()
|
|
||||||
<-workCtx.Done()
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
first := receiveFence(t, started)
|
|
||||||
fixture.deleteKeys(t)
|
|
||||||
second := receiveFence(t, started)
|
|
||||||
if second.Generation == first.Generation {
|
|
||||||
t.Fatalf("generation after Redis state loss = %q, want a new generation", second.Generation)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
waitRunner(t, done)
|
|
||||||
}
|
|
||||||
|
|
||||||
type redisFixture struct {
|
|
||||||
client *redis.Client
|
|
||||||
namespace string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRedisFixture(t *testing.T) redisFixture {
|
|
||||||
t.Helper()
|
|
||||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
|
||||||
if redisURL == "" {
|
|
||||||
t.Skip("PROXY_POOL_TEST_REDIS_URL is not set")
|
|
||||||
}
|
|
||||||
options, err := redis.ParseURL(redisURL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse PROXY_POOL_TEST_REDIS_URL: %v", err)
|
|
||||||
}
|
|
||||||
client := redis.NewClient(options)
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := client.Ping(ctx).Err(); err != nil {
|
|
||||||
_ = client.Close()
|
|
||||||
t.Fatalf("ping Redis: %v", err)
|
|
||||||
}
|
|
||||||
namespace := fmt.Sprintf("provider-it-%d-%d-%d", os.Getpid(), time.Now().UnixNano(), integrationNamespaceSequence.Add(1))
|
|
||||||
t.Cleanup(func() {
|
|
||||||
redisFixture{client: client, namespace: namespace}.deleteKeys(t)
|
|
||||||
_ = client.Close()
|
|
||||||
})
|
|
||||||
return redisFixture{client: client, namespace: namespace}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fixture redisFixture) deleteKeys(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cleanupCancel()
|
|
||||||
var cursor uint64
|
|
||||||
for {
|
|
||||||
keys, next, err := fixture.client.Scan(cleanupCtx, cursor, "pp:"+fixture.namespace+":*", 128).Result()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("scan Redis provider keys: %v", err)
|
|
||||||
}
|
|
||||||
if len(keys) > 0 {
|
|
||||||
if err := fixture.client.Unlink(cleanupCtx, keys...).Err(); err != nil {
|
|
||||||
t.Fatalf("remove Redis provider keys: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cursor = next
|
|
||||||
if cursor == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fixture redisFixture) coordinator(t *testing.T, holder string) *Adapter {
|
|
||||||
t.Helper()
|
|
||||||
adapter, err := New(fixture.client, Options{
|
|
||||||
Namespace: fixture.namespace, HolderID: holder, LeaseTTL: 600 * time.Millisecond,
|
|
||||||
RenewEvery: 150 * time.Millisecond, RetryInterval: 20 * time.Millisecond, PermitGrace: 100 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(%s): %v", holder, err)
|
|
||||||
}
|
|
||||||
return adapter
|
|
||||||
}
|
|
||||||
|
|
||||||
func receiveLeadership(t *testing.T, values <-chan leadershipFixture) leadershipFixture {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case value := <-values:
|
|
||||||
return value
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("timed out waiting for leadership")
|
|
||||||
return leadershipFixture{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type leadershipFixture struct {
|
|
||||||
holder string
|
|
||||||
fence controllerProvider.Fence
|
|
||||||
}
|
|
||||||
|
|
||||||
func receiveSession(t *testing.T, values <-chan controllerProvider.LeaderSession) controllerProvider.LeaderSession {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case value := <-values:
|
|
||||||
return value
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("timed out waiting for leader session")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func receiveFence(t *testing.T, values <-chan controllerProvider.Fence) controllerProvider.Fence {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case value := <-values:
|
|
||||||
return value
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("timed out waiting for provider fence")
|
|
||||||
return controllerProvider.Fence{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResultFixture {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case value := <-values:
|
|
||||||
return value
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("timed out waiting for request permit")
|
|
||||||
return permitResultFixture{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type permitResultFixture struct {
|
|
||||||
permit controllerProvider.RequestPermit
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitRunner(t *testing.T, done <-chan error) {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
if err != nil && err != context.Canceled {
|
|
||||||
t.Fatalf("RunLeader() error = %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("timed out waiting for RunLeader shutdown")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,51 +0,0 @@
|
|||||||
package redisprovider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
controllerProvider "proxy-pool/internal/controller/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
type keyBuilder struct {
|
|
||||||
namespace string
|
|
||||||
}
|
|
||||||
|
|
||||||
type upstreamKeys struct {
|
|
||||||
generation string
|
|
||||||
epoch string
|
|
||||||
leader string
|
|
||||||
next string
|
|
||||||
inflight string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
|
||||||
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" {
|
|
||||||
return upstreamKeys{}, controllerProvider.ErrInvalidCoordination
|
|
||||||
}
|
|
||||||
digest := digestParts(upstreamID)
|
|
||||||
prefix := "pp:" + builder.namespace + ":{provider:" + digest + "}"
|
|
||||||
return upstreamKeys{
|
|
||||||
generation: prefix + ":generation",
|
|
||||||
epoch: prefix + ":epoch",
|
|
||||||
leader: prefix + ":leader",
|
|
||||||
next: prefix + ":next-request",
|
|
||||||
inflight: prefix + ":inflight",
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (keys upstreamKeys) all() []string {
|
|
||||||
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
|
|
||||||
}
|
|
||||||
|
|
||||||
func digestParts(values ...string) string {
|
|
||||||
digest := sha256.New()
|
|
||||||
for _, value := range values {
|
|
||||||
_, _ = digest.Write([]byte(strconv.Itoa(len(value))))
|
|
||||||
_, _ = digest.Write([]byte{':'})
|
|
||||||
_, _ = digest.Write([]byte(value))
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(digest.Sum(nil))
|
|
||||||
}
|
|
||||||
@ -1,61 +0,0 @@
|
|||||||
package redisprovider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
_ "embed"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
controllerProvider "proxy-pool/internal/controller/provider"
|
|
||||||
)
|
|
||||||
|
|
||||||
type scriptReply struct {
|
|
||||||
Status string `json:"status"`
|
|
||||||
Generation string `json:"generation,omitempty"`
|
|
||||||
Epoch uint64 `json:"epoch,string,omitempty"`
|
|
||||||
WaitMS int64 `json:"waitMs,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:embed scripts/provider.lua
|
|
||||||
var providerSource string
|
|
||||||
|
|
||||||
var providerScript = redis.NewScript(providerSource)
|
|
||||||
|
|
||||||
func runScript(ctx context.Context, client redis.Scripter, keys upstreamKeys, args ...any) (scriptReply, error) {
|
|
||||||
var reply scriptReply
|
|
||||||
result, err := providerScript.Run(ctx, client, keys.all(), args...).Result()
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return reply, err
|
|
||||||
}
|
|
||||||
return reply, errors.Join(controllerProvider.ErrCoordinationUnavailable,
|
|
||||||
fmt.Errorf("run Redis provider script: %w", err))
|
|
||||||
}
|
|
||||||
var payload []byte
|
|
||||||
switch value := result.(type) {
|
|
||||||
case string:
|
|
||||||
payload = []byte(value)
|
|
||||||
case []byte:
|
|
||||||
payload = value
|
|
||||||
default:
|
|
||||||
return reply, errors.Join(controllerProvider.ErrCoordinationUnavailable,
|
|
||||||
fmt.Errorf("decode Redis provider script: unexpected reply type %T", result))
|
|
||||||
}
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
|
||||||
decoder.DisallowUnknownFields()
|
|
||||||
if err := decoder.Decode(&reply); err != nil || reply.Status == "" {
|
|
||||||
return scriptReply{}, errors.Join(controllerProvider.ErrCoordinationUnavailable,
|
|
||||||
fmt.Errorf("decode Redis provider script reply: %w", err))
|
|
||||||
}
|
|
||||||
var trailing any
|
|
||||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
||||||
return scriptReply{}, errors.Join(controllerProvider.ErrCoordinationUnavailable,
|
|
||||||
errors.New("decode Redis provider script reply: trailing value"))
|
|
||||||
}
|
|
||||||
return reply, nil
|
|
||||||
}
|
|
||||||
@ -1,158 +0,0 @@
|
|||||||
local operation = ARGV[1]
|
|
||||||
|
|
||||||
local function now_ms()
|
|
||||||
local value = redis.call('TIME')
|
|
||||||
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
|
|
||||||
end
|
|
||||||
|
|
||||||
local function reply(status, generation, epoch, wait_ms)
|
|
||||||
return cjson.encode({
|
|
||||||
status = status,
|
|
||||||
generation = generation,
|
|
||||||
epoch = tostring(epoch or '0'),
|
|
||||||
waitMs = wait_ms
|
|
||||||
})
|
|
||||||
end
|
|
||||||
|
|
||||||
local function read_leader()
|
|
||||||
local encoded = redis.call('GET', KEYS[3])
|
|
||||||
if not encoded then
|
|
||||||
return nil, nil
|
|
||||||
end
|
|
||||||
local ok, value = pcall(cjson.decode, encoded)
|
|
||||||
if not ok or type(value) ~= 'table' or type(value.generation) ~= 'string' or
|
|
||||||
type(value.holderId) ~= 'string' or type(value.token) ~= 'string' or
|
|
||||||
type(value.epoch) ~= 'string' then
|
|
||||||
return nil, 'invalid'
|
|
||||||
end
|
|
||||||
return value, nil
|
|
||||||
end
|
|
||||||
|
|
||||||
local function same_leader(value, generation, holder_id, token, epoch)
|
|
||||||
return value and value.generation == generation and value.holderId == holder_id and
|
|
||||||
value.token == token and value.epoch == tostring(epoch)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'acquire_leader' then
|
|
||||||
local generation_candidate = ARGV[2]
|
|
||||||
local holder_id = ARGV[3]
|
|
||||||
local token = ARGV[4]
|
|
||||||
local lease_ttl = tonumber(ARGV[5])
|
|
||||||
if not lease_ttl or lease_ttl <= 0 then
|
|
||||||
return reply('invalid', '', 0, 0)
|
|
||||||
end
|
|
||||||
redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
|
||||||
local generation = redis.call('GET', KEYS[1])
|
|
||||||
local current, current_error = read_leader()
|
|
||||||
if current_error then
|
|
||||||
return reply('unavailable', generation, 0, 0)
|
|
||||||
end
|
|
||||||
if current then
|
|
||||||
if current.generation == generation and current.holderId == holder_id and current.token == token then
|
|
||||||
current.expiresAtMs = now_ms() + lease_ttl
|
|
||||||
redis.call('SET', KEYS[3], cjson.encode(current), 'PX', lease_ttl)
|
|
||||||
return reply('ok', generation, current.epoch, 0)
|
|
||||||
end
|
|
||||||
local remaining = redis.call('PTTL', KEYS[3])
|
|
||||||
return reply('busy', generation, 0, math.max(remaining, 1))
|
|
||||||
end
|
|
||||||
redis.call('INCR', KEYS[2])
|
|
||||||
local epoch = redis.call('GET', KEYS[2])
|
|
||||||
local leader = {
|
|
||||||
version = 1,
|
|
||||||
generation = generation,
|
|
||||||
holderId = holder_id,
|
|
||||||
token = token,
|
|
||||||
epoch = epoch,
|
|
||||||
expiresAtMs = now_ms() + lease_ttl
|
|
||||||
}
|
|
||||||
redis.call('SET', KEYS[3], cjson.encode(leader), 'PX', lease_ttl)
|
|
||||||
return reply('ok', generation, epoch, 0)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'renew_leader' then
|
|
||||||
local generation = ARGV[2]
|
|
||||||
local holder_id = ARGV[3]
|
|
||||||
local token = ARGV[4]
|
|
||||||
local epoch = ARGV[5]
|
|
||||||
local lease_ttl = tonumber(ARGV[6])
|
|
||||||
local current, current_error = read_leader()
|
|
||||||
if current_error then
|
|
||||||
return reply('unavailable', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
if not same_leader(current, generation, holder_id, token, epoch) then
|
|
||||||
return reply('stale', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
current.expiresAtMs = now_ms() + lease_ttl
|
|
||||||
redis.call('SET', KEYS[3], cjson.encode(current), 'PX', lease_ttl)
|
|
||||||
return reply('ok', generation, epoch, 0)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'release_leader' then
|
|
||||||
local generation = ARGV[2]
|
|
||||||
local holder_id = ARGV[3]
|
|
||||||
local token = ARGV[4]
|
|
||||||
local epoch = ARGV[5]
|
|
||||||
local current, current_error = read_leader()
|
|
||||||
if current_error then
|
|
||||||
return reply('unavailable', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
if same_leader(current, generation, holder_id, token, epoch) then
|
|
||||||
redis.call('DEL', KEYS[3])
|
|
||||||
end
|
|
||||||
return reply('ok', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'acquire_fetch' then
|
|
||||||
local generation = ARGV[2]
|
|
||||||
local holder_id = ARGV[3]
|
|
||||||
local leader_token = ARGV[4]
|
|
||||||
local epoch = ARGV[5]
|
|
||||||
local permit_token = ARGV[6]
|
|
||||||
local request_interval = tonumber(ARGV[7])
|
|
||||||
local max_in_flight = tonumber(ARGV[8])
|
|
||||||
local permit_ttl = tonumber(ARGV[9])
|
|
||||||
local current, current_error = read_leader()
|
|
||||||
if current_error then
|
|
||||||
return reply('unavailable', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
if not same_leader(current, generation, holder_id, leader_token, epoch) then
|
|
||||||
return reply('stale', generation, epoch or 0, 0)
|
|
||||||
end
|
|
||||||
local now = now_ms()
|
|
||||||
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
|
|
||||||
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
|
|
||||||
if existing then
|
|
||||||
return reply('ok', generation, epoch, 0)
|
|
||||||
end
|
|
||||||
local next_request = redis.call('GET', KEYS[4])
|
|
||||||
if next_request and not tonumber(next_request) then
|
|
||||||
return reply('unavailable', generation, epoch, 0)
|
|
||||||
end
|
|
||||||
if next_request and tonumber(next_request) > now then
|
|
||||||
return reply('rate_limited', generation, epoch, tonumber(next_request) - now)
|
|
||||||
end
|
|
||||||
if redis.call('ZCARD', KEYS[5]) >= max_in_flight then
|
|
||||||
local earliest = redis.call('ZRANGE', KEYS[5], 0, 0, 'WITHSCORES')
|
|
||||||
local wait_ms = 1
|
|
||||||
if earliest[2] then
|
|
||||||
wait_ms = math.max(tonumber(earliest[2]) - now, 1)
|
|
||||||
end
|
|
||||||
return reply('at_capacity', generation, epoch, wait_ms)
|
|
||||||
end
|
|
||||||
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
|
||||||
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
|
|
||||||
if request_interval > 0 then
|
|
||||||
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
|
||||||
else
|
|
||||||
redis.call('DEL', KEYS[4])
|
|
||||||
end
|
|
||||||
return reply('ok', generation, epoch, 0)
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'release_fetch' then
|
|
||||||
redis.call('ZREM', KEYS[5], ARGV[2])
|
|
||||||
return reply('ok', '', 0, 0)
|
|
||||||
end
|
|
||||||
|
|
||||||
return reply('invalid', '', 0, 0)
|
|
||||||
@ -163,7 +163,6 @@ type Upstream struct {
|
|||||||
ProxyAuth ProxyAuth `yaml:"proxyAuth"`
|
ProxyAuth ProxyAuth `yaml:"proxyAuth"`
|
||||||
Pool Pool `yaml:"pool"`
|
Pool Pool `yaml:"pool"`
|
||||||
Capacity Capacity `yaml:"capacity"`
|
Capacity Capacity `yaml:"capacity"`
|
||||||
Refill Refill `yaml:"refill"`
|
|
||||||
Lifecycle Lifecycle `yaml:"lifecycle"`
|
Lifecycle Lifecycle `yaml:"lifecycle"`
|
||||||
Fetch Fetch `yaml:"fetch"`
|
Fetch Fetch `yaml:"fetch"`
|
||||||
Check Check `yaml:"check"`
|
Check Check `yaml:"check"`
|
||||||
@ -218,19 +217,12 @@ type Capacity struct {
|
|||||||
MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"`
|
MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Refill struct {
|
|
||||||
ReconcileInterval Duration `yaml:"reconcileInterval"`
|
|
||||||
MinimumAvailableSlots int64 `yaml:"minimumAvailableSlots"`
|
|
||||||
TargetAvailableSlots int64 `yaml:"targetAvailableSlots"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Lifecycle struct {
|
type Lifecycle struct {
|
||||||
TTL Duration `yaml:"ttl"`
|
TTL Duration `yaml:"ttl"`
|
||||||
AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"`
|
AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Fetch struct {
|
type Fetch struct {
|
||||||
EstimatedIPsPerCall int `yaml:"estimatedIPsPerCall"`
|
|
||||||
RequestInterval Duration `yaml:"requestInterval"`
|
RequestInterval Duration `yaml:"requestInterval"`
|
||||||
Timeout Duration `yaml:"timeout"`
|
Timeout Duration `yaml:"timeout"`
|
||||||
MaxAttempts int `yaml:"maxAttempts"`
|
MaxAttempts int `yaml:"maxAttempts"`
|
||||||
|
|||||||
@ -61,15 +61,10 @@ upstreams:
|
|||||||
maxSize: 100
|
maxSize: 100
|
||||||
capacity:
|
capacity:
|
||||||
maxConcurrencyPerProxy: 10
|
maxConcurrencyPerProxy: 10
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 200
|
|
||||||
targetAvailableSlots: 500
|
|
||||||
lifecycle:
|
lifecycle:
|
||||||
ttl: 120s
|
ttl: 120s
|
||||||
allocationSafetyMargin: 10s
|
allocationSafetyMargin: 10s
|
||||||
fetch:
|
fetch:
|
||||||
estimatedIPsPerCall: 20
|
|
||||||
requestInterval: 1s
|
requestInterval: 1s
|
||||||
timeout: 3s
|
timeout: 3s
|
||||||
maxAttempts: 5
|
maxAttempts: 5
|
||||||
@ -91,10 +86,7 @@ func TestLoadStrictValidConfiguration(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load(): %v", err)
|
t.Fatalf("Load(): %v", err)
|
||||||
}
|
}
|
||||||
provider := cfg.Upstreams["provider-a"]
|
if cfg.Version != 1 || cfg.Upstreams["provider-a"].Pool.MaxSize != 100 {
|
||||||
if cfg.Version != 1 || provider.Pool.MaxSize != 100 || provider.Fetch.EstimatedIPsPerCall != 20 ||
|
|
||||||
provider.Refill.ReconcileInterval.Value() != time.Second || provider.Refill.MinimumAvailableSlots != 200 ||
|
|
||||||
provider.Refill.TargetAvailableSlots != 500 {
|
|
||||||
t.Fatalf("unexpected config: %+v", cfg)
|
t.Fatalf("unexpected config: %+v", cfg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -326,31 +318,6 @@ func TestValidateAcceptsBearerListenerAuthentication(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateMetricsListener(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
cfg := mustLoadValidConfig(t)
|
|
||||||
cfg.Metrics = Metrics{Enabled: true}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
|
|
||||||
t.Fatalf("Validate(metrics without listen) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Metrics.Listen = "not-an-address"
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
|
|
||||||
t.Fatalf("Validate(invalid metrics listen) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Metrics.Listen = "127.0.0.1:70000"
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "metrics listen") {
|
|
||||||
t.Fatalf("Validate(out-of-range metrics port) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Metrics.Listen = "0.0.0.0:9090"
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(public metrics listener) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Metrics = Metrics{Enabled: false, Listen: "not-an-address"}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(disabled metrics listener) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@ -537,47 +504,6 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
|||||||
},
|
},
|
||||||
want: "fetch.maxInFlight",
|
want: "fetch.maxInFlight",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "zero estimated IPs per call",
|
|
||||||
mutate: func(cfg *Config) {
|
|
||||||
updateUpstream(cfg, func(upstream *Upstream) { upstream.Fetch.EstimatedIPsPerCall = 0 })
|
|
||||||
},
|
|
||||||
want: "estimatedIPsPerCall",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "estimated IPs exceed pool size",
|
|
||||||
mutate: func(cfg *Config) {
|
|
||||||
updateUpstream(cfg, func(upstream *Upstream) {
|
|
||||||
upstream.Fetch.EstimatedIPsPerCall = upstream.Pool.MaxSize + 1
|
|
||||||
})
|
|
||||||
},
|
|
||||||
want: "estimatedIPsPerCall",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero refill interval",
|
|
||||||
mutate: func(cfg *Config) {
|
|
||||||
updateUpstream(cfg, func(upstream *Upstream) { upstream.Refill.ReconcileInterval = 0 })
|
|
||||||
},
|
|
||||||
want: "refill.reconcileInterval",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "refill target does not exceed minimum",
|
|
||||||
mutate: func(cfg *Config) {
|
|
||||||
updateUpstream(cfg, func(upstream *Upstream) {
|
|
||||||
upstream.Refill.TargetAvailableSlots = upstream.Refill.MinimumAvailableSlots
|
|
||||||
})
|
|
||||||
},
|
|
||||||
want: "targetAvailableSlots",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "refill target exceeds theoretical capacity",
|
|
||||||
mutate: func(cfg *Config) {
|
|
||||||
updateUpstream(cfg, func(upstream *Upstream) {
|
|
||||||
upstream.Refill.TargetAvailableSlots = int64(upstream.Pool.MaxSize)*int64(upstream.Capacity.MaxConcurrencyPerProxy) + 1
|
|
||||||
})
|
|
||||||
},
|
|
||||||
want: "targetAvailableSlots",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "negative fetch response limit",
|
name: "negative fetch response limit",
|
||||||
mutate: func(cfg *Config) {
|
mutate: func(cfg *Config) {
|
||||||
|
|||||||
@ -4,7 +4,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
|
func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
|
||||||
@ -78,13 +77,8 @@ func storeTestConfig(upstreamName string) *Config {
|
|||||||
API: ProviderAPI{Auth: ProviderAuth{Type: "none"}},
|
API: ProviderAPI{Auth: ProviderAuth{Type: "none"}},
|
||||||
ProxyAuth: ProxyAuth{Type: "response"},
|
ProxyAuth: ProxyAuth{Type: "response"},
|
||||||
Pool: Pool{MaxSize: 10}, Capacity: Capacity{MaxConcurrencyPerProxy: 1},
|
Pool: Pool{MaxSize: 10}, Capacity: Capacity{MaxConcurrencyPerProxy: 1},
|
||||||
Refill: Refill{
|
|
||||||
ReconcileInterval: Duration(time.Second), MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
|
||||||
},
|
|
||||||
Lifecycle: Lifecycle{TTL: Duration(60_000_000_000), AllocationSafetyMargin: Duration(10_000_000_000)},
|
Lifecycle: Lifecycle{TTL: Duration(60_000_000_000), AllocationSafetyMargin: Duration(10_000_000_000)},
|
||||||
Fetch: Fetch{
|
Fetch: Fetch{Timeout: Duration(1_000_000_000), MaxAttempts: 1, MaxInFlight: 1},
|
||||||
EstimatedIPsPerCall: 1, Timeout: Duration(time.Second), MaxAttempts: 1, MaxInFlight: 1,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Routing: []Routing{{
|
Routing: []Routing{{
|
||||||
|
|||||||
@ -2,11 +2,9 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"net"
|
"net"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -30,14 +28,6 @@ func Validate(cfg *Config) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cfg.Metrics.Enabled {
|
|
||||||
if cfg.Metrics.Listen == "" {
|
|
||||||
return fmt.Errorf("validate metrics listen: address is required")
|
|
||||||
}
|
|
||||||
if _, err := validateListenAddress("metrics", cfg.Metrics.Listen); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fetchConfigured(cfg.Defaults.Fetch) {
|
if fetchConfigured(cfg.Defaults.Fetch) {
|
||||||
if err := validateFetch("defaults.fetch", cfg.Defaults.Fetch); err != nil {
|
if err := validateFetch("defaults.fetch", cfg.Defaults.Fetch); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -120,9 +110,9 @@ func validateListener(name string, listener Listener, security Security) error {
|
|||||||
return fmt.Errorf("validate %s limits.%s: must be non-negative", name, limit.name)
|
return fmt.Errorf("validate %s limits.%s: must be non-negative", name, limit.name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
host, err := validateListenAddress(name, listener.Listen)
|
host, _, err := net.SplitHostPort(listener.Listen)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("validate %s listen: %w", name, err)
|
||||||
}
|
}
|
||||||
if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 {
|
if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 {
|
||||||
return fmt.Errorf("validate %s: unprotected public listener is forbidden", name)
|
return fmt.Errorf("validate %s: unprotected public listener is forbidden", name)
|
||||||
@ -149,17 +139,6 @@ func validateListener(name string, listener Listener, security Security) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateListenAddress(name, address string) (string, error) {
|
|
||||||
host, port, err := net.SplitHostPort(address)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("validate %s listen: %w", name, err)
|
|
||||||
}
|
|
||||||
if _, err = strconv.ParseUint(port, 10, 16); err != nil {
|
|
||||||
return "", fmt.Errorf("validate %s listen: invalid port", name)
|
|
||||||
}
|
|
||||||
return host, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateRouting(index int, route Routing, upstreams map[string]Upstream, seen map[string]struct{}) error {
|
func validateRouting(index int, route Routing, upstreams map[string]Upstream, seen map[string]struct{}) error {
|
||||||
if route.Name == "" {
|
if route.Name == "" {
|
||||||
return fmt.Errorf("validate routing[%d]: name is required", index)
|
return fmt.Errorf("validate routing[%d]: name is required", index)
|
||||||
@ -339,22 +318,6 @@ func validateUpstream(name string, upstream Upstream) error {
|
|||||||
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
|
||||||
}
|
|
||||||
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
|
||||||
}
|
|
||||||
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
|
||||||
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
|
||||||
}
|
|
||||||
if err := requirePositive(scope+" lifecycle.ttl", upstream.Lifecycle.TTL); err != nil {
|
if err := requirePositive(scope+" lifecycle.ttl", upstream.Lifecycle.TTL); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -367,12 +330,6 @@ func validateUpstream(name string, upstream Upstream) error {
|
|||||||
if err := validateFetch(scope+" fetch", upstream.Fetch); err != nil {
|
if err := validateFetch(scope+" fetch", upstream.Fetch); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if upstream.Fetch.EstimatedIPsPerCall > upstream.Pool.MaxSize {
|
|
||||||
return fmt.Errorf("validate %s fetch.estimatedIPsPerCall: cannot exceed pool.maxSize", scope)
|
|
||||||
}
|
|
||||||
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.EstimatedIPsPerCall > upstream.Fetch.MaxTotal {
|
|
||||||
return fmt.Errorf("validate %s fetch.estimatedIPsPerCall: cannot exceed fetch.maxTotal", scope)
|
|
||||||
}
|
|
||||||
if err := validateCheck(scope+" check", upstream.Check); err != nil {
|
if err := validateCheck(scope+" check", upstream.Check); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -400,9 +357,6 @@ func validateUpstream(name string, upstream Upstream) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateFetch(scope string, fetch Fetch) error {
|
func validateFetch(scope string, fetch Fetch) error {
|
||||||
if err := requirePositive(scope+".estimatedIPsPerCall", fetch.EstimatedIPsPerCall); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := requireNonNegative(scope+".requestInterval", fetch.RequestInterval); err != nil {
|
if err := requireNonNegative(scope+".requestInterval", fetch.RequestInterval); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -468,7 +422,7 @@ func validateCheck(scope string, check Check) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetchConfigured(fetch Fetch) bool {
|
func fetchConfigured(fetch Fetch) bool {
|
||||||
return fetch.EstimatedIPsPerCall != 0 || fetch.RequestInterval != 0 || fetch.Timeout != 0 || fetch.MaxAttempts != 0 ||
|
return fetch.RequestInterval != 0 || fetch.Timeout != 0 || fetch.MaxAttempts != 0 ||
|
||||||
fetch.MaxInFlight != 0 || fetch.MaxTotal != 0 || fetch.MaxResponseBytes != 0 ||
|
fetch.MaxInFlight != 0 || fetch.MaxTotal != 0 || fetch.MaxResponseBytes != 0 ||
|
||||||
fetch.TemplateTimeout != 0 || fetch.Retry.Initial != 0 || fetch.Retry.Max != 0 ||
|
fetch.TemplateTimeout != 0 || fetch.Retry.Initial != 0 || fetch.Retry.Max != 0 ||
|
||||||
fetch.Retry.Jitter != 0
|
fetch.Retry.Jitter != 0
|
||||||
|
|||||||
@ -150,8 +150,7 @@ upstreams:
|
|||||||
proxyAuth: {type: response}
|
proxyAuth: {type: response}
|
||||||
pool: {maxSize: 2000}
|
pool: {maxSize: 2000}
|
||||||
capacity: {maxConcurrencyPerProxy: 20}
|
capacity: {maxConcurrencyPerProxy: 20}
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 100, targetAvailableSlots: 200}
|
|
||||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||||
fetch: {estimatedIPsPerCall: 100, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||||
`
|
`
|
||||||
|
|||||||
@ -178,23 +178,6 @@ func (service *ApplicationService) ReloadConfiguration(ctx context.Context, comm
|
|||||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return service.ApplyConfiguration(ctx, command, loaded)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ApplyConfiguration persists and publishes one already loaded configuration snapshot.
|
|
||||||
// Startup and runtime reload paths share this method so storage connections and the
|
|
||||||
// committed management view cannot be built from different reads of the source file.
|
|
||||||
func (service *ApplicationService) ApplyConfiguration(
|
|
||||||
ctx context.Context,
|
|
||||||
command ReloadCommand,
|
|
||||||
loaded LoadedConfiguration,
|
|
||||||
) (MutationResult, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return MutationResult{RequestID: command.RequestID}, err
|
|
||||||
}
|
|
||||||
if loaded.Value == nil || strings.TrimSpace(loaded.Source) != loaded.Source || loaded.Source == "" ||
|
if loaded.Value == nil || strings.TrimSpace(loaded.Source) != loaded.Source || loaded.Source == "" ||
|
||||||
len(loaded.Source) > adminstate.MaxSourceBytes {
|
len(loaded.Source) > adminstate.MaxSourceBytes {
|
||||||
return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration
|
return MutationResult{RequestID: command.RequestID}, ErrInvalidConfiguration
|
||||||
|
|||||||
@ -293,42 +293,6 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloading(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 30, 9, 0, 0, 0, time.UTC)
|
|
||||||
configuration := validReloadConfiguration()
|
|
||||||
publisher := &recordingConfigurationPublisher{}
|
|
||||||
state := &recordingAdminState{
|
|
||||||
mutation: adminstate.MutationResult{RequestID: "controller-startup", Changed: true, Revision: 1},
|
|
||||||
}
|
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
|
||||||
State: state,
|
|
||||||
Operations: staticOperationalStatusReader{},
|
|
||||||
Configuration: forbiddenConfigurationLoader{},
|
|
||||||
Publisher: publisher,
|
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := service.ApplyConfiguration(context.Background(), ReloadCommand{
|
|
||||||
RequestID: "controller-startup", ActorID: "proxy-controller",
|
|
||||||
}, LoadedConfiguration{Value: configuration, Source: "configs/controller.yaml"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ApplyConfiguration() error = %v", err)
|
|
||||||
}
|
|
||||||
if result.Version != 1 || !result.Changed {
|
|
||||||
t.Fatalf("ApplyConfiguration() result = %+v", result)
|
|
||||||
}
|
|
||||||
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
|
||||||
t.Fatalf("published configurations = %+v", publisher.published)
|
|
||||||
}
|
|
||||||
if state.lastConfig.Source != "configs/controller.yaml" || state.lastConfig.Actor.ID != "proxy-controller" ||
|
|
||||||
!state.lastConfig.OccurredAt.Equal(now) {
|
|
||||||
t.Fatalf("CommitConfig() metadata = %+v", state.lastConfig)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration(t *testing.T) {
|
func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@ -473,13 +437,8 @@ func validReloadUpstream(secret string) config.Upstream {
|
|||||||
API: config.ProviderAPI{Auth: config.ProviderAuth{Type: "none"}},
|
API: config.ProviderAPI{Auth: config.ProviderAuth{Type: "none"}},
|
||||||
ProxyAuth: config.ProxyAuth{Type: "static", Username: "user", Password: secret},
|
ProxyAuth: config.ProxyAuth{Type: "static", Username: "user", Password: secret},
|
||||||
Pool: config.Pool{MaxSize: 10}, Capacity: config.Capacity{MaxConcurrencyPerProxy: 2},
|
Pool: config.Pool{MaxSize: 10}, Capacity: config.Capacity{MaxConcurrencyPerProxy: 2},
|
||||||
Refill: config.Refill{
|
|
||||||
ReconcileInterval: config.Duration(time.Second), MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
|
||||||
},
|
|
||||||
Lifecycle: config.Lifecycle{TTL: config.Duration(time.Minute), AllocationSafetyMargin: config.Duration(10 * time.Second)},
|
Lifecycle: config.Lifecycle{TTL: config.Duration(time.Minute), AllocationSafetyMargin: config.Duration(10 * time.Second)},
|
||||||
Fetch: config.Fetch{
|
Fetch: config.Fetch{Timeout: config.Duration(time.Second), MaxAttempts: 2, MaxInFlight: 1},
|
||||||
EstimatedIPsPerCall: 1, Timeout: config.Duration(time.Second), MaxAttempts: 2, MaxInFlight: 1,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -541,12 +500,6 @@ type staticConfigurationLoader struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
type forbiddenConfigurationLoader struct{}
|
|
||||||
|
|
||||||
func (forbiddenConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) {
|
|
||||||
panic("ApplyConfiguration must not reload the source")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) {
|
func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (LoadedConfiguration, error) {
|
||||||
return loader.loaded, loader.err
|
return loader.loaded, loader.err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,192 +0,0 @@
|
|||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/controller/admin"
|
|
||||||
"proxy-pool/internal/controller/distribution"
|
|
||||||
"proxy-pool/internal/controller/extraction"
|
|
||||||
"proxy-pool/internal/controller/operations"
|
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
"proxy-pool/internal/platform/admission"
|
|
||||||
"proxy-pool/internal/platform/httpserver"
|
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidOptions = errors.New("invalid controller bootstrap options")
|
|
||||||
ErrStartup = errors.New("controller startup failed")
|
|
||||||
)
|
|
||||||
|
|
||||||
type Options struct {
|
|
||||||
ConfigPath string
|
|
||||||
Resolver config.Resolver
|
|
||||||
Now func() time.Time
|
|
||||||
HTTP httpserver.Options
|
|
||||||
}
|
|
||||||
|
|
||||||
type activityStore interface {
|
|
||||||
extractionDomain.Store
|
|
||||||
activitypool.StateInventoryReader
|
|
||||||
}
|
|
||||||
|
|
||||||
type ports struct {
|
|
||||||
state admin.StateRepository
|
|
||||||
activity activityStore
|
|
||||||
readiness distribution.ReadinessChecker
|
|
||||||
metricsReadiness platformMetrics.ReadinessChecker
|
|
||||||
close func() error
|
|
||||||
}
|
|
||||||
|
|
||||||
type infrastructure interface {
|
|
||||||
Open(context.Context, *config.Config) (ports, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type controllerRunner interface {
|
|
||||||
Run(context.Context) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type runtimeFactory interface {
|
|
||||||
New(*config.Config, controllerRuntime.Dependencies, controllerRuntime.Options) (controllerRunner, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Run(ctx context.Context, options Options) error {
|
|
||||||
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
|
||||||
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
|
||||||
nilInterface(options.Resolver) || nilInterface(infrastructure) || nilInterface(factory) {
|
|
||||||
return ErrInvalidOptions
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if options.Now == nil {
|
|
||||||
options.Now = time.Now
|
|
||||||
}
|
|
||||||
|
|
||||||
loader, err := admin.NewFileConfigurationLoader(options.ConfigPath, options.Resolver)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Join(ErrInvalidOptions, err)
|
|
||||||
}
|
|
||||||
loaded, err := loader.LoadConfiguration(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
configurationStore, err := config.NewStore(loaded.Value)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
opened, err := infrastructure.Open(ctx, loaded.Value)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: open infrastructure: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
if opened.close == nil {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
resultErr = errors.Join(resultErr, opened.close())
|
|
||||||
}()
|
|
||||||
|
|
||||||
dependencies := controllerRuntime.Dependencies{}
|
|
||||||
if loaded.Value.Distribution.Enabled {
|
|
||||||
if nilInterface(opened.activity) || nilInterface(opened.readiness) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
service, serviceErr := extraction.NewService(opened.activity, extractionPolicy(loaded.Value), admission.AllowAll{}, options.Now)
|
|
||||||
if serviceErr != nil {
|
|
||||||
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
|
|
||||||
}
|
|
||||||
dependencies.Extractor = service
|
|
||||||
dependencies.Readiness = opened.readiness
|
|
||||||
}
|
|
||||||
if loaded.Value.Admin.Enabled {
|
|
||||||
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
|
|
||||||
if statusErr != nil {
|
|
||||||
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
|
||||||
}
|
|
||||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
|
||||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
|
||||||
}, admin.ApplicationOptions{Now: options.Now})
|
|
||||||
if serviceErr != nil {
|
|
||||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
|
||||||
}
|
|
||||||
if _, applyErr := service.ApplyConfiguration(ctx, admin.ReloadCommand{
|
|
||||||
RequestID: "controller-startup", ActorID: "proxy-controller",
|
|
||||||
}, loaded); applyErr != nil {
|
|
||||||
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
|
||||||
}
|
|
||||||
dependencies.AdminService = service
|
|
||||||
}
|
|
||||||
if loaded.Value.Metrics.Enabled {
|
|
||||||
if nilInterface(opened.metricsReadiness) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
|
||||||
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
|
|
||||||
})
|
|
||||||
if handlerErr != nil {
|
|
||||||
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
|
|
||||||
}
|
|
||||||
dependencies.MetricsHandler = handler
|
|
||||||
}
|
|
||||||
|
|
||||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
if nilInterface(runner) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
return runner.Run(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
|
||||||
configured := configuration.Distribution.Extraction
|
|
||||||
return extraction.Policy{
|
|
||||||
MaxCountPerRequest: configured.MaxCountPerRequest,
|
|
||||||
DefaultFulfillment: extractionDomain.Fulfillment(configured.Fulfillment),
|
|
||||||
MinRemainingTTL: configured.MinRemainingTTL.Value(),
|
|
||||||
MaxHealthCheckAge: configured.MaxHealthCheckAge.Value(),
|
|
||||||
ReserveForGateway: configured.ReserveForGateway,
|
|
||||||
IdempotencyTTL: configured.IdempotencyTTL.Value(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func nilInterface(value any) bool {
|
|
||||||
if value == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
reflected := reflect.ValueOf(value)
|
|
||||||
switch reflected.Kind() {
|
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
|
||||||
return reflected.IsNil()
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type productionRuntimeFactory struct{}
|
|
||||||
|
|
||||||
func (productionRuntimeFactory) New(
|
|
||||||
configuration *config.Config,
|
|
||||||
dependencies controllerRuntime.Dependencies,
|
|
||||||
options controllerRuntime.Options,
|
|
||||||
) (controllerRunner, error) {
|
|
||||||
return controllerRuntime.New(configuration, dependencies, options)
|
|
||||||
}
|
|
||||||
@ -1,84 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/controller/admin"
|
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
|
||||||
postgresURL := os.Getenv("PROXY_POOL_TEST_POSTGRES_URL")
|
|
||||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
|
||||||
if postgresURL == "" || redisURL == "" {
|
|
||||||
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
|
||||||
}
|
|
||||||
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
|
||||||
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
|
||||||
factory := &integrationRuntimeFactory{}
|
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
|
||||||
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
|
||||||
},
|
|
||||||
}, &productionInfrastructure{}, factory)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("run() error = %v", err)
|
|
||||||
}
|
|
||||||
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
|
||||||
t.Fatalf("Admin Status = %+v", factory.status)
|
|
||||||
}
|
|
||||||
if factory.status.Upstreams[0].Name != "provider-a" || factory.status.Upstreams[0].Available != 0 ||
|
|
||||||
factory.status.Upstreams[1].Name != "provider-b" {
|
|
||||||
t.Fatalf("Admin Status upstreams = %+v", factory.status.Upstreams)
|
|
||||||
}
|
|
||||||
if factory.readyStatus != http.StatusOK || factory.metricsStatus != http.StatusOK ||
|
|
||||||
!strings.Contains(factory.metricsBody, "go_") {
|
|
||||||
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type integrationRuntimeFactory struct {
|
|
||||||
status admin.Status
|
|
||||||
readyStatus int
|
|
||||||
metricsStatus int
|
|
||||||
metricsBody string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (factory *integrationRuntimeFactory) New(
|
|
||||||
_ *config.Config,
|
|
||||||
dependencies controllerRuntime.Dependencies,
|
|
||||||
_ controllerRuntime.Options,
|
|
||||||
) (controllerRunner, error) {
|
|
||||||
return integrationRunner{run: func(ctx context.Context) error {
|
|
||||||
if err := dependencies.Readiness.Ready(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ready := httptest.NewRecorder()
|
|
||||||
dependencies.MetricsHandler.ServeHTTP(ready, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
|
||||||
factory.readyStatus = ready.Code
|
|
||||||
metrics := httptest.NewRecorder()
|
|
||||||
dependencies.MetricsHandler.ServeHTTP(metrics, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
|
||||||
factory.metricsStatus = metrics.Code
|
|
||||||
factory.metricsBody = metrics.Body.String()
|
|
||||||
status, err := dependencies.AdminService.Status(ctx)
|
|
||||||
factory.status = status
|
|
||||||
return err
|
|
||||||
}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type integrationRunner struct {
|
|
||||||
run func(context.Context) error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (runner integrationRunner) Run(ctx context.Context) error { return runner.run(ctx) }
|
|
||||||
@ -1,211 +0,0 @@
|
|||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
|
||||||
state := adminstate.NewMemoryStore()
|
|
||||||
activity := &stubActivityStore{}
|
|
||||||
closeErr := errors.New("close failed")
|
|
||||||
infrastructure := &stubInfrastructure{ports: ports{
|
|
||||||
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
|
||||||
close: func() error { return closeErr },
|
|
||||||
}}
|
|
||||||
runErr := errors.New("runtime failed")
|
|
||||||
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
|
||||||
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
|
||||||
}, infrastructure, factory)
|
|
||||||
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
|
||||||
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
|
||||||
}
|
|
||||||
if resolver.reads != 1 {
|
|
||||||
t.Fatalf("configuration reads = %d, want 1", resolver.reads)
|
|
||||||
}
|
|
||||||
if infrastructure.opens != 1 || infrastructure.configuration == nil {
|
|
||||||
t.Fatalf("infrastructure opens = %d, config = %p", infrastructure.opens, infrastructure.configuration)
|
|
||||||
}
|
|
||||||
snapshot, snapshotErr := state.Snapshot(context.Background())
|
|
||||||
if snapshotErr != nil || snapshot.Config == nil || snapshot.Config.Source != "controller.yaml" || snapshot.Revision != 1 {
|
|
||||||
t.Fatalf("management snapshot = %+v, %v", snapshot, snapshotErr)
|
|
||||||
}
|
|
||||||
if factory.configuration == nil || factory.dependencies.Extractor == nil ||
|
|
||||||
factory.dependencies.Readiness == nil || factory.dependencies.AdminService == nil ||
|
|
||||||
factory.dependencies.MetricsHandler == nil {
|
|
||||||
t.Fatalf("runtime assembly = config:%p dependencies:%+v", factory.configuration, factory.dependencies)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
valid := Options{ConfigPath: "controller.yaml", Resolver: &memoryResolver{}, Now: time.Now}
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
ctx context.Context
|
|
||||||
options Options
|
|
||||||
}{
|
|
||||||
{name: "nil context", options: valid},
|
|
||||||
{name: "missing path", ctx: context.Background(), options: Options{Resolver: valid.Resolver, Now: time.Now}},
|
|
||||||
{name: "unclean path", ctx: context.Background(), options: Options{ConfigPath: " controller.yaml", Resolver: valid.Resolver, Now: time.Now}},
|
|
||||||
{name: "missing resolver", ctx: context.Background(), options: Options{ConfigPath: "controller.yaml", Now: time.Now}},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
test := test
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
if err := run(test.ctx, test.options, &stubInfrastructure{}, &recordingRuntimeFactory{}); !errors.Is(err, ErrInvalidOptions) {
|
|
||||||
t.Fatalf("run() error = %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type memoryResolver struct {
|
|
||||||
files map[string][]byte
|
|
||||||
reads int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*memoryResolver) LookupEnv(string) (string, bool) { return "", false }
|
|
||||||
|
|
||||||
func (resolver *memoryResolver) ReadFile(path string) ([]byte, error) {
|
|
||||||
resolver.reads++
|
|
||||||
content, ok := resolver.files[path]
|
|
||||||
if !ok {
|
|
||||||
return nil, errors.New("file missing")
|
|
||||||
}
|
|
||||||
return append([]byte(nil), content...), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type stubInfrastructure struct {
|
|
||||||
ports ports
|
|
||||||
err error
|
|
||||||
opens int
|
|
||||||
configuration *config.Config
|
|
||||||
}
|
|
||||||
|
|
||||||
func (infrastructure *stubInfrastructure) Open(
|
|
||||||
_ context.Context,
|
|
||||||
configuration *config.Config,
|
|
||||||
) (ports, error) {
|
|
||||||
infrastructure.opens++
|
|
||||||
infrastructure.configuration = configuration
|
|
||||||
return infrastructure.ports, infrastructure.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type recordingRuntimeFactory struct {
|
|
||||||
configuration *config.Config
|
|
||||||
dependencies controllerRuntime.Dependencies
|
|
||||||
runner controllerRunner
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (factory *recordingRuntimeFactory) New(
|
|
||||||
configuration *config.Config,
|
|
||||||
dependencies controllerRuntime.Dependencies,
|
|
||||||
options controllerRuntime.Options,
|
|
||||||
) (controllerRunner, error) {
|
|
||||||
factory.configuration = configuration
|
|
||||||
factory.dependencies = dependencies
|
|
||||||
return factory.runner, factory.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type runnerStub struct{ err error }
|
|
||||||
|
|
||||||
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
|
||||||
|
|
||||||
type readyStub struct{}
|
|
||||||
|
|
||||||
func (readyStub) Ready(context.Context) error { return nil }
|
|
||||||
|
|
||||||
type stubActivityStore struct{}
|
|
||||||
|
|
||||||
func (*stubActivityStore) Extract(_ context.Context, command extractionDomain.Command) (extractionDomain.Result, error) {
|
|
||||||
return extractionDomain.Result{Requested: command.Requested}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*stubActivityStore) ReadStateInventory(
|
|
||||||
_ context.Context,
|
|
||||||
upstreamIDs []string,
|
|
||||||
_ time.Time,
|
|
||||||
) ([]activitypool.StateInventory, error) {
|
|
||||||
result := make([]activitypool.StateInventory, len(upstreamIDs))
|
|
||||||
for index, upstreamID := range upstreamIDs {
|
|
||||||
result[index].UpstreamID = upstreamID
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const bootstrapTestConfig = `
|
|
||||||
version: 1
|
|
||||||
security:
|
|
||||||
requireProtectionOnPublicListen: true
|
|
||||||
gateway:
|
|
||||||
enabled: false
|
|
||||||
distribution:
|
|
||||||
enabled: true
|
|
||||||
listen: 127.0.0.1:0
|
|
||||||
auth: {mode: none}
|
|
||||||
extraction:
|
|
||||||
fulfillment: partial
|
|
||||||
maxCountPerRequest: 20
|
|
||||||
minRemainingTTL: 5s
|
|
||||||
maxHealthCheckAge: 15s
|
|
||||||
reserveForGateway: 5
|
|
||||||
idempotencyTTL: 5m
|
|
||||||
admin:
|
|
||||||
enabled: true
|
|
||||||
listen: 127.0.0.1:0
|
|
||||||
auth: {mode: none}
|
|
||||||
metrics:
|
|
||||||
enabled: true
|
|
||||||
listen: 127.0.0.1:0
|
|
||||||
storage:
|
|
||||||
postgresURL: postgres://fixture
|
|
||||||
redisURL: redis://fixture
|
|
||||||
routing:
|
|
||||||
- name: extract
|
|
||||||
enabled: true
|
|
||||||
purpose: extract
|
|
||||||
upstreams: [provider-a, provider-b]
|
|
||||||
strategy: {type: sequential, switchAfterEmptyFetch: 5, endBehavior: stayLast}
|
|
||||||
onUnavailable: {action: reject}
|
|
||||||
upstreams:
|
|
||||||
provider-a: &upstream
|
|
||||||
enabled: true
|
|
||||||
exposure: [extract]
|
|
||||||
provider: {billingMode: fetch, protocols: [http]}
|
|
||||||
api:
|
|
||||||
url: https://provider.invalid/proxies
|
|
||||||
method: GET
|
|
||||||
template: '{{.}}'
|
|
||||||
auth: {type: none}
|
|
||||||
proxyAuth: {type: response}
|
|
||||||
pool: {maxSize: 100}
|
|
||||||
capacity: {maxConcurrencyPerProxy: 10}
|
|
||||||
refill: {reconcileInterval: 1s, minimumAvailableSlots: 100, targetAvailableSlots: 200}
|
|
||||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
|
||||||
fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
|
|
||||||
check:
|
|
||||||
interval: 30s
|
|
||||||
jitter: 20
|
|
||||||
maxInFlight: 100
|
|
||||||
timeout: 2s
|
|
||||||
maxAttempts: 2
|
|
||||||
maxConsecutiveFailures: 3
|
|
||||||
urls: [https://example.invalid/health]
|
|
||||||
provider-b: *upstream
|
|
||||||
`
|
|
||||||
@ -1,218 +0,0 @@
|
|||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
|
|
||||||
"proxy-pool/internal/adapters/postgresadmin"
|
|
||||||
"proxy-pool/internal/adapters/redisactivity"
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/platform/credentials"
|
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
redisNamespace = "controller"
|
|
||||||
redisOperationTTL = 30 * time.Second
|
|
||||||
redisMinimumScan = 4_096
|
|
||||||
redisCleanupLimit = 1_024
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrPostgresConfiguration = errors.New("invalid PostgreSQL configuration")
|
|
||||||
ErrPostgresUnavailable = errors.New("PostgreSQL unavailable")
|
|
||||||
ErrRedisConfiguration = errors.New("invalid Redis configuration")
|
|
||||||
ErrRedisUnavailable = errors.New("Redis unavailable")
|
|
||||||
)
|
|
||||||
|
|
||||||
type productionInfrastructure struct{}
|
|
||||||
|
|
||||||
func (*productionInfrastructure) Open(
|
|
||||||
ctx context.Context,
|
|
||||||
configuration *config.Config,
|
|
||||||
) (_ ports, resultErr error) {
|
|
||||||
if ctx == nil || configuration == nil {
|
|
||||||
return ports{}, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
var postgresPool *pgxpool.Pool
|
|
||||||
var redisClient *redis.Client
|
|
||||||
closeResources := func() error {
|
|
||||||
var closeErr error
|
|
||||||
if redisClient != nil {
|
|
||||||
closeErr = redisClient.Close()
|
|
||||||
}
|
|
||||||
if postgresPool != nil {
|
|
||||||
postgresPool.Close()
|
|
||||||
}
|
|
||||||
return closeErr
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if resultErr != nil {
|
|
||||||
_ = closeResources()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
opened := ports{close: closeResources}
|
|
||||||
if configuration.Admin.Enabled {
|
|
||||||
if strings.TrimSpace(configuration.Storage.PostgresURL) == "" {
|
|
||||||
return ports{}, ErrPostgresConfiguration
|
|
||||||
}
|
|
||||||
poolConfig, err := pgxpool.ParseConfig(configuration.Storage.PostgresURL)
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, ErrPostgresConfiguration
|
|
||||||
}
|
|
||||||
poolConfig.ConnConfig.RuntimeParams["application_name"] = "proxy-controller"
|
|
||||||
postgresPool, err = pgxpool.NewWithConfig(ctx, poolConfig)
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, ErrPostgresUnavailable
|
|
||||||
}
|
|
||||||
if err = postgresPool.Ping(ctx); err != nil {
|
|
||||||
return ports{}, contextOr(ctx, ErrPostgresUnavailable)
|
|
||||||
}
|
|
||||||
if err = postgresadmin.ApplyMigrations(ctx, postgresPool); err != nil {
|
|
||||||
return ports{}, err
|
|
||||||
}
|
|
||||||
opened.state, err = postgresadmin.New(postgresPool)
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
|
|
||||||
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
|
||||||
return ports{}, ErrRedisConfiguration
|
|
||||||
}
|
|
||||||
redisOptions, err := redis.ParseURL(configuration.Storage.RedisURL)
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, ErrRedisConfiguration
|
|
||||||
}
|
|
||||||
redisClient = redis.NewClient(redisOptions)
|
|
||||||
if err = redisClient.Ping(ctx).Err(); err != nil {
|
|
||||||
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
|
||||||
}
|
|
||||||
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, err
|
|
||||||
}
|
|
||||||
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
|
||||||
Namespace: redisNamespace,
|
|
||||||
Credentials: credentialStore,
|
|
||||||
OperationTTL: redisOperationTTL,
|
|
||||||
MaxCandidateScan: candidateScan(configuration),
|
|
||||||
CleanupLimit: redisCleanupLimit,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return ports{}, err
|
|
||||||
}
|
|
||||||
opened.activity = adapter
|
|
||||||
opened.readiness = redisReadiness{client: redisClient}
|
|
||||||
}
|
|
||||||
if configuration.Metrics.Enabled {
|
|
||||||
opened.metricsReadiness = selectMetricsReadiness(
|
|
||||||
configuration,
|
|
||||||
storeReadiness{postgres: postgresPool, redis: redisClient},
|
|
||||||
redisReadiness{client: redisClient},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return opened, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectMetricsReadiness(
|
|
||||||
configuration *config.Config,
|
|
||||||
admin, activity platformMetrics.ReadinessChecker,
|
|
||||||
) platformMetrics.ReadinessChecker {
|
|
||||||
if configuration.Distribution.Enabled {
|
|
||||||
return activity
|
|
||||||
}
|
|
||||||
if configuration.Admin.Enabled {
|
|
||||||
return admin
|
|
||||||
}
|
|
||||||
return alwaysReady{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type alwaysReady struct{}
|
|
||||||
|
|
||||||
func (alwaysReady) Ready(ctx context.Context) error {
|
|
||||||
if ctx == nil {
|
|
||||||
return ErrInvalidOptions
|
|
||||||
}
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
type storeReadiness struct {
|
|
||||||
postgres *pgxpool.Pool
|
|
||||||
redis *redis.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness storeReadiness) Ready(ctx context.Context) error {
|
|
||||||
if ctx == nil {
|
|
||||||
return ErrInvalidOptions
|
|
||||||
}
|
|
||||||
if readiness.postgres != nil {
|
|
||||||
if err := readiness.postgres.Ping(ctx); err != nil {
|
|
||||||
return contextOr(ctx, ErrPostgresUnavailable)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if readiness.redis != nil {
|
|
||||||
if err := readiness.redis.Ping(ctx).Err(); err != nil {
|
|
||||||
return contextOr(ctx, ErrRedisUnavailable)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type redisReadiness struct {
|
|
||||||
client *redis.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness redisReadiness) Ready(ctx context.Context) error {
|
|
||||||
if ctx == nil || readiness.client == nil {
|
|
||||||
return ErrRedisUnavailable
|
|
||||||
}
|
|
||||||
if err := readiness.client.Ping(ctx).Err(); err != nil {
|
|
||||||
return contextOr(ctx, ErrRedisUnavailable)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func credentialCapacity(configuration *config.Config) int {
|
|
||||||
capacity := 0
|
|
||||||
maximum := int(^uint(0) >> 1)
|
|
||||||
for _, upstream := range configuration.Upstreams {
|
|
||||||
if upstream.Pool.MaxSize <= 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if capacity > maximum-upstream.Pool.MaxSize {
|
|
||||||
return maximum
|
|
||||||
}
|
|
||||||
capacity += upstream.Pool.MaxSize
|
|
||||||
}
|
|
||||||
if capacity == 0 {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return capacity
|
|
||||||
}
|
|
||||||
|
|
||||||
func candidateScan(configuration *config.Config) int {
|
|
||||||
configured := configuration.Distribution.Extraction
|
|
||||||
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
|
||||||
return int(^uint(0) >> 1)
|
|
||||||
}
|
|
||||||
value := configured.MaxCountPerRequest + configured.ReserveForGateway
|
|
||||||
if value < redisMinimumScan {
|
|
||||||
return redisMinimumScan
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
func contextOr(ctx context.Context, fallback error) error {
|
|
||||||
if ctx != nil && ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
@ -1,96 +0,0 @@
|
|||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestProductionInfrastructureRejectsInvalidStorageWithoutLeakingURLs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
postgresSecret := "postgres-secret"
|
|
||||||
_, err := (&productionInfrastructure{}).Open(context.Background(), &config.Config{
|
|
||||||
Admin: config.Listener{Enabled: true},
|
|
||||||
Storage: config.Storage{PostgresURL: "postgres://user:" + postgresSecret + "@%zz"},
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrPostgresConfiguration) || strings.Contains(err.Error(), postgresSecret) {
|
|
||||||
t.Fatalf("Open(invalid PostgreSQL) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
redisSecret := "redis-secret"
|
|
||||||
_, err = (&productionInfrastructure{}).Open(context.Background(), &config.Config{
|
|
||||||
Distribution: config.Distribution{Listener: config.Listener{Enabled: true}},
|
|
||||||
Storage: config.Storage{RedisURL: "redis://user:" + redisSecret + "@%zz"},
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrRedisConfiguration) || strings.Contains(err.Error(), redisSecret) {
|
|
||||||
t.Fatalf("Open(invalid Redis) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectMetricsReadinessPreservesDistributionWhenAdminStoreFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
adminCalls := &atomic.Int64{}
|
|
||||||
activityCalls := &atomic.Int64{}
|
|
||||||
adminReady := readinessFunc(func(context.Context) error {
|
|
||||||
adminCalls.Add(1)
|
|
||||||
return ErrPostgresUnavailable
|
|
||||||
})
|
|
||||||
activityReady := readinessFunc(func(context.Context) error {
|
|
||||||
activityCalls.Add(1)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
selected := selectMetricsReadiness(&config.Config{
|
|
||||||
Admin: config.Listener{Enabled: true},
|
|
||||||
Distribution: config.Distribution{Listener: config.Listener{Enabled: true}},
|
|
||||||
}, adminReady, activityReady)
|
|
||||||
if err := selected.Ready(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Ready() error = %v", err)
|
|
||||||
}
|
|
||||||
if adminCalls.Load() != 0 || activityCalls.Load() != 1 {
|
|
||||||
t.Fatalf("readiness calls = admin:%d activity:%d", adminCalls.Load(), activityCalls.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectMetricsReadinessUsesAdminStoresWithoutDistribution(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
wantErr := errors.New("admin unavailable")
|
|
||||||
selected := selectMetricsReadiness(
|
|
||||||
&config.Config{Admin: config.Listener{Enabled: true}},
|
|
||||||
readinessFunc(func(context.Context) error { return wantErr }),
|
|
||||||
readinessFunc(func(context.Context) error { return nil }),
|
|
||||||
)
|
|
||||||
if err := selected.Ready(context.Background()); !errors.Is(err, wantErr) {
|
|
||||||
t.Fatalf("Ready() error = %v, want %v", err, wantErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type readinessFunc func(context.Context) error
|
|
||||||
|
|
||||||
func (function readinessFunc) Ready(ctx context.Context) error { return function(ctx) }
|
|
||||||
|
|
||||||
func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
configuration := &config.Config{
|
|
||||||
Distribution: config.Distribution{Extraction: config.Extraction{
|
|
||||||
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
|
||||||
}},
|
|
||||||
Upstreams: map[string]config.Upstream{
|
|
||||||
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
|
|
||||||
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if got := credentialCapacity(configuration); got != 5_000 {
|
|
||||||
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
|
||||||
}
|
|
||||||
if got := candidateScan(configuration); got != 5_100 {
|
|
||||||
t.Fatalf("candidateScan() = %d, want 5100", got)
|
|
||||||
}
|
|
||||||
configuration.Distribution.Extraction = config.Extraction{MaxCountPerRequest: 1}
|
|
||||||
if got := candidateScan(configuration); got != redisMinimumScan {
|
|
||||||
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,104 +0,0 @@
|
|||||||
package operations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"sort"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/controller/admin"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidReader = errors.New("invalid operational status reader")
|
|
||||||
ErrUnavailable = errors.New("operational status unavailable")
|
|
||||||
)
|
|
||||||
|
|
||||||
type ConfigurationReader interface {
|
|
||||||
Current() *config.Config
|
|
||||||
}
|
|
||||||
|
|
||||||
type Reader struct {
|
|
||||||
configuration ConfigurationReader
|
|
||||||
inventory activitypool.StateInventoryReader
|
|
||||||
now func() time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ admin.OperationalStatusReader = (*Reader)(nil)
|
|
||||||
|
|
||||||
func NewReader(
|
|
||||||
configuration ConfigurationReader,
|
|
||||||
inventory activitypool.StateInventoryReader,
|
|
||||||
now func() time.Time,
|
|
||||||
) (*Reader, error) {
|
|
||||||
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
|
|
||||||
return nil, ErrInvalidReader
|
|
||||||
}
|
|
||||||
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
|
||||||
if ctx == nil || reader == nil {
|
|
||||||
return admin.OperationalStatus{}, ErrInvalidReader
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return admin.OperationalStatus{}, err
|
|
||||||
}
|
|
||||||
configuration := reader.configuration.Current()
|
|
||||||
if configuration == nil {
|
|
||||||
return admin.OperationalStatus{}, ErrUnavailable
|
|
||||||
}
|
|
||||||
|
|
||||||
upstreamIDs := make([]string, 0, len(configuration.Upstreams))
|
|
||||||
for upstreamID := range configuration.Upstreams {
|
|
||||||
upstreamIDs = append(upstreamIDs, upstreamID)
|
|
||||||
}
|
|
||||||
sort.Strings(upstreamIDs)
|
|
||||||
inventories, err := reader.inventory.ReadStateInventory(ctx, upstreamIDs, reader.now().UTC())
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return admin.OperationalStatus{}, err
|
|
||||||
}
|
|
||||||
return admin.OperationalStatus{}, errors.Join(ErrUnavailable, err)
|
|
||||||
}
|
|
||||||
if len(inventories) != len(upstreamIDs) {
|
|
||||||
return admin.OperationalStatus{}, ErrUnavailable
|
|
||||||
}
|
|
||||||
|
|
||||||
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
|
||||||
for index, inventory := range inventories {
|
|
||||||
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
|
|
||||||
return admin.OperationalStatus{}, ErrUnavailable
|
|
||||||
}
|
|
||||||
status.Upstreams[index] = admin.UpstreamActivity{
|
|
||||||
Name: inventory.UpstreamID,
|
|
||||||
Available: inventory.Available,
|
|
||||||
Checking: inventory.Checking,
|
|
||||||
Suspect: inventory.Suspect,
|
|
||||||
Draining: inventory.Draining,
|
|
||||||
Extracted: inventory.Extracted,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return status, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func invalidInventory(inventory activitypool.StateInventory) bool {
|
|
||||||
return inventory.Fetched < 0 || inventory.Checking < 0 || inventory.Available < 0 ||
|
|
||||||
inventory.Suspect < 0 || inventory.Draining < 0 || inventory.Unhealthy < 0 || inventory.Extracted < 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func nilInterface(value any) bool {
|
|
||||||
if value == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
reflected := reflect.ValueOf(value)
|
|
||||||
switch reflected.Kind() {
|
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
|
||||||
return reflected.IsNil()
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,118 +0,0 @@
|
|||||||
package operations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
||||||
inventory := &recordingStateInventoryReader{result: []activitypool.StateInventory{
|
|
||||||
{UpstreamID: "provider-a", Fetched: 3, Checking: 2, Available: 11, Suspect: 1, Draining: 4, Extracted: 8},
|
|
||||||
{UpstreamID: "provider-b", Available: 7},
|
|
||||||
}}
|
|
||||||
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
|
||||||
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
|
||||||
}}, inventory, func() time.Time { return now })
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewReader() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
status, err := reader.ReadOperationalStatus(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadOperationalStatus() error = %v", err)
|
|
||||||
}
|
|
||||||
if len(inventory.upstreamIDs) != 2 || inventory.upstreamIDs[0] != "provider-a" || inventory.upstreamIDs[1] != "provider-b" ||
|
|
||||||
!inventory.now.Equal(now) {
|
|
||||||
t.Fatalf("ReadStateInventory() input = %+v at %v", inventory.upstreamIDs, inventory.now)
|
|
||||||
}
|
|
||||||
if status.SnapshotVersion != 0 || len(status.Workers) != 0 || len(status.Upstreams) != 2 {
|
|
||||||
t.Fatalf("operational status shape = %+v", status)
|
|
||||||
}
|
|
||||||
first := status.Upstreams[0]
|
|
||||||
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
|
||||||
first.Draining != 4 || first.Extracted != 8 {
|
|
||||||
t.Fatalf("first upstream = %+v", first)
|
|
||||||
}
|
|
||||||
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
|
||||||
t.Fatalf("second upstream = %+v", status.Upstreams[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReaderRejectsMissingOrMalformedDependencies(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
now := func() time.Time { return time.Now().UTC() }
|
|
||||||
validConfig := staticConfigurationReader{configuration: &config.Config{
|
|
||||||
Upstreams: map[string]config.Upstream{"provider-a": {}},
|
|
||||||
}}
|
|
||||||
validInventory := &recordingStateInventoryReader{result: []activitypool.StateInventory{{UpstreamID: "provider-a"}}}
|
|
||||||
|
|
||||||
if _, err := NewReader(nil, validInventory, now); !errors.Is(err, ErrInvalidReader) {
|
|
||||||
t.Fatalf("NewReader(nil config) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := NewReader(validConfig, nil, now); !errors.Is(err, ErrInvalidReader) {
|
|
||||||
t.Fatalf("NewReader(nil inventory) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := NewReader(validConfig, validInventory, nil); !errors.Is(err, ErrInvalidReader) {
|
|
||||||
t.Fatalf("NewReader(nil clock) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
malformed := &recordingStateInventoryReader{result: []activitypool.StateInventory{{UpstreamID: "wrong"}}}
|
|
||||||
reader, err := NewReader(validConfig, malformed, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewReader() error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := reader.ReadOperationalStatus(context.Background()); !errors.Is(err, ErrUnavailable) {
|
|
||||||
t.Fatalf("ReadOperationalStatus(malformed) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReaderPreservesCancellationAndClassifiesDependencyFailures(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
dependencyError := errors.New("redis down")
|
|
||||||
inventory := &recordingStateInventoryReader{err: dependencyError}
|
|
||||||
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
|
||||||
Upstreams: map[string]config.Upstream{"provider-a": {}},
|
|
||||||
}}, inventory, time.Now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewReader() error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := reader.ReadOperationalStatus(context.Background()); !errors.Is(err, ErrUnavailable) || !errors.Is(err, dependencyError) {
|
|
||||||
t.Fatalf("ReadOperationalStatus(dependency) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if _, err := reader.ReadOperationalStatus(ctx); !errors.Is(err, context.Canceled) || errors.Is(err, ErrUnavailable) {
|
|
||||||
t.Fatalf("ReadOperationalStatus(canceled) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticConfigurationReader struct {
|
|
||||||
configuration *config.Config
|
|
||||||
}
|
|
||||||
|
|
||||||
func (reader staticConfigurationReader) Current() *config.Config { return reader.configuration }
|
|
||||||
|
|
||||||
type recordingStateInventoryReader struct {
|
|
||||||
result []activitypool.StateInventory
|
|
||||||
err error
|
|
||||||
upstreamIDs []string
|
|
||||||
now time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
|
||||||
_ context.Context,
|
|
||||||
upstreamIDs []string,
|
|
||||||
now time.Time,
|
|
||||||
) ([]activitypool.StateInventory, error) {
|
|
||||||
reader.upstreamIDs = append([]string(nil), upstreamIDs...)
|
|
||||||
reader.now = now
|
|
||||||
return append([]activitypool.StateInventory(nil), reader.result...), reader.err
|
|
||||||
}
|
|
||||||
@ -13,8 +13,6 @@ var (
|
|||||||
ErrInvalidFetchCompletion = errors.New("invalid fetch completion")
|
ErrInvalidFetchCompletion = errors.New("invalid fetch completion")
|
||||||
ErrFetchPermitFinished = errors.New("fetch permit is already finished")
|
ErrFetchPermitFinished = errors.New("fetch permit is already finished")
|
||||||
ErrInvalidManagedRelease = errors.New("invalid managed proxy release")
|
ErrInvalidManagedRelease = errors.New("invalid managed proxy release")
|
||||||
ErrInvalidManagedSynchronization = errors.New("invalid managed proxy synchronization")
|
|
||||||
ErrManagedSynchronizationInFlight = errors.New("managed proxy synchronization has pending fetches")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type FetchBudgetConfig struct {
|
type FetchBudgetConfig struct {
|
||||||
@ -114,22 +112,6 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
|||||||
return b.usage
|
return b.usage
|
||||||
}
|
}
|
||||||
|
|
||||||
// SynchronizeManaged replaces the local current-inventory count with the
|
|
||||||
// authoritative activity-store observation. Pending requests and cumulative
|
|
||||||
// fetch usage remain owned by this budget.
|
|
||||||
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
|
||||||
if b == nil || managed < 0 {
|
|
||||||
return ErrInvalidManagedSynchronization
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
if b.usage.PendingExpected > 0 {
|
|
||||||
return ErrManagedSynchronizationInFlight
|
|
||||||
}
|
|
||||||
b.usage.Managed = managed
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReleaseManaged returns current-inventory capacity after extraction, expiry,
|
// ReleaseManaged returns current-inventory capacity after extraction, expiry,
|
||||||
// or removal. It deliberately does not restore the cumulative fetch quota.
|
// or removal. It deliberately does not restore the cumulative fetch quota.
|
||||||
func (b *FetchBudget) ReleaseManaged(count int) error {
|
func (b *FetchBudget) ReleaseManaged(count int) error {
|
||||||
|
|||||||
@ -61,48 +61,6 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
|
||||||
}
|
|
||||||
permit, ok, err := budget.ReserveFetch("a")
|
|
||||||
if err != nil || !ok {
|
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := budget.SynchronizeManaged(1); !errors.Is(err, ErrManagedSynchronizationInFlight) {
|
|
||||||
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
|
||||||
}
|
|
||||||
usage := budget.Snapshot()
|
|
||||||
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
|
|
||||||
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
|
|
||||||
}
|
|
||||||
if err := permit.Cancel(); err != nil {
|
|
||||||
t.Fatalf("Cancel(): %v", err)
|
|
||||||
}
|
|
||||||
if err := budget.SynchronizeManaged(1); err != nil {
|
|
||||||
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
|
||||||
}
|
|
||||||
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
|
||||||
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 1,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
|
||||||
}
|
|
||||||
if err := budget.SynchronizeManaged(-1); !errors.Is(err, ErrInvalidManagedSynchronization) {
|
|
||||||
t.Fatalf("SynchronizeManaged() error = %v, want ErrInvalidManagedSynchronization", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
|
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@ -2,8 +2,6 @@ package pool
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"math"
|
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/upstream"
|
"proxy-pool/internal/domain/upstream"
|
||||||
@ -13,9 +11,7 @@ var ErrInvalidReconcilePolicy = errors.New("invalid pool reconcile policy")
|
|||||||
|
|
||||||
type ReconcilePolicy struct {
|
type ReconcilePolicy struct {
|
||||||
MinimumAvailableSlots int64
|
MinimumAvailableSlots int64
|
||||||
TargetAvailableSlots int64
|
|
||||||
ExpectedPerFetch int
|
ExpectedPerFetch int
|
||||||
ExpectedSlotsPerFetch int64
|
|
||||||
SafetyMargin time.Duration
|
SafetyMargin time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -28,7 +24,6 @@ type ReconcileDecision struct {
|
|||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
FetchedTotal int64
|
||||||
FetchAllowance int
|
FetchAllowance int
|
||||||
EffectiveSlots int64
|
|
||||||
Triggered bool
|
Triggered bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -36,75 +31,34 @@ type Reconciler struct {
|
|||||||
policy ReconcilePolicy
|
policy ReconcilePolicy
|
||||||
budget *FetchBudget
|
budget *FetchBudget
|
||||||
notifier FetchNotifier
|
notifier FetchNotifier
|
||||||
mu sync.Mutex
|
|
||||||
refilling bool
|
|
||||||
slotsPerProxy int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNotifier) (*Reconciler, error) {
|
func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNotifier) (*Reconciler, error) {
|
||||||
if policy.MinimumAvailableSlots <= 0 || policy.TargetAvailableSlots <= policy.MinimumAvailableSlots ||
|
if policy.MinimumAvailableSlots <= 0 || policy.ExpectedPerFetch <= 0 ||
|
||||||
policy.ExpectedPerFetch <= 0 || policy.ExpectedSlotsPerFetch <= 0 ||
|
|
||||||
policy.SafetyMargin < 0 || budget == nil || notifier == nil {
|
policy.SafetyMargin < 0 || budget == nil || notifier == nil {
|
||||||
return nil, ErrInvalidReconcilePolicy
|
return nil, ErrInvalidReconcilePolicy
|
||||||
}
|
}
|
||||||
if budget.expected != policy.ExpectedPerFetch {
|
if budget.expected != policy.ExpectedPerFetch {
|
||||||
return nil, ErrInvalidReconcilePolicy
|
return nil, ErrInvalidReconcilePolicy
|
||||||
}
|
}
|
||||||
if policy.ExpectedSlotsPerFetch%int64(policy.ExpectedPerFetch) != 0 {
|
return &Reconciler{policy: policy, budget: budget, notifier: notifier}, nil
|
||||||
return nil, ErrInvalidReconcilePolicy
|
|
||||||
}
|
|
||||||
return &Reconciler{
|
|
||||||
policy: policy, budget: budget, notifier: notifier,
|
|
||||||
slotsPerProxy: policy.ExpectedSlotsPerFetch / int64(policy.ExpectedPerFetch),
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
|
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
|
||||||
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
|
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
|
||||||
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
|
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
|
||||||
usage := r.budget.Snapshot()
|
usage := r.budget.Snapshot()
|
||||||
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
|
|
||||||
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
|
|
||||||
decision := ReconcileDecision{
|
decision := ReconcileDecision{
|
||||||
AvailableSlots: availableSlots,
|
AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin),
|
||||||
PendingExpected: usage.PendingExpected,
|
PendingExpected: usage.PendingExpected,
|
||||||
FetchedTotal: usage.FetchedTotal,
|
FetchedTotal: usage.FetchedTotal,
|
||||||
FetchAllowance: r.budget.FetchAllowance(),
|
FetchAllowance: r.budget.FetchAllowance(),
|
||||||
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
|
|
||||||
}
|
}
|
||||||
r.mu.Lock()
|
if decision.AvailableSlots >= r.policy.MinimumAvailableSlots ||
|
||||||
if r.refilling {
|
decision.FetchAllowance < r.policy.ExpectedPerFetch {
|
||||||
if usage.PendingExpected == 0 && decision.AvailableSlots >= r.policy.TargetAvailableSlots {
|
|
||||||
r.refilling = false
|
|
||||||
}
|
|
||||||
} else if decision.AvailableSlots < r.policy.MinimumAvailableSlots {
|
|
||||||
r.refilling = true
|
|
||||||
}
|
|
||||||
trigger := r.refilling &&
|
|
||||||
decision.EffectiveSlots < r.policy.TargetAvailableSlots &&
|
|
||||||
decision.FetchAllowance >= r.policy.ExpectedPerFetch
|
|
||||||
r.mu.Unlock()
|
|
||||||
if !trigger {
|
|
||||||
return decision
|
return decision
|
||||||
}
|
}
|
||||||
r.notifier.Notify()
|
r.notifier.Notify()
|
||||||
decision.Triggered = true
|
decision.Triggered = true
|
||||||
return decision
|
return decision
|
||||||
}
|
}
|
||||||
|
|
||||||
func saturatingMultiply(left, right int64) int64 {
|
|
||||||
if left <= 0 || right <= 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if left > math.MaxInt64/right {
|
|
||||||
return math.MaxInt64
|
|
||||||
}
|
|
||||||
return left * right
|
|
||||||
}
|
|
||||||
|
|
||||||
func saturatingAdd(left, right int64) int64 {
|
|
||||||
if left >= math.MaxInt64-right {
|
|
||||||
return math.MaxInt64
|
|
||||||
}
|
|
||||||
return left + right
|
|
||||||
}
|
|
||||||
|
|||||||
@ -18,9 +18,7 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
|||||||
notifier := &recordingFetchNotifier{}
|
notifier := &recordingFetchNotifier{}
|
||||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||||
MinimumAvailableSlots: 5,
|
MinimumAvailableSlots: 5,
|
||||||
TargetAvailableSlots: 8,
|
|
||||||
ExpectedPerFetch: 2,
|
ExpectedPerFetch: 2,
|
||||||
ExpectedSlotsPerFetch: 2,
|
|
||||||
SafetyMargin: 10 * time.Second,
|
SafetyMargin: 10 * time.Second,
|
||||||
}, budget, notifier)
|
}, budget, notifier)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -58,8 +56,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
|||||||
defer permit.Cancel()
|
defer permit.Cancel()
|
||||||
notifier := &recordingFetchNotifier{}
|
notifier := &recordingFetchNotifier{}
|
||||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||||
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
MinimumAvailableSlots: 1, ExpectedPerFetch: 2,
|
||||||
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
|
|
||||||
}, budget, notifier)
|
}, budget, notifier)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
@ -74,79 +71,6 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPoolReconcilerUsesTargetWatermarkUntilRefillCompletes(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
|
||||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
|
||||||
}
|
|
||||||
notifier := &recordingFetchNotifier{}
|
|
||||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
|
||||||
MinimumAvailableSlots: 3,
|
|
||||||
TargetAvailableSlots: 8,
|
|
||||||
ExpectedPerFetch: 2,
|
|
||||||
ExpectedSlotsPerFetch: 2,
|
|
||||||
}, budget, notifier)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
belowMinimum := reconciler.Reconcile(now, inventoryWithSlots(now, 2))
|
|
||||||
betweenWatermarks := reconciler.Reconcile(now, inventoryWithSlots(now, 5))
|
|
||||||
atTarget := reconciler.Reconcile(now, inventoryWithSlots(now, 8))
|
|
||||||
aboveMinimumAfterCompletion := reconciler.Reconcile(now, inventoryWithSlots(now, 5))
|
|
||||||
|
|
||||||
if !belowMinimum.Triggered || !betweenWatermarks.Triggered || atTarget.Triggered || aboveMinimumAfterCompletion.Triggered {
|
|
||||||
t.Fatalf("triggered states = [%v %v %v %v], want [true true false false]",
|
|
||||||
belowMinimum.Triggered, betweenWatermarks.Triggered, atTarget.Triggered, aboveMinimumAfterCompletion.Triggered)
|
|
||||||
}
|
|
||||||
if notifier.calls != 2 {
|
|
||||||
t.Fatalf("Notify() calls = %d, want 2", notifier.calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
|
||||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
|
||||||
}
|
|
||||||
notifier := &recordingFetchNotifier{}
|
|
||||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
|
||||||
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
|
|
||||||
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 6,
|
|
||||||
}, budget, notifier)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewReconciler(): %v", err)
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 7, 30, 10, 0, 0, 0, time.UTC)
|
|
||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); !decision.Triggered {
|
|
||||||
t.Fatalf("initial Reconcile() = %+v, want trigger", decision)
|
|
||||||
}
|
|
||||||
permit, ok, err := budget.ReserveFetch("provider-a")
|
|
||||||
if err != nil || !ok {
|
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
|
||||||
}
|
|
||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
|
||||||
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
|
||||||
}
|
|
||||||
if err := permit.Complete(2, 1); err != nil {
|
|
||||||
t.Fatalf("Complete(): %v", err)
|
|
||||||
}
|
|
||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
|
||||||
t.Fatalf("post-fetch Reconcile() = %+v, want refill episode to continue", decision)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
|
|
||||||
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
|
|
||||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,
|
|
||||||
}}}
|
|
||||||
}
|
|
||||||
|
|
||||||
type recordingFetchNotifier struct{ calls int }
|
type recordingFetchNotifier struct{ calls int }
|
||||||
|
|
||||||
func (n *recordingFetchNotifier) Notify() { n.calls++ }
|
func (n *recordingFetchNotifier) Notify() { n.calls++ }
|
||||||
|
|||||||
@ -1,41 +0,0 @@
|
|||||||
package provider
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidCoordination = errors.New("invalid provider coordination")
|
|
||||||
ErrCoordinationUnavailable = errors.New("provider coordination unavailable")
|
|
||||||
ErrLeadershipLost = errors.New("provider leadership lost")
|
|
||||||
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
|
||||||
)
|
|
||||||
|
|
||||||
type CoordinationLimits struct {
|
|
||||||
RequestInterval time.Duration
|
|
||||||
MaxInFlight int
|
|
||||||
MaxAttemptDuration time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type Fence struct {
|
|
||||||
Generation string
|
|
||||||
Epoch uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
type Coordinator interface {
|
|
||||||
// RunLeader waits for leadership and runs work only while its fencing lease
|
|
||||||
// is valid. The work context is canceled before a known lease expiry.
|
|
||||||
RunLeader(context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type LeaderSession interface {
|
|
||||||
Fence() Fence
|
|
||||||
AcquireFetch(context.Context) (RequestPermit, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type RequestPermit interface {
|
|
||||||
// Release is idempotent. A failed release expires automatically in storage.
|
|
||||||
Release(context.Context) error
|
|
||||||
}
|
|
||||||
@ -26,7 +26,6 @@ type Dependencies struct {
|
|||||||
Extractor distribution.Extractor
|
Extractor distribution.Extractor
|
||||||
Readiness distribution.ReadinessChecker
|
Readiness distribution.ReadinessChecker
|
||||||
AdminService admin.Service
|
AdminService admin.Service
|
||||||
MetricsHandler http.Handler
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
@ -38,7 +37,6 @@ type Options struct {
|
|||||||
type Listeners struct {
|
type Listeners struct {
|
||||||
Distribution net.Listener
|
Distribution net.Listener
|
||||||
Admin net.Listener
|
Admin net.Listener
|
||||||
Metrics net.Listener
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Runtime struct {
|
type Runtime struct {
|
||||||
@ -48,14 +46,11 @@ type Runtime struct {
|
|||||||
adminEnabled bool
|
adminEnabled bool
|
||||||
adminAddress string
|
adminAddress string
|
||||||
adminHandler http.Handler
|
adminHandler http.Handler
|
||||||
metricsEnabled bool
|
|
||||||
metricsAddress string
|
|
||||||
metricsHandler http.Handler
|
|
||||||
httpOptions httpserver.Options
|
httpOptions httpserver.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runtime, error) {
|
func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runtime, error) {
|
||||||
if cfg == nil || (!cfg.Distribution.Enabled && !cfg.Admin.Enabled && !cfg.Metrics.Enabled) {
|
if cfg == nil || (!cfg.Distribution.Enabled && !cfg.Admin.Enabled) {
|
||||||
return nil, ErrInvalidRuntime
|
return nil, ErrInvalidRuntime
|
||||||
}
|
}
|
||||||
distributionBodyLimit, adminBodyLimit, err := resolveBodyLimits(options)
|
distributionBodyLimit, adminBodyLimit, err := resolveBodyLimits(options)
|
||||||
@ -109,15 +104,6 @@ func New(cfg *config.Config, dependencies Dependencies, options Options) (*Runti
|
|||||||
result.adminAddress = cfg.Admin.Listen
|
result.adminAddress = cfg.Admin.Listen
|
||||||
result.adminHandler = handler
|
result.adminHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Metrics.Enabled {
|
|
||||||
if strings.TrimSpace(cfg.Metrics.Listen) == "" || dependencies.MetricsHandler == nil {
|
|
||||||
return nil, ErrInvalidRuntime
|
|
||||||
}
|
|
||||||
result.metricsEnabled = true
|
|
||||||
result.metricsAddress = cfg.Metrics.Listen
|
|
||||||
result.metricsHandler = dependencies.MetricsHandler
|
|
||||||
}
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -125,7 +111,7 @@ func (runtime *Runtime) Run(ctx context.Context) error {
|
|||||||
if runtime == nil || ctx == nil {
|
if runtime == nil || ctx == nil {
|
||||||
return ErrInvalidRuntime
|
return ErrInvalidRuntime
|
||||||
}
|
}
|
||||||
bindings := make([]httpserver.Binding, 0, 3)
|
bindings := make([]httpserver.Binding, 0, 2)
|
||||||
if runtime.distributionEnabled {
|
if runtime.distributionEnabled {
|
||||||
bindings = append(bindings, httpserver.Binding{
|
bindings = append(bindings, httpserver.Binding{
|
||||||
Name: "distribution", Address: runtime.distributionAddress, Handler: runtime.distributionHandler,
|
Name: "distribution", Address: runtime.distributionAddress, Handler: runtime.distributionHandler,
|
||||||
@ -136,11 +122,6 @@ func (runtime *Runtime) Run(ctx context.Context) error {
|
|||||||
Name: "admin", Address: runtime.adminAddress, Handler: runtime.adminHandler,
|
Name: "admin", Address: runtime.adminAddress, Handler: runtime.adminHandler,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if runtime.metricsEnabled {
|
|
||||||
bindings = append(bindings, httpserver.Binding{
|
|
||||||
Name: "metrics", Address: runtime.metricsAddress, Handler: runtime.metricsHandler,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err := httpserver.ListenAndServe(ctx, runtime.httpOptions, bindings...); err != nil {
|
if err := httpserver.ListenAndServe(ctx, runtime.httpOptions, bindings...); err != nil {
|
||||||
return fmt.Errorf("run controller HTTP runtime: %w", err)
|
return fmt.Errorf("run controller HTTP runtime: %w", err)
|
||||||
}
|
}
|
||||||
@ -150,11 +131,10 @@ func (runtime *Runtime) Run(ctx context.Context) error {
|
|||||||
func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error {
|
func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error {
|
||||||
if runtime == nil || ctx == nil ||
|
if runtime == nil || ctx == nil ||
|
||||||
runtime.distributionEnabled != (listeners.Distribution != nil) ||
|
runtime.distributionEnabled != (listeners.Distribution != nil) ||
|
||||||
runtime.adminEnabled != (listeners.Admin != nil) ||
|
runtime.adminEnabled != (listeners.Admin != nil) {
|
||||||
runtime.metricsEnabled != (listeners.Metrics != nil) {
|
|
||||||
return ErrInvalidRuntime
|
return ErrInvalidRuntime
|
||||||
}
|
}
|
||||||
endpoints := make([]httpserver.Endpoint, 0, 3)
|
endpoints := make([]httpserver.Endpoint, 0, 2)
|
||||||
if runtime.distributionEnabled {
|
if runtime.distributionEnabled {
|
||||||
endpoints = append(endpoints, httpserver.Endpoint{
|
endpoints = append(endpoints, httpserver.Endpoint{
|
||||||
Name: "distribution", Listener: listeners.Distribution, Handler: runtime.distributionHandler,
|
Name: "distribution", Listener: listeners.Distribution, Handler: runtime.distributionHandler,
|
||||||
@ -165,11 +145,6 @@ func (runtime *Runtime) Serve(ctx context.Context, listeners Listeners) error {
|
|||||||
Name: "admin", Listener: listeners.Admin, Handler: runtime.adminHandler,
|
Name: "admin", Listener: listeners.Admin, Handler: runtime.adminHandler,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if runtime.metricsEnabled {
|
|
||||||
endpoints = append(endpoints, httpserver.Endpoint{
|
|
||||||
Name: "metrics", Listener: listeners.Metrics, Handler: runtime.metricsHandler,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err := httpserver.Serve(ctx, runtime.httpOptions, endpoints...); err != nil {
|
if err := httpserver.Serve(ctx, runtime.httpOptions, endpoints...); err != nil {
|
||||||
return fmt.Errorf("serve controller HTTP runtime: %w", err)
|
return fmt.Errorf("serve controller HTTP runtime: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,47 +19,34 @@ import (
|
|||||||
func TestRuntimeServesDistributionAndAdminOnIndependentListeners(t *testing.T) {
|
func TestRuntimeServesDistributionAndAdminOnIndependentListeners(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
cfg := runtimeConfig()
|
cfg := runtimeConfig()
|
||||||
cfg.Metrics = config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}
|
|
||||||
adminService := &stubAdminService{status: admin.Status{ConfigVersion: "cfg-7", SnapshotVersion: 11}}
|
adminService := &stubAdminService{status: admin.Status{ConfigVersion: "cfg-7", SnapshotVersion: 11}}
|
||||||
runtime, err := New(cfg, Dependencies{
|
runtime, err := New(cfg, Dependencies{
|
||||||
Extractor: stubExtractor{},
|
Extractor: stubExtractor{},
|
||||||
Readiness: stubReadiness{},
|
Readiness: stubReadiness{},
|
||||||
AdminService: adminService,
|
AdminService: adminService,
|
||||||
MetricsHandler: http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.URL.Path != "/readyz" {
|
|
||||||
http.NotFound(response, request)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response.WriteHeader(http.StatusOK)
|
|
||||||
}),
|
|
||||||
}, Options{HTTP: testHTTPOptions()})
|
}, Options{HTTP: testHTTPOptions()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("New() error = %v", err)
|
t.Fatalf("New() error = %v", err)
|
||||||
}
|
}
|
||||||
distributionListener := mustListen(t)
|
distributionListener := mustListen(t)
|
||||||
adminListener := mustListen(t)
|
adminListener := mustListen(t)
|
||||||
metricsListener := mustListen(t)
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
result := make(chan error, 1)
|
result := make(chan error, 1)
|
||||||
go func() {
|
go func() {
|
||||||
result <- runtime.Serve(ctx, Listeners{
|
result <- runtime.Serve(ctx, Listeners{
|
||||||
Distribution: distributionListener,
|
Distribution: distributionListener,
|
||||||
Admin: adminListener,
|
Admin: adminListener,
|
||||||
Metrics: metricsListener,
|
|
||||||
})
|
})
|
||||||
}()
|
}()
|
||||||
|
|
||||||
distributionURL := "http://" + distributionListener.Addr().String()
|
distributionURL := "http://" + distributionListener.Addr().String()
|
||||||
adminURL := "http://" + adminListener.Addr().String()
|
adminURL := "http://" + adminListener.Addr().String()
|
||||||
metricsURL := "http://" + metricsListener.Addr().String()
|
|
||||||
assertStatus(t, http.MethodGet, distributionURL+"/health/live", nil, http.StatusOK)
|
assertStatus(t, http.MethodGet, distributionURL+"/health/live", nil, http.StatusOK)
|
||||||
assertStatus(t, http.MethodGet, distributionURL+"/api/v1/status", nil, http.StatusNotFound)
|
assertStatus(t, http.MethodGet, distributionURL+"/api/v1/status", nil, http.StatusNotFound)
|
||||||
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", nil, http.StatusUnauthorized)
|
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", nil, http.StatusUnauthorized)
|
||||||
adminHeaders := http.Header{"Authorization": []string{"Bearer admin-token"}}
|
adminHeaders := http.Header{"Authorization": []string{"Bearer admin-token"}}
|
||||||
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", adminHeaders, http.StatusOK)
|
assertStatus(t, http.MethodGet, adminURL+"/api/v1/status", adminHeaders, http.StatusOK)
|
||||||
assertStatus(t, http.MethodGet, adminURL+"/health/live", adminHeaders, http.StatusNotFound)
|
assertStatus(t, http.MethodGet, adminURL+"/health/live", adminHeaders, http.StatusNotFound)
|
||||||
assertStatus(t, http.MethodGet, metricsURL+"/readyz", nil, http.StatusOK)
|
|
||||||
assertStatus(t, http.MethodGet, metricsURL+"/api/v1/status", nil, http.StatusNotFound)
|
|
||||||
if adminService.statusCalls.Load() != 1 {
|
if adminService.statusCalls.Load() != 1 {
|
||||||
t.Fatalf("admin status calls = %d, want 1", adminService.statusCalls.Load())
|
t.Fatalf("admin status calls = %d, want 1", adminService.statusCalls.Load())
|
||||||
}
|
}
|
||||||
@ -120,44 +107,6 @@ func TestServeRequiresExactlyTheEnabledListeners(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRuntimeServesMetricsOnIndependentListener(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
cfg := &config.Config{Metrics: config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}}
|
|
||||||
metricsHandler := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.URL.Path != "/readyz" {
|
|
||||||
http.NotFound(response, request)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response.WriteHeader(http.StatusOK)
|
|
||||||
})
|
|
||||||
runtime, err := New(cfg, Dependencies{MetricsHandler: metricsHandler}, Options{HTTP: testHTTPOptions()})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(metrics-only) error = %v", err)
|
|
||||||
}
|
|
||||||
listener := mustListen(t)
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
result := make(chan error, 1)
|
|
||||||
go func() { result <- runtime.Serve(ctx, Listeners{Metrics: listener}) }()
|
|
||||||
assertStatus(t, http.MethodGet, "http://"+listener.Addr().String()+"/readyz", nil, http.StatusOK)
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-result:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Serve(metrics-only) error = %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("Serve(metrics-only) did not stop")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRequiresMetricsHandlerWhenEnabled(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
cfg := &config.Config{Metrics: config.Metrics{Enabled: true, Listen: "127.0.0.1:0"}}
|
|
||||||
if _, err := New(cfg, Dependencies{}, Options{}); !errors.Is(err, ErrInvalidRuntime) {
|
|
||||||
t.Fatalf("New(metrics without handler) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runtimeConfig() *config.Config {
|
func runtimeConfig() *config.Config {
|
||||||
return &config.Config{
|
return &config.Config{
|
||||||
Distribution: config.Distribution{
|
Distribution: config.Distribution{
|
||||||
|
|||||||
@ -18,7 +18,6 @@ type Store interface {
|
|||||||
activitypool.Upserter
|
activitypool.Upserter
|
||||||
activitypool.HealthStore
|
activitypool.HealthStore
|
||||||
activitypool.InventoryReader
|
activitypool.InventoryReader
|
||||||
activitypool.StateInventoryReader
|
|
||||||
activitypool.Maintainer
|
activitypool.Maintainer
|
||||||
extractionDomain.Store
|
extractionDomain.Store
|
||||||
ownershipDomain.Repository
|
ownershipDomain.Repository
|
||||||
@ -49,9 +48,6 @@ func Run(t *testing.T, factory Factory) {
|
|||||||
t.Run("inventory and bounded maintenance", func(t *testing.T) {
|
t.Run("inventory and bounded maintenance", func(t *testing.T) {
|
||||||
runMaintenanceContract(t, newStore(t, factory))
|
runMaintenanceContract(t, newStore(t, factory))
|
||||||
})
|
})
|
||||||
t.Run("state inventory lifecycle", func(t *testing.T) {
|
|
||||||
runStateInventoryContract(t, newStore(t, factory))
|
|
||||||
})
|
|
||||||
t.Run("concurrent exclusivity", func(t *testing.T) {
|
t.Run("concurrent exclusivity", func(t *testing.T) {
|
||||||
runConcurrencyContract(t, factory)
|
runConcurrencyContract(t, factory)
|
||||||
})
|
})
|
||||||
@ -342,88 +338,6 @@ func runMaintenanceContract(t *testing.T, store Store) {
|
|||||||
assertInventory(t, store, "provider-a", now.Add(6*time.Second), 0)
|
assertInventory(t, store, "provider-a", now.Add(6*time.Second), 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runStateInventoryContract(t *testing.T, store Store) {
|
|
||||||
t.Helper()
|
|
||||||
now := contractNow()
|
|
||||||
upstreamA := "provider:a:FETCHED"
|
|
||||||
upstreamB := "provider:a"
|
|
||||||
states := []proxyDomain.State{
|
|
||||||
proxyDomain.StateFetched,
|
|
||||||
proxyDomain.StateChecking,
|
|
||||||
proxyDomain.StateAvailable,
|
|
||||||
proxyDomain.StateSuspect,
|
|
||||||
proxyDomain.StateDraining,
|
|
||||||
proxyDomain.StateUnhealthy,
|
|
||||||
proxyDomain.StateExtracted,
|
|
||||||
}
|
|
||||||
for index, state := range states {
|
|
||||||
upsertOne(t, store, upstreamA, now, time.Minute,
|
|
||||||
contractProxy(fmt.Sprintf("state-%d", index), fmt.Sprintf("192.0.2.%d", index+30), state))
|
|
||||||
}
|
|
||||||
upsertOne(t, store, upstreamB, now, time.Minute,
|
|
||||||
contractProxy("collision-control", "198.51.100.30", proxyDomain.StateFetched))
|
|
||||||
|
|
||||||
inventories, err := store.ReadStateInventory(context.Background(), []string{upstreamB, upstreamA}, now)
|
|
||||||
if err != nil || len(inventories) != 2 {
|
|
||||||
t.Fatalf("ReadStateInventory() = %+v, %v", inventories, err)
|
|
||||||
}
|
|
||||||
if inventories[0] != (activitypool.StateInventory{UpstreamID: upstreamB, Fetched: 1}) {
|
|
||||||
t.Fatalf("ReadStateInventory(collision control) = %+v", inventories[0])
|
|
||||||
}
|
|
||||||
wantAll := activitypool.StateInventory{
|
|
||||||
UpstreamID: upstreamA, Fetched: 1, Checking: 1, Available: 1, Suspect: 1,
|
|
||||||
Draining: 1, Unhealthy: 1, Extracted: 1,
|
|
||||||
}
|
|
||||||
if inventories[1] != wantAll {
|
|
||||||
t.Fatalf("ReadStateInventory(all states) = %+v, want %+v", inventories[1], wantAll)
|
|
||||||
}
|
|
||||||
|
|
||||||
transition := activitypool.HealthUpdate{
|
|
||||||
ProxyID: "state-0", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
|
|
||||||
}
|
|
||||||
if _, err := store.ApplyHealth(context.Background(), transition); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(state inventory transition): %v", err)
|
|
||||||
}
|
|
||||||
if _, err := store.ApplyHealth(context.Background(), transition); err != nil {
|
|
||||||
t.Fatalf("ApplyHealth(state inventory replay): %v", err)
|
|
||||||
}
|
|
||||||
afterTransition, err := store.ReadStateInventory(context.Background(), []string{upstreamA}, now.Add(time.Second))
|
|
||||||
if err != nil || len(afterTransition) != 1 || afterTransition[0].Fetched != 0 || afterTransition[0].Checking != 2 {
|
|
||||||
t.Fatalf("ReadStateInventory(after transition) = %+v, %v", afterTransition, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
extractCommand := extractionDomain.Command{
|
|
||||||
RequestID: "state-inventory-extract", ClientID: "client-a", Requested: 1,
|
|
||||||
IdempotencyKey: "state-inventory-extract", IdempotencyTTL: time.Minute,
|
|
||||||
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
||||||
Upstreams: []string{upstreamA},
|
|
||||||
}
|
|
||||||
result, err := store.Extract(context.Background(), extractCommand)
|
|
||||||
if err != nil || result.Returned != 1 {
|
|
||||||
t.Fatalf("Extract(state inventory) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
extractCommand.RequestID = "state-inventory-extract-replay"
|
|
||||||
if replayed, err := store.Extract(context.Background(), extractCommand); err != nil || replayed.Returned != 1 ||
|
|
||||||
replayed.Items[0].ID != result.Items[0].ID {
|
|
||||||
t.Fatalf("Extract(state inventory replay) = %+v, %v", replayed, err)
|
|
||||||
}
|
|
||||||
afterExtract, err := store.ReadStateInventory(context.Background(), []string{upstreamA}, now.Add(2*time.Second))
|
|
||||||
if err != nil || len(afterExtract) != 1 || afterExtract[0].Available != 0 || afterExtract[0].Extracted != 2 {
|
|
||||||
t.Fatalf("ReadStateInventory(after extract) = %+v, %v", afterExtract, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
afterExpiry, err := store.ReadStateInventory(context.Background(), []string{upstreamA, upstreamB}, now.Add(2*time.Minute))
|
|
||||||
if err != nil || len(afterExpiry) != 2 || afterExpiry[0] != (activitypool.StateInventory{UpstreamID: upstreamA}) ||
|
|
||||||
afterExpiry[1] != (activitypool.StateInventory{UpstreamID: upstreamB}) {
|
|
||||||
t.Fatalf("ReadStateInventory(after expiry) = %+v, %v", afterExpiry, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
empty, err := store.ReadStateInventory(context.Background(), nil, now)
|
|
||||||
if err != nil || len(empty) != 0 {
|
|
||||||
t.Fatalf("ReadStateInventory(empty) = %+v, %v", empty, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runConcurrencyContract(t *testing.T, factory Factory) {
|
func runConcurrencyContract(t *testing.T, factory Factory) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
now := contractNow()
|
now := contractNow()
|
||||||
@ -525,10 +439,6 @@ func runCancellationContract(t *testing.T, store Store) {
|
|||||||
return err
|
return err
|
||||||
}},
|
}},
|
||||||
{name: "inventory", call: func() error { _, err := store.Inventory(ctx, "provider-a", now); return err }},
|
{name: "inventory", call: func() error { _, err := store.Inventory(ctx, "provider-a", now); return err }},
|
||||||
{name: "state inventory", call: func() error {
|
|
||||||
_, err := store.ReadStateInventory(ctx, []string{"provider-a"}, now)
|
|
||||||
return err
|
|
||||||
}},
|
|
||||||
{name: "sweep", call: func() error { _, err := store.SweepExpired(ctx, now, 1); return err }},
|
{name: "sweep", call: func() error { _, err := store.SweepExpired(ctx, now, 1); return err }},
|
||||||
{name: "extract", call: func() error {
|
{name: "extract", call: func() error {
|
||||||
_, err := store.Extract(ctx, extractionDomain.Command{Requested: 1, Fulfillment: extractionDomain.Partial, Now: now})
|
_, err := store.Extract(ctx, extractionDomain.Command{Requested: 1, Fulfillment: extractionDomain.Partial, Now: now})
|
||||||
|
|||||||
@ -60,19 +60,6 @@ type Inventory struct {
|
|||||||
Managed int
|
Managed int
|
||||||
}
|
}
|
||||||
|
|
||||||
// StateInventory is a low-cardinality operational view of one upstream.
|
|
||||||
// Expired and removed entries are intentionally excluded.
|
|
||||||
type StateInventory struct {
|
|
||||||
UpstreamID string
|
|
||||||
Fetched int64
|
|
||||||
Checking int64
|
|
||||||
Available int64
|
|
||||||
Suspect int64
|
|
||||||
Draining int64
|
|
||||||
Unhealthy int64
|
|
||||||
Extracted int64
|
|
||||||
}
|
|
||||||
|
|
||||||
type HealthStore interface {
|
type HealthStore interface {
|
||||||
ApplyHealth(context.Context, HealthUpdate) (Entry, error)
|
ApplyHealth(context.Context, HealthUpdate) (Entry, error)
|
||||||
}
|
}
|
||||||
@ -81,10 +68,6 @@ type InventoryReader interface {
|
|||||||
Inventory(context.Context, string, time.Time) (Inventory, error)
|
Inventory(context.Context, string, time.Time) (Inventory, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type StateInventoryReader interface {
|
|
||||||
ReadStateInventory(context.Context, []string, time.Time) ([]StateInventory, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Maintainer interface {
|
type Maintainer interface {
|
||||||
SweepExpired(context.Context, time.Time, int) (int, error)
|
SweepExpired(context.Context, time.Time, int) (int, error)
|
||||||
}
|
}
|
||||||
@ -116,7 +99,6 @@ var (
|
|||||||
_ Upserter = (*MemoryPool)(nil)
|
_ Upserter = (*MemoryPool)(nil)
|
||||||
_ HealthStore = (*MemoryPool)(nil)
|
_ HealthStore = (*MemoryPool)(nil)
|
||||||
_ InventoryReader = (*MemoryPool)(nil)
|
_ InventoryReader = (*MemoryPool)(nil)
|
||||||
_ StateInventoryReader = (*MemoryPool)(nil)
|
|
||||||
_ Maintainer = (*MemoryPool)(nil)
|
_ Maintainer = (*MemoryPool)(nil)
|
||||||
_ extractionDomain.Store = (*MemoryPool)(nil)
|
_ extractionDomain.Store = (*MemoryPool)(nil)
|
||||||
_ ownershipDomain.Repository = (*MemoryPool)(nil)
|
_ ownershipDomain.Repository = (*MemoryPool)(nil)
|
||||||
@ -320,50 +302,6 @@ func (p *MemoryPool) Inventory(ctx context.Context, upstreamID string, now time.
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *MemoryPool) ReadStateInventory(
|
|
||||||
ctx context.Context,
|
|
||||||
upstreamIDs []string,
|
|
||||||
now time.Time,
|
|
||||||
) ([]StateInventory, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return nil, ErrInvalidInventory
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if p == nil || now.IsZero() {
|
|
||||||
return nil, ErrInvalidInventory
|
|
||||||
}
|
|
||||||
result := make([]StateInventory, len(upstreamIDs))
|
|
||||||
positions := make(map[string][]int, len(upstreamIDs))
|
|
||||||
for index, upstreamID := range upstreamIDs {
|
|
||||||
if upstreamID == "" {
|
|
||||||
return nil, ErrInvalidInventory
|
|
||||||
}
|
|
||||||
result[index].UpstreamID = upstreamID
|
|
||||||
positions[upstreamID] = append(positions[upstreamID], index)
|
|
||||||
}
|
|
||||||
if len(result) == 0 {
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, entry := range p.entries {
|
|
||||||
indexes := positions[entry.Proxy.SourceUpstream]
|
|
||||||
if len(indexes) == 0 || entry.Proxy.ExpiresAt == nil || !entry.Proxy.ExpiresAt.After(now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, index := range indexes {
|
|
||||||
incrementStateInventory(&result[index], entry.State)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *MemoryPool) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) {
|
func (p *MemoryPool) SweepExpired(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return 0, ErrInvalidMaintenance
|
return 0, ErrInvalidMaintenance
|
||||||
@ -806,28 +744,6 @@ func managedActivityState(state proxyDomain.State) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func incrementStateInventory(inventory *StateInventory, state proxyDomain.State) {
|
|
||||||
if inventory == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch state {
|
|
||||||
case proxyDomain.StateFetched:
|
|
||||||
inventory.Fetched++
|
|
||||||
case proxyDomain.StateChecking:
|
|
||||||
inventory.Checking++
|
|
||||||
case proxyDomain.StateAvailable:
|
|
||||||
inventory.Available++
|
|
||||||
case proxyDomain.StateSuspect:
|
|
||||||
inventory.Suspect++
|
|
||||||
case proxyDomain.StateDraining:
|
|
||||||
inventory.Draining++
|
|
||||||
case proxyDomain.StateUnhealthy:
|
|
||||||
inventory.Unhealthy++
|
|
||||||
case proxyDomain.StateExtracted:
|
|
||||||
inventory.Extracted++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy {
|
func cloneProxy(candidate proxyDomain.Proxy) proxyDomain.Proxy {
|
||||||
if candidate.ExpiresAt != nil {
|
if candidate.ExpiresAt != nil {
|
||||||
value := *candidate.ExpiresAt
|
value := *candidate.ExpiresAt
|
||||||
|
|||||||
@ -1,14 +0,0 @@
|
|||||||
package admission
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
// AllowAll validates the common admission contract without applying another
|
|
||||||
// quota. It is used when an outer transport boundary already owns rate limits.
|
|
||||||
type AllowAll struct{}
|
|
||||||
|
|
||||||
func (AllowAll) Admit(ctx context.Context, key string) error {
|
|
||||||
if ctx == nil || key == "" {
|
|
||||||
return ErrInvalidIdentity
|
|
||||||
}
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
package admission
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAllowAllPreservesContextAndIdentityValidation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
var admission AllowAll
|
|
||||||
if err := admission.Admit(context.Background(), "client-a"); err != nil {
|
|
||||||
t.Fatalf("Admit(valid) error = %v", err)
|
|
||||||
}
|
|
||||||
if err := admission.Admit(context.Background(), ""); !errors.Is(err, ErrInvalidIdentity) {
|
|
||||||
t.Fatalf("Admit(empty identity) error = %v", err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if err := admission.Admit(ctx, "client-a"); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Admit(canceled) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,70 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrInvalidDependencies = errors.New("invalid metrics dependencies")
|
|
||||||
|
|
||||||
type ReadinessChecker interface {
|
|
||||||
Ready(context.Context) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type Dependencies struct {
|
|
||||||
Gatherer prometheus.Gatherer
|
|
||||||
Readiness ReadinessChecker
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewHandler(dependencies Dependencies) (http.Handler, error) {
|
|
||||||
if nilInterface(dependencies.Gatherer) || nilInterface(dependencies.Readiness) {
|
|
||||||
return nil, ErrInvalidDependencies
|
|
||||||
}
|
|
||||||
metricsHandler := promhttp.HandlerFor(dependencies.Gatherer, promhttp.HandlerOpts{})
|
|
||||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.Method != http.MethodGet {
|
|
||||||
response.Header().Set("Allow", http.MethodGet)
|
|
||||||
http.Error(response, "method not allowed", http.StatusMethodNotAllowed)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch request.URL.Path {
|
|
||||||
case "/livez":
|
|
||||||
writeStatus(response, http.StatusOK, "live")
|
|
||||||
case "/readyz":
|
|
||||||
if err := dependencies.Readiness.Ready(request.Context()); err != nil {
|
|
||||||
writeStatus(response, http.StatusServiceUnavailable, "unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeStatus(response, http.StatusOK, "ready")
|
|
||||||
case "/metrics":
|
|
||||||
metricsHandler.ServeHTTP(response, request)
|
|
||||||
default:
|
|
||||||
http.NotFound(response, request)
|
|
||||||
}
|
|
||||||
}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeStatus(response http.ResponseWriter, status int, value string) {
|
|
||||||
response.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
||||||
response.Header().Set("X-Content-Type-Options", "nosniff")
|
|
||||||
response.WriteHeader(status)
|
|
||||||
_, _ = response.Write([]byte(value + "\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func nilInterface(value any) bool {
|
|
||||||
if value == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
reflected := reflect.ValueOf(value)
|
|
||||||
switch reflected.Kind() {
|
|
||||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
|
||||||
return reflected.IsNil()
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,121 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHandlerServesMetricsAndIndependentProbes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
registry := prometheus.NewRegistry()
|
|
||||||
gauge := prometheus.NewGauge(prometheus.GaugeOpts{Name: "proxy_pool_test_inventory"})
|
|
||||||
gauge.Set(7)
|
|
||||||
registry.MustRegister(gauge)
|
|
||||||
readiness := &recordingReadiness{}
|
|
||||||
handler, err := NewHandler(Dependencies{Gatherer: registry, Readiness: readiness})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewHandler() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
server := httptest.NewServer(handler)
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
assertProbe(t, http.MethodGet, server.URL+"/livez", http.StatusOK, "live")
|
|
||||||
if readiness.calls.Load() != 0 {
|
|
||||||
t.Fatalf("livez readiness calls = %d, want 0", readiness.calls.Load())
|
|
||||||
}
|
|
||||||
assertProbe(t, http.MethodGet, server.URL+"/readyz", http.StatusOK, "ready")
|
|
||||||
if readiness.calls.Load() != 1 {
|
|
||||||
t.Fatalf("readyz readiness calls = %d, want 1", readiness.calls.Load())
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := http.Get(server.URL + "/metrics")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET /metrics: %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
body, err := io.ReadAll(response.Body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadAll(/metrics): %v", err)
|
|
||||||
}
|
|
||||||
if response.StatusCode != http.StatusOK || !strings.Contains(string(body), "proxy_pool_test_inventory 7") {
|
|
||||||
t.Fatalf("GET /metrics = %d %q", response.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerFailsReadyWithoutLeakingDependencyError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
const secret = "redis://user:secret@redis:6379"
|
|
||||||
handler, err := NewHandler(Dependencies{
|
|
||||||
Gatherer: prometheus.NewRegistry(),
|
|
||||||
Readiness: &recordingReadiness{err: errors.New(secret)},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewHandler() error = %v", err)
|
|
||||||
}
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
|
||||||
if recorder.Code != http.StatusServiceUnavailable || strings.Contains(recorder.Body.String(), secret) {
|
|
||||||
t.Fatalf("GET /readyz = %d %q", recorder.Code, recorder.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerRejectsInvalidDependenciesMethodsAndPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
if _, err := NewHandler(Dependencies{}); !errors.Is(err, ErrInvalidDependencies) {
|
|
||||||
t.Fatalf("NewHandler(empty) error = %v", err)
|
|
||||||
}
|
|
||||||
handler, err := NewHandler(Dependencies{
|
|
||||||
Gatherer: prometheus.NewRegistry(), Readiness: &recordingReadiness{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewHandler() error = %v", err)
|
|
||||||
}
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/livez", nil))
|
|
||||||
if recorder.Code != http.StatusMethodNotAllowed || recorder.Header().Get("Allow") != http.MethodGet {
|
|
||||||
t.Fatalf("POST /livez = %d Allow=%q", recorder.Code, recorder.Header().Get("Allow"))
|
|
||||||
}
|
|
||||||
recorder = httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/missing", nil))
|
|
||||||
if recorder.Code != http.StatusNotFound {
|
|
||||||
t.Fatalf("GET /missing = %d", recorder.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type recordingReadiness struct {
|
|
||||||
calls atomic.Int64
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness *recordingReadiness) Ready(context.Context) error {
|
|
||||||
readiness.calls.Add(1)
|
|
||||||
return readiness.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertProbe(t *testing.T, method, target string, wantStatus int, wantBody string) {
|
|
||||||
t.Helper()
|
|
||||||
request, err := http.NewRequest(method, target, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest(): %v", err)
|
|
||||||
}
|
|
||||||
response, err := http.DefaultClient.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do(%s): %v", target, err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
body, err := io.ReadAll(response.Body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadAll(%s): %v", target, err)
|
|
||||||
}
|
|
||||||
if response.StatusCode != wantStatus || strings.TrimSpace(string(body)) != wantBody {
|
|
||||||
t.Fatalf("%s = %d %q", target, response.StatusCode, body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
43
progress.md
43
progress.md
@ -2,49 +2,6 @@
|
|||||||
|
|
||||||
## 2026-07-30
|
## 2026-07-30
|
||||||
|
|
||||||
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
|
|
||||||
`redisprovider` Adapter;Redis Lua 原子维护 generation、epoch、Leader 租约、
|
|
||||||
全局 requestInterval 与带 TTL 的 maxInFlight Permit,异常时 fail-closed。
|
|
||||||
- 真实 Redis 8.2 已验证两个 Controller 同 Upstream 只有一个 Leader、取消后
|
|
||||||
epoch 单调接管、跨实例请求限速/在途上限、Permit 幂等释放,以及 Redis 协调
|
|
||||||
状态丢失后使用新 generation 自动重建。
|
|
||||||
- 新增显式 `refill.reconcileInterval/minimumAvailableSlots/targetAvailableSlots` 与
|
|
||||||
`fetch.estimatedIPsPerCall`;20 份示例、主配置、Compose/Kubernetes 配置均通过
|
|
||||||
严格解析和理论容量边界校验。
|
|
||||||
- Pool Reconciler 已按并发槽位实现 minimum/target 迟滞并计入 pending 预估;
|
|
||||||
FetchBudget 只在没有 pending 请求时接受 Redis Managed 同步,消除 Upsert 与
|
|
||||||
Permit Complete 短窗口中的重复计数风险。
|
|
||||||
- Provider Fleet、Worker Active/Reserved 容量汇总和 Controller bootstrap 接线
|
|
||||||
尚未完成,本轮不增加 51/73 的总验收计数。
|
|
||||||
|
|
||||||
- 新增公用 `platform/metrics.NewHandler`,固定提供 `/livez`、`/readyz` 与
|
|
||||||
`/metrics`;Readiness 失败只返回脱敏状态,不泄露底层存储错误。
|
|
||||||
- Metrics 地址进入严格配置校验;Controller Runtime 支持 Metrics-only 和
|
|
||||||
Distribution/Admin/Metrics 三监听器隔离、首错联动及统一优雅停机。
|
|
||||||
- Controller bootstrap 使用同一根生命周期装配 Prometheus 默认 Gatherer,
|
|
||||||
Metrics Readiness 按启用能力低成本检查 PostgreSQL/Redis,不触发 Provider 或
|
|
||||||
扫描 Proxy 明细。
|
|
||||||
- `go test` 定向包和 `.\scripts\test-controller.ps1` 通过;真实 PostgreSQL 18 +
|
|
||||||
Redis 8.2 fixture 已验证 `/readyz` 与 Prometheus 输出,临时容器和网络已清理。
|
|
||||||
|
|
||||||
- 新增 `cmd/proxy-controller` 与公用 `controller/bootstrap.Run`;配置只加载一次,
|
|
||||||
同一快照用于存储连接、PostgreSQL 管理态提交和 HTTP Runtime 构造,避免启动
|
|
||||||
期间二次读取产生配置撕裂。
|
|
||||||
- Bootstrap 已封装 PostgreSQL Ping/迁移/pgx Adapter、Redis Ping/活动池、
|
|
||||||
Distribution/Admin 服务构造、运行错误与关闭错误合并;`main` 仅处理
|
|
||||||
`-config`、`PROXY_POOL_CONFIG`、信号上下文和退出码。
|
|
||||||
- 活动池新增公用 `StateInventoryReader`;Memory/Redis 使用同一契约,Redis
|
|
||||||
通过低基数 Hash 和五个原子 Lua 维护七类状态计数,不扫描 Proxy 明细。
|
|
||||||
- 新增 `controller/operations.Reader`,把当前配置中的 Upstream 与活动状态映射到
|
|
||||||
Admin Status;Worker/Snapshot 与 Provider 统计尚无来源时保持空/零,不用
|
|
||||||
`Managed` 冒充 `Available`。
|
|
||||||
- Redis 状态读取在有界过期清理仍有积压或检测到负计数时 fail-closed,避免
|
|
||||||
Admin 返回包含失效代理或损坏计数的成功响应。
|
|
||||||
- 本轮全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、
|
|
||||||
`go build ./cmd/proxy-controller` 与 `git diff --check` 通过;新加的 Redis
|
|
||||||
backlog/负计数场景也已通过真实 Redis 8.2 fixture。
|
|
||||||
- 新增 `test-controller.ps1` 双存储 fixture;真实 PostgreSQL 18 + Redis 8.2 已
|
|
||||||
通过迁移、启动配置提交、Redis Readiness 与 Admin Status 组合验证。
|
|
||||||
- 固定 `github.com/jackc/pgx/v5 v5.6.0`,实现封装在 `adminstate.Store` 后的
|
- 固定 `github.com/jackc/pgx/v5 v5.6.0`,实现封装在 `adminstate.Store` 后的
|
||||||
PostgreSQL 深适配器;配置、Upstream、Routing mutation 在同一事务中提交
|
PostgreSQL 深适配器;配置、Upstream、Routing mutation 在同一事务中提交
|
||||||
revision、管理状态、审计与 Outbox,数据库错误不泄漏 DSN、SQL 或参数。
|
revision、管理状态、审计与 Outbox,数据库错误不泄漏 DSN、SQL 或参数。
|
||||||
|
|||||||
@ -1,42 +0,0 @@
|
|||||||
$ErrorActionPreference = "Stop"
|
|
||||||
|
|
||||||
$repositoryRoot = Split-Path -Parent $PSScriptRoot
|
|
||||||
$composeFile = Join-Path $repositoryRoot "deploy/docker-compose.test.yml"
|
|
||||||
$composeProject = "proxy-pool-controller-test"
|
|
||||||
$previousPostgresURL = [Environment]::GetEnvironmentVariable("PROXY_POOL_TEST_POSTGRES_URL", "Process")
|
|
||||||
$previousRedisURL = [Environment]::GetEnvironmentVariable("PROXY_POOL_TEST_REDIS_URL", "Process")
|
|
||||||
|
|
||||||
try {
|
|
||||||
docker compose -p $composeProject -f $composeFile up -d --wait --wait-timeout 60 postgres redis
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "starting Controller test fixtures failed with exit code $LASTEXITCODE"
|
|
||||||
}
|
|
||||||
|
|
||||||
$env:PROXY_POOL_TEST_POSTGRES_URL = "postgres://proxy_pool_test:proxy-pool-test@127.0.0.1:15432/proxy_pool_test?sslmode=disable"
|
|
||||||
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
|
|
||||||
Push-Location $repositoryRoot
|
|
||||||
try {
|
|
||||||
go test -count=1 -tags=integration -timeout 60s ./internal/controller/bootstrap
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "Controller integration tests failed with exit code $LASTEXITCODE"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
Pop-Location
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
if ($null -eq $previousPostgresURL) {
|
|
||||||
Remove-Item Env:PROXY_POOL_TEST_POSTGRES_URL -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$env:PROXY_POOL_TEST_POSTGRES_URL = $previousPostgresURL
|
|
||||||
}
|
|
||||||
if ($null -eq $previousRedisURL) {
|
|
||||||
Remove-Item Env:PROXY_POOL_TEST_REDIS_URL -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$env:PROXY_POOL_TEST_REDIS_URL = $previousRedisURL
|
|
||||||
}
|
|
||||||
docker compose -p $composeProject -f $composeFile down --volumes --remove-orphans
|
|
||||||
}
|
|
||||||
@ -14,7 +14,7 @@ try {
|
|||||||
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
|
$env:PROXY_POOL_TEST_REDIS_URL = "redis://127.0.0.1:16379/15"
|
||||||
Push-Location $repositoryRoot
|
Push-Location $repositoryRoot
|
||||||
try {
|
try {
|
||||||
go test -count=1 -tags=integration -timeout 60s ./internal/adapters/redisactivity/... ./internal/adapters/redisprovider/...
|
go test -count=1 -tags=integration -timeout 60s ./internal/adapters/redisactivity/...
|
||||||
if ($LASTEXITCODE -ne 0) {
|
if ($LASTEXITCODE -ne 0) {
|
||||||
throw "Redis integration tests failed with exit code $LASTEXITCODE"
|
throw "Redis integration tests failed with exit code $LASTEXITCODE"
|
||||||
}
|
}
|
||||||
|
|||||||
@ -33,9 +33,6 @@
|
|||||||
和真实 PostgreSQL 18 集成测试已完成
|
和真实 PostgreSQL 18 集成测试已完成
|
||||||
12. [进行中] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界;
|
12. [进行中] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界;
|
||||||
机器契约和文档类滞后勾选已按仓库证据校正
|
机器契约和文档类滞后勾选已按仓库证据校正
|
||||||
13. [进行中] 落地 `proxy-controller` 进程装配;配置单次加载、PostgreSQL 迁移、
|
|
||||||
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
|
|
||||||
双存储 bootstrap 和探针集成已通过,Provider、业务指标与完整容器进程链仍待实现
|
|
||||||
|
|
||||||
## 串并行关系
|
## 串并行关系
|
||||||
|
|
||||||
@ -56,8 +53,7 @@
|
|||||||
|
|
||||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;Redis 8.2
|
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;Redis 8.2
|
||||||
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
|
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
|
||||||
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
|
- `cmd/proxy-*`、PostgreSQL 管理面 Adapter、Provider Leader/分布式限流、
|
||||||
Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
|
Checker 运行时、Redis 故障转移验证与代表性集群压测属于后续实施范围。
|
||||||
故障转移验证与代表性集群压测属于后续实施范围。
|
|
||||||
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
|
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
|
||||||
验收项完成率约 69.9%,不等同于生产就绪度。
|
验收项完成率约 69.9%,不等同于生产就绪度。
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user