feat: add atomic global health reducer
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run

This commit is contained in:
youfak 2026-07-31 17:55:06 +08:00
parent 6766097ea7
commit 0a904a7b6b
13 changed files with 908 additions and 35 deletions

View File

@ -273,6 +273,12 @@ Outcome 上报已完成为有界队列、批次序列/摘要栅栏和确认重
- [ ] Ensure target failures affect only the target profile. - [ ] Ensure target failures affect only the target profile.
- [ ] Add fixture target server and deterministic clock/scheduler tests. - [ ] Add fixture target server and deterministic clock/scheduler tests.
当前进度2026-07-31已新增不依赖 gRPC/Redis 的 `domain/health` Observation、
全局/Target Profile Reducer 与任务摘要幂等语义BASIC/EGRESS 结果经活动池窄接口在
Memory 和 Redis Lua 同一原子边界归并,覆盖首次 CHECKING 失败进入 UNHEALTHY、
AVAILABLE/SUSPECT 的阈值降级、精确重放、冲突拒绝与成功恢复。TARGET 仍只在纯领域
Profile 中归并尚未接入持久化、调度器、Checker RPC 或独立进程,因此本任务保持未完成。
## Task 12: Machine-readable Contracts ## Task 12: Machine-readable Contracts
**Files:** `api/openapi/proxy-pool.yaml`, `api/proto/controlplane/v1/controlplane.proto`, **Files:** `api/openapi/proxy-pool.yaml`, `api/proto/controlplane/v1/controlplane.proto`,

View File

@ -57,7 +57,9 @@
Routing 一起纳入 Snapshot checksum仅在 mTLS 和 Gateway 当前内存 View 中处理。Gateway Routing 一起纳入 Snapshot checksum仅在 mTLS 和 Gateway 当前内存 View 中处理。Gateway
Routing payload 已由配置顺序和 Admin 当前状态合成并覆盖 Snapshot checksum其运行时消费、 Routing payload 已由配置顺序和 Admin 当前状态合成并覆盖 Snapshot checksum其运行时消费、
Outcome 已实现为 Gateway 本地有界队列、微批确认重试和 Controller 的 session/sequence/ Outcome 已实现为 Gateway 本地有界队列、微批确认重试和 Controller 的 session/sequence/
摘要 Redis 栅栏;原始事件不落 Redis 或 PostgreSQL。Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前 摘要 Redis 栅栏;原始事件不落 Redis 或 PostgreSQL。Checker 已有全局健康 Reducer
与 Memory/Redis 原子状态提交基础但任务调度、Checker RPC、TARGET Profile 持久化和
独立执行进程尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Controller `session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Controller
在最近成功下发的 Snapshot `valid_until` 到达时关闭流Gateway 的公用 在最近成功下发的 Snapshot `valid_until` 到达时关闭流Gateway 的公用
`SessionSupervisor` 已实现可恢复错误的有界退避重连。Gateway 会校验并执行 Snapshot `SessionSupervisor` 已实现可恢复错误的有界退避重连。Gateway 会校验并执行 Snapshot

View File

@ -79,9 +79,9 @@
| ID | 最终需求 | 来源 | 验证证据 | | ID | 最终需求 | 来源 | 验证证据 |
|---|---|---|---| |---|---|---|---|
| HEALTH-001 | 全局健康与 Routing/目标健康分离 | 221-270, 8679-8708 | Protobuf/配置契约已定义;目标 Profile 与健康 Reducer 待实现 | | HEALTH-001 | 全局健康与 Routing/目标健康分离 | 221-270, 8679-8708 | `domain/health` 已将 BASIC/EGRESS 全局 Reducer 与 TARGET Profile Reducer 分离TARGET 持久化和 Routing 消费待实现 |
| HEALTH-002 | 健康调度有 jitter、maxInFlight 和分级频率 | 8679-8736 | 配置校验已完成;有界调度器、抖动和分级频率测试待实现 | | HEALTH-002 | 健康调度有 jitter、maxInFlight 和分级频率 | 8679-8736 | 配置校验已完成;有界调度器、抖动和分级频率测试待实现 |
| HEALTH-003 | 失败分级 SUSPECT -> UNHEALTHY -> REMOVE | 8679-8736 | Proxy 状态迁移骨架已完成;连续失败 Reducer 与活动池原子提交待实现 | | HEALTH-003 | 失败分级 SUSPECT -> UNHEALTHY -> REMOVE | 8679-8736 | 全局 Reducer 已在 Memory/Redis 活动池原子提交连续失败、精确重放和成功恢复;任务调度与 REMOVE 编排待实现 |
| SEC-001 | API 认证与 Proxy 认证分离Secret 统一脱敏 | 7528-8111, 8904-8945 | Config 脱敏、Provider Store -> SecretRef -> Gateway Resolver 跨包测试与格式化泄漏回归测试 | | SEC-001 | API 认证与 Proxy 认证分离Secret 统一脱敏 | 7528-8111, 8904-8945 | Config 脱敏、Provider Store -> SecretRef -> Gateway Resolver 跨包测试与格式化泄漏回归测试 |
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 | | SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 | | OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |

View File

@ -32,6 +32,10 @@ type proxyRecord struct {
UsableUntilMS int64 `json:"usableUntilMs"` UsableUntilMS int64 `json:"usableUntilMs"`
LastCheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"` LastCheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"`
LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"` LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"`
ConsecutiveFailures int `json:"consecutiveFailures,omitempty"`
LastHealthTaskID string `json:"lastHealthTaskId,omitempty"`
LastHealthDigest string `json:"lastHealthDigest,omitempty"`
LastHealthObservedAtMS int64 `json:"lastHealthObservedAtMs,omitempty"`
LatencyNS int64 `json:"latencyNs"` LatencyNS int64 `json:"latencyNs"`
MaxConcurrency int64 `json:"maxConcurrency"` MaxConcurrency int64 `json:"maxConcurrency"`
State string `json:"state"` State string `json:"state"`
@ -204,11 +208,16 @@ func validateProxyRecord(record proxyRecord) error {
record.Port <= 0 || record.Port > 65_535 || record.SourceUpstream == "" || record.Port <= 0 || record.Port > 65_535 || record.SourceUpstream == "" ||
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 || record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 || record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 || record.LastSuccessAtMS < 0 || record.ConsecutiveFailures < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
!validScheme(record.Scheme) || !validProxyState(record.State) || !validScheme(record.Scheme) || !validProxyState(record.State) ||
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") { record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
return ErrInvalidRecord return ErrInvalidRecord
} }
if (record.LastHealthTaskID == "") != (record.LastHealthDigest == "") ||
(record.LastHealthDigest != "" && (!validDigest(record.LastHealthDigest) || record.LastHealthObservedAtMS <= 0)) ||
record.LastHealthObservedAtMS < 0 {
return ErrInvalidRecord
}
for _, key := range record.IndexKeys { for _, key := range record.IndexKeys {
if key == "" || !strings.Contains(key, "{activity}") { if key == "" || !strings.Contains(key, "{activity}") {
return ErrInvalidRecord return ErrInvalidRecord

View File

@ -0,0 +1,74 @@
//go:build integration
package redisactivity
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/domain/activitypool"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestRedisAppliesGlobalHealthObservationAtomically(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
}); err != nil {
t.Fatalf("UpsertFetched(): %v", err)
}
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
}); err != nil {
t.Fatalf("ApplyHealth(checking): %v", err)
}
first := redisGlobalObservation("task-1", false, now.Add(2*time.Second))
failed, err := fixture.Adapter.ApplyGlobalObservation(context.Background(), activitypool.GlobalHealthCommand{
Observation: first, MaxConsecutiveFailures: 3,
})
if err != nil || failed.State != proxyDomain.StateUnhealthy || failed.GlobalHealth.ConsecutiveFailures != 1 {
t.Fatalf("ApplyGlobalObservation(initial failure) = %+v, %v", failed, err)
}
replayed, err := fixture.Adapter.ApplyGlobalObservation(context.Background(), activitypool.GlobalHealthCommand{
Observation: first, MaxConsecutiveFailures: 3,
})
if err != nil || replayed.GlobalHealth != failed.GlobalHealth {
t.Fatalf("ApplyGlobalObservation(replay) = %+v, %v", replayed, err)
}
conflicting := first
conflicting.Success = true
conflicting.FailureClass = ""
if _, err := fixture.Adapter.ApplyGlobalObservation(context.Background(), activitypool.GlobalHealthCommand{
Observation: conflicting, MaxConsecutiveFailures: 3,
}); !errors.Is(err, healthDomain.ErrConflictingObservation) {
t.Fatalf("ApplyGlobalObservation(conflict) error = %v, want ErrConflictingObservation", err)
}
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateChecking,
}); err != nil {
t.Fatalf("ApplyHealth(recheck): %v", err)
}
recovered, err := fixture.Adapter.ApplyGlobalObservation(context.Background(), activitypool.GlobalHealthCommand{
Observation: redisGlobalObservation("task-2", true, now.Add(4*time.Second)), MaxConsecutiveFailures: 3,
})
if err != nil || recovered.State != proxyDomain.StateAvailable || recovered.GlobalHealth.ConsecutiveFailures != 0 ||
recovered.GlobalHealth.LastObservedAt != now.Add(4*time.Second) {
t.Fatalf("ApplyGlobalObservation(recovery) = %+v, %v", recovered, err)
}
}
func redisGlobalObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
observation := healthDomain.Observation{
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,
Success: success, Latency: 10 * time.Millisecond, ObservedAt: observedAt,
}
if !success {
observation.FailureClass = "timeout"
}
return observation
}

View File

@ -2,11 +2,14 @@ package redisactivity
import ( import (
"context" "context"
"encoding/hex"
"proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/activitypool"
healthDomain "proxy-pool/internal/domain/health"
) )
var _ activitypool.HealthStore = (*Adapter)(nil) var _ activitypool.HealthStore = (*Adapter)(nil)
var _ activitypool.GlobalHealthStore = (*Adapter)(nil)
func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpdate) (activitypool.Entry, error) { func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpdate) (activitypool.Entry, error) {
if ctx == nil { if ctx == nil {
@ -56,3 +59,71 @@ func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpd
return activitypool.Entry{}, invalidScriptReply("unexpected health status") return activitypool.Entry{}, invalidScriptReply("unexpected health status")
} }
} }
// ApplyGlobalObservation reduces one BASIC or EGRESS observation inside the
// same Redis Lua transaction that updates state indexes and inventory.
func (a *Adapter) ApplyGlobalObservation(
ctx context.Context,
command activitypool.GlobalHealthCommand,
) (activitypool.Entry, error) {
if ctx == nil {
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
}
if err := ctx.Err(); err != nil {
return activitypool.Entry{}, err
}
normalized, err := healthDomain.NormalizeObservation(command.Observation)
if err != nil {
return activitypool.Entry{}, err
}
if a == nil || normalized.Level == healthDomain.LevelTarget || command.MaxConsecutiveFailures <= 0 {
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
}
digest, err := healthDomain.ObservationDigest(normalized)
if err != nil {
return activitypool.Entry{}, err
}
operationID, err := newOperationID()
if err != nil {
return activitypool.Entry{}, err
}
success := "0"
if normalized.Success {
success = "1"
}
result, err := runScript(ctx, a.client, healthScript, []string{
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.operation(operationID),
}, normalized.ObservedAt.UnixMilli(), "", int64(normalized.Latency),
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), normalized.ProxyID,
"global", success, command.MaxConsecutiveFailures, normalized.TaskID, hex.EncodeToString(digest[:]))
if err != nil {
return activitypool.Entry{}, err
}
var reply healthScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return activitypool.Entry{}, err
}
switch reply.Status {
case scriptNotFound:
return activitypool.Entry{}, activitypool.ErrActivityNotFound
case scriptStale:
return activitypool.Entry{}, healthDomain.ErrStaleObservation
case scriptConflict:
return activitypool.Entry{}, healthDomain.ErrConflictingObservation
case scriptInvalid:
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
case scriptOK:
if reply.Record == "" {
return activitypool.Entry{}, invalidScriptReply("global health reply omitted record")
}
record, err := decodeProxyRecord(reply.Record)
if err != nil {
return activitypool.Entry{}, invalidScriptReply("global health reply contained an invalid record")
}
return proxyRecordEntry(record), nil
default:
return activitypool.Entry{}, invalidScriptReply("unexpected global health status")
}
}

View File

@ -15,6 +15,11 @@ local latency_ns = tonumber(ARGV[3])
local cleanup_limit = tonumber(ARGV[4]) local cleanup_limit = tonumber(ARGV[4])
local operation_ttl_ms = tonumber(ARGV[5]) local operation_ttl_ms = tonumber(ARGV[5])
local proxy_id = ARGV[6] local proxy_id = ARGV[6]
local mode = ARGV[7] or 'transition'
local success = ARGV[8]
local max_consecutive_failures = tonumber(ARGV[9])
local task_id = ARGV[10]
local observation_digest = ARGV[11]
local committed = redis.call('GET', operation_key) local committed = redis.call('GET', operation_key)
if committed then if committed then
@ -194,14 +199,61 @@ if tonumber(record.expiresAtMs) <= checked_at_ms then
end end
local last_checked_at_ms = tonumber(record.lastCheckedAtMs or '0') local last_checked_at_ms = tonumber(record.lastCheckedAtMs or '0')
if checked_at_ms < last_checked_at_ms then if mode == 'global' then
if (success ~= '0' and success ~= '1') or not max_consecutive_failures or
max_consecutive_failures <= 0 or type(task_id) ~= 'string' or task_id == '' or
type(observation_digest) ~= 'string' or string.len(observation_digest) ~= 64 then
return finish({status = 'invalid'})
end
local last_task_id = record.lastHealthTaskId or ''
local last_observation_digest = record.lastHealthDigest or ''
if last_task_id == task_id then
if last_observation_digest == observation_digest then
return finish({status = 'ok', record = raw}, tonumber(record.expiresAtMs))
end
return finish({status = 'conflict'})
end
if checked_at_ms <= last_checked_at_ms then
return finish({status = 'stale'}) return finish({status = 'stale'})
end end
if checked_at_ms == last_checked_at_ms then local consecutive_failures = tonumber(record.consecutiveFailures or '0')
if not consecutive_failures or consecutive_failures < 0 then
return finish({status = 'invalid'})
end
if success == '1' then
if record.state ~= 'CHECKING' and record.state ~= 'AVAILABLE' and record.state ~= 'SUSPECT' then
return finish({status = 'invalid'})
end
next_state = 'AVAILABLE'
consecutive_failures = 0
else
consecutive_failures = consecutive_failures + 1
if record.state == 'CHECKING' then
next_state = 'UNHEALTHY'
elseif record.state == 'AVAILABLE' or record.state == 'SUSPECT' then
if consecutive_failures >= max_consecutive_failures then
next_state = 'UNHEALTHY'
else
next_state = 'SUSPECT'
end
else
return finish({status = 'invalid'})
end
end
record.consecutiveFailures = consecutive_failures
record.lastHealthTaskId = task_id
record.lastHealthDigest = observation_digest
record.lastHealthObservedAtMs = checked_at_ms
else
if checked_at_ms < last_checked_at_ms then
return finish({status = 'stale'})
end
if checked_at_ms == last_checked_at_ms then
if record.state ~= next_state then if record.state ~= next_state then
return finish({status = 'stale'}) return finish({status = 'stale'})
end end
return finish({status = 'ok', record = raw}, tonumber(record.expiresAtMs)) return finish({status = 'ok', record = raw}, tonumber(record.expiresAtMs))
end
end end
if record.state ~= next_state and not (transitions[record.state] and transitions[record.state][next_state]) then if record.state ~= next_state and not (transitions[record.state] and transitions[record.state][next_state]) then
return finish({status = 'invalid'}) return finish({status = 'invalid'})

View File

@ -227,6 +227,10 @@ for _, candidate in ipairs(candidates) do
incoming.state = current.state incoming.state = current.state
incoming.lastCheckedAtMs = current.lastCheckedAtMs incoming.lastCheckedAtMs = current.lastCheckedAtMs
incoming.lastSuccessAtMs = current.lastSuccessAtMs incoming.lastSuccessAtMs = current.lastSuccessAtMs
incoming.consecutiveFailures = current.consecutiveFailures
incoming.lastHealthTaskId = current.lastHealthTaskId
incoming.lastHealthDigest = current.lastHealthDigest
incoming.lastHealthObservedAtMs = current.lastHealthObservedAtMs
incoming.latencyNs = current.latencyNs incoming.latencyNs = current.latencyNs
incoming.ownerWorkerId = current.ownerWorkerId incoming.ownerWorkerId = current.ownerWorkerId
local encoded = cjson.encode(incoming) local encoded = cjson.encode(incoming)

View File

@ -11,6 +11,7 @@ import (
"proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction" extractionDomain "proxy-pool/internal/domain/extraction"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy" proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/platform/credentials" "proxy-pool/internal/platform/credentials"
) )
@ -252,9 +253,19 @@ func proxyRecordEntry(record proxyRecord) activitypool.Entry {
value := time.UnixMilli(record.LastSuccessAtMS).UTC() value := time.UnixMilli(record.LastSuccessAtMS).UTC()
proxy.LastSuccessAt = &value proxy.LastSuccessAt = &value
} }
global := healthDomain.GlobalState{State: proxyDomain.State(record.State)}
if record.LastHealthTaskID != "" {
digest, err := hex.DecodeString(record.LastHealthDigest)
if err == nil && len(digest) == sha256.Size && record.LastHealthObservedAtMS > 0 {
copy(global.LastObservationDigest[:], digest)
global.LastTaskID = record.LastHealthTaskID
global.LastObservedAt = time.UnixMilli(record.LastHealthObservedAtMS).UTC()
global.ConsecutiveFailures = record.ConsecutiveFailures
}
}
return activitypool.Entry{ return activitypool.Entry{
Proxy: proxy, UsableUntil: usableUntil, Proxy: proxy, UsableUntil: usableUntil,
OwnerWorkerID: record.OwnerWorkerID, State: proxyDomain.State(record.State), OwnerWorkerID: record.OwnerWorkerID, State: proxyDomain.State(record.State), GlobalHealth: global,
} }
} }

View File

@ -0,0 +1,102 @@
package activitypool
import (
"context"
"errors"
"testing"
"time"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestMemoryPoolAppliesGlobalObservationAtomically(t *testing.T) {
now := time.Date(2026, 7, 31, 11, 0, 0, 0, time.UTC)
pool := seededHealthPool(t, now)
if _, err := pool.ApplyHealth(context.Background(), HealthUpdate{
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
}); err != nil {
t.Fatalf("ApplyHealth(checking): %v", err)
}
failed, err := pool.ApplyGlobalObservation(context.Background(), GlobalHealthCommand{
Observation: healthObservation("task-1", false, now.Add(2*time.Second)), MaxConsecutiveFailures: 3,
})
if err != nil || failed.State != proxyDomain.StateUnhealthy || failed.GlobalHealth.ConsecutiveFailures != 1 {
t.Fatalf("ApplyGlobalObservation(initial failure) = %+v, %v", failed, err)
}
if _, err := pool.ApplyHealth(context.Background(), HealthUpdate{
ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateChecking,
}); err != nil {
t.Fatalf("ApplyHealth(recheck): %v", err)
}
available, err := pool.ApplyGlobalObservation(context.Background(), GlobalHealthCommand{
Observation: healthObservation("task-2", true, now.Add(4*time.Second)), MaxConsecutiveFailures: 3,
})
if err != nil || available.State != proxyDomain.StateAvailable || available.GlobalHealth.ConsecutiveFailures != 0 ||
available.Proxy.LastSuccessAt == nil || !available.Proxy.LastSuccessAt.Equal(now.Add(4*time.Second)) {
t.Fatalf("ApplyGlobalObservation(recovery) = %+v, %v", available, err)
}
}
func TestMemoryPoolGlobalObservationReplayDoesNotIncreaseFailureStreak(t *testing.T) {
now := time.Date(2026, 7, 31, 11, 0, 0, 0, time.UTC)
pool := seededHealthPool(t, now)
if _, err := pool.ApplyHealth(context.Background(), HealthUpdate{
ProxyID: "proxy-a", CheckedAt: now.Add(time.Second), NextState: proxyDomain.StateChecking,
}); err != nil {
t.Fatalf("ApplyHealth(checking): %v", err)
}
command := GlobalHealthCommand{Observation: healthObservation("task-1", false, now.Add(2*time.Second)), MaxConsecutiveFailures: 3}
first, err := pool.ApplyGlobalObservation(context.Background(), command)
if err != nil {
t.Fatalf("ApplyGlobalObservation(first): %v", err)
}
replayed, err := pool.ApplyGlobalObservation(context.Background(), command)
if err != nil || replayed.GlobalHealth != first.GlobalHealth {
t.Fatalf("ApplyGlobalObservation(replay) = %+v, %v", replayed, err)
}
conflicting := command
conflicting.Observation.Success = true
conflicting.Observation.FailureClass = ""
if _, err := pool.ApplyGlobalObservation(context.Background(), conflicting); !errors.Is(err, healthDomain.ErrConflictingObservation) {
t.Fatalf("ApplyGlobalObservation(conflict) error = %v, want ErrConflictingObservation", err)
}
}
func TestMemoryPoolRejectsTargetObservationFromGlobalStore(t *testing.T) {
now := time.Date(2026, 7, 31, 11, 0, 0, 0, time.UTC)
pool := seededHealthPool(t, now)
_, err := pool.ApplyGlobalObservation(context.Background(), GlobalHealthCommand{Observation: healthDomain.Observation{
TaskID: "task-target", ProxyID: "proxy-a", Level: healthDomain.LevelTarget, RoutingName: "route-a",
TargetURL: "https://target.example/", Success: false, FailureClass: "target_403", ObservedAt: now.Add(time.Second),
}, MaxConsecutiveFailures: 2})
if !errors.Is(err, ErrInvalidHealthUpdate) {
t.Fatalf("ApplyGlobalObservation(target) error = %v, want ErrInvalidHealthUpdate", err)
}
}
func seededHealthPool(t *testing.T, now time.Time) *MemoryPool {
t.Helper()
pool := NewMemoryPool()
_, err := pool.UpsertFetched(context.Background(), "provider-a", FetchedBatch{
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
Proxies: []proxyDomain.Proxy{{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080,
}},
})
if err != nil {
t.Fatalf("UpsertFetched(): %v", err)
}
return pool
}
func healthObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
observation := healthDomain.Observation{
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,
Success: success, Latency: 10 * time.Millisecond, ObservedAt: observedAt,
}
if !success {
observation.FailureClass = "timeout"
}
return observation
}

View File

@ -10,6 +10,7 @@ import (
"time" "time"
extractionDomain "proxy-pool/internal/domain/extraction" extractionDomain "proxy-pool/internal/domain/extraction"
healthDomain "proxy-pool/internal/domain/health"
ownershipDomain "proxy-pool/internal/domain/ownership" ownershipDomain "proxy-pool/internal/domain/ownership"
proxyDomain "proxy-pool/internal/domain/proxy" proxyDomain "proxy-pool/internal/domain/proxy"
) )
@ -77,6 +78,18 @@ type HealthStore interface {
ApplyHealth(context.Context, HealthUpdate) (Entry, error) ApplyHealth(context.Context, HealthUpdate) (Entry, error)
} }
// GlobalHealthCommand carries a BASIC or EGRESS fact to the authoritative
// activity pool. TARGET observations use a separate profile store and never
// enter this command.
type GlobalHealthCommand struct {
Observation healthDomain.Observation
MaxConsecutiveFailures int
}
type GlobalHealthStore interface {
ApplyGlobalObservation(context.Context, GlobalHealthCommand) (Entry, error)
}
type InventoryReader interface { type InventoryReader interface {
Inventory(context.Context, string, time.Time) (Inventory, error) Inventory(context.Context, string, time.Time) (Inventory, error)
} }
@ -94,6 +107,7 @@ type Entry struct {
UsableUntil time.Time UsableUntil time.Time
OwnerWorkerID string OwnerWorkerID string
State proxyDomain.State State proxyDomain.State
GlobalHealth healthDomain.GlobalState
} }
type MemoryPool struct { type MemoryPool struct {
@ -115,6 +129,7 @@ type idempotencyEntry struct {
var ( var (
_ Upserter = (*MemoryPool)(nil) _ Upserter = (*MemoryPool)(nil)
_ HealthStore = (*MemoryPool)(nil) _ HealthStore = (*MemoryPool)(nil)
_ GlobalHealthStore = (*MemoryPool)(nil)
_ InventoryReader = (*MemoryPool)(nil) _ InventoryReader = (*MemoryPool)(nil)
_ StateInventoryReader = (*MemoryPool)(nil) _ StateInventoryReader = (*MemoryPool)(nil)
_ Maintainer = (*MemoryPool)(nil) _ Maintainer = (*MemoryPool)(nil)
@ -221,6 +236,7 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
UsableUntil: usableUntil, UsableUntil: usableUntil,
OwnerWorkerID: current.OwnerWorkerID, OwnerWorkerID: current.OwnerWorkerID,
State: current.State, State: current.State,
GlobalHealth: current.GlobalHealth,
} }
continue continue
} }
@ -236,6 +252,7 @@ func (p *MemoryPool) UpsertFetched(ctx context.Context, upstreamID string, batch
Proxy: cloneProxy(candidate), Proxy: cloneProxy(candidate),
UsableUntil: usableUntil, UsableUntil: usableUntil,
State: candidate.State, State: candidate.State,
GlobalHealth: healthDomain.GlobalState{State: candidate.State},
} }
p.keyByID[candidate.ID] = key p.keyByID[candidate.ID] = key
if managedActivityState(candidate.State) { if managedActivityState(candidate.State) {
@ -284,6 +301,7 @@ func (p *MemoryPool) ApplyHealth(ctx context.Context, update HealthUpdate) (Entr
checkedAt := update.CheckedAt.UTC() checkedAt := update.CheckedAt.UTC()
entry.State = update.NextState entry.State = update.NextState
entry.Proxy.State = update.NextState entry.Proxy.State = update.NextState
entry.GlobalHealth.State = update.NextState
entry.Proxy.LastCheckedAt = &checkedAt entry.Proxy.LastCheckedAt = &checkedAt
entry.Proxy.Latency = update.Latency entry.Proxy.Latency = update.Latency
if update.NextState == proxyDomain.StateAvailable { if update.NextState == proxyDomain.StateAvailable {
@ -295,6 +313,65 @@ func (p *MemoryPool) ApplyHealth(ctx context.Context, update HealthUpdate) (Entr
return entry, nil return entry, nil
} }
// ApplyGlobalObservation atomically reduces a BASIC or EGRESS Checker fact
// against the current Proxy health state and publishes the resulting state.
func (p *MemoryPool) ApplyGlobalObservation(
ctx context.Context,
command GlobalHealthCommand,
) (Entry, error) {
if ctx == nil {
return Entry{}, ErrInvalidHealthUpdate
}
if err := ctx.Err(); err != nil {
return Entry{}, err
}
normalized, err := healthDomain.NormalizeObservation(command.Observation)
if err != nil {
return Entry{}, err
}
if normalized.Level == healthDomain.LevelTarget || command.MaxConsecutiveFailures <= 0 {
return Entry{}, ErrInvalidHealthUpdate
}
if p == nil {
return Entry{}, ErrInvalidHealthUpdate
}
p.mu.Lock()
defer p.mu.Unlock()
if err := ctx.Err(); err != nil {
return Entry{}, err
}
p.purgeExpiredLocked(normalized.ObservedAt)
entry, ok := p.entryByIDLocked(normalized.ProxyID)
if !ok {
return Entry{}, ErrActivityNotFound
}
current := entry.GlobalHealth
if current.State == "" {
current.State = entry.State
}
if current.State != entry.State {
return Entry{}, ErrInvalidHealthUpdate
}
next, err := healthDomain.ReduceGlobal(current, normalized, command.MaxConsecutiveFailures)
if err != nil {
return Entry{}, err
}
if entry.State != next.State && !proxyDomain.CanTransition(entry.State, next.State) {
return Entry{}, ErrInvalidHealthUpdate
}
entry.State = next.State
entry.Proxy.State = next.State
entry.Proxy.LastCheckedAt = timePointer(normalized.ObservedAt)
entry.Proxy.Latency = normalized.Latency
if normalized.Success {
entry.Proxy.LastSuccessAt = timePointer(normalized.ObservedAt)
}
entry.GlobalHealth = next
p.setEntryByIDLocked(normalized.ProxyID, entry)
entry.Proxy = cloneProxy(entry.Proxy)
return entry, nil
}
func (p *MemoryPool) Inventory(ctx context.Context, upstreamID string, now time.Time) (Inventory, error) { func (p *MemoryPool) Inventory(ctx context.Context, upstreamID string, now time.Time) (Inventory, error) {
result := Inventory{UpstreamID: upstreamID} result := Inventory{UpstreamID: upstreamID}
if ctx == nil { if ctx == nil {
@ -477,6 +554,7 @@ func (p *MemoryPool) Extract(ctx context.Context, command extractionDomain.Comma
entry := p.entries[key] entry := p.entries[key]
entry.State = proxyDomain.StateExtracted entry.State = proxyDomain.StateExtracted
entry.Proxy.State = proxyDomain.StateExtracted entry.Proxy.State = proxyDomain.StateExtracted
entry.GlobalHealth.State = proxyDomain.StateExtracted
p.entries[key] = entry p.entries[key] = entry
result.Items = append(result.Items, extractionCandidate(entry)) result.Items = append(result.Items, extractionCandidate(entry))
} }
@ -488,6 +566,11 @@ func (p *MemoryPool) Extract(ctx context.Context, command extractionDomain.Comma
return result, nil return result, nil
} }
func timePointer(value time.Time) *time.Time {
canonical := value.UTC()
return &canonical
}
func (p *MemoryPool) rememberExtractionLocked( func (p *MemoryPool) rememberExtractionLocked(
idempotencyKey string, idempotencyKey string,
command extractionDomain.Command, command extractionDomain.Command,

View File

@ -0,0 +1,364 @@
// Package health defines transport-free facts and deterministic health
// reduction rules shared by the Controller and Checker integration layers.
package health
import (
"crypto/sha256"
"errors"
"fmt"
"net/netip"
"net/url"
"strings"
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
)
var (
ErrInvalidObservation = errors.New("invalid health observation")
ErrInvalidGlobalState = errors.New("invalid global health state")
ErrInvalidTargetState = errors.New("invalid target health state")
ErrNonGlobalObservation = errors.New("target health observation cannot change global proxy health")
ErrNonTargetObservation = errors.New("global health observation cannot change target profile health")
ErrStaleObservation = errors.New("stale health observation")
ErrConflictingObservation = errors.New("conflicting health observation replay")
ErrInvalidFailureThreshold = errors.New("invalid health failure threshold")
ErrInvalidCheckPreparation = errors.New("invalid health check preparation")
)
type Level string
const (
LevelBasic Level = "BASIC"
LevelEgress Level = "EGRESS"
LevelTarget Level = "TARGET"
)
type TargetStatus string
const (
TargetUnknown TargetStatus = "UNKNOWN"
TargetAvailable TargetStatus = "AVAILABLE"
TargetSuspect TargetStatus = "SUSPECT"
TargetUnhealthy TargetStatus = "UNHEALTHY"
)
// Observation is an immutable result produced by a Checker. It has no
// authority to modify a Proxy until the Controller reduces and commits it.
type Observation struct {
TaskID string
ProxyID string
Level Level
RoutingName string
TargetURL string
Success bool
FailureClass string
Latency time.Duration
ObservedEgressIP string
ObservedAt time.Time
}
type TargetProfile struct {
RoutingName string
TargetURL string
}
// GlobalState is the small authoritative state needed to reduce BASIC and
// EGRESS observations. The persistence adapter owns storing it atomically.
type GlobalState struct {
State proxyDomain.State
ConsecutiveFailures int
LastTaskID string
LastObservedAt time.Time
LastObservationDigest [sha256.Size]byte
}
// TargetState is isolated by (proxy, routing, target URL). A target failure
// never changes a Proxy's GlobalState.
type TargetState struct {
Status TargetStatus
ConsecutiveFailures int
LastTaskID string
LastObservedAt time.Time
LastSuccessAt time.Time
Latency time.Duration
LastObservationDigest [sha256.Size]byte
}
// NormalizeObservation validates and canonicalizes an observation before it
// crosses an idempotency or persistence boundary.
func NormalizeObservation(value Observation) (Observation, error) {
value.TaskID = strings.TrimSpace(value.TaskID)
value.ProxyID = strings.TrimSpace(value.ProxyID)
value.RoutingName = strings.TrimSpace(value.RoutingName)
value.TargetURL = strings.TrimSpace(value.TargetURL)
value.FailureClass = strings.TrimSpace(value.FailureClass)
value.ObservedEgressIP = strings.TrimSpace(value.ObservedEgressIP)
if !validIdentifier(value.TaskID) || !validIdentifier(value.ProxyID) || value.ObservedAt.IsZero() || value.Latency < 0 {
return Observation{}, ErrInvalidObservation
}
if value.Success && value.FailureClass != "" {
return Observation{}, ErrInvalidObservation
}
if value.ObservedEgressIP != "" {
if _, err := netip.ParseAddr(value.ObservedEgressIP); err != nil {
return Observation{}, ErrInvalidObservation
}
}
switch value.Level {
case LevelBasic, LevelEgress:
if value.RoutingName != "" || value.TargetURL != "" {
return Observation{}, ErrInvalidObservation
}
case LevelTarget:
profile, err := NormalizeTargetProfile(TargetProfile{RoutingName: value.RoutingName, TargetURL: value.TargetURL})
if err != nil {
return Observation{}, err
}
value.RoutingName = profile.RoutingName
value.TargetURL = profile.TargetURL
default:
return Observation{}, ErrInvalidObservation
}
value.ObservedAt = value.ObservedAt.UTC()
return value, nil
}
func NormalizeTargetProfile(value TargetProfile) (TargetProfile, error) {
value.RoutingName = strings.TrimSpace(value.RoutingName)
value.TargetURL = strings.TrimSpace(value.TargetURL)
if !validIdentifier(value.RoutingName) || value.TargetURL == "" {
return TargetProfile{}, ErrInvalidObservation
}
parsed, err := url.Parse(value.TargetURL)
if err != nil || parsed.Scheme == "" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
return TargetProfile{}, ErrInvalidObservation
}
value.TargetURL = parsed.String()
return value, nil
}
func (value TargetProfile) Key() string {
return value.RoutingName + "\x00" + value.TargetURL
}
func ObservationDigest(value Observation) ([sha256.Size]byte, error) {
normalized, err := NormalizeObservation(value)
if err != nil {
return [sha256.Size]byte{}, err
}
payload := strings.Join([]string{
normalized.TaskID, normalized.ProxyID, string(normalized.Level), normalized.RoutingName,
normalized.TargetURL, fmt.Sprintf("%t", normalized.Success), normalized.FailureClass,
normalized.Latency.String(), normalized.ObservedEgressIP, normalized.ObservedAt.Format(time.RFC3339Nano),
}, "\x00")
return sha256.Sum256([]byte(payload)), nil
}
// BeginGlobalCheck moves only states that are deliberately unavailable during
// probing into CHECKING. AVAILABLE and SUSPECT remain serving states while a
// periodic probe is in flight.
func BeginGlobalCheck(current GlobalState) (proxyDomain.State, error) {
if err := validateGlobalState(current); err != nil {
return "", err
}
switch current.State {
case proxyDomain.StateFetched, proxyDomain.StateUnhealthy:
return proxyDomain.StateChecking, nil
case proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect:
return current.State, nil
default:
return "", ErrInvalidCheckPreparation
}
}
// ReduceGlobal reduces one BASIC or EGRESS fact. It is intentionally pure so
// Memory and Redis adapters can apply the same result atomically.
func ReduceGlobal(current GlobalState, observation Observation, maxConsecutiveFailures int) (GlobalState, error) {
if maxConsecutiveFailures <= 0 {
return GlobalState{}, ErrInvalidFailureThreshold
}
if err := validateGlobalState(current); err != nil {
return GlobalState{}, err
}
normalized, err := NormalizeObservation(observation)
if err != nil {
return GlobalState{}, err
}
if normalized.Level == LevelTarget {
return GlobalState{}, ErrNonGlobalObservation
}
digest, err := ObservationDigest(normalized)
if err != nil {
return GlobalState{}, err
}
if err := checkObservationOrder(current.LastTaskID, current.LastObservedAt, current.LastObservationDigest, normalized, digest); err != nil {
if errors.Is(err, errExactReplay) {
return current, nil
}
return GlobalState{}, err
}
next := current
next.LastTaskID = normalized.TaskID
next.LastObservedAt = normalized.ObservedAt
next.LastObservationDigest = digest
if normalized.Success {
switch current.State {
case proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect:
next.State = proxyDomain.StateAvailable
next.ConsecutiveFailures = 0
return next, nil
default:
return GlobalState{}, ErrInvalidGlobalState
}
}
switch current.State {
case proxyDomain.StateChecking:
next.ConsecutiveFailures++
next.State = proxyDomain.StateUnhealthy
return next, nil
case proxyDomain.StateAvailable, proxyDomain.StateSuspect:
next.ConsecutiveFailures++
if next.ConsecutiveFailures >= maxConsecutiveFailures {
next.State = proxyDomain.StateUnhealthy
} else {
next.State = proxyDomain.StateSuspect
}
return next, nil
default:
return GlobalState{}, ErrInvalidGlobalState
}
}
// ReduceTarget reduces one TARGET fact into only its target profile state.
// Callers must keep this result separate from activity-pool global state.
func ReduceTarget(current TargetState, observation Observation, maxConsecutiveFailures int) (TargetState, error) {
if maxConsecutiveFailures <= 0 {
return TargetState{}, ErrInvalidFailureThreshold
}
if current.Status == "" {
current.Status = TargetUnknown
}
if err := validateTargetState(current); err != nil {
return TargetState{}, err
}
normalized, err := NormalizeObservation(observation)
if err != nil {
return TargetState{}, err
}
if normalized.Level != LevelTarget {
return TargetState{}, ErrNonTargetObservation
}
digest, err := ObservationDigest(normalized)
if err != nil {
return TargetState{}, err
}
if err := checkObservationOrder(current.LastTaskID, current.LastObservedAt, current.LastObservationDigest, normalized, digest); err != nil {
if errors.Is(err, errExactReplay) {
return current, nil
}
return TargetState{}, err
}
next := current
next.LastTaskID = normalized.TaskID
next.LastObservedAt = normalized.ObservedAt
next.LastObservationDigest = digest
next.Latency = normalized.Latency
if normalized.Success {
next.Status = TargetAvailable
next.ConsecutiveFailures = 0
next.LastSuccessAt = normalized.ObservedAt
return next, nil
}
next.ConsecutiveFailures++
if next.ConsecutiveFailures >= maxConsecutiveFailures {
next.Status = TargetUnhealthy
} else {
next.Status = TargetSuspect
}
return next, nil
}
var errExactReplay = errors.New("exact health observation replay")
func checkObservationOrder(
lastTaskID string,
lastObservedAt time.Time,
lastDigest [sha256.Size]byte,
observation Observation,
digest [sha256.Size]byte,
) error {
if lastObservedAt.IsZero() {
return nil
}
if observation.ObservedAt.Before(lastObservedAt) {
return ErrStaleObservation
}
if observation.TaskID == lastTaskID {
if digest == lastDigest {
return errExactReplay
}
return ErrConflictingObservation
}
if observation.ObservedAt.Equal(lastObservedAt) {
return ErrStaleObservation
}
return nil
}
func validateGlobalState(value GlobalState) error {
if value.ConsecutiveFailures < 0 {
return ErrInvalidGlobalState
}
switch value.State {
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable,
proxyDomain.StateSuspect, proxyDomain.StateUnhealthy:
default:
return ErrInvalidGlobalState
}
if value.LastObservedAt.IsZero() {
if value.LastTaskID != "" || value.ConsecutiveFailures != 0 || value.LastObservationDigest != ([sha256.Size]byte{}) {
return ErrInvalidGlobalState
}
return nil
}
if !validIdentifier(value.LastTaskID) || value.LastObservationDigest == ([sha256.Size]byte{}) {
return ErrInvalidGlobalState
}
return nil
}
func validateTargetState(value TargetState) error {
if value.ConsecutiveFailures < 0 || value.Latency < 0 {
return ErrInvalidTargetState
}
switch value.Status {
case TargetUnknown, TargetAvailable, TargetSuspect, TargetUnhealthy:
default:
return ErrInvalidTargetState
}
if value.LastObservedAt.IsZero() {
if value.Status != TargetUnknown || value.LastTaskID != "" || value.ConsecutiveFailures != 0 ||
!value.LastSuccessAt.IsZero() || value.LastObservationDigest != ([sha256.Size]byte{}) {
return ErrInvalidTargetState
}
return nil
}
if !validIdentifier(value.LastTaskID) || value.LastObservationDigest == ([sha256.Size]byte{}) ||
(!value.LastSuccessAt.IsZero() && value.LastSuccessAt.After(value.LastObservedAt)) {
return ErrInvalidTargetState
}
return nil
}
func validIdentifier(value string) bool {
if value == "" || len(value) > 256 {
return false
}
for _, character := range value {
if character <= ' ' || character == '\x7f' {
return false
}
}
return true
}

View File

@ -0,0 +1,95 @@
package health
import (
"errors"
"testing"
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestReduceGlobalAppliesSuspectUnhealthyAndRecovery(t *testing.T) {
now := time.Date(2026, 7, 31, 10, 0, 0, 0, time.UTC)
current := GlobalState{State: proxyDomain.StateAvailable}
first, err := ReduceGlobal(current, globalObservation("task-1", false, now), 2)
if err != nil || first.State != proxyDomain.StateSuspect || first.ConsecutiveFailures != 1 {
t.Fatalf("first failure = %+v, %v; want SUSPECT / 1", first, err)
}
second, err := ReduceGlobal(first, globalObservation("task-2", false, now.Add(time.Second)), 2)
if err != nil || second.State != proxyDomain.StateUnhealthy || second.ConsecutiveFailures != 2 {
t.Fatalf("second failure = %+v, %v; want UNHEALTHY / 2", second, err)
}
if next, err := BeginGlobalCheck(second); err != nil || next != proxyDomain.StateChecking {
t.Fatalf("BeginGlobalCheck(unhealthy) = %s, %v; want CHECKING", next, err)
}
recovering := second
recovering.State = proxyDomain.StateChecking
recovered, err := ReduceGlobal(recovering, globalObservation("task-3", true, now.Add(2*time.Second)), 2)
if err != nil || recovered.State != proxyDomain.StateAvailable || recovered.ConsecutiveFailures != 0 {
t.Fatalf("recovery = %+v, %v; want AVAILABLE / 0", recovered, err)
}
}
func TestReduceGlobalMarksFailedInitialCheckUnhealthy(t *testing.T) {
now := time.Date(2026, 7, 31, 10, 0, 0, 0, time.UTC)
next, err := ReduceGlobal(GlobalState{State: proxyDomain.StateChecking}, globalObservation("task-1", false, now), 3)
if err != nil || next.State != proxyDomain.StateUnhealthy || next.ConsecutiveFailures != 1 {
t.Fatalf("initial failure = %+v, %v; want UNHEALTHY / 1", next, err)
}
}
func TestReduceGlobalRejectsStaleAndConflictingReplays(t *testing.T) {
now := time.Date(2026, 7, 31, 10, 0, 0, 0, time.UTC)
first, err := ReduceGlobal(GlobalState{State: proxyDomain.StateAvailable}, globalObservation("task-1", false, now), 2)
if err != nil {
t.Fatalf("ReduceGlobal(first): %v", err)
}
replayed, err := ReduceGlobal(first, globalObservation("task-1", false, now), 2)
if err != nil || replayed != first {
t.Fatalf("ReduceGlobal(replay) = %+v, %v; want unchanged", replayed, err)
}
if _, err := ReduceGlobal(first, globalObservation("task-1", true, now), 2); !errors.Is(err, ErrConflictingObservation) {
t.Fatalf("conflicting replay error = %v, want ErrConflictingObservation", err)
}
if _, err := ReduceGlobal(first, globalObservation("task-2", true, now.Add(-time.Second)), 2); !errors.Is(err, ErrStaleObservation) {
t.Fatalf("stale observation error = %v, want ErrStaleObservation", err)
}
}
func TestReduceTargetNeverChangesGlobalState(t *testing.T) {
now := time.Date(2026, 7, 31, 10, 0, 0, 0, time.UTC)
observation := Observation{
TaskID: "task-target-1", ProxyID: "proxy-a", Level: LevelTarget, RoutingName: "route-a",
TargetURL: "https://target.example/path?volatile=true", Success: false, FailureClass: "target_403",
Latency: 20 * time.Millisecond, ObservedAt: now,
}
target, err := ReduceTarget(TargetState{}, observation, 2)
if err != nil || target.Status != TargetSuspect || target.ConsecutiveFailures != 1 {
t.Fatalf("ReduceTarget(first) = %+v, %v; want SUSPECT / 1", target, err)
}
if _, err := ReduceGlobal(GlobalState{State: proxyDomain.StateAvailable}, observation, 2); !errors.Is(err, ErrNonGlobalObservation) {
t.Fatalf("ReduceGlobal(target) error = %v, want ErrNonGlobalObservation", err)
}
profile, err := NormalizeTargetProfile(TargetProfile{RoutingName: observation.RoutingName, TargetURL: observation.TargetURL})
if err != nil || profile.TargetURL != "https://target.example/path?volatile=true" {
t.Fatalf("NormalizeTargetProfile() = %+v, %v", profile, err)
}
}
func TestNormalizeObservationRejectsCrossLevelFields(t *testing.T) {
now := time.Date(2026, 7, 31, 10, 0, 0, 0, time.UTC)
_, err := NormalizeObservation(Observation{
TaskID: "task-a", ProxyID: "proxy-a", Level: LevelBasic, RoutingName: "route-a", Success: true, ObservedAt: now,
})
if !errors.Is(err, ErrInvalidObservation) {
t.Fatalf("NormalizeObservation() error = %v, want ErrInvalidObservation", err)
}
}
func globalObservation(taskID string, success bool, observedAt time.Time) Observation {
observation := Observation{TaskID: taskID, ProxyID: "proxy-a", Level: LevelBasic, Success: success, ObservedAt: observedAt}
if !success {
observation.FailureClass = "timeout"
}
return observation
}