fix: close expired worker snapshot streams
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run

This commit is contained in:
youfak 2026-07-31 15:02:25 +08:00
parent baefaf6dbf
commit c2d0c08fcb
5 changed files with 110 additions and 5 deletions

View File

@ -92,6 +92,11 @@ Gateway 接收完整快照时必须拒绝缺失、格式错误或已到期的 `v
保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图 保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图
发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。 发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。
Controller 只会下发尚未到期的完整快照,并在最近一次成功下发快照的
`valid_until` 到达时结束 `WatchSnapshots` 流。Gateway 的会话调用方必须在流结束后
按带 jitter 的有界退避重建 Register/Watch 会话;在新快照通过校验并原子替换前,
旧视图仍按其整体有效期 fail-closed。
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成 验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成
热路径只读取一次原子指针。 热路径只读取一次原子指针。

View File

@ -214,8 +214,9 @@ WorkerControlPlane 现已接入 Controller 生命周期Register、ACK 和 Run
Snapshot 并保持连接Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。 Snapshot 并保持连接Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的 按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成, 已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成,
重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Routing payload、凭据分发、 重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Worker 服务端会在最近完整
Gateway 命令与 Outcome 上报仍未实现。 Snapshot 的 `valid_until` 到达时结束流,令 Gateway 调用方可按退避策略重建会话;
Routing payload、凭据分发、Gateway 命令与 Outcome 上报仍未实现。
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider 已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖 Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖

View File

@ -53,8 +53,9 @@
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
索引构建无凭据引用的已归属 Proxy payload并以租约收紧可用期Routing payload、 索引构建无凭据引用的已归属 Proxy payload并以租约收紧可用期Routing payload、
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前 凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Gateway 会 `session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Controller
校验并执行 Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。 在最近成功下发的 Snapshot `valid_until` 到达时关闭流Gateway 会校验并执行
Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、 - `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schemapgx 公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schemapgx
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"errors" "errors"
"time"
controlplanev1 "proxy-pool/gen/controlplane/v1" controlplanev1 "proxy-pool/gen/controlplane/v1"
"proxy-pool/internal/domain/workerruntime" "proxy-pool/internal/domain/workerruntime"
@ -78,10 +79,17 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
if err != nil { if err != nil {
return grpcError(err) return grpcError(err)
} }
var expiryTimer *time.Timer
var expiry <-chan time.Time
defer func() {
stopSnapshotExpiryTimer(expiryTimer)
}()
for { for {
select { select {
case <-stream.Context().Done(): case <-stream.Context().Done():
return stream.Context().Err() return stream.Context().Err()
case <-expiry:
return nil
case snapshot, ok := <-updates: case snapshot, ok := <-updates:
if !ok { if !ok {
return nil return nil
@ -92,6 +100,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil { if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
return err return err
} }
expiryTimer, expiry = resetSnapshotExpiryTimer(expiryTimer, snapshot.GetValidUntil().AsTime())
} }
} }
} }
@ -99,7 +108,8 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error { func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error {
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size || if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil || snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil { snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil ||
!snapshot.GetValidUntil().AsTime().After(time.Now().UTC()) {
return ErrInvalidCommand return ErrInvalidCommand
} }
var checksum [sha256.Size]byte var checksum [sha256.Size]byte
@ -109,6 +119,37 @@ func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, session
}) })
} }
func resetSnapshotExpiryTimer(timer *time.Timer, deadline time.Time) (*time.Timer, <-chan time.Time) {
delay := time.Until(deadline)
if delay <= 0 {
delay = time.Nanosecond
}
if timer == nil {
timer = time.NewTimer(delay)
return timer, timer.C
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timer.Reset(delay)
return timer, timer.C
}
func stopSnapshotExpiryTimer(timer *time.Timer) {
if timer == nil {
return
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
}
func (handler *GRPCHandler) AcknowledgeSnapshot(ctx context.Context, request *controlplanev1.AcknowledgeSnapshotRequest) (*emptypb.Empty, error) { func (handler *GRPCHandler) AcknowledgeSnapshot(ctx context.Context, request *controlplanev1.AcknowledgeSnapshotRequest) (*emptypb.Empty, error) {
if request == nil || handler == nil || handler.service == nil || handler.identity == nil { if request == nil || handler == nil || handler.service == nil || handler.identity == nil {
return nil, grpcError(ErrInvalidCommand) return nil, grpcError(ErrInvalidCommand)

View File

@ -2,6 +2,7 @@ package worker
import ( import (
"context" "context"
"io"
"net" "net"
"testing" "testing"
"time" "time"
@ -90,6 +91,52 @@ func TestGRPCHandlerStreamsIssuedFullSnapshots(t *testing.T) {
} }
} }
func TestGRPCHandlerClosesSnapshotStreamAtValidityDeadline(t *testing.T) {
checksum := make([]byte, 32)
checksum[0] = 1
full := &controlplanev1.WorkerSnapshot{
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
GeneratedAt: timestamppb.New(time.Now()), ValidUntil: timestamppb.New(time.Now().Add(100 * time.Millisecond)),
}
service := &grpcServiceStub{}
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, holdingSnapshotSource{snapshot: full})
defer cleanup()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
stream, err := client.WatchSnapshots(ctx, &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
if err != nil {
t.Fatalf("WatchSnapshots(): %v", err)
}
if received, err := stream.Recv(); err != nil || received.GetFull().GetVersion() != 3 {
t.Fatalf("first Recv() = %+v, %v", received, err)
}
if _, err := stream.Recv(); err != io.EOF {
t.Fatalf("Recv(after validity deadline) error = %v, want EOF", err)
}
}
func TestGRPCHandlerRejectsExpiredSnapshot(t *testing.T) {
checksum := make([]byte, 32)
checksum[0] = 1
full := &controlplanev1.WorkerSnapshot{
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
GeneratedAt: timestamppb.New(time.Now().Add(-time.Minute)), ValidUntil: timestamppb.New(time.Now().Add(-time.Second)),
}
service := &grpcServiceStub{}
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, snapshotSourceStub{snapshots: []*controlplanev1.WorkerSnapshot{full}})
defer cleanup()
stream, err := client.WatchSnapshots(context.Background(), &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
if err != nil {
t.Fatalf("WatchSnapshots(): %v", err)
}
if _, err := stream.Recv(); status.Code(err) != codes.InvalidArgument {
t.Fatalf("Recv() code = %s, want InvalidArgument; error=%v", status.Code(err), err)
}
if service.issued.Version != 0 {
t.Fatalf("expired snapshot was issued: %+v", service.issued)
}
}
func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) { func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) {
checksum := make([]byte, 32) checksum := make([]byte, 32)
checksum[0] = 1 checksum[0] = 1
@ -157,6 +204,16 @@ type snapshotSourceStub struct {
snapshots []*controlplanev1.WorkerSnapshot snapshots []*controlplanev1.WorkerSnapshot
} }
type holdingSnapshotSource struct {
snapshot *controlplanev1.WorkerSnapshot
}
func (source holdingSnapshotSource) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) {
updates := make(chan *controlplanev1.WorkerSnapshot, 1)
updates <- source.snapshot
return updates, nil
}
func (source snapshotSourceStub) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) { func (source snapshotSourceStub) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) {
updates := make(chan *controlplanev1.WorkerSnapshot, len(source.snapshots)) updates := make(chan *controlplanev1.WorkerSnapshot, len(source.snapshots))
for _, snapshot := range source.snapshots { for _, snapshot := range source.snapshots {