fix: close expired worker snapshot streams
This commit is contained in:
parent
baefaf6dbf
commit
c2d0c08fcb
@ -92,6 +92,11 @@ Gateway 接收完整快照时必须拒绝缺失、格式错误或已到期的 `v
|
|||||||
保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图
|
保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图
|
||||||
发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。
|
发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。
|
||||||
|
|
||||||
|
Controller 只会下发尚未到期的完整快照,并在最近一次成功下发快照的
|
||||||
|
`valid_until` 到达时结束 `WatchSnapshots` 流。Gateway 的会话调用方必须在流结束后
|
||||||
|
按带 jitter 的有界退避重建 Register/Watch 会话;在新快照通过校验并原子替换前,
|
||||||
|
旧视图仍按其整体有效期 fail-closed。
|
||||||
|
|
||||||
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
|
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
|
||||||
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成,
|
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成,
|
||||||
热路径只读取一次原子指针。
|
热路径只读取一次原子指针。
|
||||||
|
|||||||
@ -214,8 +214,9 @@ WorkerControlPlane 现已接入 Controller 生命周期:Register、ACK 和 Run
|
|||||||
Snapshot 并保持连接;Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
|
Snapshot 并保持连接;Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
|
||||||
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
|
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
|
||||||
已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成,
|
已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成,
|
||||||
重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Routing payload、凭据分发、
|
重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Worker 服务端会在最近完整
|
||||||
Gateway 命令与 Outcome 上报仍未实现。
|
Snapshot 的 `valid_until` 到达时结束流,令 Gateway 调用方可按退避策略重建会话;
|
||||||
|
Routing payload、凭据分发、Gateway 命令与 Outcome 上报仍未实现。
|
||||||
|
|
||||||
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
||||||
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
||||||
|
|||||||
@ -53,8 +53,9 @@
|
|||||||
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
||||||
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
||||||
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
|
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
|
||||||
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Gateway 会
|
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Controller
|
||||||
校验并执行 Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。
|
在最近成功下发的 Snapshot `valid_until` 到达时关闭流,Gateway 会校验并执行
|
||||||
|
Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。
|
||||||
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
||||||
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
||||||
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
||||||
"proxy-pool/internal/domain/workerruntime"
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
@ -78,10 +79,17 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return grpcError(err)
|
return grpcError(err)
|
||||||
}
|
}
|
||||||
|
var expiryTimer *time.Timer
|
||||||
|
var expiry <-chan time.Time
|
||||||
|
defer func() {
|
||||||
|
stopSnapshotExpiryTimer(expiryTimer)
|
||||||
|
}()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-stream.Context().Done():
|
case <-stream.Context().Done():
|
||||||
return stream.Context().Err()
|
return stream.Context().Err()
|
||||||
|
case <-expiry:
|
||||||
|
return nil
|
||||||
case snapshot, ok := <-updates:
|
case snapshot, ok := <-updates:
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
@ -92,6 +100,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
|
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
expiryTimer, expiry = resetSnapshotExpiryTimer(expiryTimer, snapshot.GetValidUntil().AsTime())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -99,7 +108,8 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error {
|
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error {
|
||||||
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
|
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
|
||||||
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
|
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
|
||||||
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil {
|
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil ||
|
||||||
|
!snapshot.GetValidUntil().AsTime().After(time.Now().UTC()) {
|
||||||
return ErrInvalidCommand
|
return ErrInvalidCommand
|
||||||
}
|
}
|
||||||
var checksum [sha256.Size]byte
|
var checksum [sha256.Size]byte
|
||||||
@ -109,6 +119,37 @@ func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, session
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resetSnapshotExpiryTimer(timer *time.Timer, deadline time.Time) (*time.Timer, <-chan time.Time) {
|
||||||
|
delay := time.Until(deadline)
|
||||||
|
if delay <= 0 {
|
||||||
|
delay = time.Nanosecond
|
||||||
|
}
|
||||||
|
if timer == nil {
|
||||||
|
timer = time.NewTimer(delay)
|
||||||
|
return timer, timer.C
|
||||||
|
}
|
||||||
|
if !timer.Stop() {
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
timer.Reset(delay)
|
||||||
|
return timer, timer.C
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopSnapshotExpiryTimer(timer *time.Timer) {
|
||||||
|
if timer == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !timer.Stop() {
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (handler *GRPCHandler) AcknowledgeSnapshot(ctx context.Context, request *controlplanev1.AcknowledgeSnapshotRequest) (*emptypb.Empty, error) {
|
func (handler *GRPCHandler) AcknowledgeSnapshot(ctx context.Context, request *controlplanev1.AcknowledgeSnapshotRequest) (*emptypb.Empty, error) {
|
||||||
if request == nil || handler == nil || handler.service == nil || handler.identity == nil {
|
if request == nil || handler == nil || handler.service == nil || handler.identity == nil {
|
||||||
return nil, grpcError(ErrInvalidCommand)
|
return nil, grpcError(ErrInvalidCommand)
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package worker
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -90,6 +91,52 @@ func TestGRPCHandlerStreamsIssuedFullSnapshots(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGRPCHandlerClosesSnapshotStreamAtValidityDeadline(t *testing.T) {
|
||||||
|
checksum := make([]byte, 32)
|
||||||
|
checksum[0] = 1
|
||||||
|
full := &controlplanev1.WorkerSnapshot{
|
||||||
|
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
|
||||||
|
GeneratedAt: timestamppb.New(time.Now()), ValidUntil: timestamppb.New(time.Now().Add(100 * time.Millisecond)),
|
||||||
|
}
|
||||||
|
service := &grpcServiceStub{}
|
||||||
|
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, holdingSnapshotSource{snapshot: full})
|
||||||
|
defer cleanup()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
stream, err := client.WatchSnapshots(ctx, &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WatchSnapshots(): %v", err)
|
||||||
|
}
|
||||||
|
if received, err := stream.Recv(); err != nil || received.GetFull().GetVersion() != 3 {
|
||||||
|
t.Fatalf("first Recv() = %+v, %v", received, err)
|
||||||
|
}
|
||||||
|
if _, err := stream.Recv(); err != io.EOF {
|
||||||
|
t.Fatalf("Recv(after validity deadline) error = %v, want EOF", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGRPCHandlerRejectsExpiredSnapshot(t *testing.T) {
|
||||||
|
checksum := make([]byte, 32)
|
||||||
|
checksum[0] = 1
|
||||||
|
full := &controlplanev1.WorkerSnapshot{
|
||||||
|
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
|
||||||
|
GeneratedAt: timestamppb.New(time.Now().Add(-time.Minute)), ValidUntil: timestamppb.New(time.Now().Add(-time.Second)),
|
||||||
|
}
|
||||||
|
service := &grpcServiceStub{}
|
||||||
|
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, snapshotSourceStub{snapshots: []*controlplanev1.WorkerSnapshot{full}})
|
||||||
|
defer cleanup()
|
||||||
|
stream, err := client.WatchSnapshots(context.Background(), &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WatchSnapshots(): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := stream.Recv(); status.Code(err) != codes.InvalidArgument {
|
||||||
|
t.Fatalf("Recv() code = %s, want InvalidArgument; error=%v", status.Code(err), err)
|
||||||
|
}
|
||||||
|
if service.issued.Version != 0 {
|
||||||
|
t.Fatalf("expired snapshot was issued: %+v", service.issued)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) {
|
func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) {
|
||||||
checksum := make([]byte, 32)
|
checksum := make([]byte, 32)
|
||||||
checksum[0] = 1
|
checksum[0] = 1
|
||||||
@ -157,6 +204,16 @@ type snapshotSourceStub struct {
|
|||||||
snapshots []*controlplanev1.WorkerSnapshot
|
snapshots []*controlplanev1.WorkerSnapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type holdingSnapshotSource struct {
|
||||||
|
snapshot *controlplanev1.WorkerSnapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source holdingSnapshotSource) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) {
|
||||||
|
updates := make(chan *controlplanev1.WorkerSnapshot, 1)
|
||||||
|
updates <- source.snapshot
|
||||||
|
return updates, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (source snapshotSourceStub) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) {
|
func (source snapshotSourceStub) Watch(_ context.Context, _ SnapshotWatchRequest) (<-chan *controlplanev1.WorkerSnapshot, error) {
|
||||||
updates := make(chan *controlplanev1.WorkerSnapshot, len(source.snapshots))
|
updates := make(chan *controlplanev1.WorkerSnapshot, len(source.snapshots))
|
||||||
for _, snapshot := range source.snapshots {
|
for _, snapshot := range source.snapshots {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user