fix: stop gateway dispatch after snapshot expiry
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run

This commit is contained in:
youfak 2026-07-31 14:43:25 +08:00
parent 6f3a92170d
commit baefaf6dbf
7 changed files with 107 additions and 24 deletions

View File

@ -88,6 +88,10 @@ Worker 自报的超前版本或 epoch 也必须拒绝。
`usable_until` 作为最后可分配时刻;达到该时间后即使尚未到 `expires_at`
也不得再为新请求选择该 Proxy。
Gateway 接收完整快照时必须拒绝缺失、格式错误或已到期的 `valid_until`,并将其
保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图
发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成
热路径只读取一次原子指针。

View File

@ -383,7 +383,8 @@ flowchart LR
`UsableUntil` 和版本校验和。Worker 在 `UsableUntil` 到达后立即停止新分配,
不等待供应商硬过期时间 `ExpiresAt`
- Worker 断开控制面后在 `maxStaleAge` 内使用最后快照;超限停止接收新流量,
已有隧道排空。
已有隧道排空。Gateway 会把 Controller Snapshot 的整体 `valid_until` 保存在
本地视图;该期限到达后本地调度不再为新请求分配 Proxy。
## 12. 存储与一致性

View File

@ -53,7 +53,8 @@
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
索引构建无凭据引用的已归属 Proxy payload并以租约收紧可用期Routing payload、
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Gateway 会
校验并执行 Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schemapgx
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox

View File

@ -108,6 +108,10 @@ func (watcher *SnapshotWatcher) applyFull(full *controlplanev1.WorkerSnapshot) e
if full == nil || full.GetVersion() == 0 || full.GetOwnershipEpoch() == 0 {
return ErrInvalidSnapshotWatcher
}
validUntil, err := requiredFutureTimestamp(full.GetValidUntil(), time.Now().UTC())
if err != nil {
return err
}
checksum, err := snapshotwire.Checksum(full)
if err != nil {
return err
@ -121,7 +125,7 @@ func (watcher *SnapshotWatcher) applyFull(full *controlplanev1.WorkerSnapshot) e
}
envelope := snapshot.Envelope{
ClusterID: watcher.options.ClusterID, WorkerID: watcher.options.WorkerID,
Epoch: full.GetOwnershipEpoch(), Version: full.GetVersion(), Full: true, Proxies: proxies,
Epoch: full.GetOwnershipEpoch(), Version: full.GetVersion(), Full: true, ValidUntil: validUntil, Proxies: proxies,
}
envelope.Checksum = snapshot.Checksum(proxies)
return watcher.store.Apply(envelope)
@ -192,6 +196,17 @@ func wireTimestamp(value *timestamppb.Timestamp) (*time.Time, error) {
return &converted, nil
}
func requiredFutureTimestamp(value *timestamppb.Timestamp, now time.Time) (time.Time, error) {
if value == nil || value.CheckValid() != nil || now.IsZero() {
return time.Time{}, ErrInvalidSnapshotWatcher
}
converted := value.AsTime().UTC()
if !converted.After(now) {
return time.Time{}, ErrInvalidSnapshotWatcher
}
return converted, nil
}
type generatedSnapshotRPCClient struct {
client controlplanev1.WorkerControlPlaneClient
}

View File

@ -34,7 +34,8 @@ func TestSnapshotWatcherAppliesVerifiedFullSnapshotAndAcknowledges(t *testing.T)
t.Fatalf("Watch(): %v", err)
}
view := store.Current()
if view == nil || view.Version != 1 || view.Epoch != 7 || len(view.Entries) != 1 || view.Entries[0].Proxy.ID != "proxy-a" {
if view == nil || view.Version != 1 || view.Epoch != 7 || len(view.Entries) != 1 || view.Entries[0].Proxy.ID != "proxy-a" ||
!view.ValidUntil.Equal(full.GetValidUntil().AsTime()) {
t.Fatalf("snapshot view = %+v", view)
}
if client.watch.GetSessionId() != "session-a" || client.ack.GetVersion() != 1 || !client.ack.GetApplied() || string(client.ack.GetChecksum()) != string(full.GetChecksum()) {
@ -58,6 +59,43 @@ func TestSnapshotWatcherRejectsChecksumAndAcknowledgesFailure(t *testing.T) {
}
}
func TestSnapshotWatcherRejectsMissingOverallValidityDeadline(t *testing.T) {
store := snapshot.NewStore("cluster-a", "worker-a")
full := &controlplanev1.WorkerSnapshot{Version: 1, OwnershipEpoch: 7, GeneratedAt: timestamppb.New(time.Now())}
setSnapshotChecksum(t, full)
client := &snapshotClientStub{stream: &snapshotStreamStub{values: []*controlplanev1.SnapshotEnvelope{{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: full}}}}}
watcher, err := NewSnapshotWatcher(client, store, SnapshotWatcherOptions{ClusterID: "cluster-a", WorkerID: "worker-a"})
if err != nil {
t.Fatalf("NewSnapshotWatcher(): %v", err)
}
if err := watcher.Watch(context.Background(), "session-a"); err == nil {
t.Fatal("Watch() error = nil, want missing validity rejection")
}
if client.ack == nil || client.ack.GetApplied() || client.ack.GetErrorCode() != "snapshot_apply_failed" {
t.Fatalf("negative acknowledgement = %+v", client.ack)
}
}
func TestSnapshotWatcherRejectsExpiredOverallValidityDeadline(t *testing.T) {
store := snapshot.NewStore("cluster-a", "worker-a")
full := &controlplanev1.WorkerSnapshot{
Version: 1, OwnershipEpoch: 7, GeneratedAt: timestamppb.New(time.Now().Add(-time.Minute)),
ValidUntil: timestamppb.New(time.Now().Add(-time.Second)),
}
setSnapshotChecksum(t, full)
client := &snapshotClientStub{stream: &snapshotStreamStub{values: []*controlplanev1.SnapshotEnvelope{{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: full}}}}}
watcher, err := NewSnapshotWatcher(client, store, SnapshotWatcherOptions{ClusterID: "cluster-a", WorkerID: "worker-a"})
if err != nil {
t.Fatalf("NewSnapshotWatcher(): %v", err)
}
if err := watcher.Watch(context.Background(), "session-a"); err == nil {
t.Fatal("Watch() error = nil, want expired validity rejection")
}
if client.ack == nil || client.ack.GetApplied() {
t.Fatalf("negative acknowledgement = %+v", client.ack)
}
}
type snapshotClientStub struct {
stream SnapshotStream
watch *controlplanev1.WatchSnapshotsRequest

View File

@ -50,6 +50,27 @@ func TestAcquireFiltersAndReservesLocalCapacity(t *testing.T) {
}
}
func TestAcquireRejectsSnapshotAfterOverallValidityDeadline(t *testing.T) {
now := time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC)
expiresAt := now.Add(time.Hour)
store := snapshot.NewStore("cluster-a", "worker-a")
envelope := snapshot.Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
ValidUntil: now.Add(time.Second),
Proxies: []proxyDomain.Proxy{{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable,
MaxConcurrency: 1, ExpiresAt: &expiresAt,
}},
}
envelope.Checksum = snapshot.Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(): %v", err)
}
if _, err := New(store).Acquire(Request{Now: now.Add(time.Second), Scheme: proxyDomain.SchemeHTTP}); !errors.Is(err, ErrNoCandidate) {
t.Fatalf("Acquire(at validity deadline) error = %v, want ErrNoCandidate", err)
}
}
func TestAcquireNeverOversubscribesSnapshotProxy(t *testing.T) {
store := snapshot.NewStore("cluster-a", "worker-a")
proxies := []proxyDomain.Proxy{{ID: "p1", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable, MaxConcurrency: 8}}

View File

@ -98,6 +98,7 @@ type Envelope struct {
Version uint64
Full bool
Checksum string
ValidUntil time.Time
Proxies []proxyDomain.Proxy
}
@ -112,6 +113,7 @@ type View struct {
Epoch uint64
Version uint64
Checksum string
ValidUntil time.Time
Entries []Entry
all []int
@ -293,6 +295,7 @@ func (s *Store) Apply(envelope Envelope) error {
Epoch: envelope.Epoch,
Version: envelope.Version,
Checksum: envelope.Checksum,
ValidUntil: envelope.ValidUntil.UTC(),
Entries: entries,
}
next.buildIndexes()
@ -304,7 +307,7 @@ func (v *View) Select(query Query) Selection {
if query.Now.IsZero() {
query.Now = time.Now().UTC()
}
if v == nil {
if v == nil || (!v.ValidUntil.IsZero() && !v.ValidUntil.After(query.Now)) {
return Selection{}
}