fix: stop gateway dispatch after snapshot expiry
This commit is contained in:
parent
6f3a92170d
commit
baefaf6dbf
@ -88,6 +88,10 @@ Worker 自报的超前版本或 epoch 也必须拒绝。
|
|||||||
`usable_until` 作为最后可分配时刻;达到该时间后即使尚未到 `expires_at`,
|
`usable_until` 作为最后可分配时刻;达到该时间后即使尚未到 `expires_at`,
|
||||||
也不得再为新请求选择该 Proxy。
|
也不得再为新请求选择该 Proxy。
|
||||||
|
|
||||||
|
Gateway 接收完整快照时必须拒绝缺失、格式错误或已到期的 `valid_until`,并将其
|
||||||
|
保存在本地不可变视图。该整体期限到达后,调度直接按无候选处理,不再使用旧视图
|
||||||
|
发起新的上游连接,也不查询 Redis 或 PostgreSQL 补偿。
|
||||||
|
|
||||||
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
|
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
|
||||||
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成,
|
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成,
|
||||||
热路径只读取一次原子指针。
|
热路径只读取一次原子指针。
|
||||||
|
|||||||
@ -383,7 +383,8 @@ flowchart LR
|
|||||||
`UsableUntil` 和版本校验和。Worker 在 `UsableUntil` 到达后立即停止新分配,
|
`UsableUntil` 和版本校验和。Worker 在 `UsableUntil` 到达后立即停止新分配,
|
||||||
不等待供应商硬过期时间 `ExpiresAt`。
|
不等待供应商硬过期时间 `ExpiresAt`。
|
||||||
- Worker 断开控制面后在 `maxStaleAge` 内使用最后快照;超限停止接收新流量,
|
- Worker 断开控制面后在 `maxStaleAge` 内使用最后快照;超限停止接收新流量,
|
||||||
已有隧道排空。
|
已有隧道排空。Gateway 会把 Controller Snapshot 的整体 `valid_until` 保存在
|
||||||
|
本地视图;该期限到达后本地调度不再为新请求分配 Proxy。
|
||||||
|
|
||||||
## 12. 存储与一致性
|
## 12. 存储与一致性
|
||||||
|
|
||||||
|
|||||||
@ -53,7 +53,8 @@
|
|||||||
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
||||||
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
||||||
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
|
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
|
||||||
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。
|
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。Gateway 会
|
||||||
|
校验并执行 Snapshot 整体 `valid_until`,过期视图不再分配新 Proxy。
|
||||||
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
||||||
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
||||||
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
||||||
|
|||||||
@ -108,6 +108,10 @@ func (watcher *SnapshotWatcher) applyFull(full *controlplanev1.WorkerSnapshot) e
|
|||||||
if full == nil || full.GetVersion() == 0 || full.GetOwnershipEpoch() == 0 {
|
if full == nil || full.GetVersion() == 0 || full.GetOwnershipEpoch() == 0 {
|
||||||
return ErrInvalidSnapshotWatcher
|
return ErrInvalidSnapshotWatcher
|
||||||
}
|
}
|
||||||
|
validUntil, err := requiredFutureTimestamp(full.GetValidUntil(), time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
checksum, err := snapshotwire.Checksum(full)
|
checksum, err := snapshotwire.Checksum(full)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -121,7 +125,7 @@ func (watcher *SnapshotWatcher) applyFull(full *controlplanev1.WorkerSnapshot) e
|
|||||||
}
|
}
|
||||||
envelope := snapshot.Envelope{
|
envelope := snapshot.Envelope{
|
||||||
ClusterID: watcher.options.ClusterID, WorkerID: watcher.options.WorkerID,
|
ClusterID: watcher.options.ClusterID, WorkerID: watcher.options.WorkerID,
|
||||||
Epoch: full.GetOwnershipEpoch(), Version: full.GetVersion(), Full: true, Proxies: proxies,
|
Epoch: full.GetOwnershipEpoch(), Version: full.GetVersion(), Full: true, ValidUntil: validUntil, Proxies: proxies,
|
||||||
}
|
}
|
||||||
envelope.Checksum = snapshot.Checksum(proxies)
|
envelope.Checksum = snapshot.Checksum(proxies)
|
||||||
return watcher.store.Apply(envelope)
|
return watcher.store.Apply(envelope)
|
||||||
@ -192,6 +196,17 @@ func wireTimestamp(value *timestamppb.Timestamp) (*time.Time, error) {
|
|||||||
return &converted, nil
|
return &converted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requiredFutureTimestamp(value *timestamppb.Timestamp, now time.Time) (time.Time, error) {
|
||||||
|
if value == nil || value.CheckValid() != nil || now.IsZero() {
|
||||||
|
return time.Time{}, ErrInvalidSnapshotWatcher
|
||||||
|
}
|
||||||
|
converted := value.AsTime().UTC()
|
||||||
|
if !converted.After(now) {
|
||||||
|
return time.Time{}, ErrInvalidSnapshotWatcher
|
||||||
|
}
|
||||||
|
return converted, nil
|
||||||
|
}
|
||||||
|
|
||||||
type generatedSnapshotRPCClient struct {
|
type generatedSnapshotRPCClient struct {
|
||||||
client controlplanev1.WorkerControlPlaneClient
|
client controlplanev1.WorkerControlPlaneClient
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,7 +34,8 @@ func TestSnapshotWatcherAppliesVerifiedFullSnapshotAndAcknowledges(t *testing.T)
|
|||||||
t.Fatalf("Watch(): %v", err)
|
t.Fatalf("Watch(): %v", err)
|
||||||
}
|
}
|
||||||
view := store.Current()
|
view := store.Current()
|
||||||
if view == nil || view.Version != 1 || view.Epoch != 7 || len(view.Entries) != 1 || view.Entries[0].Proxy.ID != "proxy-a" {
|
if view == nil || view.Version != 1 || view.Epoch != 7 || len(view.Entries) != 1 || view.Entries[0].Proxy.ID != "proxy-a" ||
|
||||||
|
!view.ValidUntil.Equal(full.GetValidUntil().AsTime()) {
|
||||||
t.Fatalf("snapshot view = %+v", view)
|
t.Fatalf("snapshot view = %+v", view)
|
||||||
}
|
}
|
||||||
if client.watch.GetSessionId() != "session-a" || client.ack.GetVersion() != 1 || !client.ack.GetApplied() || string(client.ack.GetChecksum()) != string(full.GetChecksum()) {
|
if client.watch.GetSessionId() != "session-a" || client.ack.GetVersion() != 1 || !client.ack.GetApplied() || string(client.ack.GetChecksum()) != string(full.GetChecksum()) {
|
||||||
@ -58,6 +59,43 @@ func TestSnapshotWatcherRejectsChecksumAndAcknowledgesFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSnapshotWatcherRejectsMissingOverallValidityDeadline(t *testing.T) {
|
||||||
|
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||||
|
full := &controlplanev1.WorkerSnapshot{Version: 1, OwnershipEpoch: 7, GeneratedAt: timestamppb.New(time.Now())}
|
||||||
|
setSnapshotChecksum(t, full)
|
||||||
|
client := &snapshotClientStub{stream: &snapshotStreamStub{values: []*controlplanev1.SnapshotEnvelope{{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: full}}}}}
|
||||||
|
watcher, err := NewSnapshotWatcher(client, store, SnapshotWatcherOptions{ClusterID: "cluster-a", WorkerID: "worker-a"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSnapshotWatcher(): %v", err)
|
||||||
|
}
|
||||||
|
if err := watcher.Watch(context.Background(), "session-a"); err == nil {
|
||||||
|
t.Fatal("Watch() error = nil, want missing validity rejection")
|
||||||
|
}
|
||||||
|
if client.ack == nil || client.ack.GetApplied() || client.ack.GetErrorCode() != "snapshot_apply_failed" {
|
||||||
|
t.Fatalf("negative acknowledgement = %+v", client.ack)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSnapshotWatcherRejectsExpiredOverallValidityDeadline(t *testing.T) {
|
||||||
|
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||||
|
full := &controlplanev1.WorkerSnapshot{
|
||||||
|
Version: 1, OwnershipEpoch: 7, GeneratedAt: timestamppb.New(time.Now().Add(-time.Minute)),
|
||||||
|
ValidUntil: timestamppb.New(time.Now().Add(-time.Second)),
|
||||||
|
}
|
||||||
|
setSnapshotChecksum(t, full)
|
||||||
|
client := &snapshotClientStub{stream: &snapshotStreamStub{values: []*controlplanev1.SnapshotEnvelope{{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: full}}}}}
|
||||||
|
watcher, err := NewSnapshotWatcher(client, store, SnapshotWatcherOptions{ClusterID: "cluster-a", WorkerID: "worker-a"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSnapshotWatcher(): %v", err)
|
||||||
|
}
|
||||||
|
if err := watcher.Watch(context.Background(), "session-a"); err == nil {
|
||||||
|
t.Fatal("Watch() error = nil, want expired validity rejection")
|
||||||
|
}
|
||||||
|
if client.ack == nil || client.ack.GetApplied() {
|
||||||
|
t.Fatalf("negative acknowledgement = %+v", client.ack)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type snapshotClientStub struct {
|
type snapshotClientStub struct {
|
||||||
stream SnapshotStream
|
stream SnapshotStream
|
||||||
watch *controlplanev1.WatchSnapshotsRequest
|
watch *controlplanev1.WatchSnapshotsRequest
|
||||||
|
|||||||
@ -50,6 +50,27 @@ func TestAcquireFiltersAndReservesLocalCapacity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAcquireRejectsSnapshotAfterOverallValidityDeadline(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC)
|
||||||
|
expiresAt := now.Add(time.Hour)
|
||||||
|
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||||
|
envelope := snapshot.Envelope{
|
||||||
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
||||||
|
ValidUntil: now.Add(time.Second),
|
||||||
|
Proxies: []proxyDomain.Proxy{{
|
||||||
|
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable,
|
||||||
|
MaxConcurrency: 1, ExpiresAt: &expiresAt,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
envelope.Checksum = snapshot.Checksum(envelope.Proxies)
|
||||||
|
if err := store.Apply(envelope); err != nil {
|
||||||
|
t.Fatalf("Apply(): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := New(store).Acquire(Request{Now: now.Add(time.Second), Scheme: proxyDomain.SchemeHTTP}); !errors.Is(err, ErrNoCandidate) {
|
||||||
|
t.Fatalf("Acquire(at validity deadline) error = %v, want ErrNoCandidate", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAcquireNeverOversubscribesSnapshotProxy(t *testing.T) {
|
func TestAcquireNeverOversubscribesSnapshotProxy(t *testing.T) {
|
||||||
store := snapshot.NewStore("cluster-a", "worker-a")
|
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||||
proxies := []proxyDomain.Proxy{{ID: "p1", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable, MaxConcurrency: 8}}
|
proxies := []proxyDomain.Proxy{{ID: "p1", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable, MaxConcurrency: 8}}
|
||||||
|
|||||||
@ -92,13 +92,14 @@ type runtimeRegistration struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Envelope struct {
|
type Envelope struct {
|
||||||
ClusterID string
|
ClusterID string
|
||||||
WorkerID string
|
WorkerID string
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
Version uint64
|
Version uint64
|
||||||
Full bool
|
Full bool
|
||||||
Checksum string
|
Checksum string
|
||||||
Proxies []proxyDomain.Proxy
|
ValidUntil time.Time
|
||||||
|
Proxies []proxyDomain.Proxy
|
||||||
}
|
}
|
||||||
|
|
||||||
type Entry struct {
|
type Entry struct {
|
||||||
@ -107,12 +108,13 @@ type Entry struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type View struct {
|
type View struct {
|
||||||
ClusterID string
|
ClusterID string
|
||||||
WorkerID string
|
WorkerID string
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
Version uint64
|
Version uint64
|
||||||
Checksum string
|
Checksum string
|
||||||
Entries []Entry
|
ValidUntil time.Time
|
||||||
|
Entries []Entry
|
||||||
|
|
||||||
all []int
|
all []int
|
||||||
byScheme map[proxyDomain.Scheme][]int
|
byScheme map[proxyDomain.Scheme][]int
|
||||||
@ -288,12 +290,13 @@ func (s *Store) Apply(envelope Envelope) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
next := &View{
|
next := &View{
|
||||||
ClusterID: envelope.ClusterID,
|
ClusterID: envelope.ClusterID,
|
||||||
WorkerID: envelope.WorkerID,
|
WorkerID: envelope.WorkerID,
|
||||||
Epoch: envelope.Epoch,
|
Epoch: envelope.Epoch,
|
||||||
Version: envelope.Version,
|
Version: envelope.Version,
|
||||||
Checksum: envelope.Checksum,
|
Checksum: envelope.Checksum,
|
||||||
Entries: entries,
|
ValidUntil: envelope.ValidUntil.UTC(),
|
||||||
|
Entries: entries,
|
||||||
}
|
}
|
||||||
next.buildIndexes()
|
next.buildIndexes()
|
||||||
s.current.Store(next)
|
s.current.Store(next)
|
||||||
@ -304,7 +307,7 @@ func (v *View) Select(query Query) Selection {
|
|||||||
if query.Now.IsZero() {
|
if query.Now.IsZero() {
|
||||||
query.Now = time.Now().UTC()
|
query.Now = time.Now().UTC()
|
||||||
}
|
}
|
||||||
if v == nil {
|
if v == nil || (!v.ValidUntil.IsZero() && !v.ValidUntil.After(query.Now)) {
|
||||||
return Selection{}
|
return Selection{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user