fix: fence snapshot issuance by worker session
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run

This commit is contained in:
youfak 2026-07-31 14:36:00 +08:00
parent 51fef78368
commit 6f3a92170d
13 changed files with 256 additions and 41 deletions

View File

@ -32,6 +32,10 @@ Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有
增量、Gateway 进程装配、Outcome 与 Checker 闭环尚未实现。`ReportOutcomes` 仍明确 增量、Gateway 进程装配、Outcome 与 Checker 闭环尚未实现。`ReportOutcomes` 仍明确
返回 `Unimplemented``100,000 QPS` 仍是未验证的设计目标。 返回 `Unimplemented``100,000 QPS` 仍是未验证的设计目标。
`WatchSnapshots` 建立时校验当前 session每次签发快照引用时也把 `session_id`
交给 Redis 原子校验。重复 Register 会同时清除旧 Runtime 和已签发引用,因此迟到的
旧 Stream 既不能覆盖新 session 的引用,也不会向旧连接发送未获授权的快照。
## 2. Worker 会话 ## 2. Worker 会话
```mermaid ```mermaid
@ -54,6 +58,10 @@ sequenceDiagram
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程 `worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。 继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
同一 `worker_id` 重注册会替换 session并清除旧 Runtime 与已签发 Snapshot
Reference。旧 Stream 即使在替换后仍收到上游更新,其签发操作也会以
`FailedPrecondition` 结束,不能影响新 session 的 ACK 基线。
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。 `ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须 相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用 拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用

View File

@ -213,7 +213,9 @@ WorkerControlPlane 现已接入 Controller 生命周期Register、ACK 和 Run
身份、消息/流限制和有界停机已实现。`WatchSnapshots` 会发送当前 epoch 的基础完整 身份、消息/流限制和有界停机已实现。`WatchSnapshots` 会发送当前 epoch 的基础完整
Snapshot 并保持连接Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。 Snapshot 并保持连接Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的 按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
已归属 Proxy payload。Routing payload、凭据分发、Gateway 命令与 Outcome 上报仍未实现。 已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成,
重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Routing payload、凭据分发、
Gateway 命令与 Outcome 上报仍未实现。
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider 已新增公用 `domain/activitypool` 契约及并发安全内存参考实现Provider
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖 Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖

View File

@ -52,7 +52,8 @@
plaintext fixture 与 SPIFFE mTLS 服务端;基础 Snapshot 流和 Gateway 的 plaintext fixture 与 SPIFFE mTLS 服务端;基础 Snapshot 流和 Gateway 的
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
索引构建无凭据引用的已归属 Proxy payload并以租约收紧可用期Routing payload、 索引构建无凭据引用的已归属 Proxy payload并以租约收紧可用期Routing payload、
凭据分发、Outcome 和 Checker 尚未闭环。 凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、 - `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schemapgx 公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schemapgx
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox

View File

@ -16,6 +16,7 @@ const (
runtimeReplaceSession = "replace_session" runtimeReplaceSession = "replace_session"
runtimeCurrentEpoch = "current_epoch" runtimeCurrentEpoch = "current_epoch"
runtimeOpenSession = "open_session" runtimeOpenSession = "open_session"
runtimeValidateSession = "validate_session"
runtimeRecordSnapshot = "record_snapshot" runtimeRecordSnapshot = "record_snapshot"
runtimeAcknowledge = "acknowledge_snapshot" runtimeAcknowledge = "acknowledge_snapshot"
runtimeReplaceReport = "replace_report" runtimeReplaceReport = "replace_report"
@ -44,6 +45,18 @@ type runtimeSnapshotReferenceWire struct {
Checksum string `json:"checksum"` Checksum string `json:"checksum"`
} }
type runtimeSnapshotIssueWire struct {
Version int `json:"version"`
SessionID string `json:"sessionId"`
Reference runtimeSnapshotReferenceWire `json:"reference"`
}
type runtimeSessionValidationWire struct {
Version int `json:"version"`
WorkerID string `json:"workerId"`
SessionID string `json:"sessionId"`
}
type runtimeAcknowledgementWire struct { type runtimeAcknowledgementWire struct {
Version int `json:"version"` Version int `json:"version"`
WorkerID string `json:"workerId"` WorkerID string `json:"workerId"`
@ -140,15 +153,46 @@ func (a *Adapter) OpenSession(ctx context.Context, session workerruntime.Session
return invalidScriptReply("unexpected worker open session reply") return invalidScriptReply("unexpected worker open session reply")
} }
func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, reference workerruntime.SnapshotReference, ttl time.Duration) error { func (a *Adapter) ValidateSession(ctx context.Context, workerID, sessionID string) error {
if err := validateRuntimeCall(ctx, a); err != nil {
return err
}
if !workerruntime.ValidIdentifier(workerID) || !workerruntime.ValidIdentifier(sessionID) {
return workerruntime.ErrInvalidSession
}
payload, err := json.Marshal(runtimeSessionValidationWire{
Version: runtimeWireVersion, WorkerID: workerID, SessionID: sessionID,
})
if err != nil {
return workerruntime.ErrInvalidSession
}
reply, err := a.runRuntime(ctx, runtimeValidateSession, 0, payload, "")
if err != nil {
return err
}
switch reply.Status {
case scriptOK:
return nil
case scriptInvalid:
return workerruntime.ErrInvalidSession
case scriptUnavailable:
return workerruntime.ErrStaleSession
default:
return invalidScriptReply("unexpected worker session validation reply")
}
}
func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, sessionID string, reference workerruntime.SnapshotReference, ttl time.Duration) error {
if err := validateRuntimeCall(ctx, a); err != nil { if err := validateRuntimeCall(ctx, a); err != nil {
return err return err
} }
normalized, err := workerruntime.NormalizeSnapshotReference(reference) normalized, err := workerruntime.NormalizeSnapshotReference(reference)
if err != nil || ttl <= 0 { if err != nil || !workerruntime.ValidIdentifier(sessionID) || ttl <= 0 {
return workerruntime.ErrInvalidSnapshotReference return workerruntime.ErrInvalidSnapshotReference
} }
payload, err := json.Marshal(referenceWire(normalized)) payload, err := json.Marshal(runtimeSnapshotIssueWire{
Version: runtimeWireVersion, SessionID: sessionID, Reference: referenceWire(normalized),
})
if err != nil { if err != nil {
return workerruntime.ErrInvalidSnapshotReference return workerruntime.ErrInvalidSnapshotReference
} }
@ -167,6 +211,8 @@ func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, reference workerrunt
return workerruntime.ErrConflictingSnapshotReference return workerruntime.ErrConflictingSnapshotReference
case scriptSnapshotMismatch: case scriptSnapshotMismatch:
return workerruntime.ErrSnapshotMismatch return workerruntime.ErrSnapshotMismatch
case scriptUnavailable:
return workerruntime.ErrStaleSession
default: default:
return invalidScriptReply("unexpected worker snapshot reference reply") return invalidScriptReply("unexpected worker snapshot reference reply")
} }

View File

@ -148,20 +148,44 @@ if operation == 'open_session' then
end end
redis.call('HDEL', runtime_key, session.workerId) redis.call('HDEL', runtime_key, session.workerId)
redis.call('ZREM', runtime_expiry_key, session.workerId) redis.call('ZREM', runtime_expiry_key, session.workerId)
redis.call('HDEL', snapshots_key, session.workerId)
redis.call('ZREM', snapshot_expiry_key, session.workerId)
session.expiresAtMs = now + ttl_ms session.expiresAtMs = now + ttl_ms
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session)) redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId) redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
return reply('ok') return reply('ok')
end end
if operation == 'validate_session' then
local validation = decode_table(payload)
if not validation or validation.version ~= 1 or type(validation.workerId) ~= 'string' or
validation.workerId == '' or type(validation.sessionId) ~= 'string' or validation.sessionId == '' then
return reply('invalid')
end
local session = decode_table(redis.call('HGET', sessions_key, validation.workerId))
if not valid_control_session(session) or session.workerId ~= validation.workerId or
session.sessionId ~= validation.sessionId or type(session.expiresAtMs) ~= 'number' or
session.expiresAtMs <= now then
return reply('unavailable')
end
return reply('ok')
end
if operation == 'record_snapshot' then if operation == 'record_snapshot' then
if not ttl_ms or ttl_ms <= 0 then if not ttl_ms or ttl_ms <= 0 then
return reply('invalid') return reply('invalid')
end end
local reference = decode_table(payload) local issue = decode_table(payload)
if not valid_reference(reference) then if not issue or issue.version ~= 1 or type(issue.sessionId) ~= 'string' or issue.sessionId == '' or
not valid_reference(issue.reference) then
return reply('invalid') return reply('invalid')
end end
local reference = issue.reference
local session = decode_table(redis.call('HGET', sessions_key, reference.workerId))
if not valid_control_session(session) or session.workerId ~= reference.workerId or session.sessionId ~= issue.sessionId or
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
return reply('unavailable')
end
local epoch = redis.call('GET', epoch_key) local epoch = redis.call('GET', epoch_key)
if not epoch then if not epoch then
epoch = '1' epoch = '1'

View File

@ -68,6 +68,9 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
(len(request.GetLastChecksum()) != 0 && len(request.GetLastChecksum()) != sha256.Size) { (len(request.GetLastChecksum()) != 0 && len(request.GetLastChecksum()) != sha256.Size) {
return grpcError(ErrInvalidCommand) return grpcError(ErrInvalidCommand)
} }
if err := handler.service.ValidateSession(stream.Context(), request.GetWorkerId(), request.GetSessionId()); err != nil {
return grpcError(err)
}
updates, err := handler.snapshots.Watch(stream.Context(), SnapshotWatchRequest{ updates, err := handler.snapshots.Watch(stream.Context(), SnapshotWatchRequest{
WorkerID: request.GetWorkerId(), SessionID: request.GetSessionId(), LastAppliedVersion: request.GetLastAppliedVersion(), WorkerID: request.GetWorkerId(), SessionID: request.GetSessionId(), LastAppliedVersion: request.GetLastAppliedVersion(),
LastChecksum: append([]byte(nil), request.GetLastChecksum()...), LastChecksum: append([]byte(nil), request.GetLastChecksum()...),
@ -83,7 +86,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
if !ok { if !ok {
return nil return nil
} }
if err := handler.issueSnapshot(stream.Context(), request.GetWorkerId(), snapshot); err != nil { if err := handler.issueSnapshot(stream.Context(), request.GetWorkerId(), request.GetSessionId(), snapshot); err != nil {
return grpcError(err) return grpcError(err)
} }
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil { if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
@ -93,7 +96,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
} }
} }
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID string, snapshot *controlplanev1.WorkerSnapshot) error { func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error {
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size || if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil || snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil { snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil {
@ -101,7 +104,7 @@ func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID string,
} }
var checksum [sha256.Size]byte var checksum [sha256.Size]byte
copy(checksum[:], snapshot.GetChecksum()) copy(checksum[:], snapshot.GetChecksum())
return handler.service.IssueSnapshot(ctx, workerruntime.SnapshotReference{ return handler.service.IssueSnapshot(ctx, sessionID, workerruntime.SnapshotReference{
WorkerID: workerID, Version: snapshot.GetVersion(), OwnershipEpoch: snapshot.GetOwnershipEpoch(), Checksum: checksum, WorkerID: workerID, Version: snapshot.GetVersion(), OwnershipEpoch: snapshot.GetOwnershipEpoch(), Checksum: checksum,
}) })
} }

View File

@ -85,6 +85,31 @@ func TestGRPCHandlerStreamsIssuedFullSnapshots(t *testing.T) {
if service.issued.WorkerID != "worker-a" || service.issued.Version != 3 || service.issued.Checksum[0] != 1 { if service.issued.WorkerID != "worker-a" || service.issued.Version != 3 || service.issued.Checksum[0] != 1 {
t.Fatalf("issued snapshot = %+v", service.issued) t.Fatalf("issued snapshot = %+v", service.issued)
} }
if service.issuedSessionID != "session-a" {
t.Fatalf("issued session = %q, want session-a", service.issuedSessionID)
}
}
func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) {
checksum := make([]byte, 32)
checksum[0] = 1
service := &grpcServiceStub{issueErr: workerruntime.ErrStaleSession}
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, snapshotSourceStub{snapshots: []*controlplanev1.WorkerSnapshot{{
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
GeneratedAt: timestamppb.New(time.Now()), ValidUntil: timestamppb.New(time.Now().Add(time.Minute)),
}}})
defer cleanup()
stream, err := client.WatchSnapshots(context.Background(), &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
if err != nil {
t.Fatalf("WatchSnapshots(): %v", err)
}
_, err = stream.Recv()
if status.Code(err) != codes.FailedPrecondition {
t.Fatalf("Recv() error = %v, want FailedPrecondition", err)
}
if service.issued.Version != 0 || service.issuedSessionID != "" {
t.Fatalf("stale session issued snapshot = %+v for %q", service.issued, service.issuedSessionID)
}
} }
type grpcServiceStub struct { type grpcServiceStub struct {
@ -95,14 +120,24 @@ type grpcServiceStub struct {
report workerruntime.Report report workerruntime.Report
reportErr error reportErr error
issued workerruntime.SnapshotReference issued workerruntime.SnapshotReference
issuedSessionID string
validateErr error
issueErr error
} }
func (stub *grpcServiceStub) Register(context.Context, RegisterCommand) (Registration, error) { func (stub *grpcServiceStub) Register(context.Context, RegisterCommand) (Registration, error) {
return stub.registration, stub.registerErr return stub.registration, stub.registerErr
} }
func (stub *grpcServiceStub) CurrentOwnershipEpoch(context.Context) (uint64, error) { return 9, nil } func (stub *grpcServiceStub) CurrentOwnershipEpoch(context.Context) (uint64, error) { return 9, nil }
func (stub *grpcServiceStub) IssueSnapshot(_ context.Context, reference workerruntime.SnapshotReference) error { func (stub *grpcServiceStub) ValidateSession(context.Context, string, string) error {
return stub.validateErr
}
func (stub *grpcServiceStub) IssueSnapshot(_ context.Context, sessionID string, reference workerruntime.SnapshotReference) error {
if stub.issueErr != nil {
return stub.issueErr
}
stub.issued = reference stub.issued = reference
stub.issuedSessionID = sessionID
return nil return nil
} }
func (stub *grpcServiceStub) Acknowledge(_ context.Context, acknowledgement SnapshotAcknowledgement) error { func (stub *grpcServiceStub) Acknowledge(_ context.Context, acknowledgement SnapshotAcknowledgement) error {

View File

@ -64,7 +64,8 @@ type Options struct {
type Service interface { type Service interface {
Register(context.Context, RegisterCommand) (Registration, error) Register(context.Context, RegisterCommand) (Registration, error)
CurrentOwnershipEpoch(context.Context) (uint64, error) CurrentOwnershipEpoch(context.Context) (uint64, error)
IssueSnapshot(context.Context, workerruntime.SnapshotReference) error ValidateSession(context.Context, string, string) error
IssueSnapshot(context.Context, string, workerruntime.SnapshotReference) error
Acknowledge(context.Context, SnapshotAcknowledgement) error Acknowledge(context.Context, SnapshotAcknowledgement) error
ReportRuntime(context.Context, workerruntime.Report) (RuntimeDecision, error) ReportRuntime(context.Context, workerruntime.Report) (RuntimeDecision, error)
} }
@ -83,7 +84,20 @@ func (service *service) CurrentOwnershipEpoch(ctx context.Context) (uint64, erro
return epoch, nil return epoch, nil
} }
func (service *service) IssueSnapshot(ctx context.Context, reference workerruntime.SnapshotReference) error { func (service *service) ValidateSession(ctx context.Context, workerID, sessionID string) error {
if ctx == nil || !workerruntime.ValidIdentifier(workerID) || !workerruntime.ValidIdentifier(sessionID) {
return ErrInvalidCommand
}
if err := ctx.Err(); err != nil {
return err
}
if err := service.store.ValidateSession(ctx, workerID, sessionID); err != nil {
return classifyStoreError(err)
}
return nil
}
func (service *service) IssueSnapshot(ctx context.Context, sessionID string, reference workerruntime.SnapshotReference) error {
if ctx == nil { if ctx == nil {
return ErrInvalidCommand return ErrInvalidCommand
} }
@ -91,10 +105,10 @@ func (service *service) IssueSnapshot(ctx context.Context, reference workerrunti
return err return err
} }
normalized, err := workerruntime.NormalizeSnapshotReference(reference) normalized, err := workerruntime.NormalizeSnapshotReference(reference)
if err != nil { if err != nil || !workerruntime.ValidIdentifier(sessionID) {
return errors.Join(ErrInvalidCommand, err) return errors.Join(ErrInvalidCommand, err)
} }
if err := service.store.RecordIssuedSnapshot(ctx, normalized, service.options.SessionTTL); err != nil { if err := service.store.RecordIssuedSnapshot(ctx, sessionID, normalized, service.options.SessionTTL); err != nil {
return classifyStoreError(err) return classifyStoreError(err)
} }
return nil return nil

View File

@ -35,7 +35,7 @@ func TestServiceRegistersAcknowledgesAndReportsRuntime(t *testing.T) {
WorkerID: "worker-a", Version: 7, OwnershipEpoch: registered.OwnershipEpoch, WorkerID: "worker-a", Version: 7, OwnershipEpoch: registered.OwnershipEpoch,
Checksum: sha256.Sum256([]byte("snapshot-7")), Checksum: sha256.Sum256([]byte("snapshot-7")),
} }
if err := service.IssueSnapshot(context.Background(), reference); err != nil { if err := service.IssueSnapshot(context.Background(), registered.SessionID, reference); err != nil {
t.Fatalf("IssueSnapshot(): %v", err) t.Fatalf("IssueSnapshot(): %v", err)
} }
if err := service.Acknowledge(context.Background(), SnapshotAcknowledgement{ if err := service.Acknowledge(context.Background(), SnapshotAcknowledgement{
@ -90,7 +90,10 @@ func (unavailableStore) CurrentOwnershipEpoch(context.Context) (uint64, error) {
func (unavailableStore) OpenSession(context.Context, workerruntime.Session, time.Duration) error { func (unavailableStore) OpenSession(context.Context, workerruntime.Session, time.Duration) error {
return errors.New("redis unavailable") return errors.New("redis unavailable")
} }
func (unavailableStore) RecordIssuedSnapshot(context.Context, workerruntime.SnapshotReference, time.Duration) error { func (unavailableStore) ValidateSession(context.Context, string, string) error {
return errors.New("redis unavailable")
}
func (unavailableStore) RecordIssuedSnapshot(context.Context, string, workerruntime.SnapshotReference, time.Duration) error {
return errors.New("redis unavailable") return errors.New("redis unavailable")
} }
func (unavailableStore) AcknowledgeSnapshot(context.Context, workerruntime.SnapshotAcknowledgement, time.Duration) error { func (unavailableStore) AcknowledgeSnapshot(context.Context, workerruntime.SnapshotAcknowledgement, time.Duration) error {

View File

@ -24,6 +24,7 @@ func Run(t *testing.T, factory Factory) {
t.Helper() t.Helper()
t.Run("acknowledged runtime lifecycle", func(t *testing.T) { runLifecycle(t, newFixture(t, factory)) }) t.Run("acknowledged runtime lifecycle", func(t *testing.T) { runLifecycle(t, newFixture(t, factory)) })
t.Run("negative acknowledgement fences runtime", func(t *testing.T) { runNegativeAck(t, newFixture(t, factory)) }) t.Run("negative acknowledgement fences runtime", func(t *testing.T) { runNegativeAck(t, newFixture(t, factory)) })
t.Run("session replacement fences issued snapshots", func(t *testing.T) { runSessionReplacementFence(t, newFixture(t, factory)) })
} }
func runLifecycle(t *testing.T, fixture Fixture) { func runLifecycle(t *testing.T, fixture Fixture) {
@ -39,7 +40,7 @@ func runLifecycle(t *testing.T, fixture Fixture) {
if err := fixture.Store.ReplaceRuntime(ctx, report, fixture.TTL); !errors.Is(err, workerruntime.ErrSnapshotMismatch) { if err := fixture.Store.ReplaceRuntime(ctx, report, fixture.TTL); !errors.Is(err, workerruntime.ErrSnapshotMismatch) {
t.Fatalf("ReplaceRuntime(before ACK) error = %v", err) t.Fatalf("ReplaceRuntime(before ACK) error = %v", err)
} }
if err := fixture.Store.RecordIssuedSnapshot(ctx, reference, fixture.TTL); err != nil { if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", reference, fixture.TTL); err != nil {
t.Fatalf("RecordIssuedSnapshot(): %v", err) t.Fatalf("RecordIssuedSnapshot(): %v", err)
} }
ack := workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true} ack := workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
@ -69,14 +70,14 @@ func runNegativeAck(t *testing.T, fixture Fixture) {
t.Fatalf("CurrentOwnershipEpoch(): %v", err) t.Fatalf("CurrentOwnershipEpoch(): %v", err)
} }
first := snapshot(7, epoch, "snapshot-7") first := snapshot(7, epoch, "snapshot-7")
if err := fixture.Store.RecordIssuedSnapshot(ctx, first, fixture.TTL); err != nil { if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", first, fixture.TTL); err != nil {
t.Fatalf("RecordIssuedSnapshot(first): %v", err) t.Fatalf("RecordIssuedSnapshot(first): %v", err)
} }
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, fixture.TTL); err != nil { if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, fixture.TTL); err != nil {
t.Fatalf("AcknowledgeSnapshot(first): %v", err) t.Fatalf("AcknowledgeSnapshot(first): %v", err)
} }
second := snapshot(8, epoch, "snapshot-8") second := snapshot(8, epoch, "snapshot-8")
if err := fixture.Store.RecordIssuedSnapshot(ctx, second, fixture.TTL); err != nil { if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", second, fixture.TTL); err != nil {
t.Fatalf("RecordIssuedSnapshot(second): %v", err) t.Fatalf("RecordIssuedSnapshot(second): %v", err)
} }
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{ if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{
@ -89,6 +90,33 @@ func runNegativeAck(t *testing.T, fixture Fixture) {
} }
} }
func runSessionReplacementFence(t *testing.T, fixture Fixture) {
t.Helper()
ctx := context.Background()
open(t, fixture.Store, fixture.TTL)
epoch, err := fixture.Store.CurrentOwnershipEpoch(ctx)
if err != nil {
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
}
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", snapshot(3, epoch, "old"), fixture.TTL); err != nil {
t.Fatalf("RecordIssuedSnapshot(old): %v", err)
}
if err := fixture.Store.OpenSession(ctx, workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b", Zone: "zone-a", ProtocolVersion: 1,
}, fixture.TTL); err != nil {
t.Fatalf("OpenSession(replacement): %v", err)
}
if err := fixture.Store.ValidateSession(ctx, "worker-a", "session-a"); !errors.Is(err, workerruntime.ErrStaleSession) {
t.Fatalf("ValidateSession(old): %v, want ErrStaleSession", err)
}
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", snapshot(4, epoch, "stale"), fixture.TTL); !errors.Is(err, workerruntime.ErrStaleSession) {
t.Fatalf("RecordIssuedSnapshot(stale): %v, want ErrStaleSession", err)
}
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-b", snapshot(1, epoch, "new"), fixture.TTL); err != nil {
t.Fatalf("RecordIssuedSnapshot(new): %v", err)
}
}
func newFixture(t *testing.T, factory Factory) Fixture { func newFixture(t *testing.T, factory Factory) Fixture {
t.Helper() t.Helper()
fixture := factory(t) fixture := factory(t)

View File

@ -84,27 +84,52 @@ func (store *MemoryStore) OpenSession(ctx context.Context, session Session, ttl
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
delete(store.reports, normalized.WorkerID) delete(store.reports, normalized.WorkerID)
delete(store.references, normalized.WorkerID)
store.sessions[normalized.WorkerID] = memorySession{value: normalized, expiresAt: now.Add(ttl)} store.sessions[normalized.WorkerID] = memorySession{value: normalized, expiresAt: now.Add(ttl)}
return nil return nil
} }
func (store *MemoryStore) RecordIssuedSnapshot(ctx context.Context, reference SnapshotReference, ttl time.Duration) error { func (store *MemoryStore) ValidateSession(ctx context.Context, workerID, sessionID string) error {
if ctx == nil || store == nil || ttl <= 0 { if ctx == nil || store == nil || !ValidIdentifier(workerID) || !ValidIdentifier(sessionID) {
return ErrInvalidSnapshotReference return ErrInvalidSession
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return err return err
} }
normalized, err := NormalizeSnapshotReference(reference)
if err != nil {
return err
}
now, err := store.currentTime() now, err := store.currentTime()
if err != nil { if err != nil {
return err return err
} }
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
session, exists := store.sessions[workerID]
if !exists || !session.expiresAt.After(now) || session.value.SessionID != sessionID {
return ErrStaleSession
}
return nil
}
func (store *MemoryStore) RecordIssuedSnapshot(ctx context.Context, sessionID string, reference SnapshotReference, ttl time.Duration) error {
if ctx == nil || store == nil || ttl <= 0 {
return ErrInvalidSnapshotReference
}
if err := ctx.Err(); err != nil {
return err
}
normalized, err := NormalizeSnapshotReference(reference)
if err != nil || !ValidIdentifier(sessionID) {
return ErrInvalidSnapshotReference
}
now, err := store.currentTime()
if err != nil {
return err
}
store.mu.Lock()
defer store.mu.Unlock()
session, exists := store.sessions[normalized.WorkerID]
if !exists || !session.expiresAt.After(now) || session.value.SessionID != sessionID {
return ErrStaleSession
}
if normalized.OwnershipEpoch != store.epoch { if normalized.OwnershipEpoch != store.epoch {
return ErrSnapshotMismatch return ErrSnapshotMismatch
} }

View File

@ -22,7 +22,7 @@ func TestMemoryStoreRequiresAcknowledgedSnapshotForRuntime(t *testing.T) {
t.Fatalf("ReplaceRuntime(before ACK) error = %v, want ErrSnapshotMismatch", err) t.Fatalf("ReplaceRuntime(before ACK) error = %v, want ErrSnapshotMismatch", err)
} }
reference := controlReference(7, epoch, "snapshot-7") reference := controlReference(7, epoch, "snapshot-7")
if err := store.RecordIssuedSnapshot(ctx, reference, time.Minute); err != nil { if err := store.RecordIssuedSnapshot(ctx, "session-a", reference, time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(): %v", err) t.Fatalf("RecordIssuedSnapshot(): %v", err)
} }
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{ if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{
@ -45,7 +45,7 @@ func TestMemoryStoreNegativeAcknowledgementFencesDelayedRuntime(t *testing.T) {
t.Fatalf("CurrentOwnershipEpoch(): %v", err) t.Fatalf("CurrentOwnershipEpoch(): %v", err)
} }
first := controlReference(7, epoch, "snapshot-7") first := controlReference(7, epoch, "snapshot-7")
if err := store.RecordIssuedSnapshot(ctx, first, time.Minute); err != nil { if err := store.RecordIssuedSnapshot(ctx, "session-a", first, time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(first): %v", err) t.Fatalf("RecordIssuedSnapshot(first): %v", err)
} }
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, time.Minute); err != nil { if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, time.Minute); err != nil {
@ -56,7 +56,7 @@ func TestMemoryStoreNegativeAcknowledgementFencesDelayedRuntime(t *testing.T) {
} }
second := controlReference(8, epoch, "snapshot-8") second := controlReference(8, epoch, "snapshot-8")
if err := store.RecordIssuedSnapshot(ctx, second, time.Minute); err != nil { if err := store.RecordIssuedSnapshot(ctx, "session-a", second, time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(second): %v", err) t.Fatalf("RecordIssuedSnapshot(second): %v", err)
} }
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: second, Applied: false, ErrorCode: "apply_failed"}, time.Minute); err != nil { if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: second, Applied: false, ErrorCode: "apply_failed"}, time.Minute); err != nil {
@ -83,7 +83,7 @@ func TestMemoryStoreAcknowledgementReplayPreservesRuntimeFence(t *testing.T) {
t.Fatalf("CurrentOwnershipEpoch(): %v", err) t.Fatalf("CurrentOwnershipEpoch(): %v", err)
} }
reference := controlReference(7, epoch, "snapshot-7") reference := controlReference(7, epoch, "snapshot-7")
if err := store.RecordIssuedSnapshot(ctx, reference, time.Minute); err != nil { if err := store.RecordIssuedSnapshot(ctx, "session-a", reference, time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(): %v", err) t.Fatalf("RecordIssuedSnapshot(): %v", err)
} }
ack := SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true} ack := SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
@ -109,18 +109,43 @@ func TestMemoryStoreRejectsConflictingSnapshotReference(t *testing.T) {
now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC) now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now) store := newRuntimeStore(t, &now)
ctx := context.Background() ctx := context.Background()
openControlSession(t, store, "session-a", time.Minute)
epoch, err := store.CurrentOwnershipEpoch(ctx) epoch, err := store.CurrentOwnershipEpoch(ctx)
if err != nil { if err != nil {
t.Fatalf("CurrentOwnershipEpoch(): %v", err) t.Fatalf("CurrentOwnershipEpoch(): %v", err)
} }
if err := store.RecordIssuedSnapshot(ctx, controlReference(7, epoch, "first"), time.Minute); err != nil { if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(7, epoch, "first"), time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(first): %v", err) t.Fatalf("RecordIssuedSnapshot(first): %v", err)
} }
if err := store.RecordIssuedSnapshot(ctx, controlReference(7, epoch, "second"), time.Minute); !errors.Is(err, ErrConflictingSnapshotReference) { if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(7, epoch, "second"), time.Minute); !errors.Is(err, ErrConflictingSnapshotReference) {
t.Fatalf("RecordIssuedSnapshot(conflict): %v, want ErrConflictingSnapshotReference", err) t.Fatalf("RecordIssuedSnapshot(conflict): %v, want ErrConflictingSnapshotReference", err)
} }
} }
func TestMemoryStoreFencesIssuedSnapshotsAfterSessionReplacement(t *testing.T) {
now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
openControlSession(t, store, "session-a", time.Minute)
epoch, err := store.CurrentOwnershipEpoch(ctx)
if err != nil {
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
}
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(3, epoch, "old"), time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(old): %v", err)
}
openControlSession(t, store, "session-b", time.Minute)
if err := store.ValidateSession(ctx, "worker-a", "session-a"); !errors.Is(err, ErrStaleSession) {
t.Fatalf("ValidateSession(old) = %v, want ErrStaleSession", err)
}
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(4, epoch, "stale"), time.Minute); !errors.Is(err, ErrStaleSession) {
t.Fatalf("RecordIssuedSnapshot(stale) = %v, want ErrStaleSession", err)
}
if err := store.RecordIssuedSnapshot(ctx, "session-b", controlReference(1, epoch, "new"), time.Minute); err != nil {
t.Fatalf("RecordIssuedSnapshot(new): %v", err)
}
}
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) { func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC) now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now) store := newRuntimeStore(t, &now)

View File

@ -91,7 +91,8 @@ type SessionWriter interface {
type ControlStore interface { type ControlStore interface {
CurrentOwnershipEpoch(context.Context) (uint64, error) CurrentOwnershipEpoch(context.Context) (uint64, error)
OpenSession(context.Context, Session, time.Duration) error OpenSession(context.Context, Session, time.Duration) error
RecordIssuedSnapshot(context.Context, SnapshotReference, time.Duration) error ValidateSession(context.Context, string, string) error
RecordIssuedSnapshot(context.Context, string, SnapshotReference, time.Duration) error
AcknowledgeSnapshot(context.Context, SnapshotAcknowledgement, time.Duration) error AcknowledgeSnapshot(context.Context, SnapshotAcknowledgement, time.Duration) error
ReplaceRuntime(context.Context, Report, time.Duration) error ReplaceRuntime(context.Context, Report, time.Duration) error
} }