fix: fence snapshot issuance by worker session
This commit is contained in:
parent
51fef78368
commit
6f3a92170d
@ -32,6 +32,10 @@ Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有
|
|||||||
增量、Gateway 进程装配、Outcome 与 Checker 闭环尚未实现。`ReportOutcomes` 仍明确
|
增量、Gateway 进程装配、Outcome 与 Checker 闭环尚未实现。`ReportOutcomes` 仍明确
|
||||||
返回 `Unimplemented`;`100,000 QPS` 仍是未验证的设计目标。
|
返回 `Unimplemented`;`100,000 QPS` 仍是未验证的设计目标。
|
||||||
|
|
||||||
|
`WatchSnapshots` 建立时校验当前 session;每次签发快照引用时也把 `session_id`
|
||||||
|
交给 Redis 原子校验。重复 Register 会同时清除旧 Runtime 和已签发引用,因此迟到的
|
||||||
|
旧 Stream 既不能覆盖新 session 的引用,也不会向旧连接发送未获授权的快照。
|
||||||
|
|
||||||
## 2. Worker 会话
|
## 2. Worker 会话
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@ -54,6 +58,10 @@ sequenceDiagram
|
|||||||
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
||||||
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
||||||
|
|
||||||
|
同一 `worker_id` 重注册会替换 session,并清除旧 Runtime 与已签发 Snapshot
|
||||||
|
Reference。旧 Stream 即使在替换后仍收到上游更新,其签发操作也会以
|
||||||
|
`FailedPrecondition` 结束,不能影响新 session 的 ACK 基线。
|
||||||
|
|
||||||
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
|
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
|
||||||
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
|
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
|
||||||
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用
|
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用
|
||||||
|
|||||||
@ -213,7 +213,9 @@ WorkerControlPlane 现已接入 Controller 生命周期:Register、ACK 和 Run
|
|||||||
身份、消息/流限制和有界停机已实现。`WatchSnapshots` 会发送当前 epoch 的基础完整
|
身份、消息/流限制和有界停机已实现。`WatchSnapshots` 会发送当前 epoch 的基础完整
|
||||||
Snapshot 并保持连接;Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
|
Snapshot 并保持连接;Gateway 已具备 Register/Watch/ACK/Runtime 会话协调组件。
|
||||||
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
|
按 Worker 的可下发 ownership 索引已进入 Redis 原子脚本,并可构建无凭据引用的
|
||||||
已归属 Proxy payload。Routing payload、凭据分发、Gateway 命令与 Outcome 上报仍未实现。
|
已归属 Proxy payload。Snapshot 签发与 session 匹配在同一 Redis Lua 操作中完成,
|
||||||
|
重注册会清除旧引用,避免迟到 Stream 覆盖新 session。Routing payload、凭据分发、
|
||||||
|
Gateway 命令与 Outcome 上报仍未实现。
|
||||||
|
|
||||||
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
已新增公用 `domain/activitypool` 契约及并发安全内存参考实现,Provider
|
||||||
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
Reconciler 通过 `UpsertFetched` 写入带供应商 TTL 和分配安全余量的批次;已覆盖
|
||||||
|
|||||||
@ -52,7 +52,8 @@
|
|||||||
plaintext fixture 与 SPIFFE mTLS 服务端;基础 Snapshot 流和 Gateway 的
|
plaintext fixture 与 SPIFFE mTLS 服务端;基础 Snapshot 流和 Gateway 的
|
||||||
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
Register/Watch/ACK/Runtime 会话协调已实现。Redis 以 Worker 可下发 ownership
|
||||||
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
索引构建无凭据引用的已归属 Proxy payload,并以租约收紧可用期;Routing payload、
|
||||||
凭据分发、Outcome 和 Checker 尚未闭环。
|
凭据分发、Outcome 和 Checker 尚未闭环。Snapshot 签发在 Redis 中原子匹配当前
|
||||||
|
`session_id`,重注册会清除旧引用,迟到旧 Stream 不会覆盖新 session。
|
||||||
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
- `PostgreSQL 管理面`:已定义 `adminstate` 事务 seam、并发安全 MemoryStore、
|
||||||
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
公用契约、100 并发 Routing CAS、租约 Outbox 和只含六张管理表的 Schema;pgx
|
||||||
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
Adapter 已在真实 PostgreSQL 18 上通过同一契约、迁移幂等、审计/Outbox
|
||||||
|
|||||||
@ -16,6 +16,7 @@ const (
|
|||||||
runtimeReplaceSession = "replace_session"
|
runtimeReplaceSession = "replace_session"
|
||||||
runtimeCurrentEpoch = "current_epoch"
|
runtimeCurrentEpoch = "current_epoch"
|
||||||
runtimeOpenSession = "open_session"
|
runtimeOpenSession = "open_session"
|
||||||
|
runtimeValidateSession = "validate_session"
|
||||||
runtimeRecordSnapshot = "record_snapshot"
|
runtimeRecordSnapshot = "record_snapshot"
|
||||||
runtimeAcknowledge = "acknowledge_snapshot"
|
runtimeAcknowledge = "acknowledge_snapshot"
|
||||||
runtimeReplaceReport = "replace_report"
|
runtimeReplaceReport = "replace_report"
|
||||||
@ -44,6 +45,18 @@ type runtimeSnapshotReferenceWire struct {
|
|||||||
Checksum string `json:"checksum"`
|
Checksum string `json:"checksum"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type runtimeSnapshotIssueWire struct {
|
||||||
|
Version int `json:"version"`
|
||||||
|
SessionID string `json:"sessionId"`
|
||||||
|
Reference runtimeSnapshotReferenceWire `json:"reference"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runtimeSessionValidationWire struct {
|
||||||
|
Version int `json:"version"`
|
||||||
|
WorkerID string `json:"workerId"`
|
||||||
|
SessionID string `json:"sessionId"`
|
||||||
|
}
|
||||||
|
|
||||||
type runtimeAcknowledgementWire struct {
|
type runtimeAcknowledgementWire struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
WorkerID string `json:"workerId"`
|
WorkerID string `json:"workerId"`
|
||||||
@ -140,15 +153,46 @@ func (a *Adapter) OpenSession(ctx context.Context, session workerruntime.Session
|
|||||||
return invalidScriptReply("unexpected worker open session reply")
|
return invalidScriptReply("unexpected worker open session reply")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, reference workerruntime.SnapshotReference, ttl time.Duration) error {
|
func (a *Adapter) ValidateSession(ctx context.Context, workerID, sessionID string) error {
|
||||||
|
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !workerruntime.ValidIdentifier(workerID) || !workerruntime.ValidIdentifier(sessionID) {
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(runtimeSessionValidationWire{
|
||||||
|
Version: runtimeWireVersion, WorkerID: workerID, SessionID: sessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
}
|
||||||
|
reply, err := a.runRuntime(ctx, runtimeValidateSession, 0, payload, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch reply.Status {
|
||||||
|
case scriptOK:
|
||||||
|
return nil
|
||||||
|
case scriptInvalid:
|
||||||
|
return workerruntime.ErrInvalidSession
|
||||||
|
case scriptUnavailable:
|
||||||
|
return workerruntime.ErrStaleSession
|
||||||
|
default:
|
||||||
|
return invalidScriptReply("unexpected worker session validation reply")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, sessionID string, reference workerruntime.SnapshotReference, ttl time.Duration) error {
|
||||||
if err := validateRuntimeCall(ctx, a); err != nil {
|
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
normalized, err := workerruntime.NormalizeSnapshotReference(reference)
|
normalized, err := workerruntime.NormalizeSnapshotReference(reference)
|
||||||
if err != nil || ttl <= 0 {
|
if err != nil || !workerruntime.ValidIdentifier(sessionID) || ttl <= 0 {
|
||||||
return workerruntime.ErrInvalidSnapshotReference
|
return workerruntime.ErrInvalidSnapshotReference
|
||||||
}
|
}
|
||||||
payload, err := json.Marshal(referenceWire(normalized))
|
payload, err := json.Marshal(runtimeSnapshotIssueWire{
|
||||||
|
Version: runtimeWireVersion, SessionID: sessionID, Reference: referenceWire(normalized),
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return workerruntime.ErrInvalidSnapshotReference
|
return workerruntime.ErrInvalidSnapshotReference
|
||||||
}
|
}
|
||||||
@ -167,6 +211,8 @@ func (a *Adapter) RecordIssuedSnapshot(ctx context.Context, reference workerrunt
|
|||||||
return workerruntime.ErrConflictingSnapshotReference
|
return workerruntime.ErrConflictingSnapshotReference
|
||||||
case scriptSnapshotMismatch:
|
case scriptSnapshotMismatch:
|
||||||
return workerruntime.ErrSnapshotMismatch
|
return workerruntime.ErrSnapshotMismatch
|
||||||
|
case scriptUnavailable:
|
||||||
|
return workerruntime.ErrStaleSession
|
||||||
default:
|
default:
|
||||||
return invalidScriptReply("unexpected worker snapshot reference reply")
|
return invalidScriptReply("unexpected worker snapshot reference reply")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -148,20 +148,44 @@ if operation == 'open_session' then
|
|||||||
end
|
end
|
||||||
redis.call('HDEL', runtime_key, session.workerId)
|
redis.call('HDEL', runtime_key, session.workerId)
|
||||||
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
||||||
|
redis.call('HDEL', snapshots_key, session.workerId)
|
||||||
|
redis.call('ZREM', snapshot_expiry_key, session.workerId)
|
||||||
session.expiresAtMs = now + ttl_ms
|
session.expiresAtMs = now + ttl_ms
|
||||||
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
|
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
|
||||||
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
|
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
|
||||||
return reply('ok')
|
return reply('ok')
|
||||||
end
|
end
|
||||||
|
|
||||||
|
if operation == 'validate_session' then
|
||||||
|
local validation = decode_table(payload)
|
||||||
|
if not validation or validation.version ~= 1 or type(validation.workerId) ~= 'string' or
|
||||||
|
validation.workerId == '' or type(validation.sessionId) ~= 'string' or validation.sessionId == '' then
|
||||||
|
return reply('invalid')
|
||||||
|
end
|
||||||
|
local session = decode_table(redis.call('HGET', sessions_key, validation.workerId))
|
||||||
|
if not valid_control_session(session) or session.workerId ~= validation.workerId or
|
||||||
|
session.sessionId ~= validation.sessionId or type(session.expiresAtMs) ~= 'number' or
|
||||||
|
session.expiresAtMs <= now then
|
||||||
|
return reply('unavailable')
|
||||||
|
end
|
||||||
|
return reply('ok')
|
||||||
|
end
|
||||||
|
|
||||||
if operation == 'record_snapshot' then
|
if operation == 'record_snapshot' then
|
||||||
if not ttl_ms or ttl_ms <= 0 then
|
if not ttl_ms or ttl_ms <= 0 then
|
||||||
return reply('invalid')
|
return reply('invalid')
|
||||||
end
|
end
|
||||||
local reference = decode_table(payload)
|
local issue = decode_table(payload)
|
||||||
if not valid_reference(reference) then
|
if not issue or issue.version ~= 1 or type(issue.sessionId) ~= 'string' or issue.sessionId == '' or
|
||||||
|
not valid_reference(issue.reference) then
|
||||||
return reply('invalid')
|
return reply('invalid')
|
||||||
end
|
end
|
||||||
|
local reference = issue.reference
|
||||||
|
local session = decode_table(redis.call('HGET', sessions_key, reference.workerId))
|
||||||
|
if not valid_control_session(session) or session.workerId ~= reference.workerId or session.sessionId ~= issue.sessionId or
|
||||||
|
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
|
||||||
|
return reply('unavailable')
|
||||||
|
end
|
||||||
local epoch = redis.call('GET', epoch_key)
|
local epoch = redis.call('GET', epoch_key)
|
||||||
if not epoch then
|
if not epoch then
|
||||||
epoch = '1'
|
epoch = '1'
|
||||||
|
|||||||
@ -68,6 +68,9 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
(len(request.GetLastChecksum()) != 0 && len(request.GetLastChecksum()) != sha256.Size) {
|
(len(request.GetLastChecksum()) != 0 && len(request.GetLastChecksum()) != sha256.Size) {
|
||||||
return grpcError(ErrInvalidCommand)
|
return grpcError(ErrInvalidCommand)
|
||||||
}
|
}
|
||||||
|
if err := handler.service.ValidateSession(stream.Context(), request.GetWorkerId(), request.GetSessionId()); err != nil {
|
||||||
|
return grpcError(err)
|
||||||
|
}
|
||||||
updates, err := handler.snapshots.Watch(stream.Context(), SnapshotWatchRequest{
|
updates, err := handler.snapshots.Watch(stream.Context(), SnapshotWatchRequest{
|
||||||
WorkerID: request.GetWorkerId(), SessionID: request.GetSessionId(), LastAppliedVersion: request.GetLastAppliedVersion(),
|
WorkerID: request.GetWorkerId(), SessionID: request.GetSessionId(), LastAppliedVersion: request.GetLastAppliedVersion(),
|
||||||
LastChecksum: append([]byte(nil), request.GetLastChecksum()...),
|
LastChecksum: append([]byte(nil), request.GetLastChecksum()...),
|
||||||
@ -83,7 +86,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := handler.issueSnapshot(stream.Context(), request.GetWorkerId(), snapshot); err != nil {
|
if err := handler.issueSnapshot(stream.Context(), request.GetWorkerId(), request.GetSessionId(), snapshot); err != nil {
|
||||||
return grpcError(err)
|
return grpcError(err)
|
||||||
}
|
}
|
||||||
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
|
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: snapshot}}); err != nil {
|
||||||
@ -93,7 +96,7 @@ func (handler *GRPCHandler) WatchSnapshots(request *controlplanev1.WatchSnapshot
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID string, snapshot *controlplanev1.WorkerSnapshot) error {
|
func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID, sessionID string, snapshot *controlplanev1.WorkerSnapshot) error {
|
||||||
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
|
if snapshot == nil || snapshot.GetVersion() == 0 || snapshot.GetOwnershipEpoch() == 0 || len(snapshot.GetChecksum()) != sha256.Size ||
|
||||||
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
|
snapshot.GetGeneratedAt() == nil || snapshot.GetGeneratedAt().CheckValid() != nil ||
|
||||||
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil {
|
snapshot.GetValidUntil() == nil || snapshot.GetValidUntil().CheckValid() != nil {
|
||||||
@ -101,7 +104,7 @@ func (handler *GRPCHandler) issueSnapshot(ctx context.Context, workerID string,
|
|||||||
}
|
}
|
||||||
var checksum [sha256.Size]byte
|
var checksum [sha256.Size]byte
|
||||||
copy(checksum[:], snapshot.GetChecksum())
|
copy(checksum[:], snapshot.GetChecksum())
|
||||||
return handler.service.IssueSnapshot(ctx, workerruntime.SnapshotReference{
|
return handler.service.IssueSnapshot(ctx, sessionID, workerruntime.SnapshotReference{
|
||||||
WorkerID: workerID, Version: snapshot.GetVersion(), OwnershipEpoch: snapshot.GetOwnershipEpoch(), Checksum: checksum,
|
WorkerID: workerID, Version: snapshot.GetVersion(), OwnershipEpoch: snapshot.GetOwnershipEpoch(), Checksum: checksum,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -85,6 +85,31 @@ func TestGRPCHandlerStreamsIssuedFullSnapshots(t *testing.T) {
|
|||||||
if service.issued.WorkerID != "worker-a" || service.issued.Version != 3 || service.issued.Checksum[0] != 1 {
|
if service.issued.WorkerID != "worker-a" || service.issued.Version != 3 || service.issued.Checksum[0] != 1 {
|
||||||
t.Fatalf("issued snapshot = %+v", service.issued)
|
t.Fatalf("issued snapshot = %+v", service.issued)
|
||||||
}
|
}
|
||||||
|
if service.issuedSessionID != "session-a" {
|
||||||
|
t.Fatalf("issued session = %q, want session-a", service.issuedSessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGRPCHandlerDoesNotDeliverSnapshotWhenSessionBecomesStale(t *testing.T) {
|
||||||
|
checksum := make([]byte, 32)
|
||||||
|
checksum[0] = 1
|
||||||
|
service := &grpcServiceStub{issueErr: workerruntime.ErrStaleSession}
|
||||||
|
client, cleanup := grpcWorkerClient(t, service, allowIdentity{}, snapshotSourceStub{snapshots: []*controlplanev1.WorkerSnapshot{{
|
||||||
|
Version: 3, OwnershipEpoch: 9, Checksum: checksum,
|
||||||
|
GeneratedAt: timestamppb.New(time.Now()), ValidUntil: timestamppb.New(time.Now().Add(time.Minute)),
|
||||||
|
}}})
|
||||||
|
defer cleanup()
|
||||||
|
stream, err := client.WatchSnapshots(context.Background(), &controlplanev1.WatchSnapshotsRequest{WorkerId: "worker-a", SessionId: "session-a"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WatchSnapshots(): %v", err)
|
||||||
|
}
|
||||||
|
_, err = stream.Recv()
|
||||||
|
if status.Code(err) != codes.FailedPrecondition {
|
||||||
|
t.Fatalf("Recv() error = %v, want FailedPrecondition", err)
|
||||||
|
}
|
||||||
|
if service.issued.Version != 0 || service.issuedSessionID != "" {
|
||||||
|
t.Fatalf("stale session issued snapshot = %+v for %q", service.issued, service.issuedSessionID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type grpcServiceStub struct {
|
type grpcServiceStub struct {
|
||||||
@ -95,14 +120,24 @@ type grpcServiceStub struct {
|
|||||||
report workerruntime.Report
|
report workerruntime.Report
|
||||||
reportErr error
|
reportErr error
|
||||||
issued workerruntime.SnapshotReference
|
issued workerruntime.SnapshotReference
|
||||||
|
issuedSessionID string
|
||||||
|
validateErr error
|
||||||
|
issueErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (stub *grpcServiceStub) Register(context.Context, RegisterCommand) (Registration, error) {
|
func (stub *grpcServiceStub) Register(context.Context, RegisterCommand) (Registration, error) {
|
||||||
return stub.registration, stub.registerErr
|
return stub.registration, stub.registerErr
|
||||||
}
|
}
|
||||||
func (stub *grpcServiceStub) CurrentOwnershipEpoch(context.Context) (uint64, error) { return 9, nil }
|
func (stub *grpcServiceStub) CurrentOwnershipEpoch(context.Context) (uint64, error) { return 9, nil }
|
||||||
func (stub *grpcServiceStub) IssueSnapshot(_ context.Context, reference workerruntime.SnapshotReference) error {
|
func (stub *grpcServiceStub) ValidateSession(context.Context, string, string) error {
|
||||||
|
return stub.validateErr
|
||||||
|
}
|
||||||
|
func (stub *grpcServiceStub) IssueSnapshot(_ context.Context, sessionID string, reference workerruntime.SnapshotReference) error {
|
||||||
|
if stub.issueErr != nil {
|
||||||
|
return stub.issueErr
|
||||||
|
}
|
||||||
stub.issued = reference
|
stub.issued = reference
|
||||||
|
stub.issuedSessionID = sessionID
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (stub *grpcServiceStub) Acknowledge(_ context.Context, acknowledgement SnapshotAcknowledgement) error {
|
func (stub *grpcServiceStub) Acknowledge(_ context.Context, acknowledgement SnapshotAcknowledgement) error {
|
||||||
|
|||||||
@ -64,7 +64,8 @@ type Options struct {
|
|||||||
type Service interface {
|
type Service interface {
|
||||||
Register(context.Context, RegisterCommand) (Registration, error)
|
Register(context.Context, RegisterCommand) (Registration, error)
|
||||||
CurrentOwnershipEpoch(context.Context) (uint64, error)
|
CurrentOwnershipEpoch(context.Context) (uint64, error)
|
||||||
IssueSnapshot(context.Context, workerruntime.SnapshotReference) error
|
ValidateSession(context.Context, string, string) error
|
||||||
|
IssueSnapshot(context.Context, string, workerruntime.SnapshotReference) error
|
||||||
Acknowledge(context.Context, SnapshotAcknowledgement) error
|
Acknowledge(context.Context, SnapshotAcknowledgement) error
|
||||||
ReportRuntime(context.Context, workerruntime.Report) (RuntimeDecision, error)
|
ReportRuntime(context.Context, workerruntime.Report) (RuntimeDecision, error)
|
||||||
}
|
}
|
||||||
@ -83,7 +84,20 @@ func (service *service) CurrentOwnershipEpoch(ctx context.Context) (uint64, erro
|
|||||||
return epoch, nil
|
return epoch, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (service *service) IssueSnapshot(ctx context.Context, reference workerruntime.SnapshotReference) error {
|
func (service *service) ValidateSession(ctx context.Context, workerID, sessionID string) error {
|
||||||
|
if ctx == nil || !workerruntime.ValidIdentifier(workerID) || !workerruntime.ValidIdentifier(sessionID) {
|
||||||
|
return ErrInvalidCommand
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := service.store.ValidateSession(ctx, workerID, sessionID); err != nil {
|
||||||
|
return classifyStoreError(err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (service *service) IssueSnapshot(ctx context.Context, sessionID string, reference workerruntime.SnapshotReference) error {
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return ErrInvalidCommand
|
return ErrInvalidCommand
|
||||||
}
|
}
|
||||||
@ -91,10 +105,10 @@ func (service *service) IssueSnapshot(ctx context.Context, reference workerrunti
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
normalized, err := workerruntime.NormalizeSnapshotReference(reference)
|
normalized, err := workerruntime.NormalizeSnapshotReference(reference)
|
||||||
if err != nil {
|
if err != nil || !workerruntime.ValidIdentifier(sessionID) {
|
||||||
return errors.Join(ErrInvalidCommand, err)
|
return errors.Join(ErrInvalidCommand, err)
|
||||||
}
|
}
|
||||||
if err := service.store.RecordIssuedSnapshot(ctx, normalized, service.options.SessionTTL); err != nil {
|
if err := service.store.RecordIssuedSnapshot(ctx, sessionID, normalized, service.options.SessionTTL); err != nil {
|
||||||
return classifyStoreError(err)
|
return classifyStoreError(err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@ -35,7 +35,7 @@ func TestServiceRegistersAcknowledgesAndReportsRuntime(t *testing.T) {
|
|||||||
WorkerID: "worker-a", Version: 7, OwnershipEpoch: registered.OwnershipEpoch,
|
WorkerID: "worker-a", Version: 7, OwnershipEpoch: registered.OwnershipEpoch,
|
||||||
Checksum: sha256.Sum256([]byte("snapshot-7")),
|
Checksum: sha256.Sum256([]byte("snapshot-7")),
|
||||||
}
|
}
|
||||||
if err := service.IssueSnapshot(context.Background(), reference); err != nil {
|
if err := service.IssueSnapshot(context.Background(), registered.SessionID, reference); err != nil {
|
||||||
t.Fatalf("IssueSnapshot(): %v", err)
|
t.Fatalf("IssueSnapshot(): %v", err)
|
||||||
}
|
}
|
||||||
if err := service.Acknowledge(context.Background(), SnapshotAcknowledgement{
|
if err := service.Acknowledge(context.Background(), SnapshotAcknowledgement{
|
||||||
@ -90,7 +90,10 @@ func (unavailableStore) CurrentOwnershipEpoch(context.Context) (uint64, error) {
|
|||||||
func (unavailableStore) OpenSession(context.Context, workerruntime.Session, time.Duration) error {
|
func (unavailableStore) OpenSession(context.Context, workerruntime.Session, time.Duration) error {
|
||||||
return errors.New("redis unavailable")
|
return errors.New("redis unavailable")
|
||||||
}
|
}
|
||||||
func (unavailableStore) RecordIssuedSnapshot(context.Context, workerruntime.SnapshotReference, time.Duration) error {
|
func (unavailableStore) ValidateSession(context.Context, string, string) error {
|
||||||
|
return errors.New("redis unavailable")
|
||||||
|
}
|
||||||
|
func (unavailableStore) RecordIssuedSnapshot(context.Context, string, workerruntime.SnapshotReference, time.Duration) error {
|
||||||
return errors.New("redis unavailable")
|
return errors.New("redis unavailable")
|
||||||
}
|
}
|
||||||
func (unavailableStore) AcknowledgeSnapshot(context.Context, workerruntime.SnapshotAcknowledgement, time.Duration) error {
|
func (unavailableStore) AcknowledgeSnapshot(context.Context, workerruntime.SnapshotAcknowledgement, time.Duration) error {
|
||||||
|
|||||||
@ -24,6 +24,7 @@ func Run(t *testing.T, factory Factory) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
t.Run("acknowledged runtime lifecycle", func(t *testing.T) { runLifecycle(t, newFixture(t, factory)) })
|
t.Run("acknowledged runtime lifecycle", func(t *testing.T) { runLifecycle(t, newFixture(t, factory)) })
|
||||||
t.Run("negative acknowledgement fences runtime", func(t *testing.T) { runNegativeAck(t, newFixture(t, factory)) })
|
t.Run("negative acknowledgement fences runtime", func(t *testing.T) { runNegativeAck(t, newFixture(t, factory)) })
|
||||||
|
t.Run("session replacement fences issued snapshots", func(t *testing.T) { runSessionReplacementFence(t, newFixture(t, factory)) })
|
||||||
}
|
}
|
||||||
|
|
||||||
func runLifecycle(t *testing.T, fixture Fixture) {
|
func runLifecycle(t *testing.T, fixture Fixture) {
|
||||||
@ -39,7 +40,7 @@ func runLifecycle(t *testing.T, fixture Fixture) {
|
|||||||
if err := fixture.Store.ReplaceRuntime(ctx, report, fixture.TTL); !errors.Is(err, workerruntime.ErrSnapshotMismatch) {
|
if err := fixture.Store.ReplaceRuntime(ctx, report, fixture.TTL); !errors.Is(err, workerruntime.ErrSnapshotMismatch) {
|
||||||
t.Fatalf("ReplaceRuntime(before ACK) error = %v", err)
|
t.Fatalf("ReplaceRuntime(before ACK) error = %v", err)
|
||||||
}
|
}
|
||||||
if err := fixture.Store.RecordIssuedSnapshot(ctx, reference, fixture.TTL); err != nil {
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", reference, fixture.TTL); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
||||||
}
|
}
|
||||||
ack := workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
|
ack := workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
|
||||||
@ -69,14 +70,14 @@ func runNegativeAck(t *testing.T, fixture Fixture) {
|
|||||||
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
}
|
}
|
||||||
first := snapshot(7, epoch, "snapshot-7")
|
first := snapshot(7, epoch, "snapshot-7")
|
||||||
if err := fixture.Store.RecordIssuedSnapshot(ctx, first, fixture.TTL); err != nil {
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", first, fixture.TTL); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
||||||
}
|
}
|
||||||
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, fixture.TTL); err != nil {
|
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, fixture.TTL); err != nil {
|
||||||
t.Fatalf("AcknowledgeSnapshot(first): %v", err)
|
t.Fatalf("AcknowledgeSnapshot(first): %v", err)
|
||||||
}
|
}
|
||||||
second := snapshot(8, epoch, "snapshot-8")
|
second := snapshot(8, epoch, "snapshot-8")
|
||||||
if err := fixture.Store.RecordIssuedSnapshot(ctx, second, fixture.TTL); err != nil {
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", second, fixture.TTL); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(second): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(second): %v", err)
|
||||||
}
|
}
|
||||||
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{
|
if err := fixture.Store.AcknowledgeSnapshot(ctx, workerruntime.SnapshotAcknowledgement{
|
||||||
@ -89,6 +90,33 @@ func runNegativeAck(t *testing.T, fixture Fixture) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runSessionReplacementFence(t *testing.T, fixture Fixture) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
open(t, fixture.Store, fixture.TTL)
|
||||||
|
epoch, err := fixture.Store.CurrentOwnershipEpoch(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", snapshot(3, epoch, "old"), fixture.TTL); err != nil {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(old): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Store.OpenSession(ctx, workerruntime.Session{
|
||||||
|
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b", Zone: "zone-a", ProtocolVersion: 1,
|
||||||
|
}, fixture.TTL); err != nil {
|
||||||
|
t.Fatalf("OpenSession(replacement): %v", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Store.ValidateSession(ctx, "worker-a", "session-a"); !errors.Is(err, workerruntime.ErrStaleSession) {
|
||||||
|
t.Fatalf("ValidateSession(old): %v, want ErrStaleSession", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-a", snapshot(4, epoch, "stale"), fixture.TTL); !errors.Is(err, workerruntime.ErrStaleSession) {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(stale): %v, want ErrStaleSession", err)
|
||||||
|
}
|
||||||
|
if err := fixture.Store.RecordIssuedSnapshot(ctx, "session-b", snapshot(1, epoch, "new"), fixture.TTL); err != nil {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(new): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func newFixture(t *testing.T, factory Factory) Fixture {
|
func newFixture(t *testing.T, factory Factory) Fixture {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
fixture := factory(t)
|
fixture := factory(t)
|
||||||
|
|||||||
@ -84,27 +84,52 @@ func (store *MemoryStore) OpenSession(ctx context.Context, session Session, ttl
|
|||||||
store.mu.Lock()
|
store.mu.Lock()
|
||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
delete(store.reports, normalized.WorkerID)
|
delete(store.reports, normalized.WorkerID)
|
||||||
|
delete(store.references, normalized.WorkerID)
|
||||||
store.sessions[normalized.WorkerID] = memorySession{value: normalized, expiresAt: now.Add(ttl)}
|
store.sessions[normalized.WorkerID] = memorySession{value: normalized, expiresAt: now.Add(ttl)}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (store *MemoryStore) RecordIssuedSnapshot(ctx context.Context, reference SnapshotReference, ttl time.Duration) error {
|
func (store *MemoryStore) ValidateSession(ctx context.Context, workerID, sessionID string) error {
|
||||||
if ctx == nil || store == nil || ttl <= 0 {
|
if ctx == nil || store == nil || !ValidIdentifier(workerID) || !ValidIdentifier(sessionID) {
|
||||||
return ErrInvalidSnapshotReference
|
return ErrInvalidSession
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
normalized, err := NormalizeSnapshotReference(reference)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
now, err := store.currentTime()
|
now, err := store.currentTime()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
store.mu.Lock()
|
store.mu.Lock()
|
||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
|
session, exists := store.sessions[workerID]
|
||||||
|
if !exists || !session.expiresAt.After(now) || session.value.SessionID != sessionID {
|
||||||
|
return ErrStaleSession
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *MemoryStore) RecordIssuedSnapshot(ctx context.Context, sessionID string, reference SnapshotReference, ttl time.Duration) error {
|
||||||
|
if ctx == nil || store == nil || ttl <= 0 {
|
||||||
|
return ErrInvalidSnapshotReference
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
normalized, err := NormalizeSnapshotReference(reference)
|
||||||
|
if err != nil || !ValidIdentifier(sessionID) {
|
||||||
|
return ErrInvalidSnapshotReference
|
||||||
|
}
|
||||||
|
now, err := store.currentTime()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
store.mu.Lock()
|
||||||
|
defer store.mu.Unlock()
|
||||||
|
session, exists := store.sessions[normalized.WorkerID]
|
||||||
|
if !exists || !session.expiresAt.After(now) || session.value.SessionID != sessionID {
|
||||||
|
return ErrStaleSession
|
||||||
|
}
|
||||||
if normalized.OwnershipEpoch != store.epoch {
|
if normalized.OwnershipEpoch != store.epoch {
|
||||||
return ErrSnapshotMismatch
|
return ErrSnapshotMismatch
|
||||||
}
|
}
|
||||||
|
|||||||
@ -22,7 +22,7 @@ func TestMemoryStoreRequiresAcknowledgedSnapshotForRuntime(t *testing.T) {
|
|||||||
t.Fatalf("ReplaceRuntime(before ACK) error = %v, want ErrSnapshotMismatch", err)
|
t.Fatalf("ReplaceRuntime(before ACK) error = %v, want ErrSnapshotMismatch", err)
|
||||||
}
|
}
|
||||||
reference := controlReference(7, epoch, "snapshot-7")
|
reference := controlReference(7, epoch, "snapshot-7")
|
||||||
if err := store.RecordIssuedSnapshot(ctx, reference, time.Minute); err != nil {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", reference, time.Minute); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
||||||
}
|
}
|
||||||
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{
|
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{
|
||||||
@ -45,7 +45,7 @@ func TestMemoryStoreNegativeAcknowledgementFencesDelayedRuntime(t *testing.T) {
|
|||||||
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
}
|
}
|
||||||
first := controlReference(7, epoch, "snapshot-7")
|
first := controlReference(7, epoch, "snapshot-7")
|
||||||
if err := store.RecordIssuedSnapshot(ctx, first, time.Minute); err != nil {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", first, time.Minute); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
||||||
}
|
}
|
||||||
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, time.Minute); err != nil {
|
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: first, Applied: true}, time.Minute); err != nil {
|
||||||
@ -56,7 +56,7 @@ func TestMemoryStoreNegativeAcknowledgementFencesDelayedRuntime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
second := controlReference(8, epoch, "snapshot-8")
|
second := controlReference(8, epoch, "snapshot-8")
|
||||||
if err := store.RecordIssuedSnapshot(ctx, second, time.Minute); err != nil {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", second, time.Minute); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(second): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(second): %v", err)
|
||||||
}
|
}
|
||||||
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: second, Applied: false, ErrorCode: "apply_failed"}, time.Minute); err != nil {
|
if err := store.AcknowledgeSnapshot(ctx, SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: second, Applied: false, ErrorCode: "apply_failed"}, time.Minute); err != nil {
|
||||||
@ -83,7 +83,7 @@ func TestMemoryStoreAcknowledgementReplayPreservesRuntimeFence(t *testing.T) {
|
|||||||
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
}
|
}
|
||||||
reference := controlReference(7, epoch, "snapshot-7")
|
reference := controlReference(7, epoch, "snapshot-7")
|
||||||
if err := store.RecordIssuedSnapshot(ctx, reference, time.Minute); err != nil {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", reference, time.Minute); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(): %v", err)
|
||||||
}
|
}
|
||||||
ack := SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
|
ack := SnapshotAcknowledgement{WorkerID: "worker-a", SessionID: "session-a", Reference: reference, Applied: true}
|
||||||
@ -109,18 +109,43 @@ func TestMemoryStoreRejectsConflictingSnapshotReference(t *testing.T) {
|
|||||||
now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC)
|
||||||
store := newRuntimeStore(t, &now)
|
store := newRuntimeStore(t, &now)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
openControlSession(t, store, "session-a", time.Minute)
|
||||||
epoch, err := store.CurrentOwnershipEpoch(ctx)
|
epoch, err := store.CurrentOwnershipEpoch(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
}
|
}
|
||||||
if err := store.RecordIssuedSnapshot(ctx, controlReference(7, epoch, "first"), time.Minute); err != nil {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(7, epoch, "first"), time.Minute); err != nil {
|
||||||
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
t.Fatalf("RecordIssuedSnapshot(first): %v", err)
|
||||||
}
|
}
|
||||||
if err := store.RecordIssuedSnapshot(ctx, controlReference(7, epoch, "second"), time.Minute); !errors.Is(err, ErrConflictingSnapshotReference) {
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(7, epoch, "second"), time.Minute); !errors.Is(err, ErrConflictingSnapshotReference) {
|
||||||
t.Fatalf("RecordIssuedSnapshot(conflict): %v, want ErrConflictingSnapshotReference", err)
|
t.Fatalf("RecordIssuedSnapshot(conflict): %v, want ErrConflictingSnapshotReference", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreFencesIssuedSnapshotsAfterSessionReplacement(t *testing.T) {
|
||||||
|
now := time.Date(2026, 7, 31, 9, 0, 0, 0, time.UTC)
|
||||||
|
store := newRuntimeStore(t, &now)
|
||||||
|
ctx := context.Background()
|
||||||
|
openControlSession(t, store, "session-a", time.Minute)
|
||||||
|
epoch, err := store.CurrentOwnershipEpoch(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CurrentOwnershipEpoch(): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(3, epoch, "old"), time.Minute); err != nil {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(old): %v", err)
|
||||||
|
}
|
||||||
|
openControlSession(t, store, "session-b", time.Minute)
|
||||||
|
if err := store.ValidateSession(ctx, "worker-a", "session-a"); !errors.Is(err, ErrStaleSession) {
|
||||||
|
t.Fatalf("ValidateSession(old) = %v, want ErrStaleSession", err)
|
||||||
|
}
|
||||||
|
if err := store.RecordIssuedSnapshot(ctx, "session-a", controlReference(4, epoch, "stale"), time.Minute); !errors.Is(err, ErrStaleSession) {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(stale) = %v, want ErrStaleSession", err)
|
||||||
|
}
|
||||||
|
if err := store.RecordIssuedSnapshot(ctx, "session-b", controlReference(1, epoch, "new"), time.Minute); err != nil {
|
||||||
|
t.Fatalf("RecordIssuedSnapshot(new): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
|
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
|
||||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||||
store := newRuntimeStore(t, &now)
|
store := newRuntimeStore(t, &now)
|
||||||
|
|||||||
@ -91,7 +91,8 @@ type SessionWriter interface {
|
|||||||
type ControlStore interface {
|
type ControlStore interface {
|
||||||
CurrentOwnershipEpoch(context.Context) (uint64, error)
|
CurrentOwnershipEpoch(context.Context) (uint64, error)
|
||||||
OpenSession(context.Context, Session, time.Duration) error
|
OpenSession(context.Context, Session, time.Duration) error
|
||||||
RecordIssuedSnapshot(context.Context, SnapshotReference, time.Duration) error
|
ValidateSession(context.Context, string, string) error
|
||||||
|
RecordIssuedSnapshot(context.Context, string, SnapshotReference, time.Duration) error
|
||||||
AcknowledgeSnapshot(context.Context, SnapshotAcknowledgement, time.Duration) error
|
AcknowledgeSnapshot(context.Context, SnapshotAcknowledgement, time.Duration) error
|
||||||
ReplaceRuntime(context.Context, Report, time.Duration) error
|
ReplaceRuntime(context.Context, Report, time.Duration) error
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user