package contracttest import ( "context" "errors" "fmt" "strings" "sync" "sync/atomic" "testing" "time" "proxy-pool/internal/domain/adminstate" ) type Factory func(*testing.T) adminstate.Store func Run(t *testing.T, factory Factory) { t.Helper() t.Run("config transaction and immutability", func(t *testing.T) { runConfigContract(t, factory(t)) }) t.Run("upstream idempotency", func(t *testing.T) { runUpstreamContract(t, factory(t)) }) t.Run("routing compare and swap", func(t *testing.T) { runRoutingContract(t, factory(t)) }) t.Run("routing no-op audit", func(t *testing.T) { runRoutingNoOpContract(t, factory(t)) }) t.Run("routing terminal stop compare and swap", func(t *testing.T) { runRoutingDisableContract(t, factory(t)) }) t.Run("audit pagination and fields", func(t *testing.T) { runAuditContract(t, factory(t)) }) t.Run("outbox lease and acknowledgement", func(t *testing.T) { runOutboxContract(t, factory(t)) }) t.Run("outbox acknowledgement is atomic", func(t *testing.T) { runAtomicAcknowledgeContract(t, factory(t)) }) t.Run("context cancellation", func(t *testing.T) { runContextContract(t, factory(t)) }) } func runConfigContract(t *testing.T, store adminstate.Store) { t.Helper() command := configCommand("req-config", "cfg-1", strings.Repeat("ab", adminstate.SHA256HexBytes/2)) result, err := store.CommitConfig(context.Background(), command) if err != nil || !result.Changed || result.Revision != 1 || result.RequestID != command.RequestID { t.Fatalf("CommitConfig(first) = %+v, %v", result, err) } snapshot, err := store.Snapshot(context.Background()) if err != nil || snapshot.Revision != 1 || snapshot.Config == nil || snapshot.Config.ConfigVersion != "cfg-1" || len(snapshot.Upstreams) != 2 || len(snapshot.Routings) != 1 { t.Fatalf("Snapshot() = %+v, %v", snapshot, err) } snapshot.Config.ConfigVersion = "mutated" snapshot.Upstreams[0].Name = "mutated" snapshot.Routings[0].Upstreams[0] = "mutated" again, err := store.Snapshot(context.Background()) if err != nil || again.Config.ConfigVersion != "cfg-1" || again.Upstreams[0].Name == "mutated" || again.Routings[0].Upstreams[0] == "mutated" { t.Fatalf("Snapshot() leaked mutable state: %+v, %v", again, err) } command.RequestID = "req-config-replay" command.OccurredAt = command.OccurredAt.Add(time.Second) command.Checksum = strings.ToUpper(command.Checksum) replayed, err := store.CommitConfig(context.Background(), command) if err != nil || replayed.Changed || replayed.Revision != 1 { t.Fatalf("CommitConfig(replay) = %+v, %v", replayed, err) } audits, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10}) if err != nil || len(audits) != 2 || !audits[0].Changed || audits[1].Changed || audits[0].Revision != 1 || audits[1].Revision != 1 { t.Fatalf("ReadAudit() = %+v, %v", audits, err) } conflict := command conflict.RequestID = "req-config-conflict" conflict.Checksum = strings.Repeat("b", adminstate.SHA256HexBytes) if _, err := store.CommitConfig(context.Background(), conflict); !errors.Is(err, adminstate.ErrConflict) { t.Fatalf("CommitConfig(conflict) error = %v", err) } audits, err = store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10}) if err != nil || len(audits) != 2 { t.Fatalf("ReadAudit(after conflict) = %+v, %v", audits, err) } invalid := command invalid.RequestID = "req-config-invalid" invalid.Routings[0].CurrentUpstream = "missing" if _, err := store.CommitConfig(context.Background(), invalid); !errors.Is(err, adminstate.ErrInvalidCommand) { t.Fatalf("CommitConfig(invalid) error = %v", err) } final, err := store.Snapshot(context.Background()) if err != nil || final.Revision != 1 || final.Config.ConfigVersion != "cfg-1" { t.Fatalf("Snapshot(after invalid) = %+v, %v", final, err) } } func runUpstreamContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) result, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-disable", Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "provider-a", Enabled: false, }) if err != nil || !result.Changed || result.Revision != 2 { t.Fatalf("SetUpstreamEnabled(disable) = %+v, %v", result, err) } replayed, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-disable-replay", Actor: contractActor(), OccurredAt: now.Add(2 * time.Second), Name: "provider-a", Enabled: false, }) if err != nil || replayed.Changed || replayed.Revision != 2 { t.Fatalf("SetUpstreamEnabled(replay) = %+v, %v", replayed, err) } if _, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-missing", Actor: contractActor(), OccurredAt: now.Add(3 * time.Second), Name: "missing", Enabled: true, }); !errors.Is(err, adminstate.ErrNotFound) { t.Fatalf("SetUpstreamEnabled(missing) error = %v", err) } snapshot, err := store.Snapshot(context.Background()) if err != nil || snapshot.Revision != 2 || upstreamEnabled(snapshot, "provider-a") { t.Fatalf("Snapshot() = %+v, %v", snapshot, err) } audits, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10}) if err != nil || len(audits) != 3 || !audits[1].Changed || audits[2].Changed { t.Fatalf("ReadAudit() = %+v, %v", audits, err) } events, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: now.Add(4 * time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(events) != 2 || events[0].Revision != 1 || events[1].Revision != 2 { t.Fatalf("Claim() = %+v, %v", events, err) } } func runRoutingContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) if _, err := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{ RequestID: "req-bad-target", Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "checkout", ExpectedCurrent: "provider-a", Target: "provider-c", }); !errors.Is(err, adminstate.ErrInvalidCommand) { t.Fatalf("SwitchRouting(bad target) error = %v", err) } start := make(chan struct{}) var changed atomic.Int64 var conflicts atomic.Int64 var unexpectedMu sync.Mutex var unexpected []error var workers sync.WaitGroup for index := range 100 { workers.Add(1) go func(index int) { defer workers.Done() <-start result, err := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{ RequestID: fmt.Sprintf("req-switch-%03d", index), Actor: contractActor(), OccurredAt: now.Add(2 * time.Second), Name: "checkout", ExpectedCurrent: "provider-a", Target: "provider-b", Reason: "capacity", }) switch { case err == nil && result.Changed: changed.Add(1) case errors.Is(err, adminstate.ErrConflict): conflicts.Add(1) default: unexpectedMu.Lock() unexpected = append(unexpected, err) unexpectedMu.Unlock() } }(index) } close(start) workers.Wait() if changed.Load() != 1 || conflicts.Load() != 99 || len(unexpected) != 0 { t.Fatalf("concurrent switch changed=%d conflicts=%d unexpected=%v", changed.Load(), conflicts.Load(), unexpected) } snapshot, err := store.Snapshot(context.Background()) if err != nil || snapshot.Revision != 2 || snapshot.Routings[0].CurrentUpstream != "provider-b" { t.Fatalf("Snapshot() = %+v, %v", snapshot, err) } } func runRoutingNoOpContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) result, err := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{ RequestID: "req-switch-noop", Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "checkout", ExpectedCurrent: "provider-a", Target: "provider-a", Reason: "already selected", }) if err != nil || result.Changed || result.Revision != 1 { t.Fatalf("SwitchRouting(no-op) = %+v, %v", result, err) } audits, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10}) if err != nil || len(audits) != 2 || audits[1].Action != adminstate.ActionSwitchRoute || audits[1].Changed || audits[1].Revision != 1 || audits[1].Reason != "already selected" { t.Fatalf("ReadAudit(no-op) = %+v, %v", audits, err) } events, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: now.Add(2 * time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(events) != 1 || events[0].Type != "config.committed" { t.Fatalf("Claim(after no-op) = %+v, %v", events, err) } } func runRoutingDisableContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) start := make(chan struct{}) var changed atomic.Int64 var noops atomic.Int64 var unexpectedMu sync.Mutex var unexpected []error var workers sync.WaitGroup for index := range 100 { workers.Add(1) go func(index int) { defer workers.Done() <-start result, err := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{ RequestID: fmt.Sprintf("req-stop-%03d", index), Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results", }) switch { case err == nil && result.Changed: changed.Add(1) case err == nil && !result.Changed && result.Revision == 2: noops.Add(1) default: unexpectedMu.Lock() unexpected = append(unexpected, err) unexpectedMu.Unlock() } }(index) } close(start) workers.Wait() if changed.Load() != 1 || noops.Load() != 99 || len(unexpected) != 0 { t.Fatalf("concurrent DisableRouting changed=%d noops=%d unexpected=%v", changed.Load(), noops.Load(), unexpected) } snapshot, err := store.Snapshot(context.Background()) if err != nil || snapshot.Revision != 2 || routingEnabled(snapshot, "checkout") || snapshot.Routings[0].CurrentUpstream != "provider-a" { t.Fatalf("Snapshot(after DisableRouting) = %+v, %v", snapshot, err) } audits, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 200}) if err != nil || len(audits) != 101 { t.Fatalf("ReadAudit(after DisableRouting) = %d records, %v", len(audits), err) } var changedStops int for _, audit := range audits { if audit.Action == adminstate.ActionDisableRoute && audit.Changed { changedStops++ if audit.ResourceType != "routing" || audit.ResourceName != "checkout" || audit.Revision != 2 || audit.Reason != "terminal empty results" { t.Fatalf("changed disable audit = %+v", audit) } } } if changedStops != 1 { t.Fatalf("changed disable audits = %d, want 1", changedStops) } events, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: now.Add(2 * time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(events) != 2 || events[1].Type != "routing.disabled" || events[1].AggregateType != "routing" || events[1].AggregateID != "checkout" || events[1].Revision != 2 { t.Fatalf("Claim(after DisableRouting) = %+v, %v", events, err) } before := len(audits) if _, err := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{ RequestID: "req-stale-stop", Actor: contractActor(), OccurredAt: now.Add(3 * time.Second), Name: "checkout", ExpectedCurrent: "provider-b", }); !errors.Is(err, adminstate.ErrConflict) { t.Fatalf("DisableRouting(stale) error = %v, want ErrConflict", err) } if _, err := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{ RequestID: "req-missing-stop", Actor: contractActor(), OccurredAt: now.Add(3 * time.Second), Name: "missing", ExpectedCurrent: "provider-a", }); !errors.Is(err, adminstate.ErrNotFound) { t.Fatalf("DisableRouting(missing) error = %v, want ErrNotFound", err) } audits, err = store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 200}) if err != nil || len(audits) != before { t.Fatalf("ReadAudit(after rejected DisableRouting) = %d records, %v; want %d", len(audits), err, before) } } func runAuditContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) _, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-disable", Actor: adminstate.Actor{ID: "admin-b", SourceIP: "::ffff:192.0.2.11"}, OccurredAt: now.Add(time.Second), Name: "provider-a", Enabled: false, }) if err != nil { t.Fatalf("SetUpstreamEnabled(): %v", err) } first, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 1}) if err != nil || len(first) != 1 { t.Fatalf("ReadAudit(first page) = %+v, %v", first, err) } if first[0].ID == 0 || first[0].RequestID != "req-config" || first[0].Actor != contractActor() || first[0].Action != adminstate.ActionCommitConfig || first[0].ResourceType != "config" || first[0].ResourceName != "cfg-1" || !first[0].Changed || first[0].Revision != 1 || !first[0].OccurredAt.Equal(now) { t.Fatalf("first audit record = %+v", first[0]) } second, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{AfterID: first[0].ID, Limit: 1}) if err != nil || len(second) != 1 || second[0].ID <= first[0].ID || second[0].RequestID != "req-disable" || second[0].Actor.ID != "admin-b" || second[0].Actor.SourceIP != "192.0.2.11" || second[0].Action != adminstate.ActionSetUpstream || second[0].ResourceType != "upstream" || second[0].ResourceName != "provider-a" || !second[0].Changed || second[0].Revision != 2 || !second[0].OccurredAt.Equal(now.Add(time.Second)) { t.Fatalf("ReadAudit(second page) = %+v, %v", second, err) } empty, err := store.ReadAudit(context.Background(), adminstate.AuditQuery{AfterID: second[0].ID, Limit: 1}) if err != nil || len(empty) != 0 { t.Fatalf("ReadAudit(after end) = %+v, %v", empty, err) } } func runOutboxContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) result, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-disable", Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "provider-a", Enabled: false, }) if err != nil || !result.Changed { t.Fatalf("SetUpstreamEnabled() = %+v, %v", result, err) } first, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: now.Add(2 * time.Second), Limit: 1, Lease: time.Minute, }) if err != nil || len(first) != 1 || first[0].ID != 1 { t.Fatalf("Claim(first) = %+v, %v", first, err) } first[0].Payload[0] = 'X' second, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-b", Now: now.Add(2 * time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(second) != 1 || second[0].ID != 2 { t.Fatalf("Claim(second) = %+v, %v", second, err) } if err := store.Acknowledge(context.Background(), adminstate.AcknowledgeCommand{ ConsumerID: "publisher-b", Now: now.Add(3 * time.Second), EventIDs: []uint64{1}, }); !errors.Is(err, adminstate.ErrConflict) { t.Fatalf("Acknowledge(wrong owner) error = %v", err) } reclaimed, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-b", Now: now.Add(2*time.Minute + time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(reclaimed) != 2 || reclaimed[0].ID != 1 || reclaimed[1].ID != 2 || reclaimed[0].Payload[0] == 'X' { t.Fatalf("Claim(reclaimed) = %+v, %v", reclaimed, err) } if err := store.Acknowledge(context.Background(), adminstate.AcknowledgeCommand{ ConsumerID: "publisher-b", Now: now.Add(2*time.Minute + 2*time.Second), EventIDs: []uint64{1, 2}, }); err != nil { t.Fatalf("Acknowledge(valid): %v", err) } empty, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-c", Now: now.Add(4 * time.Minute), Limit: 10, Lease: time.Minute, }) if err != nil || len(empty) != 0 { t.Fatalf("Claim(after ACK) = %+v, %v", empty, err) } } func runAtomicAcknowledgeContract(t *testing.T, store adminstate.Store) { t.Helper() now := contractNow() commit(t, store, configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes))) if _, err := store.SetUpstreamEnabled(context.Background(), adminstate.SetUpstreamCommand{ RequestID: "req-disable", Actor: contractActor(), OccurredAt: now.Add(time.Second), Name: "provider-a", Enabled: false, }); err != nil { t.Fatalf("SetUpstreamEnabled(): %v", err) } events, err := store.Claim(context.Background(), adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: now.Add(2 * time.Second), Limit: 10, Lease: time.Minute, }) if err != nil || len(events) != 2 { t.Fatalf("Claim() = %+v, %v", events, err) } if err := store.Acknowledge(context.Background(), adminstate.AcknowledgeCommand{ ConsumerID: "publisher-a", Now: now.Add(3 * time.Second), EventIDs: []uint64{events[0].ID, events[1].ID + 1000}, }); !errors.Is(err, adminstate.ErrNotFound) { t.Fatalf("Acknowledge(partially invalid) error = %v", err) } for _, event := range events { if err := store.Acknowledge(context.Background(), adminstate.AcknowledgeCommand{ ConsumerID: "publisher-a", Now: now.Add(4 * time.Second), EventIDs: []uint64{event.ID}, }); err != nil { t.Fatalf("Acknowledge(%d) after failed batch: %v", event.ID, err) } } } func runContextContract(t *testing.T, store adminstate.Store) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) cancel() command := configCommand("req-config", "cfg-1", strings.Repeat("a", adminstate.SHA256HexBytes)) if _, err := store.CommitConfig(ctx, command); !errors.Is(err, context.Canceled) { t.Fatalf("CommitConfig(canceled) error = %v", err) } if _, err := store.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{ RequestID: "req-upstream", Actor: contractActor(), OccurredAt: contractNow(), Name: "provider-a", Enabled: false, }); !errors.Is(err, context.Canceled) { t.Fatalf("SetUpstreamEnabled(canceled) error = %v", err) } if _, err := store.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{ RequestID: "req-switch", Actor: contractActor(), OccurredAt: contractNow(), Name: "checkout", ExpectedCurrent: "provider-a", Target: "provider-b", }); !errors.Is(err, context.Canceled) { t.Fatalf("SwitchRouting(canceled) error = %v", err) } if _, err := store.DisableRouting(ctx, adminstate.DisableRoutingCommand{ RequestID: "req-stop", Actor: contractActor(), OccurredAt: contractNow(), Name: "checkout", ExpectedCurrent: "provider-a", }); !errors.Is(err, context.Canceled) { t.Fatalf("DisableRouting(canceled) error = %v", err) } if _, err := store.Snapshot(ctx); !errors.Is(err, context.Canceled) { t.Fatalf("Snapshot(canceled) error = %v", err) } if _, err := store.ReadAudit(ctx, adminstate.AuditQuery{Limit: 1}); !errors.Is(err, context.Canceled) { t.Fatalf("ReadAudit(canceled) error = %v", err) } if _, err := store.Claim(ctx, adminstate.ClaimCommand{ ConsumerID: "publisher-a", Now: contractNow(), Limit: 1, Lease: time.Second, }); !errors.Is(err, context.Canceled) { t.Fatalf("Claim(canceled) error = %v", err) } if err := store.Acknowledge(ctx, adminstate.AcknowledgeCommand{ ConsumerID: "publisher-a", Now: contractNow(), EventIDs: []uint64{1}, }); !errors.Is(err, context.Canceled) { t.Fatalf("Acknowledge(canceled) error = %v", err) } } func configCommand(requestID, version, checksum string) adminstate.CommitConfigCommand { return adminstate.CommitConfigCommand{ RequestID: requestID, Actor: contractActor(), OccurredAt: contractNow(), ConfigVersion: version, Checksum: checksum, Source: "configs/proxy-pool.yaml", Upstreams: []adminstate.UpstreamDefinition{ {Name: "provider-a", Enabled: true}, {Name: "provider-b", Enabled: true}, }, Routings: []adminstate.RoutingDefinition{{ Name: "checkout", Enabled: true, Upstreams: []string{"provider-a", "provider-b"}, CurrentUpstream: "provider-a", }}, } } func contractActor() adminstate.Actor { return adminstate.Actor{ID: "admin-a", SourceIP: "192.0.2.10"} } func contractNow() time.Time { return time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC) } func commit(t *testing.T, store adminstate.Store, command adminstate.CommitConfigCommand) { t.Helper() if result, err := store.CommitConfig(context.Background(), command); err != nil || !result.Changed { t.Fatalf("CommitConfig() = %+v, %v", result, err) } } func upstreamEnabled(snapshot adminstate.Snapshot, name string) bool { for _, upstream := range snapshot.Upstreams { if upstream.Name == name { return upstream.Enabled } } return false } func routingEnabled(snapshot adminstate.Snapshot, name string) bool { for _, routing := range snapshot.Routings { if routing.Name == name { return routing.Enabled } } return false }