//go:build integration package redisactivity import ( "context" "errors" "fmt" "sync" "testing" "time" "proxy-pool/internal/domain/activitypool" extractionDomain "proxy-pool/internal/domain/extraction" ownershipDomain "proxy-pool/internal/domain/ownership" proxyDomain "proxy-pool/internal/domain/proxy" ) func TestRedisOwnershipLifecycle(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, testProxy("proxy-a", "192.0.2.10")) assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute) if err != nil || assigned.Epoch == 0 || assigned.Version != 1 || assigned.Draining { t.Fatalf("Assign() = %+v, %v", assigned, err) } assertRedisKeysHaveTTL(t, fixture, fixture.Adapter.keys.owners, fixture.Adapter.keys.ownerExpiry, ) if ttl, err := fixture.Client.PTTL(context.Background(), fixture.Adapter.keys.epoch).Result(); err != nil || ttl != -1 { t.Fatalf("PTTL(%s) = %s, %v; want persistent key", fixture.Adapter.keys.epoch, ttl, err) } if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current != assigned { t.Fatalf("Get(assigned) = %+v, %t, %v", current, ok, err) } assertRedisWorkerOwned(t, fixture, "worker-a", "proxy-a") blocked, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{ RequestID: "req-owned", ClientID: "client-a", Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second), }) if err != nil || blocked.Returned != 0 { t.Fatalf("Extract(owned) = %+v, %v", blocked, err) } renewed, err := fixture.Adapter.Renew(context.Background(), now.Add(30*time.Second), "proxy-a", "worker-a", assigned.Epoch, 5*time.Minute) if err != nil || renewed.Version != assigned.Version+1 || renewed.Epoch != assigned.Epoch || !renewed.ExpiresAt.Equal(now.Add(2*time.Minute)) { t.Fatalf("Renew() = %+v, %v", renewed, err) } if score, err := fixture.Client.ZScore(context.Background(), fixture.Adapter.keys.workerOwned("worker-a"), "proxy-a").Result(); err != nil || int64(score) != renewed.ExpiresAt.UnixMilli() { t.Fatalf("worker-owned score = %f, %v; want %d", score, err, renewed.ExpiresAt.UnixMilli()) } if _, err := fixture.Adapter.Renew(context.Background(), now.Add(31*time.Second), "proxy-a", "worker-a", assigned.Epoch+1, time.Minute); !errors.Is(err, ownershipDomain.ErrStaleAssignment) { t.Fatalf("Renew(stale epoch) error = %v", err) } draining, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch) if err != nil || !draining.Draining || draining.Version != renewed.Version+1 { t.Fatalf("BeginDrain() = %+v, %v", draining, err) } assertRedisWorkerOwned(t, fixture, "worker-a") pending, err := fixture.Adapter.PendingDrains(context.Background(), "worker-a", 1) if err != nil || len(pending) != 1 || pending[0].ProxyID != "proxy-a" || pending[0].AssignmentEpoch != assigned.Epoch || pending[0].RequiredSnapshotEpoch <= assigned.Epoch { t.Fatalf("PendingDrains() = %+v, %v", pending, err) } if tickets, err := fixture.Client.ZRange(context.Background(), fixture.Adapter.keys.workerDraining("worker-a"), 0, -1).Result(); err != nil || len(tickets) != 1 || tickets[0] != "proxy-a" { t.Fatalf("worker-draining = %v, %v", tickets, err) } replayed, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch) if err != nil || replayed != draining { t.Fatalf("BeginDrain(replay) = %+v, %v", replayed, err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 1, 0); !errors.Is(err, ownershipDomain.ErrDrainNotReady) { t.Fatalf("AcknowledgeDrain(active) error = %v", err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 1); !errors.Is(err, ownershipDomain.ErrDrainNotReady) { t.Fatalf("AcknowledgeDrain(reserved) error = %v", err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); err != nil { t.Fatalf("AcknowledgeDrain(): %v", err) } assertRedisWorkerOwned(t, fixture, "worker-a") assertRedisKeysHaveTTL(t, fixture, fixture.Adapter.keys.available, fixture.Adapter.keys.protocol("http"), fixture.Adapter.keys.region("cn"), fixture.Adapter.keys.carrier("ct"), fixture.Adapter.keys.upstream("provider-a"), ) if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || ok { t.Fatalf("Get(after ACK) = %+v, %t, %v", current, ok, err) } if pending, err := fixture.Adapter.PendingDrains(context.Background(), "worker-a", 1); err != nil || len(pending) != 0 { t.Fatalf("PendingDrains(after ACK) = %+v, %v", pending, err) } restored, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{ RequestID: "req-restored", ClientID: "client-a", Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(40 * time.Second), }) if err != nil || restored.Returned != 1 || restored.Items[0].ID != "proxy-a" { t.Fatalf("Extract(after ACK) = %+v, %v", restored, err) } } func TestRedisOwnershipRejectsInvalidDrainAndStaleAssignment(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute, testProxy("proxy-a", "192.0.2.10")) assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 20*time.Second) if err != nil { t.Fatalf("Assign(): %v", err) } if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), "proxy-a", "worker-b", time.Minute); !errors.Is(err, ownershipDomain.ErrAlreadyOwned) { t.Fatalf("Assign(already owned) error = %v", err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); !errors.Is(err, ownershipDomain.ErrNotDraining) { t.Fatalf("AcknowledgeDrain(not draining) error = %v", err) } if _, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-b", assigned.Epoch); !errors.Is(err, ownershipDomain.ErrStaleAssignment) { t.Fatalf("BeginDrain(stale worker) error = %v", err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch+1, 0, 0); !errors.Is(err, ownershipDomain.ErrStaleAssignment) { t.Fatalf("AcknowledgeDrain(stale epoch) error = %v", err) } } func TestRedisOwnershipExpireIsLimitedAndAllowsTakeover(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() for index := range 2 { proxyID := fmt.Sprintf("proxy-%d", index) seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, testProxy(proxyID, fmt.Sprintf("192.0.2.%d", index+10))) if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), proxyID, "worker-a", time.Second); err != nil { t.Fatalf("Assign(%s): %v", proxyID, err) } } first, _, err := fixture.Adapter.Get(context.Background(), "proxy-0") if err != nil { t.Fatalf("Get(proxy-0): %v", err) } expired, err := fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1) if err != nil || len(expired) != 1 { t.Fatalf("Expire(first) = %+v, %v", expired, err) } expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1) if err != nil || len(expired) != 1 { t.Fatalf("Expire(second) = %+v, %v", expired, err) } expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1) if err != nil || len(expired) != 0 { t.Fatalf("Expire(empty) = %+v, %v", expired, err) } takeover, err := fixture.Adapter.Assign(context.Background(), now.Add(5*time.Second), "proxy-0", "worker-b", time.Minute) if err != nil || takeover.Epoch <= first.Epoch { t.Fatalf("Assign(takeover) = %+v, %v; old epoch=%d", takeover, err, first.Epoch) } directFixture := newRedisTestFixture(t) seedRedisAvailable(t, directFixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, testProxy("direct", "192.0.2.30")) old, err := directFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "direct", "worker-a", time.Second) if err != nil { t.Fatalf("Assign(direct old): %v", err) } direct, err := directFixture.Adapter.Assign(context.Background(), now.Add(4*time.Second), "direct", "worker-b", time.Minute) if err != nil || direct.WorkerID != "worker-b" || direct.Epoch <= old.Epoch { t.Fatalf("Assign(direct takeover) = %+v, %v; old=%+v", direct, err, old) } } func TestRedisOwnershipAndExtractionHaveOneWinner(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() for iteration := range 100 { proxyID := fmt.Sprintf("race-%d", iteration) candidate := testProxy(proxyID, fmt.Sprintf("198.51.100.%d", iteration+1)) candidate.State = proxyDomain.StateAvailable if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 200, Proxies: []proxyDomain.Proxy{candidate}, }); err != nil { t.Fatalf("iteration %d UpsertFetched(): %v", iteration, err) } var workers sync.WaitGroup ownershipWon := make(chan bool, 1) extractionWon := make(chan bool, 1) errorsCh := make(chan error, 2) workers.Add(2) go func() { defer workers.Done() _, err := fixture.Adapter.Assign(context.Background(), now.Add(time.Second), proxyID, "worker-a", time.Minute) if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) { errorsCh <- err } ownershipWon <- err == nil }() go func() { defer workers.Done() result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{ RequestID: fmt.Sprintf("req-race-%d", iteration), ClientID: "client-a", Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second), Upstreams: []string{"provider-a"}, }) if err != nil { errorsCh <- err } extractionWon <- err == nil && result.Returned == 1 && result.Items[0].ID == proxyID }() workers.Wait() close(errorsCh) for err := range errorsCh { t.Fatalf("iteration %d race error: %v", iteration, err) } winners := 0 if <-ownershipWon { winners++ } if <-extractionWon { winners++ } if winners != 1 { t.Fatalf("iteration %d winners = %d, want 1", iteration, winners) } } } func TestRedisOwnedProxyIsNotReintroducedByHealth(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute, testProxy("proxy-a", "192.0.2.10")) if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 30*time.Second); err != nil { t.Fatalf("Assign(): %v", err) } if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{ ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateSuspect, }); err != nil { t.Fatalf("ApplyHealth(suspect): %v", err) } if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{ ProxyID: "proxy-a", CheckedAt: now.Add(4 * time.Second), NextState: proxyDomain.StateAvailable, }); err != nil { t.Fatalf("ApplyHealth(available): %v", err) } result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{ RequestID: "req-owned-health", ClientID: "client-a", Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(5 * time.Second), }) if err != nil || result.Returned != 0 { t.Fatalf("Extract(owned after health) = %+v, %v", result, err) } } func TestRedisInventoryTracksExtractionWithoutOwnershipWrites(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() for index := range 2 { seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute, testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10))) } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(2*time.Second), 2) result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{ RequestID: "req-inventory", ClientID: "client-a", Requested: 1, Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second), }) if err != nil || result.Returned != 1 { t.Fatalf("Extract() = %+v, %v", result, err) } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(3*time.Second), 1) remainingID := "proxy-0" if result.Items[0].ID == remainingID { remainingID = "proxy-1" } if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), remainingID, "worker-a", 10*time.Second); err != nil { t.Fatalf("Assign(remaining): %v", err) } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(4*time.Second), 1) } func TestRedisInventoryPerformsBoundedExpiryCleanup(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() for _, candidate := range []struct { proxy proxyDomain.Proxy ttl time.Duration }{ {proxy: testProxy("short", "192.0.2.10"), ttl: 5 * time.Second}, {proxy: testProxy("long", "192.0.2.11"), ttl: time.Minute}, } { if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: candidate.ttl, MaxSize: 10, Proxies: []proxyDomain.Proxy{candidate.proxy}, }); err != nil { t.Fatalf("UpsertFetched(%s): %v", candidate.proxy.ID, err) } } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 1) if exists, err := fixture.Client.HExists(context.Background(), fixture.Adapter.keys.records, "short").Result(); err != nil || exists { t.Fatalf("short record after Inventory cleanup = %t, %v", exists, err) } } func TestRedisSweepExpiredIsLimitedAndCleansOwnership(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() for index := range 2 { candidate := testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10)) if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: 5 * time.Second, MaxSize: 10, Proxies: []proxyDomain.Proxy{candidate}, }); err != nil { t.Fatalf("UpsertFetched(proxy-%d): %v", index, err) } } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(time.Second), 2) removed, err := fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1) if err != nil || removed != 1 { t.Fatalf("SweepExpired(first) = %d, %v", removed, err) } if remaining, err := fixture.Client.ZCard(context.Background(), fixture.Adapter.keys.expiry).Result(); err != nil || remaining != 1 { t.Fatalf("expiry index after limited sweep = %d, %v; want 1", remaining, err) } removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1) if err != nil || removed != 1 { t.Fatalf("SweepExpired(second) = %d, %v", removed, err) } assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 0) removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1) if err != nil || removed != 0 { t.Fatalf("SweepExpired(empty) = %d, %v", removed, err) } ownedFixture := newRedisTestFixture(t) seedRedisAvailable(t, ownedFixture.Adapter, "provider-a", now, now.Add(time.Second), 5*time.Second, testProxy("owned", "192.0.2.30")) if _, err := ownedFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "owned", "worker-a", time.Minute); err != nil { t.Fatalf("Assign(owned): %v", err) } if removed, err := ownedFixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 1 { t.Fatalf("SweepExpired(owned) = %d, %v", removed, err) } if assignment, ok, err := ownedFixture.Adapter.Get(context.Background(), "owned"); err != nil || ok { t.Fatalf("Get(swept owned) = %+v, %t, %v", assignment, ok, err) } assertRedisInventory(t, ownedFixture.Adapter, "provider-a", now.Add(6*time.Second), 0) for _, hashKey := range []string{ ownedFixture.Adapter.keys.records, ownedFixture.Adapter.keys.idkeys, ownedFixture.Adapter.keys.owners, } { if exists, err := ownedFixture.Client.HExists(context.Background(), hashKey, "owned").Result(); err != nil || exists { t.Fatalf("hash %s retained owned proxy = %t, %v", hashKey, exists, err) } } if count, err := ownedFixture.Client.HLen(context.Background(), ownedFixture.Adapter.keys.unique).Result(); err != nil || count != 0 { t.Fatalf("unique mappings after sweep = %d, %v", count, err) } for _, sortedSet := range []string{ ownedFixture.Adapter.keys.expiry, ownedFixture.Adapter.keys.available, ownedFixture.Adapter.keys.ownerExpiry, ownedFixture.Adapter.keys.protocol("http"), ownedFixture.Adapter.keys.region("cn"), ownedFixture.Adapter.keys.carrier("ct"), ownedFixture.Adapter.keys.upstream("provider-a"), ownedFixture.Adapter.keys.workerOwned("worker-a"), } { if count, err := ownedFixture.Client.ZCard(context.Background(), sortedSet).Result(); err != nil || count != 0 { t.Fatalf("sorted set %s entries after sweep = %d, %v", sortedSet, count, err) } } } func TestRedisOwnershipAndMaintenanceValidateInputs(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() if _, err := fixture.Adapter.Assign(nil, now, "proxy-a", "worker-a", time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("Assign(nil context) error = %v", err) } if _, err := fixture.Adapter.Renew(context.Background(), now, "proxy-a", "worker-a", 0, time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("Renew(zero epoch) error = %v", err) } if _, err := fixture.Adapter.BeginDrain(context.Background(), "", "worker-a", 1); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("BeginDrain(empty proxy) error = %v", err) } if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", 1, -1, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("AcknowledgeDrain(negative active) error = %v", err) } if _, _, err := fixture.Adapter.Get(context.Background(), ""); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("Get(empty proxy) error = %v", err) } if _, err := fixture.Adapter.Expire(context.Background(), now, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("Expire(zero limit) error = %v", err) } if _, err := fixture.Adapter.Inventory(context.Background(), "", now); !errors.Is(err, activitypool.ErrInvalidInventory) { t.Fatalf("Inventory(empty upstream) error = %v", err) } if _, err := fixture.Adapter.SweepExpired(context.Background(), now, 0); !errors.Is(err, activitypool.ErrInvalidMaintenance) { t.Fatalf("SweepExpired(zero limit) error = %v", err) } } func TestRedisReadWorkerSnapshotUsesWorkerOwnedIndex(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() candidate := testProxy("proxy-a", "192.0.2.10") candidate.MaxConcurrency = 7 seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, candidate) assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute) if err != nil { t.Fatalf("Assign(): %v", err) } proxies, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1) if err != nil || len(proxies) != 1 { t.Fatalf("ReadWorkerSnapshot() = %+v, %v", proxies, err) } if proxies[0].Proxy.ID != "proxy-a" || proxies[0].OwnershipEpoch != assignment.Epoch || !proxies[0].LeaseExpiresAt.Equal(assignment.ExpiresAt) { t.Fatalf("snapshot proxy = %+v, want assignment %+v", proxies[0], assignment) } other, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-b", 1) if err != nil || len(other) != 0 { t.Fatalf("ReadWorkerSnapshot(other) = %+v, %v", other, err) } if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{ ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateSuspect, }); err != nil { t.Fatalf("ApplyHealth(suspect): %v", err) } proxies, err = fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1) if err != nil || len(proxies) != 0 { t.Fatalf("ReadWorkerSnapshot(suspect) = %+v, %v", proxies, err) } if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{ ProxyID: "proxy-a", CheckedAt: now.Add(4 * time.Second), NextState: proxyDomain.StateAvailable, }); err != nil { t.Fatalf("ApplyHealth(available): %v", err) } proxies, err = fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1) if err != nil || len(proxies) != 1 { t.Fatalf("ReadWorkerSnapshot(available) = %+v, %v", proxies, err) } if _, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) { t.Fatalf("ReadWorkerSnapshot(zero limit) error = %v", err) } } func assertRedisInventory(t *testing.T, adapter *Adapter, upstreamID string, now time.Time, want int) { t.Helper() inventory, err := adapter.Inventory(context.Background(), upstreamID, now) if err != nil || inventory.UpstreamID != upstreamID || inventory.Managed != want { t.Fatalf("Inventory(%s) = %+v, %v; want %d", upstreamID, inventory, err, want) } } func assertRedisWorkerOwned(t *testing.T, fixture redisTestFixture, workerID string, want ...string) { t.Helper() actual, err := fixture.Client.ZRange(context.Background(), fixture.Adapter.keys.workerOwned(workerID), 0, -1).Result() if err != nil { t.Fatalf("ZRange(worker-owned %s): %v", workerID, err) } if len(actual) != len(want) { t.Fatalf("worker-owned %s = %v, want %v", workerID, actual, want) } for index, proxyID := range want { if actual[index] != proxyID { t.Fatalf("worker-owned %s = %v, want %v", workerID, actual, want) } } } func assertRedisKeysHaveTTL(t *testing.T, fixture redisTestFixture, keys ...string) { t.Helper() for _, key := range keys { ttl, err := fixture.Client.PTTL(context.Background(), key).Result() if err != nil || ttl <= 0 { t.Fatalf("PTTL(%s) = %s, %v; want positive TTL", key, ttl, err) } } }