500 lines
23 KiB
Go
500 lines
23 KiB
Go
//go:build integration
|
|
|
|
package redisactivity
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"proxy-pool/internal/domain/activitypool"
|
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
ownershipDomain "proxy-pool/internal/domain/ownership"
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
)
|
|
|
|
func TestRedisOwnershipLifecycle(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
testProxy("proxy-a", "192.0.2.10"))
|
|
|
|
assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
|
|
if err != nil || assigned.Epoch == 0 || assigned.Version != 1 || assigned.Draining {
|
|
t.Fatalf("Assign() = %+v, %v", assigned, err)
|
|
}
|
|
assertRedisKeysHaveTTL(t, fixture,
|
|
fixture.Adapter.keys.owners,
|
|
fixture.Adapter.keys.ownerExpiry,
|
|
)
|
|
if ttl, err := fixture.Client.PTTL(context.Background(), fixture.Adapter.keys.epoch).Result(); err != nil || ttl != -1 {
|
|
t.Fatalf("PTTL(%s) = %s, %v; want persistent key", fixture.Adapter.keys.epoch, ttl, err)
|
|
}
|
|
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current != assigned {
|
|
t.Fatalf("Get(assigned) = %+v, %t, %v", current, ok, err)
|
|
}
|
|
assertRedisWorkerOwned(t, fixture, "worker-a", "proxy-a")
|
|
blocked, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
RequestID: "req-owned", ClientID: "client-a", Requested: 1,
|
|
Fulfillment: extractionDomain.Partial, Now: now.Add(3 * time.Second),
|
|
})
|
|
if err != nil || blocked.Returned != 0 {
|
|
t.Fatalf("Extract(owned) = %+v, %v", blocked, err)
|
|
}
|
|
|
|
renewed, err := fixture.Adapter.Renew(context.Background(), now.Add(30*time.Second),
|
|
"proxy-a", "worker-a", assigned.Epoch, 5*time.Minute)
|
|
if err != nil || renewed.Version != assigned.Version+1 || renewed.Epoch != assigned.Epoch ||
|
|
!renewed.ExpiresAt.Equal(now.Add(2*time.Minute)) {
|
|
t.Fatalf("Renew() = %+v, %v", renewed, err)
|
|
}
|
|
if score, err := fixture.Client.ZScore(context.Background(), fixture.Adapter.keys.workerOwned("worker-a"), "proxy-a").Result(); err != nil || int64(score) != renewed.ExpiresAt.UnixMilli() {
|
|
t.Fatalf("worker-owned score = %f, %v; want %d", score, err, renewed.ExpiresAt.UnixMilli())
|
|
}
|
|
if _, err := fixture.Adapter.Renew(context.Background(), now.Add(31*time.Second),
|
|
"proxy-a", "worker-a", assigned.Epoch+1, time.Minute); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
t.Fatalf("Renew(stale epoch) error = %v", err)
|
|
}
|
|
|
|
draining, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch)
|
|
if err != nil || !draining.Draining || draining.Version != renewed.Version+1 {
|
|
t.Fatalf("BeginDrain() = %+v, %v", draining, err)
|
|
}
|
|
assertRedisWorkerOwned(t, fixture, "worker-a")
|
|
pending, err := fixture.Adapter.PendingDrains(context.Background(), "worker-a", 1)
|
|
if err != nil || len(pending) != 1 || pending[0].ProxyID != "proxy-a" ||
|
|
pending[0].AssignmentEpoch != assigned.Epoch || pending[0].RequiredSnapshotEpoch <= assigned.Epoch {
|
|
t.Fatalf("PendingDrains() = %+v, %v", pending, err)
|
|
}
|
|
if tickets, err := fixture.Client.ZRange(context.Background(), fixture.Adapter.keys.workerDraining("worker-a"), 0, -1).Result(); err != nil || len(tickets) != 1 || tickets[0] != "proxy-a" {
|
|
t.Fatalf("worker-draining = %v, %v", tickets, err)
|
|
}
|
|
pending[0].Barrier = ownershipDomain.SnapshotBarrier{
|
|
SessionID: "session-a", Version: 1, OwnershipEpoch: pending[0].RequiredSnapshotEpoch,
|
|
Checksum: [32]byte{1},
|
|
}
|
|
if err := fixture.Adapter.BindDrainBarrier(context.Background(), pending[0]); err != nil {
|
|
t.Fatalf("BindDrainBarrier(): %v", err)
|
|
}
|
|
if bound, err := fixture.Adapter.PendingDrains(context.Background(), "worker-a", 1); err != nil || len(bound) != 1 || bound[0].Barrier != pending[0].Barrier {
|
|
t.Fatalf("PendingDrains(bound) = %+v, %v", bound, err)
|
|
}
|
|
replayed, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch)
|
|
if err != nil || replayed != draining {
|
|
t.Fatalf("BeginDrain(replay) = %+v, %v", replayed, err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 1, 0); !errors.Is(err, ownershipDomain.ErrDrainNotReady) {
|
|
t.Fatalf("AcknowledgeDrain(active) error = %v", err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 1); !errors.Is(err, ownershipDomain.ErrDrainNotReady) {
|
|
t.Fatalf("AcknowledgeDrain(reserved) error = %v", err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); err != nil {
|
|
t.Fatalf("AcknowledgeDrain(): %v", err)
|
|
}
|
|
assertRedisWorkerOwned(t, fixture, "worker-a")
|
|
assertRedisKeysHaveTTL(t, fixture,
|
|
fixture.Adapter.keys.available,
|
|
fixture.Adapter.keys.protocol("http"),
|
|
fixture.Adapter.keys.region("cn"),
|
|
fixture.Adapter.keys.carrier("ct"),
|
|
fixture.Adapter.keys.upstream("provider-a"),
|
|
)
|
|
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || ok {
|
|
t.Fatalf("Get(after ACK) = %+v, %t, %v", current, ok, err)
|
|
}
|
|
if pending, err := fixture.Adapter.PendingDrains(context.Background(), "worker-a", 1); err != nil || len(pending) != 0 {
|
|
t.Fatalf("PendingDrains(after ACK) = %+v, %v", pending, err)
|
|
}
|
|
restored, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
RequestID: "req-restored", ClientID: "client-a", Requested: 1,
|
|
Fulfillment: extractionDomain.Partial, Now: now.Add(40 * time.Second),
|
|
})
|
|
if err != nil || restored.Returned != 1 || restored.Items[0].ID != "proxy-a" {
|
|
t.Fatalf("Extract(after ACK) = %+v, %v", restored, err)
|
|
}
|
|
}
|
|
|
|
func TestRedisOwnershipRejectsInvalidDrainAndStaleAssignment(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
testProxy("proxy-a", "192.0.2.10"))
|
|
assigned, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 20*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("Assign(): %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), "proxy-a", "worker-b", time.Minute); !errors.Is(err, ownershipDomain.ErrAlreadyOwned) {
|
|
t.Fatalf("Assign(already owned) error = %v", err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch, 0, 0); !errors.Is(err, ownershipDomain.ErrNotDraining) {
|
|
t.Fatalf("AcknowledgeDrain(not draining) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.BeginDrain(context.Background(), "proxy-a", "worker-b", assigned.Epoch); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
t.Fatalf("BeginDrain(stale worker) error = %v", err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", assigned.Epoch+1, 0, 0); !errors.Is(err, ownershipDomain.ErrStaleAssignment) {
|
|
t.Fatalf("AcknowledgeDrain(stale epoch) error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRedisOwnershipExpireIsLimitedAndAllowsTakeover(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
for index := range 2 {
|
|
proxyID := fmt.Sprintf("proxy-%d", index)
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
testProxy(proxyID, fmt.Sprintf("192.0.2.%d", index+10)))
|
|
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), proxyID, "worker-a", time.Second); err != nil {
|
|
t.Fatalf("Assign(%s): %v", proxyID, err)
|
|
}
|
|
}
|
|
first, _, err := fixture.Adapter.Get(context.Background(), "proxy-0")
|
|
if err != nil {
|
|
t.Fatalf("Get(proxy-0): %v", err)
|
|
}
|
|
expired, err := fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
if err != nil || len(expired) != 1 {
|
|
t.Fatalf("Expire(first) = %+v, %v", expired, err)
|
|
}
|
|
expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
if err != nil || len(expired) != 1 {
|
|
t.Fatalf("Expire(second) = %+v, %v", expired, err)
|
|
}
|
|
expired, err = fixture.Adapter.Expire(context.Background(), now.Add(4*time.Second), 1)
|
|
if err != nil || len(expired) != 0 {
|
|
t.Fatalf("Expire(empty) = %+v, %v", expired, err)
|
|
}
|
|
takeover, err := fixture.Adapter.Assign(context.Background(), now.Add(5*time.Second), "proxy-0", "worker-b", time.Minute)
|
|
if err != nil || takeover.Epoch <= first.Epoch {
|
|
t.Fatalf("Assign(takeover) = %+v, %v; old epoch=%d", takeover, err, first.Epoch)
|
|
}
|
|
|
|
directFixture := newRedisTestFixture(t)
|
|
seedRedisAvailable(t, directFixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
|
testProxy("direct", "192.0.2.30"))
|
|
old, err := directFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "direct", "worker-a", time.Second)
|
|
if err != nil {
|
|
t.Fatalf("Assign(direct old): %v", err)
|
|
}
|
|
direct, err := directFixture.Adapter.Assign(context.Background(), now.Add(4*time.Second), "direct", "worker-b", time.Minute)
|
|
if err != nil || direct.WorkerID != "worker-b" || direct.Epoch <= old.Epoch {
|
|
t.Fatalf("Assign(direct takeover) = %+v, %v; old=%+v", direct, err, old)
|
|
}
|
|
}
|
|
|
|
func TestRedisOwnershipAndExtractionHaveOneWinner(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
for iteration := range 100 {
|
|
proxyID := fmt.Sprintf("race-%d", iteration)
|
|
candidate := testProxy(proxyID, fmt.Sprintf("198.51.100.%d", iteration+1))
|
|
candidate.State = proxyDomain.StateAvailable
|
|
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
ObservedAt: now, ConfiguredTTL: 10 * time.Minute, MaxSize: 200,
|
|
Proxies: []proxyDomain.Proxy{candidate},
|
|
}); err != nil {
|
|
t.Fatalf("iteration %d UpsertFetched(): %v", iteration, err)
|
|
}
|
|
|
|
var workers sync.WaitGroup
|
|
ownershipWon := make(chan bool, 1)
|
|
extractionWon := make(chan bool, 1)
|
|
errorsCh := make(chan error, 2)
|
|
workers.Add(2)
|
|
go func() {
|
|
defer workers.Done()
|
|
_, err := fixture.Adapter.Assign(context.Background(), now.Add(time.Second), proxyID, "worker-a", time.Minute)
|
|
if err != nil && !errors.Is(err, ownershipDomain.ErrOwnershipUnavailable) {
|
|
errorsCh <- err
|
|
}
|
|
ownershipWon <- err == nil
|
|
}()
|
|
go func() {
|
|
defer workers.Done()
|
|
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
RequestID: fmt.Sprintf("req-race-%d", iteration), ClientID: "client-a", Requested: 1,
|
|
Fulfillment: extractionDomain.Partial, Now: now.Add(time.Second),
|
|
Upstreams: []string{"provider-a"},
|
|
})
|
|
if err != nil {
|
|
errorsCh <- err
|
|
}
|
|
extractionWon <- err == nil && result.Returned == 1 && result.Items[0].ID == proxyID
|
|
}()
|
|
workers.Wait()
|
|
close(errorsCh)
|
|
for err := range errorsCh {
|
|
t.Fatalf("iteration %d race error: %v", iteration, err)
|
|
}
|
|
winners := 0
|
|
if <-ownershipWon {
|
|
winners++
|
|
}
|
|
if <-extractionWon {
|
|
winners++
|
|
}
|
|
if winners != 1 {
|
|
t.Fatalf("iteration %d winners = %d, want 1", iteration, winners)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRedisOwnedProxyIsNotReintroducedByHealth(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
testProxy("proxy-a", "192.0.2.10"))
|
|
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", 30*time.Second); err != nil {
|
|
t.Fatalf("Assign(): %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateSuspect,
|
|
}); err != nil {
|
|
t.Fatalf("ApplyHealth(suspect): %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
ProxyID: "proxy-a", CheckedAt: now.Add(4 * time.Second), NextState: proxyDomain.StateAvailable,
|
|
}); err != nil {
|
|
t.Fatalf("ApplyHealth(available): %v", err)
|
|
}
|
|
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
RequestID: "req-owned-health", ClientID: "client-a", Requested: 1,
|
|
Fulfillment: extractionDomain.Partial, Now: now.Add(5 * time.Second),
|
|
})
|
|
if err != nil || result.Returned != 0 {
|
|
t.Fatalf("Extract(owned after health) = %+v, %v", result, err)
|
|
}
|
|
}
|
|
|
|
func TestRedisInventoryTracksExtractionWithoutOwnershipWrites(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
for index := range 2 {
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10)))
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(2*time.Second), 2)
|
|
result, err := fixture.Adapter.Extract(context.Background(), extractionDomain.Command{
|
|
RequestID: "req-inventory", ClientID: "client-a", Requested: 1,
|
|
Fulfillment: extractionDomain.Partial, Now: now.Add(2 * time.Second),
|
|
})
|
|
if err != nil || result.Returned != 1 {
|
|
t.Fatalf("Extract() = %+v, %v", result, err)
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(3*time.Second), 1)
|
|
remainingID := "proxy-0"
|
|
if result.Items[0].ID == remainingID {
|
|
remainingID = "proxy-1"
|
|
}
|
|
if _, err := fixture.Adapter.Assign(context.Background(), now.Add(3*time.Second), remainingID, "worker-a", 10*time.Second); err != nil {
|
|
t.Fatalf("Assign(remaining): %v", err)
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(4*time.Second), 1)
|
|
}
|
|
|
|
func TestRedisInventoryPerformsBoundedExpiryCleanup(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
for _, candidate := range []struct {
|
|
proxy proxyDomain.Proxy
|
|
ttl time.Duration
|
|
}{
|
|
{proxy: testProxy("short", "192.0.2.10"), ttl: 5 * time.Second},
|
|
{proxy: testProxy("long", "192.0.2.11"), ttl: time.Minute},
|
|
} {
|
|
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
ObservedAt: now, ConfiguredTTL: candidate.ttl, MaxSize: 10,
|
|
Proxies: []proxyDomain.Proxy{candidate.proxy},
|
|
}); err != nil {
|
|
t.Fatalf("UpsertFetched(%s): %v", candidate.proxy.ID, err)
|
|
}
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 1)
|
|
if exists, err := fixture.Client.HExists(context.Background(), fixture.Adapter.keys.records, "short").Result(); err != nil || exists {
|
|
t.Fatalf("short record after Inventory cleanup = %t, %v", exists, err)
|
|
}
|
|
}
|
|
|
|
func TestRedisSweepExpiredIsLimitedAndCleansOwnership(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
for index := range 2 {
|
|
candidate := testProxy(fmt.Sprintf("proxy-%d", index), fmt.Sprintf("192.0.2.%d", index+10))
|
|
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
ObservedAt: now, ConfiguredTTL: 5 * time.Second, MaxSize: 10,
|
|
Proxies: []proxyDomain.Proxy{candidate},
|
|
}); err != nil {
|
|
t.Fatalf("UpsertFetched(proxy-%d): %v", index, err)
|
|
}
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(time.Second), 2)
|
|
removed, err := fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
if err != nil || removed != 1 {
|
|
t.Fatalf("SweepExpired(first) = %d, %v", removed, err)
|
|
}
|
|
if remaining, err := fixture.Client.ZCard(context.Background(), fixture.Adapter.keys.expiry).Result(); err != nil || remaining != 1 {
|
|
t.Fatalf("expiry index after limited sweep = %d, %v; want 1", remaining, err)
|
|
}
|
|
removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
if err != nil || removed != 1 {
|
|
t.Fatalf("SweepExpired(second) = %d, %v", removed, err)
|
|
}
|
|
assertRedisInventory(t, fixture.Adapter, "provider-a", now.Add(6*time.Second), 0)
|
|
removed, err = fixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1)
|
|
if err != nil || removed != 0 {
|
|
t.Fatalf("SweepExpired(empty) = %d, %v", removed, err)
|
|
}
|
|
|
|
ownedFixture := newRedisTestFixture(t)
|
|
seedRedisAvailable(t, ownedFixture.Adapter, "provider-a", now, now.Add(time.Second), 5*time.Second,
|
|
testProxy("owned", "192.0.2.30"))
|
|
if _, err := ownedFixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "owned", "worker-a", time.Minute); err != nil {
|
|
t.Fatalf("Assign(owned): %v", err)
|
|
}
|
|
if removed, err := ownedFixture.Adapter.SweepExpired(context.Background(), now.Add(6*time.Second), 1); err != nil || removed != 1 {
|
|
t.Fatalf("SweepExpired(owned) = %d, %v", removed, err)
|
|
}
|
|
if assignment, ok, err := ownedFixture.Adapter.Get(context.Background(), "owned"); err != nil || ok {
|
|
t.Fatalf("Get(swept owned) = %+v, %t, %v", assignment, ok, err)
|
|
}
|
|
assertRedisInventory(t, ownedFixture.Adapter, "provider-a", now.Add(6*time.Second), 0)
|
|
for _, hashKey := range []string{
|
|
ownedFixture.Adapter.keys.records,
|
|
ownedFixture.Adapter.keys.idkeys,
|
|
ownedFixture.Adapter.keys.owners,
|
|
} {
|
|
if exists, err := ownedFixture.Client.HExists(context.Background(), hashKey, "owned").Result(); err != nil || exists {
|
|
t.Fatalf("hash %s retained owned proxy = %t, %v", hashKey, exists, err)
|
|
}
|
|
}
|
|
if count, err := ownedFixture.Client.HLen(context.Background(), ownedFixture.Adapter.keys.unique).Result(); err != nil || count != 0 {
|
|
t.Fatalf("unique mappings after sweep = %d, %v", count, err)
|
|
}
|
|
for _, sortedSet := range []string{
|
|
ownedFixture.Adapter.keys.expiry,
|
|
ownedFixture.Adapter.keys.available,
|
|
ownedFixture.Adapter.keys.ownerExpiry,
|
|
ownedFixture.Adapter.keys.protocol("http"),
|
|
ownedFixture.Adapter.keys.region("cn"),
|
|
ownedFixture.Adapter.keys.carrier("ct"),
|
|
ownedFixture.Adapter.keys.upstream("provider-a"),
|
|
ownedFixture.Adapter.keys.workerOwned("worker-a"),
|
|
} {
|
|
if count, err := ownedFixture.Client.ZCard(context.Background(), sortedSet).Result(); err != nil || count != 0 {
|
|
t.Fatalf("sorted set %s entries after sweep = %d, %v", sortedSet, count, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRedisOwnershipAndMaintenanceValidateInputs(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
if _, err := fixture.Adapter.Assign(nil, now, "proxy-a", "worker-a", time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("Assign(nil context) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.Renew(context.Background(), now, "proxy-a", "worker-a", 0, time.Minute); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("Renew(zero epoch) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.BeginDrain(context.Background(), "", "worker-a", 1); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("BeginDrain(empty proxy) error = %v", err)
|
|
}
|
|
if err := fixture.Adapter.AcknowledgeDrain(context.Background(), "proxy-a", "worker-a", 1, -1, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("AcknowledgeDrain(negative active) error = %v", err)
|
|
}
|
|
if _, _, err := fixture.Adapter.Get(context.Background(), ""); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("Get(empty proxy) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.Expire(context.Background(), now, 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("Expire(zero limit) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.Inventory(context.Background(), "", now); !errors.Is(err, activitypool.ErrInvalidInventory) {
|
|
t.Fatalf("Inventory(empty upstream) error = %v", err)
|
|
}
|
|
if _, err := fixture.Adapter.SweepExpired(context.Background(), now, 0); !errors.Is(err, activitypool.ErrInvalidMaintenance) {
|
|
t.Fatalf("SweepExpired(zero limit) error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRedisReadWorkerSnapshotUsesWorkerOwnedIndex(t *testing.T) {
|
|
fixture := newRedisTestFixture(t)
|
|
now := redisTestNow()
|
|
candidate := testProxy("proxy-a", "192.0.2.10")
|
|
candidate.MaxConcurrency = 7
|
|
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, candidate)
|
|
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
|
|
if err != nil {
|
|
t.Fatalf("Assign(): %v", err)
|
|
}
|
|
proxies, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1)
|
|
if err != nil || len(proxies) != 1 {
|
|
t.Fatalf("ReadWorkerSnapshot() = %+v, %v", proxies, err)
|
|
}
|
|
if proxies[0].Proxy.ID != "proxy-a" || proxies[0].OwnershipEpoch != assignment.Epoch ||
|
|
!proxies[0].LeaseExpiresAt.Equal(assignment.ExpiresAt) {
|
|
t.Fatalf("snapshot proxy = %+v, want assignment %+v", proxies[0], assignment)
|
|
}
|
|
other, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-b", 1)
|
|
if err != nil || len(other) != 0 {
|
|
t.Fatalf("ReadWorkerSnapshot(other) = %+v, %v", other, err)
|
|
}
|
|
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
ProxyID: "proxy-a", CheckedAt: now.Add(3 * time.Second), NextState: proxyDomain.StateSuspect,
|
|
}); err != nil {
|
|
t.Fatalf("ApplyHealth(suspect): %v", err)
|
|
}
|
|
proxies, err = fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1)
|
|
if err != nil || len(proxies) != 0 {
|
|
t.Fatalf("ReadWorkerSnapshot(suspect) = %+v, %v", proxies, err)
|
|
}
|
|
if _, err := fixture.Adapter.ApplyHealth(context.Background(), activitypool.HealthUpdate{
|
|
ProxyID: "proxy-a", CheckedAt: now.Add(4 * time.Second), NextState: proxyDomain.StateAvailable,
|
|
}); err != nil {
|
|
t.Fatalf("ApplyHealth(available): %v", err)
|
|
}
|
|
proxies, err = fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 1)
|
|
if err != nil || len(proxies) != 1 {
|
|
t.Fatalf("ReadWorkerSnapshot(available) = %+v, %v", proxies, err)
|
|
}
|
|
if _, err := fixture.Adapter.ReadWorkerSnapshot(context.Background(), "worker-a", 0); !errors.Is(err, ownershipDomain.ErrInvalidOwnership) {
|
|
t.Fatalf("ReadWorkerSnapshot(zero limit) error = %v", err)
|
|
}
|
|
}
|
|
|
|
func assertRedisInventory(t *testing.T, adapter *Adapter, upstreamID string, now time.Time, want int) {
|
|
t.Helper()
|
|
inventory, err := adapter.Inventory(context.Background(), upstreamID, now)
|
|
if err != nil || inventory.UpstreamID != upstreamID || inventory.Managed != want {
|
|
t.Fatalf("Inventory(%s) = %+v, %v; want %d", upstreamID, inventory, err, want)
|
|
}
|
|
}
|
|
|
|
func assertRedisWorkerOwned(t *testing.T, fixture redisTestFixture, workerID string, want ...string) {
|
|
t.Helper()
|
|
actual, err := fixture.Client.ZRange(context.Background(), fixture.Adapter.keys.workerOwned(workerID), 0, -1).Result()
|
|
if err != nil {
|
|
t.Fatalf("ZRange(worker-owned %s): %v", workerID, err)
|
|
}
|
|
if len(actual) != len(want) {
|
|
t.Fatalf("worker-owned %s = %v, want %v", workerID, actual, want)
|
|
}
|
|
for index, proxyID := range want {
|
|
if actual[index] != proxyID {
|
|
t.Fatalf("worker-owned %s = %v, want %v", workerID, actual, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertRedisKeysHaveTTL(t *testing.T, fixture redisTestFixture, keys ...string) {
|
|
t.Helper()
|
|
for _, key := range keys {
|
|
ttl, err := fixture.Client.PTTL(context.Background(), key).Result()
|
|
if err != nil || ttl <= 0 {
|
|
t.Fatalf("PTTL(%s) = %s, %v; want positive TTL", key, ttl, err)
|
|
}
|
|
}
|
|
}
|