//go:build integration package redisactivity import ( "context" "errors" "testing" "time" controllerHealth "proxy-pool/internal/controller/health" "proxy-pool/internal/domain/activitypool" healthDomain "proxy-pool/internal/domain/health" proxyDomain "proxy-pool/internal/domain/proxy" ) func TestRedisHealthTasksLeaseAndRescheduleBasicChecks(t *testing.T) { fixture := newRedisTestFixture(t) ctx := context.Background() now := time.Now().UTC() if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1, Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")}, }); err != nil { t.Fatalf("UpsertFetched(): %v", err) } candidates, err := fixture.Adapter.DueCandidates(ctx, now, 1) if err != nil || len(candidates) != 1 || candidates[0].ProxyID != "proxy-a" || candidates[0].State != proxyDomain.StateFetched { t.Fatalf("DueCandidates() = (%+v, %v)", candidates, err) } planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{ Interval: 30 * time.Second, MaxInFlight: 1, Timeout: 5 * time.Second, MaxAttempts: 1, }) if err != nil { t.Fatalf("newHealthTaskPlanner(): %v", err) } plannedAt := now plans, err := planner.Plan(plannedAt, 0, 1, candidates) if err != nil || len(plans) != 1 { t.Fatalf("Plan() = (%+v, %v)", plans, err) } if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 { t.Fatalf("Offer() = (%d, %v)", offered, err) } claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{ CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic}, }) if err != nil || len(claimed) != 1 || claimed[0].ProxyID != "proxy-a" || claimed[0].LeaseToken == "" { t.Fatalf("Claim() = (%+v, %v)", claimed, err) } observation := healthDomain.Observation{ TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelBasic, Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(), } if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("AuthorizeObservation(): %v", err) } updated, err := fixture.Adapter.ApplyGlobalObservation(ctx, activitypool.GlobalHealthCommand{ Observation: observation, MaxConsecutiveFailures: 2, }) if err != nil || updated.State != proxyDomain.StateAvailable { t.Fatalf("ApplyGlobalObservation() = (%+v, %v)", updated, err) } if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("CompleteObservation(): %v", err) } next, err := fixture.Adapter.DueCandidates(ctx, plans[0].NextDue, 1) if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].State != proxyDomain.StateAvailable { t.Fatalf("DueCandidates(next) = (%+v, %v)", next, err) } } func TestRedisHealthTaskSweepReleasesLeaseAndUpstreamCapacity(t *testing.T) { fixture := newRedisTestFixture(t) ctx := context.Background() now := time.Now().UTC() if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: time.Second, MaxSize: 1, Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")}, }); err != nil { t.Fatalf("UpsertFetched(): %v", err) } candidates, err := fixture.Adapter.DueCandidatesForUpstream(ctx, "provider-a", now, 1) if err != nil || len(candidates) != 1 { t.Fatalf("DueCandidatesForUpstream() = (%+v, %v)", candidates, err) } planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{ Interval: 30 * time.Second, MaxInFlight: 1, Timeout: 10 * time.Second, MaxAttempts: 1, }) if err != nil { t.Fatalf("NewPlanner(): %v", err) } plans, err := planner.Plan(now, 0, 1, candidates) if err != nil || len(plans) != 1 { t.Fatalf("Plan() = (%+v, %v)", plans, err) } if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 { t.Fatalf("Offer() = (%d, %v)", offered, err) } claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{ CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic}, }) if err != nil || len(claimed) != 1 { t.Fatalf("Claim() = (%+v, %v)", claimed, err) } expiredAt := now.Add(2 * time.Second) if removed, err := fixture.Adapter.SweepExpired(ctx, expiredAt, 1); err != nil || removed != 1 { t.Fatalf("SweepExpired() = (%d, %v)", removed, err) } if inFlight, err := fixture.Adapter.InFlightForUpstream(ctx, "provider-a", expiredAt); err != nil || inFlight != 0 { t.Fatalf("InFlightForUpstream() = (%d, %v)", inFlight, err) } observation := healthDomain.Observation{ TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelBasic, Success: true, Latency: time.Millisecond, ObservedAt: now, } if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, expiredAt); !errors.Is(err, healthDomain.ErrTaskNotFound) { t.Fatalf("AuthorizeObservation(after sweep) error = %v, want ErrTaskNotFound", err) } } func TestRedisHealthTasksLeaseAndRescheduleEgressChecks(t *testing.T) { fixture := newRedisTestFixture(t) ctx := context.Background() now := time.Now().UTC() const targetURL = "https://egress.example/identity" if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1, Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")}, }); err != nil { t.Fatalf("UpsertFetched(): %v", err) } candidates, err := fixture.Adapter.DueEgressCandidatesForUpstream(ctx, "provider-a", targetURL, now, 1) if err != nil || len(candidates) != 1 || candidates[0].Level != healthDomain.LevelEgress || candidates[0].TargetURL != targetURL { t.Fatalf("DueEgressCandidatesForUpstream() = (%+v, %v)", candidates, err) } planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{ Interval: 30 * time.Second, MaxInFlight: 2, Timeout: 5 * time.Second, MaxAttempts: 1, }) if err != nil { t.Fatalf("NewPlanner(): %v", err) } plans, err := planner.Plan(now, 0, 1, candidates) if err != nil || len(plans) != 1 { t.Fatalf("Plan() = (%+v, %v)", plans, err) } if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 { t.Fatalf("Offer() = (%d, %v)", offered, err) } claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{ CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelEgress}, }) if err != nil || len(claimed) != 1 || claimed[0].Level != healthDomain.LevelEgress || claimed[0].TargetURL != targetURL { t.Fatalf("Claim() = (%+v, %v)", claimed, err) } observation := healthDomain.Observation{ TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelEgress, Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(), } if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("AuthorizeObservation() = %v", err) } if _, err := fixture.Adapter.ApplyGlobalObservation(ctx, activitypool.GlobalHealthCommand{ Observation: observation, MaxConsecutiveFailures: 2, }); err != nil { t.Fatalf("ApplyGlobalObservation() = %v", err) } if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("CompleteObservation() = %v", err) } next, err := fixture.Adapter.DueEgressCandidatesForUpstream(ctx, "provider-a", targetURL, plans[0].NextDue, 1) if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].TargetURL != targetURL { t.Fatalf("DueEgressCandidatesForUpstream(next) = (%+v, %v)", next, err) } } func TestRedisHealthTasksLeaseAndRescheduleTargetChecks(t *testing.T) { fixture := newRedisTestFixture(t) ctx := context.Background() now := time.Now().UTC() const routingName = "checkout" const targetURL = "https://checkout.example/health" if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{ ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1, Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")}, }); err != nil { t.Fatalf("UpsertFetched(): %v", err) } candidates, err := fixture.Adapter.DueTargetCandidatesForUpstream(ctx, "provider-a", routingName, targetURL, now, 1) if err != nil || len(candidates) != 1 || candidates[0].Level != healthDomain.LevelTarget || candidates[0].RoutingName != routingName || candidates[0].TargetURL != targetURL { t.Fatalf("DueTargetCandidatesForUpstream() = (%+v, %v)", candidates, err) } planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{ Interval: 30 * time.Second, MaxInFlight: 2, Timeout: 5 * time.Second, MaxAttempts: 1, }) if err != nil { t.Fatalf("NewPlanner(): %v", err) } plans, err := planner.Plan(now, 0, 1, candidates) if err != nil || len(plans) != 1 { t.Fatalf("Plan() = (%+v, %v)", plans, err) } if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 { t.Fatalf("Offer() = (%d, %v)", offered, err) } claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{ CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelTarget}, }) if err != nil || len(claimed) != 1 || claimed[0].Level != healthDomain.LevelTarget || claimed[0].RoutingName != routingName || claimed[0].TargetURL != targetURL { t.Fatalf("Claim() = (%+v, %v)", claimed, err) } observation := healthDomain.Observation{ TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelTarget, RoutingName: routingName, TargetURL: targetURL, Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(), } if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("AuthorizeObservation() = %v", err) } state, err := fixture.Adapter.ApplyTargetObservation(ctx, activitypool.TargetHealthCommand{ Observation: observation, MaxConsecutiveFailures: 2, }) if err != nil || state.Status != healthDomain.TargetAvailable { t.Fatalf("ApplyTargetObservation() = (%+v, %v)", state, err) } if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil { t.Fatalf("CompleteObservation() = %v", err) } next, err := fixture.Adapter.DueTargetCandidatesForUpstream(ctx, "provider-a", routingName, targetURL, plans[0].NextDue, 1) if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].RoutingName != routingName || next[0].TargetURL != targetURL { t.Fatalf("DueTargetCandidatesForUpstream(next) = (%+v, %v)", next, err) } }