proxy-pool/internal/adapters/redisactivity/health_tasks_integration_test.go

//go:build integration

package redisactivity

import (
	"context"
	"errors"
	"testing"
	"time"

	controllerHealth "proxy-pool/internal/controller/health"
	"proxy-pool/internal/domain/activitypool"
	healthDomain "proxy-pool/internal/domain/health"
	proxyDomain "proxy-pool/internal/domain/proxy"
)

func TestRedisHealthTasksLeaseAndRescheduleBasicChecks(t *testing.T) {
	fixture := newRedisTestFixture(t)
	ctx := context.Background()
	now := time.Now().UTC()
	if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{
		ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
		Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
	}); err != nil {
		t.Fatalf("UpsertFetched(): %v", err)
	}

	candidates, err := fixture.Adapter.DueCandidates(ctx, now, 1)
	if err != nil || len(candidates) != 1 || candidates[0].ProxyID != "proxy-a" || candidates[0].State != proxyDomain.StateFetched {
		t.Fatalf("DueCandidates() = (%+v, %v)", candidates, err)
	}
	planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{
		Interval: 30 * time.Second, MaxInFlight: 1, Timeout: 5 * time.Second, MaxAttempts: 1,
	})
	if err != nil {
		t.Fatalf("newHealthTaskPlanner(): %v", err)
	}
	plannedAt := now
	plans, err := planner.Plan(plannedAt, 0, 1, candidates)
	if err != nil || len(plans) != 1 {
		t.Fatalf("Plan() = (%+v, %v)", plans, err)
	}
	if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 {
		t.Fatalf("Offer() = (%d, %v)", offered, err)
	}

	claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{
		CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
		SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic},
	})
	if err != nil || len(claimed) != 1 || claimed[0].ProxyID != "proxy-a" || claimed[0].LeaseToken == "" {
		t.Fatalf("Claim() = (%+v, %v)", claimed, err)
	}
	observation := healthDomain.Observation{
		TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelBasic,
		Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(),
	}
	if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("AuthorizeObservation(): %v", err)
	}
	updated, err := fixture.Adapter.ApplyGlobalObservation(ctx, activitypool.GlobalHealthCommand{
		Observation: observation, MaxConsecutiveFailures: 2,
	})
	if err != nil || updated.State != proxyDomain.StateAvailable {
		t.Fatalf("ApplyGlobalObservation() = (%+v, %v)", updated, err)
	}
	if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("CompleteObservation(): %v", err)
	}

	next, err := fixture.Adapter.DueCandidates(ctx, plans[0].NextDue, 1)
	if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].State != proxyDomain.StateAvailable {
		t.Fatalf("DueCandidates(next) = (%+v, %v)", next, err)
	}
}

func TestRedisHealthTaskSweepReleasesLeaseAndUpstreamCapacity(t *testing.T) {
	fixture := newRedisTestFixture(t)
	ctx := context.Background()
	now := time.Now().UTC()
	if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{
		ObservedAt: now, ConfiguredTTL: time.Second, MaxSize: 1,
		Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
	}); err != nil {
		t.Fatalf("UpsertFetched(): %v", err)
	}
	candidates, err := fixture.Adapter.DueCandidatesForUpstream(ctx, "provider-a", now, 1)
	if err != nil || len(candidates) != 1 {
		t.Fatalf("DueCandidatesForUpstream() = (%+v, %v)", candidates, err)
	}
	planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{
		Interval: 30 * time.Second, MaxInFlight: 1, Timeout: 10 * time.Second, MaxAttempts: 1,
	})
	if err != nil {
		t.Fatalf("NewPlanner(): %v", err)
	}
	plans, err := planner.Plan(now, 0, 1, candidates)
	if err != nil || len(plans) != 1 {
		t.Fatalf("Plan() = (%+v, %v)", plans, err)
	}
	if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 {
		t.Fatalf("Offer() = (%d, %v)", offered, err)
	}
	claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{
		CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
		SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic},
	})
	if err != nil || len(claimed) != 1 {
		t.Fatalf("Claim() = (%+v, %v)", claimed, err)
	}
	expiredAt := now.Add(2 * time.Second)
	if removed, err := fixture.Adapter.SweepExpired(ctx, expiredAt, 1); err != nil || removed != 1 {
		t.Fatalf("SweepExpired() = (%d, %v)", removed, err)
	}
	if inFlight, err := fixture.Adapter.InFlightForUpstream(ctx, "provider-a", expiredAt); err != nil || inFlight != 0 {
		t.Fatalf("InFlightForUpstream() = (%d, %v)", inFlight, err)
	}
	observation := healthDomain.Observation{
		TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelBasic,
		Success: true, Latency: time.Millisecond, ObservedAt: now,
	}
	if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, expiredAt); !errors.Is(err, healthDomain.ErrTaskNotFound) {
		t.Fatalf("AuthorizeObservation(after sweep) error = %v, want ErrTaskNotFound", err)
	}
}

func TestRedisHealthTasksLeaseAndRescheduleEgressChecks(t *testing.T) {
	fixture := newRedisTestFixture(t)
	ctx := context.Background()
	now := time.Now().UTC()
	const targetURL = "https://egress.example/identity"
	if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{
		ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
		Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
	}); err != nil {
		t.Fatalf("UpsertFetched(): %v", err)
	}
	candidates, err := fixture.Adapter.DueEgressCandidatesForUpstream(ctx, "provider-a", targetURL, now, 1)
	if err != nil || len(candidates) != 1 || candidates[0].Level != healthDomain.LevelEgress || candidates[0].TargetURL != targetURL {
		t.Fatalf("DueEgressCandidatesForUpstream() = (%+v, %v)", candidates, err)
	}
	planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{
		Interval: 30 * time.Second, MaxInFlight: 2, Timeout: 5 * time.Second, MaxAttempts: 1,
	})
	if err != nil {
		t.Fatalf("NewPlanner(): %v", err)
	}
	plans, err := planner.Plan(now, 0, 1, candidates)
	if err != nil || len(plans) != 1 {
		t.Fatalf("Plan() = (%+v, %v)", plans, err)
	}
	if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 {
		t.Fatalf("Offer() = (%d, %v)", offered, err)
	}
	claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{
		CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
		SupportedLevels: []healthDomain.Level{healthDomain.LevelEgress},
	})
	if err != nil || len(claimed) != 1 || claimed[0].Level != healthDomain.LevelEgress || claimed[0].TargetURL != targetURL {
		t.Fatalf("Claim() = (%+v, %v)", claimed, err)
	}
	observation := healthDomain.Observation{
		TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelEgress,
		Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(),
	}
	if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("AuthorizeObservation() = %v", err)
	}
	if _, err := fixture.Adapter.ApplyGlobalObservation(ctx, activitypool.GlobalHealthCommand{
		Observation: observation, MaxConsecutiveFailures: 2,
	}); err != nil {
		t.Fatalf("ApplyGlobalObservation() = %v", err)
	}
	if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("CompleteObservation() = %v", err)
	}
	next, err := fixture.Adapter.DueEgressCandidatesForUpstream(ctx, "provider-a", targetURL, plans[0].NextDue, 1)
	if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].TargetURL != targetURL {
		t.Fatalf("DueEgressCandidatesForUpstream(next) = (%+v, %v)", next, err)
	}
}

func TestRedisHealthTasksLeaseAndRescheduleTargetChecks(t *testing.T) {
	fixture := newRedisTestFixture(t)
	ctx := context.Background()
	now := time.Now().UTC()
	const routingName = "checkout"
	const targetURL = "https://checkout.example/health"
	if _, err := fixture.Adapter.UpsertFetched(ctx, "provider-a", activitypool.FetchedBatch{
		ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
		Proxies: []proxyDomain.Proxy{testProxy("proxy-a", "192.0.2.10")},
	}); err != nil {
		t.Fatalf("UpsertFetched(): %v", err)
	}
	candidates, err := fixture.Adapter.DueTargetCandidatesForUpstream(ctx, "provider-a", routingName, targetURL, now, 1)
	if err != nil || len(candidates) != 1 || candidates[0].Level != healthDomain.LevelTarget ||
		candidates[0].RoutingName != routingName || candidates[0].TargetURL != targetURL {
		t.Fatalf("DueTargetCandidatesForUpstream() = (%+v, %v)", candidates, err)
	}
	planner, err := controllerHealth.NewPlanner(controllerHealth.SchedulePolicy{
		Interval: 30 * time.Second, MaxInFlight: 2, Timeout: 5 * time.Second, MaxAttempts: 1,
	})
	if err != nil {
		t.Fatalf("NewPlanner(): %v", err)
	}
	plans, err := planner.Plan(now, 0, 1, candidates)
	if err != nil || len(plans) != 1 {
		t.Fatalf("Plan() = (%+v, %v)", plans, err)
	}
	if offered, err := fixture.Adapter.Offer(ctx, plans); err != nil || offered != 1 {
		t.Fatalf("Offer() = (%d, %v)", offered, err)
	}
	claimed, err := fixture.Adapter.Claim(ctx, healthDomain.TaskClaim{
		CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
		SupportedLevels: []healthDomain.Level{healthDomain.LevelTarget},
	})
	if err != nil || len(claimed) != 1 || claimed[0].Level != healthDomain.LevelTarget ||
		claimed[0].RoutingName != routingName || claimed[0].TargetURL != targetURL {
		t.Fatalf("Claim() = (%+v, %v)", claimed, err)
	}
	observation := healthDomain.Observation{
		TaskID: claimed[0].TaskID, ProxyID: claimed[0].ProxyID, Level: healthDomain.LevelTarget,
		RoutingName: routingName, TargetURL: targetURL, Success: true, Latency: time.Millisecond, ObservedAt: time.Now().UTC(),
	}
	if err := fixture.Adapter.AuthorizeObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("AuthorizeObservation() = %v", err)
	}
	state, err := fixture.Adapter.ApplyTargetObservation(ctx, activitypool.TargetHealthCommand{
		Observation: observation, MaxConsecutiveFailures: 2,
	})
	if err != nil || state.Status != healthDomain.TargetAvailable {
		t.Fatalf("ApplyTargetObservation() = (%+v, %v)", state, err)
	}
	if err := fixture.Adapter.CompleteObservation(ctx, "checker-a", claimed[0].LeaseToken, observation, time.Now().UTC()); err != nil {
		t.Fatalf("CompleteObservation() = %v", err)
	}
	next, err := fixture.Adapter.DueTargetCandidatesForUpstream(ctx, "provider-a", routingName, targetURL, plans[0].NextDue, 1)
	if err != nil || len(next) != 1 || next[0].ProxyID != "proxy-a" || next[0].RoutingName != routingName || next[0].TargetURL != targetURL {
		t.Fatalf("DueTargetCandidatesForUpstream(next) = (%+v, %v)", next, err)
	}
}