package bootstrap import ( "context" "errors" "strings" "testing" "time" "proxy-pool/internal/config" "proxy-pool/internal/controller/admin" controllerHealth "proxy-pool/internal/controller/health" "proxy-pool/internal/controller/pool" "proxy-pool/internal/controller/provider" controllerRuntime "proxy-pool/internal/controller/runtime" controllerWorker "proxy-pool/internal/controller/worker" "proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/adminstate" extractionDomain "proxy-pool/internal/domain/extraction" healthDomain "proxy-pool/internal/domain/health" "proxy-pool/internal/domain/upstream" "proxy-pool/internal/domain/workerruntime" "proxy-pool/internal/platform/admission" "proxy-pool/internal/platform/credentials" ) var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef") func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) { t.Parallel() resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}} state := adminstate.NewMemoryStore() activity := &stubActivityStore{} credentialStore, err := credentials.NewMemoryStore(10) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } closeErr := errors.New("close failed") infrastructure := &stubInfrastructure{ports: ports{ state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{}, admission: admission.AllowAll{}, coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return closeErr }, }} runErr := errors.New("runtime failed") factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}} now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() err = run(ctx, Options{ ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now }, FingerprintKey: bootstrapTestFingerprintKey, }, infrastructure, factory) if !errors.Is(err, runErr) || !errors.Is(err, closeErr) { t.Fatalf("run() error = %v, want runtime and close errors", err) } if resolver.reads != 1 { t.Fatalf("configuration reads = %d, want 1", resolver.reads) } if infrastructure.opens != 1 || infrastructure.configuration == nil { t.Fatalf("infrastructure opens = %d, config = %p", infrastructure.opens, infrastructure.configuration) } snapshot, snapshotErr := state.Snapshot(context.Background()) if snapshotErr != nil || snapshot.Config == nil || snapshot.Config.Source != "controller.yaml" || snapshot.Revision != 1 { t.Fatalf("management snapshot = %+v, %v", snapshot, snapshotErr) } if factory.configuration == nil || factory.dependencies.Extractor == nil || factory.dependencies.Readiness == nil || factory.dependencies.AdminService == nil || factory.dependencies.MetricsHandler == nil { t.Fatalf("runtime assembly = config:%p dependencies:%+v", factory.configuration, factory.dependencies) } } func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) { t.Parallel() valid := Options{ConfigPath: "controller.yaml", Resolver: &memoryResolver{}, Now: time.Now} tests := []struct { name string ctx context.Context options Options }{ {name: "nil context", options: valid}, {name: "missing path", ctx: context.Background(), options: Options{Resolver: valid.Resolver, Now: time.Now}}, {name: "unclean path", ctx: context.Background(), options: Options{ConfigPath: " controller.yaml", Resolver: valid.Resolver, Now: time.Now}}, {name: "missing resolver", ctx: context.Background(), options: Options{ConfigPath: "controller.yaml", Now: time.Now}}, } for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { t.Parallel() if err := run(test.ctx, test.options, &stubInfrastructure{}, &recordingRuntimeFactory{}); !errors.Is(err, ErrInvalidOptions) { t.Fatalf("run() error = %v", err) } }) } } func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) { for name, key := range map[string][]byte{ "missing": nil, "short": []byte("too-short"), } { t.Run(name, func(t *testing.T) { infrastructure := &stubInfrastructure{} err := run(context.Background(), Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}, Now: time.Now, FingerprintKey: key, }, infrastructure, &recordingRuntimeFactory{}) if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) { t.Fatalf("run() error = %v", err) } if infrastructure.opens != 0 { t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens) } }) } } func TestRunRejectsMissingDistributionAdmissionDependency(t *testing.T) { t.Parallel() credentialStore, err := credentials.NewMemoryStore(10) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } infrastructure := &stubInfrastructure{ports: ports{ state: adminstate.NewMemoryStore(), activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{}, coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return nil }, }} factory := &recordingRuntimeFactory{runner: runnerStub{err: errors.New("runtime should not start")}} err = run(context.Background(), Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}, Now: time.Now, FingerprintKey: bootstrapTestFingerprintKey, }, infrastructure, factory) if !errors.Is(err, ErrStartup) || !errors.Is(err, ErrInvalidOptions) { t.Fatalf("run() error = %v, want ErrStartup and ErrInvalidOptions", err) } if factory.configuration != nil { t.Fatal("runtime factory called without admission dependency") } } func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) { source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false") source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false") source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false") credentialStore, err := credentials.NewMemoryStore(10) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } infrastructure := &stubInfrastructure{ports: ports{ activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return nil }, }} factory := &recordingRuntimeFactory{} ctx, cancel := context.WithCancel(context.Background()) time.AfterFunc(20*time.Millisecond, cancel) err = run(ctx, Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}, Now: time.Now, }, infrastructure, factory) if !errors.Is(err, context.Canceled) { t.Fatalf("run(provider only) error = %v, want context cancellation", err) } if factory.configuration != nil { t.Fatal("HTTP runtime factory was called for Provider-only configuration") } } func TestRunRequiresWorkerControlStoreWhenControlPlaneEnabled(t *testing.T) { source := bootstrapControlPlaneConfig() credentialStore, err := credentials.NewMemoryStore(10) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } infrastructure := &stubInfrastructure{ports: ports{ activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return nil }, }} err = run(context.Background(), Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}, Now: time.Now, }, infrastructure, &recordingRuntimeFactory{}) if !errors.Is(err, ErrStartup) || !errors.Is(err, ErrInvalidOptions) { t.Fatalf("run() error = %v, want startup invalid options", err) } } func TestRunStartsWorkerControlPlaneWithoutHTTPRuntime(t *testing.T) { store, err := workerruntime.NewMemoryStore(time.Now) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } credentialStore, err := credentials.NewMemoryStore(10) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } infrastructure := &stubInfrastructure{ports: ports{ activity: &stubActivityStore{}, workerStore: store, coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return nil }, }} ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() workerFactory := &recordingWorkerRuntimeFactory{runner: runnerFunc(func(ctx context.Context) error { <-ctx.Done() return ctx.Err() })} err = runWithWorkerFactory(ctx, Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapControlPlaneConfig())}}, Now: time.Now, }, infrastructure, &recordingRuntimeFactory{}, workerFactory) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("run() error = %v, want context deadline exceeded", err) } if workerFactory.controlPlane.Listen != "127.0.0.1:0" || workerFactory.service == nil || workerFactory.options.Checker == nil { t.Fatalf("worker runtime factory = controlPlane:%+v service:%T checker:%T", workerFactory.controlPlane, workerFactory.service, workerFactory.options.Checker) } } func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) { state := adminstate.NewMemoryStore() credentialStore, err := credentials.NewMemoryStore(200) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } started := make(chan string, 2) stopped := make(chan string, 2) infrastructure := &stubInfrastructure{ports: ports{ state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{}, admission: admission.AllowAll{}, coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error { started <- upstreamID <-ctx.Done() stopped <- upstreamID return ctx.Err() }), credentials: credentialStore, close: func() error { return nil }, }} wantErr := errors.New("test HTTP runtime stopped") factory := runtimeFactoryFunc(func( _ *config.Config, dependencies controllerRuntime.Dependencies, _ controllerRuntime.Options, ) (controllerRunner, error) { return runnerFunc(func(ctx context.Context) error { for { select { case upstreamID := <-started: if upstreamID != "provider-a" { continue } if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{ RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false, }); err != nil { return err } for { select { case stoppedID := <-stopped: if stoppedID == "provider-a" { return wantErr } case <-ctx.Done(): return ctx.Err() } } case <-ctx.Done(): return ctx.Err() } } }), nil }) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() err = run(ctx, Options{ ConfigPath: "controller.yaml", Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}, Now: time.Now, FingerprintKey: bootstrapTestFingerprintKey, }, infrastructure, factory) if !errors.Is(err, wantErr) { t.Fatalf("run() error = %v, want %v", err, wantErr) } } func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) { configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) if err != nil { t.Fatalf("config.Load(): %v", err) } disabled := configuration.Upstreams["provider-b"] disabled.Enabled = false configuration.Upstreams["provider-b"] = disabled stats, err := provider.NewStatsRecorder(2) if err != nil { t.Fatalf("provider.NewStatsRecorder(): %v", err) } stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError}) stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError}) retainProviderStats(configuration, stats) stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError}) got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"}) if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 { t.Fatalf("Provider stats after retention = %+v", got) } } func TestNewHealthSchedulersCreatesOneRunnerPerEnabledUpstream(t *testing.T) { configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) if err != nil { t.Fatalf("config.Load(): %v", err) } disabled := configuration.Upstreams["provider-b"] disabled.Enabled = false configuration.Upstreams["provider-b"] = disabled runners, err := newHealthSchedulers(configuration, healthTaskRuntimeStub{}, time.Now) if err != nil || len(runners) != 1 { t.Fatalf("newHealthSchedulers() = (%d runners, %v)", len(runners), err) } } type memoryResolver struct { files map[string][]byte reads int } func (*memoryResolver) LookupEnv(string) (string, bool) { return "", false } func (resolver *memoryResolver) ReadFile(path string) ([]byte, error) { resolver.reads++ content, ok := resolver.files[path] if !ok { return nil, errors.New("file missing") } return append([]byte(nil), content...), nil } type stubInfrastructure struct { ports ports err error opens int configuration *config.Config } func (infrastructure *stubInfrastructure) Open( _ context.Context, configuration *config.Config, ) (ports, error) { infrastructure.opens++ infrastructure.configuration = configuration return infrastructure.ports, infrastructure.err } type recordingRuntimeFactory struct { configuration *config.Config dependencies controllerRuntime.Dependencies runner controllerRunner err error } func (factory *recordingRuntimeFactory) New( configuration *config.Config, dependencies controllerRuntime.Dependencies, options controllerRuntime.Options, ) (controllerRunner, error) { factory.configuration = configuration factory.dependencies = dependencies return factory.runner, factory.err } type runnerStub struct{ err error } func (runner runnerStub) Run(context.Context) error { return runner.err } type runnerFunc func(context.Context) error func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) } type runtimeFactoryFunc func( *config.Config, controllerRuntime.Dependencies, controllerRuntime.Options, ) (controllerRunner, error) func (factory runtimeFactoryFunc) New( configuration *config.Config, dependencies controllerRuntime.Dependencies, options controllerRuntime.Options, ) (controllerRunner, error) { return factory(configuration, dependencies, options) } type recordingWorkerRuntimeFactory struct { controlPlane config.ControlPlane service controllerWorker.Service options controllerWorker.ServerOptions runner controllerRunner err error } func (factory *recordingWorkerRuntimeFactory) New( controlPlane config.ControlPlane, service controllerWorker.Service, options controllerWorker.ServerOptions, ) (controllerRunner, error) { factory.controlPlane = controlPlane factory.service = service factory.options = options return factory.runner, factory.err } type readyStub struct{} func (readyStub) Ready(context.Context) error { return nil } type stubActivityStore struct{} type healthTaskRuntimeStub struct{} func (healthTaskRuntimeStub) Offer(context.Context, []healthDomain.PlannedTask) (int, error) { return 0, nil } func (healthTaskRuntimeStub) Claim(context.Context, healthDomain.TaskClaim) ([]healthDomain.LeasedTask, error) { return nil, nil } func (healthTaskRuntimeStub) AuthorizeObservation( context.Context, string, string, healthDomain.Observation, time.Time, ) error { return nil } func (healthTaskRuntimeStub) CompleteObservation( context.Context, string, string, healthDomain.Observation, time.Time, ) error { return nil } func (healthTaskRuntimeStub) InFlightForUpstream(context.Context, string, time.Time) (int, error) { return 0, nil } func (healthTaskRuntimeStub) DueCandidatesForUpstream( context.Context, string, time.Time, int, ) ([]controllerHealth.Candidate, error) { return nil, nil } func (*stubActivityStore) Extract(_ context.Context, command extractionDomain.Command) (extractionDomain.Result, error) { return extractionDomain.Result{Requested: command.Requested}, nil } func (*stubActivityStore) ReadStateInventory( _ context.Context, upstreamIDs []string, _ time.Time, ) ([]activitypool.StateInventory, error) { result := make([]activitypool.StateInventory, len(upstreamIDs)) for index, upstreamID := range upstreamIDs { result[index].UpstreamID = upstreamID } return result, nil } func (*stubActivityStore) UpsertFetched( _ context.Context, _ string, batch activitypool.FetchedBatch, ) (activitypool.UpsertResult, error) { return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil } func (*stubActivityStore) ApplyGlobalObservation( context.Context, activitypool.GlobalHealthCommand, ) (activitypool.Entry, error) { return activitypool.Entry{}, nil } func (*stubActivityStore) ApplyTargetObservation( context.Context, activitypool.TargetHealthCommand, ) (healthDomain.TargetState, error) { return healthDomain.TargetState{Status: healthDomain.TargetUnknown}, nil } func (*stubActivityStore) UpstreamForProxy(context.Context, string, time.Time) (string, error) { return "provider-a", nil } func (*stubActivityStore) ReadInventory( context.Context, string, time.Duration, ) (pool.InventorySnapshot, error) { return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil } type coordinatorStub struct{} func (coordinatorStub) RunLeader( ctx context.Context, _ string, _ provider.CoordinationLimits, _ func(context.Context, provider.LeaderSession) error, ) error { <-ctx.Done() return ctx.Err() } type coordinatorFunc func(context.Context, string) error func (run coordinatorFunc) RunLeader( ctx context.Context, upstreamID string, _ provider.CoordinationLimits, _ func(context.Context, provider.LeaderSession) error, ) error { return run(ctx, upstreamID) } const bootstrapTestConfig = ` version: 1 security: requireProtectionOnPublicListen: true gateway: enabled: false distribution: enabled: true listen: 127.0.0.1:0 auth: {mode: none} extraction: fulfillment: partial maxCountPerRequest: 20 minRemainingTTL: 5s maxHealthCheckAge: 15s reserveForGateway: 5 idempotencyTTL: 5m admin: enabled: true listen: 127.0.0.1:0 auth: {mode: none} metrics: enabled: true listen: 127.0.0.1:0 storage: postgresURL: postgres://fixture redisURL: redis://fixture routing: - name: extract enabled: true purpose: extract upstreams: [provider-a, provider-b] strategy: {type: sequential, switchAfterEmptyFetch: 5, endBehavior: stayLast} onUnavailable: {action: reject} upstreams: provider-a: &upstream enabled: true exposure: [extract] provider: {billingMode: fetch, protocols: [http]} api: url: https://provider.invalid/proxies method: GET template: '{{.}}' auth: {type: none} proxyAuth: {type: response} pool: {maxSize: 100} capacity: {maxConcurrencyPerProxy: 10} refill: {reconcileInterval: 1s, minimumAvailableSlots: 100, targetAvailableSlots: 200} lifecycle: {ttl: 2m, allocationSafetyMargin: 10s} fetch: {estimatedIPsPerCall: 10, requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000} check: interval: 30s jitter: 20 maxInFlight: 100 timeout: 2s maxAttempts: 2 maxConsecutiveFailures: 3 urls: [https://example.invalid/health] provider-b: *upstream ` func bootstrapControlPlaneConfig() string { source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false") source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false") source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false") return source + ` controlPlane: enabled: true listen: 127.0.0.1:0 protocolVersion: 1 heartbeatInterval: 10s sessionTTL: 30s maxStaleAge: 10s maxMessageBytes: 1048576 maxRuntimeCounters: 100 maxConcurrentStreams: 10 tls: {mode: disabled} ` }