diff --git a/cmd/proxy-controller/main.go b/cmd/proxy-controller/main.go index 28d79f1..b9a8806 100644 --- a/cmd/proxy-controller/main.go +++ b/cmd/proxy-controller/main.go @@ -15,7 +15,10 @@ import ( "proxy-pool/internal/controller/bootstrap" ) -const configEnvironment = "PROXY_POOL_CONFIG" +const ( + configEnvironment = "PROXY_POOL_CONFIG" + fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY" +) type environmentLookup func(string) string type controllerRun func(context.Context, bootstrap.Options) error @@ -54,7 +57,13 @@ func execute( return 2 } - err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}}) + var fingerprintKey []byte + if getenv != nil { + fingerprintKey = []byte(getenv(fingerprintKeyEnvironment)) + } + err := run(ctx, bootstrap.Options{ + ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey, + }) if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) { return 0 } diff --git a/cmd/proxy-controller/main_test.go b/cmd/proxy-controller/main_test.go index 5581e25..b3bf0b1 100644 --- a/cmd/proxy-controller/main_test.go +++ b/cmd/proxy-controller/main_test.go @@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) { if name == configEnvironment { return "environment.yaml" } + if name == fingerprintKeyEnvironment { + return "0123456789abcdef0123456789abcdef" + } return "" }, func(_ context.Context, options bootstrap.Options) error { received = options return nil }, &bytes.Buffer{}) - if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil { + if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 { t.Fatalf("execute() = %d, options = %+v", code, received) } } @@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) { func TestExecuteFallsBackToEnvironment(t *testing.T) { t.Parallel() var received bootstrap.Options - code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func( + code := execute(context.Background(), nil, func(name string) string { + if name == configEnvironment { + return "environment.yaml" + } + return "0123456789abcdef0123456789abcdef" + }, func( _ context.Context, options bootstrap.Options, ) error { diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 7171403..efcc839 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,5 +1,13 @@ name: proxy-pool +x-app-environment: &app-environment + PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml + PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD} + PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN} + PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN} + PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN} + PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN} + x-app: &app build: context: .. @@ -9,13 +17,7 @@ x-app: &app networks: [frontend, backend] volumes: - ./config/local.yaml:/etc/proxy-pool/config.yaml:ro - environment: - PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml - PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD} - PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN} - PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN} - PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN} - PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN} + environment: *app-environment stop_grace_period: 45s services: @@ -44,6 +46,9 @@ services: controller: <<: *app command: ["proxy-controller"] + environment: + <<: *app-environment + PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY} depends_on: postgres: condition: service_healthy diff --git a/deploy/kubernetes/base/secret.example.yaml b/deploy/kubernetes/base/secret.example.yaml index ead02fc..5891577 100644 --- a/deploy/kubernetes/base/secret.example.yaml +++ b/deploy/kubernetes/base/secret.example.yaml @@ -9,8 +9,8 @@ stringData: PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN + PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0 PROVIDER_A_TOKEN: PROVIDER_A_TOKEN PROVIDER_B_TOKEN: PROVIDER_B_TOKEN - diff --git a/internal/adapters/providerapi/template_parser.go b/internal/adapters/providerapi/template_parser.go index 5dcb4db..1471e4e 100644 --- a/internal/adapters/providerapi/template_parser.go +++ b/internal/adapters/providerapi/template_parser.go @@ -17,11 +17,12 @@ import ( ) const ( - defaultTemplateTimeout = 100 * time.Millisecond - defaultTemplateMaxBytes = int64(1 << 20) - defaultMaxCandidates = 10_000 - maxRegexPatterns = 64 - maxRegexPatternBytes = 1024 + defaultTemplateTimeout = 100 * time.Millisecond + defaultTemplateMaxBytes = int64(1 << 20) + defaultMaxCandidates = 10_000 + credentialReleaseTimeout = time.Second + maxRegexPatterns = 64 + maxRegexPatternBytes = 1024 ) type TemplateParser struct { @@ -137,7 +138,14 @@ func NewTemplateParser( return parser, nil } -func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) { +func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) { + storedCredentials := make([]credentials.Reference, 0) + defer func() { + if resultErr == nil { + return + } + p.releaseCredentials(storedCredentials) + }() if err := ctx.Err(); err != nil { return nil, err } @@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain. if len(tokens) > p.maxCandidates { return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)} } - proxies := make([]proxyDomain.Proxy, 0, len(tokens)) + proxies = make([]proxyDomain.Proxy, 0, len(tokens)) credentialIndexes := make(map[string]int) for _, token := range tokens { candidate, credential, ok := p.parseCandidate(token) @@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain. } candidate.SecretRef = reference.SecretRef candidate.CredentialVersion = reference.CredentialVersion + storedCredentials = append(storedCredentials, reference) credentialKey = candidateCredentialKey(candidate) if index, exists := credentialIndexes[credentialKey]; exists { proxies[index] = candidate @@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain. if len(tokens) > 0 && len(proxies) == 0 { return nil, ErrInvalidProxyOutput } + p.releaseUnusedCredentials(storedCredentials, proxies) return proxies, nil } +func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) { + references := make([]credentials.Reference, 0, len(candidates)) + for _, candidate := range candidates { + if candidate.SecretRef == "" || candidate.CredentialVersion == "" { + continue + } + references = append(references, credentials.Reference{ + SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion, + }) + } + p.releaseCredentials(references) +} + +func (p *TemplateParser) releaseUnusedCredentials( + stored []credentials.Reference, + candidates []proxyDomain.Proxy, +) { + retained := make(map[credentials.Reference]int, len(candidates)) + for _, candidate := range candidates { + if candidate.SecretRef == "" || candidate.CredentialVersion == "" { + continue + } + retained[credentials.Reference{ + SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion, + }]++ + } + unused := make([]credentials.Reference, 0, len(stored)) + for _, reference := range stored { + if retained[reference] > 0 { + retained[reference]-- + continue + } + unused = append(unused, reference) + } + p.releaseCredentials(unused) +} + +func (p *TemplateParser) releaseCredentials(references []credentials.Reference) { + if p == nil { + return + } + releaser, ok := p.credentialStore.(credentials.Releaser) + if !ok { + return + } + releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout) + defer cancel() + for _, reference := range references { + if releaseCtx.Err() != nil { + return + } + _ = releaser.Release(releaseCtx, reference) + } +} + func (p *TemplateParser) regexFind(pattern, value string) (string, error) { compiled, err := p.compileRegex(pattern) if err != nil { diff --git a/internal/adapters/providerapi/template_parser_test.go b/internal/adapters/providerapi/template_parser_test.go index e6bd536..dce7ac0 100644 --- a/internal/adapters/providerapi/template_parser_test.go +++ b/internal/adapters/providerapi/template_parser_test.go @@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T) } func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) { - store, err := credentials.NewMemoryStore(1) + store, err := credentials.NewMemoryStore(2) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } @@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi if len(proxies) != 2 { t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies)) } - if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef { - t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef) + if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef { + t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef) + } + if proxies[0].CredentialVersion != proxies[1].CredentialVersion { + t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion) } } @@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T) } } +func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) { + store, err := credentials.NewMemoryStore(1) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + parser, err := newTemplateParser("provider-a", config.Upstream{ + Provider: config.Provider{Protocols: []string{"http"}}, + API: config.ProviderAPI{Template: strings.Join([]string{ + "http://alice:first-password@192.0.2.10:8080", + "http://bob:second-password@192.0.2.11:8080", + }, "\n")}, + ProxyAuth: config.ProxyAuth{Type: "response"}, + Pool: config.Pool{MaxSize: 2}, + Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)}, + }, store) + if err != nil { + t.Fatalf("newTemplateParser(): %v", err) + } + + if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) { + t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err) + } + if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil { + t.Fatalf("Put(after failed parse): %v", err) + } +} + func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) { store, err := credentials.NewMemoryStore(2) if err != nil { @@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing } func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) { - store, err := credentials.NewMemoryStore(1) + store, err := credentials.NewMemoryStore(2) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } @@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing. if value.Password != "new-secret" { t.Fatalf("resolved latest password mismatch") } + parser.ReleaseCandidates(proxies) + if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil { + t.Fatalf("Put(after releasing latest candidates): %v", err) + } +} + +func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) { + store, err := credentials.NewMemoryStore(1) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + parser, err := newTemplateParser("provider-a", config.Upstream{ + Provider: config.Provider{Protocols: []string{"http"}}, + API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"}, + ProxyAuth: config.ProxyAuth{Type: "response"}, + Pool: config.Pool{MaxSize: 1}, + Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)}, + }, store) + if err != nil { + t.Fatalf("NewTemplateParser(): %v", err) + } + + proxies, err := parser.Parse(context.Background(), nil) + if err != nil { + t.Fatalf("Parse(): %v", err) + } + parser.ReleaseCandidates(proxies) + if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil { + t.Fatalf("Put(after ReleaseCandidates): %v", err) + } } func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) { diff --git a/internal/adapters/redisactivity/adapter.go b/internal/adapters/redisactivity/adapter.go index 10ba4df..0fdaf69 100644 --- a/internal/adapters/redisactivity/adapter.go +++ b/internal/adapters/redisactivity/adapter.go @@ -29,10 +29,11 @@ type Options struct { } type Adapter struct { - client redis.Scripter - credentials credentials.Store - keys keyspace - options Options + client redis.Scripter + credentials credentials.Store + credentialReleaser credentials.Releaser + keys keyspace + options Options } func New(client redis.Scripter, options Options) (*Adapter, error) { @@ -49,12 +50,14 @@ func New(client redis.Scripter, options Options) (*Adapter, error) { options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 { return nil, ErrInvalidOptions } - return &Adapter{ + adapter := &Adapter{ client: client, credentials: options.Credentials, keys: newKeyspace(options.Namespace), options: options, - }, nil + } + adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser) + return adapter, nil } func (a *Adapter) Format(state fmt.State, _ rune) { diff --git a/internal/adapters/redisactivity/testredis_test.go b/internal/adapters/redisactivity/testredis_test.go index 95b6a03..c1c6fb2 100644 --- a/internal/adapters/redisactivity/testredis_test.go +++ b/internal/adapters/redisactivity/testredis_test.go @@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) { } func newRedisTestFixture(t *testing.T) redisTestFixture { + return newRedisFixtureWithCredentialCapacity(t, 10_000) +} + +func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture { t.Helper() redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL") if redisURL == "" { @@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture { _ = client.Close() t.Fatalf("ping test Redis: %v", err) } - credentialStore, err := credentials.NewMemoryStore(10_000) + credentialStore, err := credentials.NewMemoryStore(credentialCapacity) if err != nil { _ = client.Close() t.Fatalf("NewMemoryStore(): %v", err) diff --git a/internal/adapters/redisactivity/upsert.go b/internal/adapters/redisactivity/upsert.go index 3071620..9f43618 100644 --- a/internal/adapters/redisactivity/upsert.go +++ b/internal/adapters/redisactivity/upsert.go @@ -15,7 +15,10 @@ import ( "proxy-pool/internal/platform/credentials" ) -const maxUpsertScriptBatch = 256 +const ( + maxUpsertScriptBatch = 256 + transientCredentialReleaseTimeout = time.Second +) type upsertCandidate struct { ProxyID string `json:"proxyId"` @@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil) func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) { var result activitypool.UpsertResult + if a != nil { + defer a.releaseTransientCredentials(batch.Proxies) + } if ctx == nil { return result, activitypool.ErrInvalidBatch } @@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac return result, nil } +func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) { + if a.credentialReleaser == nil { + return + } + releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout) + defer cancel() + for _, candidate := range proxies { + if releaseCtx.Err() != nil { + return + } + if candidate.SecretRef == "" || candidate.CredentialVersion == "" { + continue + } + _ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{ + SecretRef: candidate.SecretRef, + CredentialVersion: candidate.CredentialVersion, + }) + } +} + func (a *Adapter) prepareUpsertCandidate( ctx context.Context, upstreamID string, diff --git a/internal/adapters/redisactivity/upsert_integration_test.go b/internal/adapters/redisactivity/upsert_integration_test.go index cd9f788..9b6c2b0 100644 --- a/internal/adapters/redisactivity/upsert_integration_test.go +++ b/internal/adapters/redisactivity/upsert_integration_test.go @@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) { } } +func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) { + fixture := newRedisFixtureWithCredentialCapacity(t, 1) + first := testProxy("proxy-a", "192.0.2.10") + firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{ + Username: "first", Password: "first-password", + }) + if err != nil { + t.Fatalf("Put(first credential): %v", err) + } + first.SecretRef = firstReference.SecretRef + first.CredentialVersion = firstReference.CredentialVersion + if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{ + ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2, + Proxies: []proxyDomain.Proxy{first}, + }); err != nil { + t.Fatalf("UpsertFetched(first): %v", err) + } + + if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{ + Username: "second", Password: "second-password", + }); err != nil { + t.Fatalf("Put(second credential after upsert): %v", err) + } +} + func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) { fixture := newRedisTestFixture(t) now := redisTestNow() diff --git a/internal/adapters/redisprovider/adapter.go b/internal/adapters/redisprovider/adapter.go index 4722ae6..586d04b 100644 --- a/internal/adapters/redisprovider/adapter.go +++ b/internal/adapters/redisprovider/adapter.go @@ -19,8 +19,6 @@ import ( var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) -const maximumLuaInteger = int64(1<<53 - 1) - type Options struct { Namespace string HolderID string @@ -57,7 +55,8 @@ func (adapter *Adapter) RunLeader( ) error { if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 || - limits.MaxTotal < 0 || limits.MaxTotal > maximumLuaInteger || + limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger || + int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger || limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace { return controllerProvider.ErrInvalidCoordination } @@ -222,7 +221,7 @@ func (session *leaderSession) Fence() controllerProvider.Fence { func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) { if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil || - expected <= 0 || int64(expected) > maximumLuaInteger { + expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger { return nil, false, controllerProvider.ErrInvalidCoordination } operationCtx, cancel := context.WithCancel(ctx) @@ -295,7 +294,7 @@ type requestPermit struct { } func (permit *requestPermit) Complete(ctx context.Context, fetched int) error { - if fetched < 0 || int64(fetched) > maximumLuaInteger { + if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger { return controllerProvider.ErrInvalidCoordination } return permit.finish(ctx, "complete_fetch", fetched) diff --git a/internal/config/config.go b/internal/config/config.go index 0c080d1..190065f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,6 +5,12 @@ import ( "time" ) +const ( + MaximumPoolSize = 1_000_000 + MaximumExactCounter = int64(1<<53 - 1) + MaximumUpstreams = 4_096 +) + type Duration time.Duration func (d *Duration) UnmarshalText(text []byte) error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b8e4ebc..992e873 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) { }, want: "estimatedIPsPerCall", }, + { + name: "pool exceeds runtime scan bound", + mutate: func(cfg *Config) { + updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 }) + }, + want: "pool.maxSize", + }, { name: "zero refill interval", mutate: func(cfg *Config) { @@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) { } } +func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) { + cfg := mustLoadValidConfig(t) + for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ { + cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{} + } + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") { + t.Fatalf("Validate(too many upstreams) error = %v", err) + } +} + +func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) { + if strconv.IntSize < 64 { + t.Skip("64-bit int is required for values above the Redis exact range") + } + tests := []struct { + name string + mutate func(*Upstream) + }{ + {name: "fetch total", mutate: func(upstream *Upstream) { + upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1 + }}, + {name: "proxy concurrency", mutate: func(upstream *Upstream) { + upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1 + }}, + {name: "minimum slots", mutate: func(upstream *Upstream) { + upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1 + upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2 + }}, + {name: "target slots", mutate: func(upstream *Upstream) { + upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + upstream.Pool.MaxSize = 1 + upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1 + }}, + {name: "theoretical slots", mutate: func(upstream *Upstream) { + upstream.Pool.MaxSize = 2 + upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1) + upstream.Refill.MinimumAvailableSlots = 1 + upstream.Refill.TargetAvailableSlots = 2 + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := mustLoadValidConfig(t) + updateUpstream(cfg, test.mutate) + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") { + t.Fatalf("Validate(inexact counter) error = %v", err) + } + }) + } +} + func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) { configured := strings.Replace(validConfig, ` auth: mode: none`, ` auth: diff --git a/internal/config/fingerprint.go b/internal/config/fingerprint.go new file mode 100644 index 0000000..631aecf --- /dev/null +++ b/internal/config/fingerprint.go @@ -0,0 +1,30 @@ +package config + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" +) + +const MinimumFingerprintKeyBytes = 32 + +var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint") + +// Fingerprint returns a keyed digest of one fully resolved configuration. +// Only the digest is persisted; the resolved configuration and key remain local. +func Fingerprint(configuration *Config, key []byte) (string, error) { + if configuration == nil || len(key) < MinimumFingerprintKeyBytes { + return "", ErrInvalidFingerprint + } + encoded, err := json.Marshal(configuration) + if err != nil { + return "", errors.Join(ErrInvalidFingerprint, err) + } + digest := hmac.New(sha256.New, key) + if _, err := digest.Write(encoded); err != nil { + return "", errors.Join(ErrInvalidFingerprint, err) + } + return hex.EncodeToString(digest.Sum(nil)), nil +} diff --git a/internal/config/fingerprint_test.go b/internal/config/fingerprint_test.go new file mode 100644 index 0000000..3f6b868 --- /dev/null +++ b/internal/config/fingerprint_test.go @@ -0,0 +1,68 @@ +package config + +import "testing" + +var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef") + +func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) { + first := storeTestConfig("provider-a") + upstream := first.Upstreams["provider-a"] + upstream.ProxyAuth.Password = "secret-a" + first.Upstreams["provider-a"] = upstream + + stable, err := Fingerprint(first, testFingerprintKey) + if err != nil { + t.Fatalf("Fingerprint(first): %v", err) + } + again, err := Fingerprint(first, testFingerprintKey) + if err != nil || again != stable { + t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable) + } + + rotated := storeTestConfig("provider-a") + upstream = rotated.Upstreams["provider-a"] + upstream.ProxyAuth.Password = "secret-b" + rotated.Upstreams["provider-a"] = upstream + changed, err := Fingerprint(rotated, testFingerprintKey) + if err != nil { + t.Fatalf("Fingerprint(rotated): %v", err) + } + if changed == stable { + t.Fatal("Fingerprint did not change after secret rotation") + } + if len(changed) != 64 { + t.Fatalf("Fingerprint length = %d, want 64", len(changed)) + } +} + +func TestFingerprintChangesWithIndependentKey(t *testing.T) { + configuration := storeTestConfig("provider-a") + first, err := Fingerprint(configuration, testFingerprintKey) + if err != nil { + t.Fatalf("Fingerprint(first key): %v", err) + } + second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210")) + if err != nil { + t.Fatalf("Fingerprint(second key): %v", err) + } + if first == second { + t.Fatal("Fingerprint did not change with independent key") + } +} + +func TestFingerprintRejectsInvalidInputs(t *testing.T) { + for name, test := range map[string]struct { + configuration *Config + key []byte + }{ + "nil configuration": {key: testFingerprintKey}, + "missing key": {configuration: storeTestConfig("provider-a")}, + "short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")}, + } { + t.Run(name, func(t *testing.T) { + if _, err := Fingerprint(test.configuration, test.key); err == nil { + t.Fatal("Fingerprint() succeeded") + } + }) + } +} diff --git a/internal/config/store.go b/internal/config/store.go index ca3a460..e3c5ae6 100644 --- a/internal/config/store.go +++ b/internal/config/store.go @@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store") // Store publishes complete validated configurations with one atomic pointer swap. type Store struct { - current atomic.Pointer[Config] + current atomic.Pointer[publishedConfiguration] +} + +type publishedConfiguration struct { + value Config + revision uint64 } func NewStore(initial *Config) (*Store, error) { @@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) { return nil, errors.Join(ErrInvalidStore, err) } store := &Store{} - store.Publish(initial) + store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)}) return store, nil } @@ -25,19 +30,38 @@ func (store *Store) Current() *Config { if store == nil { return nil } - current := store.current.Load() - if current == nil { + published := store.current.Load() + if published == nil { return nil } - cloned := cloneConfig(*current) + cloned := cloneConfig(published.value) return &cloned } -// Publish accepts a non-nil configuration already validated by the caller. -func (store *Store) Publish(configuration *Config) { - if store == nil || configuration == nil { - return +func (store *Store) Revision() uint64 { + if store == nil { + return 0 + } + published := store.current.Load() + if published == nil { + return 0 + } + return published.revision +} + +// PublishRevision publishes only a strictly newer authoritative revision. +func (store *Store) PublishRevision(configuration *Config, revision uint64) bool { + if store == nil || configuration == nil || revision == 0 { + return false + } + for { + current := store.current.Load() + if current != nil && revision <= current.revision { + return false + } + next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision} + if store.current.CompareAndSwap(current, next) { + return true + } } - cloned := cloneConfig(*configuration) - store.current.Store(&cloned) } diff --git a/internal/config/store_test.go b/internal/config/store_test.go index fe75ac2..9cb2bde 100644 --- a/internal/config/store_test.go +++ b/internal/config/store_test.go @@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) { } next := storeTestConfig("provider-b") - store.Publish(next) + if !store.PublishRevision(next, 1) { + t.Fatal("PublishRevision() rejected newer configuration") + } next.Routing[0].Upstreams[0] = "mutated" if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" { t.Fatalf("Publish() retained caller state: %q", got) @@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) { if index%2 == 1 { name = "provider-b" } - store.Publish(storeTestConfig(name)) + store.PublishRevision(storeTestConfig(name), uint64(index+1)) }(index) go func() { defer wait.Done() @@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) { wait.Wait() } +func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) { + t.Parallel() + store, err := NewStore(storeTestConfig("provider-a")) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published { + t.Fatal("PublishRevision(newer) rejected") + } + if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published { + t.Fatal("PublishRevision(stale) succeeded") + } + if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published { + t.Fatal("PublishRevision(equal) succeeded") + } + if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" { + t.Fatalf("Current() upstream = %q, want provider-b", got) + } + if got := store.Revision(); got != 2 { + t.Fatalf("Revision() = %d, want 2", got) + } +} + func storeTestConfig(upstreamName string) *Config { return &Config{ Version: 1, diff --git a/internal/config/validate.go b/internal/config/validate.go index 964afae..5e70589 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -46,6 +46,9 @@ func Validate(cfg *Config) error { if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil { return err } + if len(cfg.Upstreams) > MaximumUpstreams { + return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams) + } enabledUpstreams := 0 for name, upstream := range cfg.Upstreams { if upstream.Enabled { @@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error { if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil { return err } + if upstream.Pool.MaxSize > MaximumPoolSize { + return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize) + } if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil { return err } + if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter { + return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope) + } if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize { return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope) } if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil { return err } + if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter { + return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope) + } if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil { return err } if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil { return err } + if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter { + return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope) + } if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots { return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope) } + if upstream.Refill.TargetAvailableSlots > MaximumExactCounter { + return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope) + } if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) { return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope) } theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy) + if theoreticalSlots > MaximumExactCounter { + return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope) + } if upstream.Refill.TargetAvailableSlots > theoreticalSlots { return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope) } @@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error { if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil { return err } + if int64(fetch.MaxInFlight) > MaximumExactCounter { + return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope) + } if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil { return err } diff --git a/internal/controller/admin/service.go b/internal/controller/admin/service.go index eb0706c..e5eb126 100644 --- a/internal/controller/admin/service.go +++ b/internal/controller/admin/service.go @@ -2,9 +2,6 @@ package admin import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" "sort" "strings" @@ -29,9 +26,15 @@ type ConfigurationLoader interface { LoadConfiguration(context.Context) (LoadedConfiguration, error) } -// ConfigurationPublisher must atomically publish an already validated configuration. +// ConfigurationPublisher publishes only a newer authoritative configuration revision. type ConfigurationPublisher interface { - Publish(*config.Config) + PublishRevision(*config.Config, uint64) bool +} + +type RuntimeController interface { + Notify() + ValidateConfiguration(context.Context, *config.Config) error + ValidateUpstream(context.Context, string) error } var _ ConfigurationPublisher = (*config.Store)(nil) @@ -41,10 +44,12 @@ type ApplicationDependencies struct { Operations OperationalStatusReader Configuration ConfigurationLoader Publisher ConfigurationPublisher + Runtime RuntimeController } type ApplicationOptions struct { - Now func() time.Time + Now func() time.Time + FingerprintKey []byte } type OperationalStatus struct { @@ -70,30 +75,40 @@ type LoadedConfiguration struct { } type ApplicationService struct { - state StateRepository - operations OperationalStatusReader - configuration ConfigurationLoader - publisher ConfigurationPublisher - now func() time.Time + state StateRepository + operations OperationalStatusReader + configuration ConfigurationLoader + publisher ConfigurationPublisher + runtime RuntimeController + now func() time.Time + fingerprintKey []byte } var _ Service = (*ApplicationService)(nil) func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) { if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) || - nilInterface(dependencies.Publisher) || options.Now == nil { + nilInterface(dependencies.Publisher) || options.Now == nil || + len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes { return nil, ErrInvalidApplicationService } return &ApplicationService{ - state: dependencies.State, - operations: dependencies.Operations, - configuration: dependencies.Configuration, - publisher: dependencies.Publisher, - now: options.Now, + state: dependencies.State, + operations: dependencies.Operations, + configuration: dependencies.Configuration, + publisher: dependencies.Publisher, + runtime: dependencies.Runtime, + now: options.Now, + fingerprintKey: append([]byte(nil), options.FingerprintKey...), }, nil } func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) { + if command.Enabled && service.runtime != nil { + if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil { + return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err) + } + } result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{ RequestID: command.RequestID, Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP}, @@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma Name: command.Name, Enabled: command.Enabled, }) + if err == nil && service.runtime != nil { + service.runtime.Notify() + } return mutationResult(result), mapAdminStateError(err) } @@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration( if err := config.Validate(loaded.Value); err != nil { return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err) } + if service.runtime != nil { + if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil { + return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err) + } + } - managementView := loaded.Value.Redacted() - encoded, err := json.Marshal(managementView) + checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey) if err != nil { return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err) } - digest := sha256.Sum256(encoded) - checksum := hex.EncodeToString(digest[:]) current, err := service.state.Snapshot(ctx) if err != nil { @@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration( if err != nil { return mutationResult(result), mapAdminStateError(err) } - service.publisher.Publish(loaded.Value) + if result.Revision == 0 { + return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService) + } + published := service.publisher.PublishRevision(loaded.Value, result.Revision) + if published && service.runtime != nil { + service.runtime.Notify() + } return mutationResult(result), nil } diff --git a/internal/controller/admin/service_test.go b/internal/controller/admin/service_test.go index f4f116d..7dcaddd 100644 --- a/internal/controller/admin/service_test.go +++ b/internal/controller/admin/service_test.go @@ -3,6 +3,7 @@ package admin import ( "context" "errors" + "sync/atomic" "testing" "time" @@ -10,6 +11,12 @@ import ( "proxy-pool/internal/domain/adminstate" ) +var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef") + +func applicationTestOptions(now func() time.Time) ApplicationOptions { + return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey} +} + func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) { t.Parallel() now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC) @@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) { Message: "enabled", }, } + runtime := &recordingRuntimeNotifier{} service, err := NewApplicationService(ApplicationDependencies{ State: state, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: func() time.Time { return now }}) + Runtime: runtime, + }, applicationTestOptions(func() time.Time { return now })) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) { }) { t.Fatalf("admin state command = %+v", state.lastUpstream) } + if runtime.notifications != 1 { + t.Fatalf("runtime notifications = %d, want 1", runtime.notifications) + } +} + +func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) { + t.Parallel() + wantErr := errors.New("invalid Provider template") + state := &recordingAdminState{} + runtime := &recordingRuntimeNotifier{validationErr: wantErr} + service, err := NewApplicationService(ApplicationDependencies{ + State: state, Operations: staticOperationalStatusReader{}, + Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, + Runtime: runtime, + }, applicationTestOptions(time.Now)) + if err != nil { + t.Fatalf("NewApplicationService(): %v", err) + } + + _, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{ + RequestID: "req-enable", Name: "provider-a", Enabled: true, + }) + if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) { + t.Fatalf("SetUpstreamEnabled() error = %v", err) + } + if state.lastUpstream != (adminstate.SetUpstreamCommand{}) { + t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream) + } } func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) { @@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) { dependencies ApplicationDependencies options ApplicationOptions }{ - {name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}}, - {name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}}, - {name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}}, - {name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}}, + {name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)}, + {name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)}, + {name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)}, + {name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)}, {name: "clock", dependencies: valid}, + {name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}}, } for _, test := range tests { test := test @@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) { func() ApplicationDependencies { value := valid; value.State = state; return value }(), func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(), } { - if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) { + if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) { t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService) } } @@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) { t.Parallel() now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60)) state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}} - service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }}) + service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now })) result, err := service.SwitchRouting(context.Background(), SwitchCommand{ RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7", @@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots( service, err := NewApplicationService(ApplicationDependencies{ State: state, Operations: operations, Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) { service, err := NewApplicationService(ApplicationDependencies{ State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) { service, err = NewApplicationService(ApplicationDependencies{ State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure}, Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) { service, err = NewApplicationService(ApplicationDependencies{ State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled}, Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC) configuration := validReloadConfiguration() publisher := &recordingConfigurationPublisher{} + runtime := &recordingRuntimeNotifier{} state := &recordingAdminState{ mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42}, snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{ @@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{ Value: configuration, Source: "configs/proxy-pool.yaml", }}, - Publisher: publisher, - }, ApplicationOptions{Now: func() time.Time { return now }}) + Publisher: publisher, Runtime: runtime, + }, applicationTestOptions(func() time.Time { return now })) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi if len(publisher.published) != 1 || publisher.published[0] != configuration { t.Fatalf("published configurations = %+v", publisher.published) } + if runtime.notifications != 1 { + t.Fatalf("runtime notifications = %d, want 1", runtime.notifications) + } command := state.lastConfig if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) || !command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" { @@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin Operations: staticOperationalStatusReader{}, Configuration: forbiddenConfigurationLoader{}, Publisher: publisher, - }, ApplicationOptions{Now: func() time.Time { return now }}) + }, applicationTestOptions(func() time.Time { return now })) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration service, err := NewApplicationService(ApplicationDependencies{ State: state, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) { Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{ Value: validReloadConfiguration(), Source: "config.yaml", }}, Publisher: publisher, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) { } } +func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) { + publisher := &recordingConfigurationPublisher{} + service, err := NewApplicationService(ApplicationDependencies{ + State: &recordingAdminState{}, Operations: staticOperationalStatusReader{}, + Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{ + Value: validReloadConfiguration(), Source: "config.yaml", + }}, Publisher: publisher, + }, applicationTestOptions(time.Now)) + if err != nil { + t.Fatalf("NewApplicationService(): %v", err) + } + + _, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"}) + if !errors.Is(err, ErrUnavailable) { + t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err) + } + if len(publisher.published) != 0 { + t.Fatalf("published configurations = %d, want 0", len(publisher.published)) + } +} + +func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) { + store, err := config.NewStore(configWithOnlyUpstream("provider-a")) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + state := &orderedCommitState{ + firstCommitted: make(chan struct{}), + releaseFirst: make(chan struct{}), + } + service, err := NewApplicationService(ApplicationDependencies{ + State: state, Operations: staticOperationalStatusReader{}, + Configuration: staticConfigurationLoader{}, Publisher: store, + }, applicationTestOptions(time.Now)) + if err != nil { + t.Fatalf("NewApplicationService(): %v", err) + } + + firstDone := make(chan error, 1) + go func() { + _, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{ + Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml", + }) + firstDone <- applyErr + }() + select { + case <-state.firstCommitted: + case <-time.After(time.Second): + t.Fatal("first commit did not reach delayed return") + } + + if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{ + Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml", + }); err != nil { + t.Fatalf("ApplyConfiguration(new): %v", err) + } + close(state.releaseFirst) + if err := <-firstDone; err != nil { + t.Fatalf("ApplyConfiguration(old): %v", err) + } + + if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" { + t.Fatalf("published upstream = %q, want provider-c", got) + } + if got := store.Revision(); got != 2 { + t.Fatalf("published revision = %d, want 2", got) + } +} + func TestApplicationServiceReloadPreservesCancellation(t *testing.T) { t.Parallel() service, err := NewApplicationService(ApplicationDependencies{ State: &recordingAdminState{}, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{err: context.Canceled}, Publisher: &recordingConfigurationPublisher{}, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) { } } -func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) { +func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) { t.Parallel() state := &recordingAdminState{} publisher := &recordingConfigurationPublisher{} var commands []adminstate.CommitConfigCommand state.onCommit = func(command adminstate.CommitConfigCommand) { commands = append(commands, command) + state.mutation = adminstate.MutationResult{ + RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)), + } } for _, secret := range []string{"secret-a", "secret-b"} { configuration := validReloadConfiguration() @@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) { Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{ Value: configuration, Source: "config.yaml", }}, Publisher: publisher, - }, ApplicationOptions{Now: time.Now}) + }, applicationTestOptions(time.Now)) if err != nil { t.Fatalf("NewApplicationService() error = %v", err) } @@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) { t.Fatalf("ReloadConfiguration() error = %v", err) } } - if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion { - t.Fatalf("secret rotation changed public management digest: %+v", commands) + if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion { + t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands) } if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" { t.Fatalf("secret rotation was not published: %+v", publisher.published) @@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config { } } +func configWithOnlyUpstream(name string) *config.Config { + configuration := validReloadConfiguration() + configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")} + configuration.Routing = []config.Routing{{ + Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name}, + Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"}, + }} + return configuration +} + func validReloadUpstream(secret string) config.Upstream { return config.Upstream{ Enabled: true, Exposure: []string{"gateway"}, @@ -505,6 +629,33 @@ type recordingAdminState struct { onCommit func(adminstate.CommitConfigCommand) } +type orderedCommitState struct { + next atomic.Uint64 + firstCommitted chan struct{} + releaseFirst chan struct{} +} + +func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) { + return adminstate.MutationResult{}, nil +} + +func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) { + return adminstate.MutationResult{}, nil +} + +func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) { + revision := state.next.Add(1) + if revision == 1 { + close(state.firstCommitted) + <-state.releaseFirst + } + return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil +} + +func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) { + return adminstate.Snapshot{}, nil +} + func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) { state.lastUpstream = command return state.mutation, state.err @@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load type recordingConfigurationPublisher struct { published []*config.Config + revisions []uint64 } -func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) { - publisher.published = append(publisher.published, configuration) +type recordingRuntimeNotifier struct { + notifications int + validationErr error +} + +func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ } + +func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error { + return notifier.validationErr +} + +func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error { + return notifier.validationErr +} + +func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool { + publisher.published = append(publisher.published, configuration) + publisher.revisions = append(publisher.revisions, revision) + return true } diff --git a/internal/controller/bootstrap/bootstrap.go b/internal/controller/bootstrap/bootstrap.go index 57a3fb5..526005a 100644 --- a/internal/controller/bootstrap/bootstrap.go +++ b/internal/controller/bootstrap/bootstrap.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "strings" "time" @@ -15,11 +16,15 @@ import ( "proxy-pool/internal/controller/distribution" "proxy-pool/internal/controller/extraction" "proxy-pool/internal/controller/operations" + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/controller/provider" controllerRuntime "proxy-pool/internal/controller/runtime" "proxy-pool/internal/domain/activitypool" extractionDomain "proxy-pool/internal/domain/extraction" "proxy-pool/internal/platform/admission" + "proxy-pool/internal/platform/credentials" "proxy-pool/internal/platform/httpserver" + "proxy-pool/internal/platform/lifecycle" platformMetrics "proxy-pool/internal/platform/metrics" ) @@ -29,14 +34,19 @@ var ( ) type Options struct { - ConfigPath string - Resolver config.Resolver - Now func() time.Time - HTTP httpserver.Options + ConfigPath string + Resolver config.Resolver + Now func() time.Time + HTTP httpserver.Options + HolderID string + RedisNamespace string + FingerprintKey []byte } type activityStore interface { extractionDomain.Store + activitypool.Upserter + pool.InventoryReader activitypool.StateInventoryReader } @@ -45,6 +55,9 @@ type ports struct { activity activityStore readiness distribution.ReadinessChecker metricsReadiness platformMetrics.ReadinessChecker + coordinator provider.Coordinator + credentials credentials.Store + providerResults provider.ResultRecorder close func() error } @@ -61,7 +74,9 @@ type runtimeFactory interface { } func Run(ctx context.Context, options Options) error { - return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{}) + return run(ctx, options, &productionInfrastructure{ + holderID: options.HolderID, namespace: options.RedisNamespace, + }, productionRuntimeFactory{}) } func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) { @@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa if err != nil { return fmt.Errorf("%w: load configuration: %w", ErrStartup, err) } + if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes { + return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint) + } configurationStore, err := config.NewStore(loaded.Value) if err != nil { return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err) @@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa defer func() { resultErr = errors.Join(resultErr, opened.close()) }() + var providerState providerStateReader + if loaded.Value.Admin.Enabled { + providerState = opened.state + } + supervisor, err := newProviderSupervisor( + configurationStore, + providerState, + func(name string, upstream config.Upstream) (lifecycle.Runner, error) { + buildRuntime, err := providerRuntimeBuilder(opened) + if err != nil { + return nil, err + } + return buildRuntime(name, upstream) + }, + func(ctx context.Context, configuration *config.Config) error { + return prepareProviderConfiguration(ctx, configuration, opened.credentials) + }, + func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) }, + loader, + options.FingerprintKey, + providerSupervisorInterval, + ) + if err != nil { + return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err) + } dependencies := controllerRuntime.Dependencies{} if loaded.Value.Distribution.Enabled { @@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa if nilInterface(opened.state) || nilInterface(opened.activity) { return errors.Join(ErrStartup, ErrInvalidOptions) } - statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now) + var providerStats []provider.StatsReader + if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) { + providerStats = append(providerStats, stats) + } + statusReader, statusErr := operations.NewReader( + configurationStore, + opened.activity, + options.Now, + providerStats..., + ) if statusErr != nil { return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr) } service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{ State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore, - }, admin.ApplicationOptions{Now: options.Now}) + Runtime: supervisor, + }, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey}) if serviceErr != nil { return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr) } @@ -146,14 +199,68 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa dependencies.MetricsHandler = handler } - runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP}) + runners := make([]lifecycle.Runner, 0, 2) + if hasHTTPRuntime(loaded.Value) { + runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP}) + if err != nil { + return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err) + } + if nilInterface(runner) { + return errors.Join(ErrStartup, ErrInvalidOptions) + } + runners = append(runners, runner) + } + runners = append(runners, supervisor) + group, err := lifecycle.NewGroup(runners...) if err != nil { - return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err) + return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err) } - if nilInterface(runner) { - return errors.Join(ErrStartup, ErrInvalidOptions) + return group.Run(ctx) +} + +func prepareProviderConfiguration( + ctx context.Context, + configuration *config.Config, + credentialStore credentials.Store, +) error { + if ctx == nil || configuration == nil { + return ErrProviderRuntime } - return runner.Run(ctx) + if err := ctx.Err(); err != nil { + return err + } + if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) { + return nil + } + ensurer, ok := credentialStore.(credentials.CapacityEnsurer) + if !ok || nilInterface(credentialStore) { + return ErrProviderRuntime + } + if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil { + return err + } + return nil +} + +func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) { + if configuration == nil { + return + } + retainer, ok := results.(provider.StatsRetainer) + if !ok || nilInterface(retainer) { + return + } + names := make([]string, 0, len(configuration.Upstreams)) + for name := range configuration.Upstreams { + names = append(names, name) + } + sort.Strings(names) + retainer.RetainProviderStats(names) +} + +func hasHTTPRuntime(configuration *config.Config) bool { + return configuration != nil && + (configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled) } func extractionPolicy(configuration *config.Config) extraction.Policy { diff --git a/internal/controller/bootstrap/bootstrap_integration_test.go b/internal/controller/bootstrap/bootstrap_integration_test.go index fddb0fb..a72010b 100644 --- a/internal/controller/bootstrap/bootstrap_integration_test.go +++ b/internal/controller/bootstrap/bootstrap_integration_test.go @@ -4,16 +4,25 @@ package bootstrap import ( "context" + "errors" "net/http" "net/http/httptest" "os" + "strconv" "strings" + "sync/atomic" "testing" "time" + "github.com/redis/go-redis/v9" + + "proxy-pool/internal/adapters/redisactivity" "proxy-pool/internal/config" "proxy-pool/internal/controller/admin" + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/controller/provider" controllerRuntime "proxy-pool/internal/controller/runtime" + "proxy-pool/internal/platform/credentials" ) func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) { @@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes if postgresURL == "" || redisURL == "" { t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required") } + var providerCalls atomic.Int64 + providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + providerCalls.Add(1) + _, _ = writer.Write([]byte("http://192.0.2.10:8080")) + })) + defer providerServer.Close() + namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10) + inventory := newIntegrationInventoryReader(t, redisURL, namespace) + source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL) source = strings.ReplaceAll(source, "redis://fixture", redisURL) + source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL) resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}} - factory := &integrationRuntimeFactory{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + providerResults := make(chan provider.Result, 8) + factory := &integrationRuntimeFactory{ + cancel: cancel, inventory: inventory, providerResults: providerResults, + } + infrastructure := &integrationInfrastructure{ + productionInfrastructure: productionInfrastructure{namespace: namespace}, + results: providerResultRecorder(func(result provider.Result) { providerResults <- result }), + } - err := run(context.Background(), Options{ + err := run(ctx, Options{ ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC) }, - }, &productionInfrastructure{}, factory) - if err != nil { - t.Fatalf("run() error = %v", err) + FingerprintKey: bootstrapTestFingerprintKey, + }, infrastructure, factory) + if !errors.Is(err, context.Canceled) { + t.Fatalf("run() error = %v, want context cancellation", err) } if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 { t.Fatalf("Admin Status = %+v", factory.status) @@ -46,13 +75,19 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes !strings.Contains(factory.metricsBody, "go_") { t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody) } + if providerCalls.Load() == 0 { + t.Fatal("production Provider HTTP adapter was not called") + } } type integrationRuntimeFactory struct { - status admin.Status - readyStatus int - metricsStatus int - metricsBody string + status admin.Status + readyStatus int + metricsStatus int + metricsBody string + cancel context.CancelFunc + inventory pool.InventoryReader + providerResults <-chan provider.Result } func (factory *integrationRuntimeFactory) New( @@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New( factory.metricsBody = metrics.Body.String() status, err := dependencies.AdminService.Status(ctx) factory.status = status - return err + if err != nil { + return err + } + if err := waitForProviderInventory( + ctx, + factory.inventory, + factory.providerResults, + []string{"provider-a", "provider-b"}, + ); err != nil { + return err + } + factory.cancel() + <-ctx.Done() + return ctx.Err() }}, nil } +func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader { + t.Helper() + options, err := redis.ParseURL(redisURL) + if err != nil { + t.Fatalf("redis.ParseURL(): %v", err) + } + client := redis.NewClient(options) + t.Cleanup(func() { _ = client.Close() }) + credentialStore, err := credentials.NewMemoryStore(200) + if err != nil { + t.Fatalf("credentials.NewMemoryStore(): %v", err) + } + reader, err := redisactivity.New(client, redisactivity.Options{ + Namespace: namespace, Credentials: credentialStore, + OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan, + MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit, + }) + if err != nil { + t.Fatalf("redisactivity.New(): %v", err) + } + return reader +} + +func waitForProviderInventory( + ctx context.Context, + inventory pool.InventoryReader, + results <-chan provider.Result, + upstreamIDs []string, +) error { + deadline, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + var latestResult provider.Result + for { + var inventoryErr error + for _, upstreamID := range upstreamIDs { + snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0) + inventoryErr = errors.Join(inventoryErr, err) + if err == nil && snapshot.Managed > 0 { + return nil + } + } + select { + case <-deadline.Done(): + return errors.Join( + errors.New("wait for Provider Redis inventory"), + deadline.Err(), + inventoryErr, + latestResult.Err, + ) + case result := <-results: + latestResult = result + case <-ticker.C: + } + } +} + +type integrationInfrastructure struct { + productionInfrastructure + results provider.ResultRecorder +} + +func (infrastructure *integrationInfrastructure) Open( + ctx context.Context, + configuration *config.Config, +) (ports, error) { + opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration) + opened.providerResults = infrastructure.results + return opened, err +} + +type providerResultRecorder func(provider.Result) + +func (record providerResultRecorder) Record(result provider.Result) { record(result) } + type integrationRunner struct { run func(context.Context) error } diff --git a/internal/controller/bootstrap/bootstrap_test.go b/internal/controller/bootstrap/bootstrap_test.go index 440f62c..af3e701 100644 --- a/internal/controller/bootstrap/bootstrap_test.go +++ b/internal/controller/bootstrap/bootstrap_test.go @@ -3,32 +3,48 @@ package bootstrap import ( "context" "errors" + "strings" "testing" "time" "proxy-pool/internal/config" + "proxy-pool/internal/controller/admin" + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/controller/provider" controllerRuntime "proxy-pool/internal/controller/runtime" "proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/adminstate" extractionDomain "proxy-pool/internal/domain/extraction" + "proxy-pool/internal/domain/upstream" + "proxy-pool/internal/platform/credentials" ) +var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef") + func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) { t.Parallel() resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}} state := adminstate.NewMemoryStore() activity := &stubActivityStore{} + credentialStore, err := credentials.NewMemoryStore(10) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } closeErr := errors.New("close failed") infrastructure := &stubInfrastructure{ports: ports{ state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{}, + coordinator: coordinatorStub{}, credentials: credentialStore, close: func() error { return closeErr }, }} runErr := errors.New("runtime failed") factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}} now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC) - err := run(context.Background(), Options{ + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + err = run(ctx, Options{ ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now }, + FingerprintKey: bootstrapTestFingerprintKey, }, infrastructure, factory) if !errors.Is(err, runErr) || !errors.Is(err, closeErr) { t.Fatalf("run() error = %v, want runtime and close errors", err) @@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) { } } +func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) { + for name, key := range map[string][]byte{ + "missing": nil, + "short": []byte("too-short"), + } { + t.Run(name, func(t *testing.T) { + infrastructure := &stubInfrastructure{} + err := run(context.Background(), Options{ + ConfigPath: "controller.yaml", + Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}, + Now: time.Now, + FingerprintKey: key, + }, infrastructure, &recordingRuntimeFactory{}) + if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) { + t.Fatalf("run() error = %v", err) + } + if infrastructure.opens != 0 { + t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens) + } + }) + } +} + +func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) { + source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false") + source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false") + source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false") + credentialStore, err := credentials.NewMemoryStore(10) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + infrastructure := &stubInfrastructure{ports: ports{ + activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore, + close: func() error { return nil }, + }} + factory := &recordingRuntimeFactory{} + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(20*time.Millisecond, cancel) + err = run(ctx, Options{ + ConfigPath: "controller.yaml", + Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}, + Now: time.Now, + }, infrastructure, factory) + if !errors.Is(err, context.Canceled) { + t.Fatalf("run(provider only) error = %v, want context cancellation", err) + } + if factory.configuration != nil { + t.Fatal("HTTP runtime factory was called for Provider-only configuration") + } +} + +func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) { + state := adminstate.NewMemoryStore() + credentialStore, err := credentials.NewMemoryStore(200) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + started := make(chan string, 2) + stopped := make(chan string, 2) + infrastructure := &stubInfrastructure{ports: ports{ + state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{}, + coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error { + started <- upstreamID + <-ctx.Done() + stopped <- upstreamID + return ctx.Err() + }), + credentials: credentialStore, + close: func() error { return nil }, + }} + wantErr := errors.New("test HTTP runtime stopped") + factory := runtimeFactoryFunc(func( + _ *config.Config, + dependencies controllerRuntime.Dependencies, + _ controllerRuntime.Options, + ) (controllerRunner, error) { + return runnerFunc(func(ctx context.Context) error { + for { + select { + case upstreamID := <-started: + if upstreamID != "provider-a" { + continue + } + if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{ + RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false, + }); err != nil { + return err + } + for { + select { + case stoppedID := <-stopped: + if stoppedID == "provider-a" { + return wantErr + } + case <-ctx.Done(): + return ctx.Err() + } + } + case <-ctx.Done(): + return ctx.Err() + } + } + }), nil + }) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + err = run(ctx, Options{ + ConfigPath: "controller.yaml", + Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}, + Now: time.Now, + FingerprintKey: bootstrapTestFingerprintKey, + }, infrastructure, factory) + if !errors.Is(err, wantErr) { + t.Fatalf("run() error = %v, want %v", err, wantErr) + } +} + +func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + disabled := configuration.Upstreams["provider-b"] + disabled.Enabled = false + configuration.Upstreams["provider-b"] = disabled + stats, err := provider.NewStatsRecorder(2) + if err != nil { + t.Fatalf("provider.NewStatsRecorder(): %v", err) + } + stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError}) + stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError}) + + retainProviderStats(configuration, stats) + stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError}) + got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"}) + if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 { + t.Fatalf("Provider stats after retention = %+v", got) + } +} + type memoryResolver struct { files map[string][]byte reads int @@ -127,6 +284,24 @@ type runnerStub struct{ err error } func (runner runnerStub) Run(context.Context) error { return runner.err } +type runnerFunc func(context.Context) error + +func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) } + +type runtimeFactoryFunc func( + *config.Config, + controllerRuntime.Dependencies, + controllerRuntime.Options, +) (controllerRunner, error) + +func (factory runtimeFactoryFunc) New( + configuration *config.Config, + dependencies controllerRuntime.Dependencies, + options controllerRuntime.Options, +) (controllerRunner, error) { + return factory(configuration, dependencies, options) +} + type readyStub struct{} func (readyStub) Ready(context.Context) error { return nil } @@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory( return result, nil } +func (*stubActivityStore) UpsertFetched( + _ context.Context, + _ string, + batch activitypool.FetchedBatch, +) (activitypool.UpsertResult, error) { + return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil +} + +func (*stubActivityStore) ReadInventory( + context.Context, + string, + time.Duration, +) (pool.InventorySnapshot, error) { + return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil +} + +type coordinatorStub struct{} + +func (coordinatorStub) RunLeader( + ctx context.Context, + _ string, + _ provider.CoordinationLimits, + _ func(context.Context, provider.LeaderSession) error, +) error { + <-ctx.Done() + return ctx.Err() +} + +type coordinatorFunc func(context.Context, string) error + +func (run coordinatorFunc) RunLeader( + ctx context.Context, + upstreamID string, + _ provider.CoordinationLimits, + _ func(context.Context, provider.LeaderSession) error, +) error { + return run(ctx, upstreamID) +} + const bootstrapTestConfig = ` version: 1 security: diff --git a/internal/controller/bootstrap/infrastructure.go b/internal/controller/bootstrap/infrastructure.go index 3df49d3..8f1c4e8 100644 --- a/internal/controller/bootstrap/infrastructure.go +++ b/internal/controller/bootstrap/infrastructure.go @@ -2,6 +2,8 @@ package bootstrap import ( "context" + "crypto/rand" + "encoding/hex" "errors" "strings" "time" @@ -11,16 +13,23 @@ import ( "proxy-pool/internal/adapters/postgresadmin" "proxy-pool/internal/adapters/redisactivity" + "proxy-pool/internal/adapters/redisprovider" "proxy-pool/internal/config" + controllerProvider "proxy-pool/internal/controller/provider" "proxy-pool/internal/platform/credentials" platformMetrics "proxy-pool/internal/platform/metrics" ) const ( - redisNamespace = "controller" - redisOperationTTL = 30 * time.Second - redisMinimumScan = 4_096 - redisCleanupLimit = 1_024 + redisNamespace = "controller" + redisOperationTTL = 30 * time.Second + redisMinimumScan = 4_096 + redisMaximumScan = config.MaximumPoolSize + redisCleanupLimit = 1_024 + providerLeaseTTL = 15 * time.Second + providerRenewEvery = 3 * time.Second + providerRetryInterval = 100 * time.Millisecond + providerPermitGrace = 5 * time.Second ) var ( @@ -30,15 +39,22 @@ var ( ErrRedisUnavailable = errors.New("Redis unavailable") ) -type productionInfrastructure struct{} +type productionInfrastructure struct { + holderID string + namespace string +} -func (*productionInfrastructure) Open( +func (infrastructure *productionInfrastructure) Open( ctx context.Context, configuration *config.Config, ) (_ ports, resultErr error) { if ctx == nil || configuration == nil { return ports{}, ErrInvalidOptions } + namespace, err := resolveRedisNamespace(infrastructure.namespace) + if err != nil { + return ports{}, err + } var postgresPool *pgxpool.Pool var redisClient *redis.Client closeResources := func() error { @@ -83,7 +99,8 @@ func (*productionInfrastructure) Open( } } - if configuration.Distribution.Enabled || configuration.Admin.Enabled { + providersEnabled := hasEnabledUpstream(configuration) + if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled { if strings.TrimSpace(configuration.Storage.RedisURL) == "" { return ports{}, ErrRedisConfiguration } @@ -95,17 +112,17 @@ func (*productionInfrastructure) Open( if err = redisClient.Ping(ctx).Err(); err != nil { return ports{}, contextOr(ctx, ErrRedisUnavailable) } - credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration)) + credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration)) if err != nil { return ports{}, err } adapter, err := redisactivity.New(redisClient, redisactivity.Options{ - Namespace: redisNamespace, + Namespace: namespace, Credentials: credentialStore, OperationTTL: redisOperationTTL, MaxCandidateScan: candidateScan(configuration), MaxRuntimeCounters: credentialCapacity(configuration), - MaxInventoryScan: credentialCapacity(configuration), + MaxInventoryScan: maxInventoryScan(configuration), CleanupLimit: redisCleanupLimit, }) if err != nil { @@ -113,6 +130,26 @@ func (*productionInfrastructure) Open( } opened.activity = adapter opened.readiness = redisReadiness{client: redisClient} + opened.credentials = credentialStore + if providersEnabled { + stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams) + if statsErr != nil { + return ports{}, statsErr + } + opened.providerResults = stats + holderID, holderErr := resolveHolderID(infrastructure.holderID) + if holderErr != nil { + return ports{}, holderErr + } + opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{ + Namespace: namespace, HolderID: holderID, + LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery, + RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace, + }) + if err != nil { + return ports{}, err + } + } } if configuration.Metrics.Enabled { opened.metricsReadiness = selectMetricsReadiness( @@ -124,11 +161,21 @@ func (*productionInfrastructure) Open( return opened, nil } +func resolveRedisNamespace(configured string) (string, error) { + if strings.TrimSpace(configured) != configured { + return "", ErrInvalidOptions + } + if configured == "" { + return redisNamespace, nil + } + return configured, nil +} + func selectMetricsReadiness( configuration *config.Config, admin, activity platformMetrics.ReadinessChecker, ) platformMetrics.ReadinessChecker { - if configuration.Distribution.Enabled { + if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) { return activity } if configuration.Admin.Enabled { @@ -186,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int { capacity := 0 maximum := int(^uint(0) >> 1) for _, upstream := range configuration.Upstreams { - if upstream.Pool.MaxSize <= 0 { + if !upstream.Enabled || upstream.Pool.MaxSize <= 0 { continue } if capacity > maximum-upstream.Pool.MaxSize { @@ -200,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int { return capacity } +func providerCredentialCapacity(configuration *config.Config) int { + capacity := 0 + maximum := int(^uint(0) >> 1) + for _, upstream := range configuration.Upstreams { + if upstream.Pool.MaxSize <= 0 { + continue + } + maxInFlight := upstream.Fetch.MaxInFlight + if maxInFlight <= 0 { + maxInFlight = 1 + } + if upstream.Pool.MaxSize > maximum/maxInFlight { + return maximum + } + leases := upstream.Pool.MaxSize * maxInFlight + if capacity > maximum-leases { + return maximum + } + capacity += leases + } + if capacity == 0 { + return 1 + } + return capacity +} + +func maxInventoryScan(_ *config.Config) int { + return config.MaximumPoolSize +} + +func hasEnabledUpstream(configuration *config.Config) bool { + if configuration == nil { + return false + } + for _, upstream := range configuration.Upstreams { + if upstream.Enabled { + return true + } + } + return false +} + +func resolveHolderID(configured string) (string, error) { + if strings.TrimSpace(configured) != configured { + return "", ErrInvalidOptions + } + if configured != "" { + return configured, nil + } + var entropy [16]byte + if _, err := rand.Read(entropy[:]); err != nil { + return "", errors.Join(ErrStartup, err) + } + return "controller-" + hex.EncodeToString(entropy[:]), nil +} + func candidateScan(configuration *config.Config) int { configured := configuration.Distribution.Extraction if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway { diff --git a/internal/controller/bootstrap/infrastructure_test.go b/internal/controller/bootstrap/infrastructure_test.go index e16e4fe..bfa9d0c 100644 --- a/internal/controller/bootstrap/infrastructure_test.go +++ b/internal/controller/bootstrap/infrastructure_test.go @@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) { MaxCountPerRequest: 100, ReserveForGateway: 5_000, }}, Upstreams: map[string]config.Upstream{ - "provider-a": {Pool: config.Pool{MaxSize: 3_000}}, - "provider-b": {Pool: config.Pool{MaxSize: 2_000}}, + "provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}}, + "provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}}, + "disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}}, }, } if got := credentialCapacity(configuration); got != 5_000 { t.Fatalf("credentialCapacity() = %d, want 5000", got) } + if got := providerCredentialCapacity(configuration); got != 158_000 { + t.Fatalf("providerCredentialCapacity() = %d, want 158000", got) + } if got := candidateScan(configuration); got != 5_100 { t.Fatalf("candidateScan() = %d, want 5100", got) } @@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) { t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan) } } + +func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) { + t.Parallel() + configuration := &config.Config{Upstreams: map[string]config.Upstream{ + "provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}}, + }} + + if got := maxInventoryScan(configuration); got != config.MaximumPoolSize { + t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize) + } +} diff --git a/internal/controller/bootstrap/provider.go b/internal/controller/bootstrap/provider.go new file mode 100644 index 0000000..6f9f0df --- /dev/null +++ b/internal/controller/bootstrap/provider.go @@ -0,0 +1,126 @@ +package bootstrap + +import ( + "errors" + "math" + "sort" + "strings" + "time" + + "proxy-pool/internal/adapters/providerapi" + "proxy-pool/internal/config" + "proxy-pool/internal/controller/pool" + controllerProvider "proxy-pool/internal/controller/provider" +) + +var ErrProviderRuntime = errors.New("invalid Provider runtime configuration") + +func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) { + if configuration == nil { + return nil, ErrProviderRuntime + } + names := make([]string, 0, len(configuration.Upstreams)) + for name, upstream := range configuration.Upstreams { + if upstream.Enabled { + names = append(names, name) + } + } + if len(names) == 0 { + return nil, nil + } + builder, err := providerRuntimeBuilder(opened) + if err != nil { + return nil, err + } + sort.Strings(names) + runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names)) + for _, name := range names { + runtime, err := builder(name, configuration.Upstreams[name]) + if err != nil { + return nil, err + } + runtimes = append(runtimes, runtime) + } + return controllerProvider.NewFleet(runtimes...) +} + +type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error) + +func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) { + if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) { + return nil, ErrProviderRuntime + } + results := opened.providerResults + if nilInterface(results) { + results = discardProviderResults{} + } + return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) { + upstream.Enabled = true + mapped, err := providerRuntimeConfig(name, upstream) + if err != nil { + return nil, err + } + adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil) + if err != nil { + return nil, err + } + parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials) + if err != nil { + return nil, err + } + return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{ + Coordinator: opened.coordinator, + Inventory: opened.activity, + Adapter: adapter, + Parser: parser, + Activity: opened.activity, + Results: results, + }) + }, nil +} + +func providerRuntimeConfig( + upstreamID string, + upstream config.Upstream, +) (controllerProvider.UpstreamRuntimeConfig, error) { + if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled || + upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 || + upstream.Pool.MaxSize > config.MaximumPoolSize || + int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger || + int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger || + int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger || + int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) { + return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime + } + expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy) + return controllerProvider.UpstreamRuntimeConfig{ + Provider: controllerProvider.Config{ + UpstreamID: upstreamID, + RequestInterval: time.Duration(upstream.Fetch.RequestInterval), + Timeout: time.Duration(upstream.Fetch.Timeout), + MaxAttempts: upstream.Fetch.MaxAttempts, + MaxInFlight: upstream.Fetch.MaxInFlight, + MaxTotal: int64(upstream.Fetch.MaxTotal), + MaxSize: upstream.Pool.MaxSize, + TTL: time.Duration(upstream.Lifecycle.TTL), + AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin), + Retry: controllerProvider.RetryConfig{ + Initial: time.Duration(upstream.Fetch.Retry.Initial), + Max: time.Duration(upstream.Fetch.Retry.Max), + Jitter: upstream.Fetch.Retry.Jitter, + }, + }, + ReconcilePolicy: pool.ReconcilePolicy{ + MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots, + TargetAvailableSlots: upstream.Refill.TargetAvailableSlots, + ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall, + ExpectedSlotsPerFetch: expectedSlots, + SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin), + }, + ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval), + }, nil +} + +type discardProviderResults struct{} + +func (discardProviderResults) Record(controllerProvider.Result) {} diff --git a/internal/controller/bootstrap/provider_supervisor.go b/internal/controller/bootstrap/provider_supervisor.go new file mode 100644 index 0000000..a893628 --- /dev/null +++ b/internal/controller/bootstrap/provider_supervisor.go @@ -0,0 +1,375 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + "sync" + "time" + + "proxy-pool/internal/config" + "proxy-pool/internal/controller/admin" + "proxy-pool/internal/domain/adminstate" + "proxy-pool/internal/platform/lifecycle" +) + +const providerSupervisorInterval = time.Second + +var ( + ErrProviderSupervisor = errors.New("invalid Provider supervisor") + errProviderManagementStateUnavailable = errors.New("Provider management state unavailable") + errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale") + errProviderConfigurationPending = errors.New("Provider configuration synchronization pending") +) + +type providerConfigurationStore interface { + Current() *config.Config + Revision() uint64 + PublishRevision(*config.Config, uint64) bool +} + +type providerConfigurationSource interface { + LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) +} + +type providerStateReader interface { + Snapshot(context.Context) (adminstate.Snapshot, error) +} + +type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error) +type providerConfigurationPreparer func(context.Context, *config.Config) error +type providerConfigurationObserver func(*config.Config) + +type providerSupervisor struct { + configuration providerConfigurationStore + state providerStateReader + build providerRunnerBuilder + prepare providerConfigurationPreparer + observe providerConfigurationObserver + source providerConfigurationSource + fingerprintKey []byte + interval time.Duration + notify chan struct{} +} + +type runningProvider struct { + configuration config.Upstream + cancel context.CancelFunc + done chan struct{} +} + +func newProviderSupervisor( + configuration providerConfigurationStore, + state providerStateReader, + build providerRunnerBuilder, + prepare providerConfigurationPreparer, + observe providerConfigurationObserver, + source providerConfigurationSource, + fingerprintKey []byte, + interval time.Duration, +) (*providerSupervisor, error) { + if nilInterface(configuration) || build == nil || interval <= 0 || + (!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) { + return nil, ErrProviderSupervisor + } + return &providerSupervisor{ + configuration: configuration, + state: state, + build: build, + prepare: prepare, + observe: observe, + source: source, + fingerprintKey: append([]byte(nil), fingerprintKey...), + interval: interval, + notify: make(chan struct{}, 1), + }, nil +} + +func (supervisor *providerSupervisor) Notify() { + if supervisor == nil || supervisor.notify == nil { + return + } + select { + case supervisor.notify <- struct{}{}: + default: + } +} + +func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error { + if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil { + return ErrProviderSupervisor + } + if err := supervisor.validateConfiguration(ctx, configuration); err != nil { + return errors.Join(ErrProviderSupervisor, err) + } + return nil +} + +func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error { + if err := ctx.Err(); err != nil { + return err + } + if err := config.Validate(configuration); err != nil { + return err + } + if supervisor.prepare != nil { + if err := supervisor.prepare(ctx, configuration); err != nil { + return err + } + } + names := make([]string, 0, len(configuration.Upstreams)) + for name, upstream := range configuration.Upstreams { + if upstream.Enabled { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + if err := ctx.Err(); err != nil { + return err + } + if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil { + return err + } + } + return nil +} + +func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error { + if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil { + return ErrProviderSupervisor + } + if err := ctx.Err(); err != nil { + return err + } + configuration := supervisor.configuration.Current() + if configuration == nil { + return ErrProviderSupervisor + } + if supervisor.prepare != nil { + if err := supervisor.prepare(ctx, configuration); err != nil { + return errors.Join(ErrProviderSupervisor, err) + } + } + upstream, exists := configuration.Upstreams[name] + if !exists { + return ErrProviderSupervisor + } + upstream.Enabled = true + if _, err := supervisor.build(name, upstream); err != nil { + return errors.Join(ErrProviderSupervisor, err) + } + return nil +} + +func (supervisor *providerSupervisor) Run(ctx context.Context) error { + if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) || + supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil { + return ErrProviderSupervisor + } + active := make(map[string]*runningProvider) + failures := make(chan error, 1) + defer stopAllProviders(active) + ticker := time.NewTicker(supervisor.interval) + defer ticker.Stop() + + for { + if err := supervisor.reconcile(ctx, active, failures); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-failures: + return err + case <-supervisor.notify: + case <-ticker.C: + } + } +} + +func (supervisor *providerSupervisor) reconcile( + ctx context.Context, + active map[string]*runningProvider, + failures chan<- error, +) error { + desired, err := supervisor.desired(ctx) + if err != nil { + if errors.Is(err, errProviderManagementStateUnavailable) || + errors.Is(err, errProviderManagementSnapshotStale) { + return nil + } + if errors.Is(err, errProviderConfigurationPending) { + stopAllProviders(active) + clear(active) + return nil + } + return err + } + names := make([]string, 0, len(desired)) + prepared := make(map[string]lifecycle.Runner) + for name, upstream := range desired { + names = append(names, name) + current := active[name] + if current != nil && reflect.DeepEqual(current.configuration, upstream) { + continue + } + runner, buildErr := supervisor.build(name, upstream) + if buildErr != nil || nilInterface(runner) { + return errors.Join(ErrProviderSupervisor, buildErr) + } + prepared[name] = runner + } + sort.Strings(names) + + for name, current := range active { + if _, keep := desired[name]; !keep { + stopProvider(current) + delete(active, name) + } + } + for _, name := range names { + runner := prepared[name] + if runner == nil { + continue + } + if current := active[name]; current != nil { + stopProvider(current) + } + active[name] = startProvider(ctx, name, desired[name], runner, failures) + } + return nil +} + +func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) { + configuration := supervisor.configuration.Current() + if configuration == nil { + return nil, ErrProviderSupervisor + } + enabled := make(map[string]bool, len(configuration.Upstreams)) + var snapshot adminstate.Snapshot + if !nilInterface(supervisor.state) { + var err error + snapshot, err = supervisor.state.Snapshot(ctx) + if err != nil { + return nil, errors.Join(errProviderManagementStateUnavailable, err) + } + configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot) + if err != nil { + return nil, err + } + for _, upstream := range snapshot.Upstreams { + enabled[upstream.Name] = upstream.Enabled + } + } + if supervisor.prepare != nil { + if err := supervisor.prepare(ctx, configuration); err != nil { + return nil, errors.Join(ErrProviderSupervisor, err) + } + } + if supervisor.observe != nil { + supervisor.observe(configuration) + } + desired := make(map[string]config.Upstream) + for name, upstream := range configuration.Upstreams { + isEnabled := upstream.Enabled + if !nilInterface(supervisor.state) { + isEnabled = enabled[name] + } + if isEnabled { + upstream.Enabled = true + desired[name] = upstream + } + } + return desired, nil +} + +func (supervisor *providerSupervisor) synchronizeConfiguration( + ctx context.Context, + current *config.Config, + snapshot adminstate.Snapshot, +) (*config.Config, error) { + if snapshot.Config == nil || snapshot.Config.Checksum == "" { + return current, nil + } + if supervisor.configuration.Revision() > snapshot.Config.Revision { + return nil, errProviderManagementSnapshotStale + } + checksum, err := config.Fingerprint(current, supervisor.fingerprintKey) + if err != nil { + return nil, errors.Join(ErrProviderSupervisor, err) + } + if checksum == snapshot.Config.Checksum { + supervisor.configuration.PublishRevision(current, snapshot.Config.Revision) + return current, nil + } + if nilInterface(supervisor.source) { + return nil, errProviderConfigurationPending + } + loaded, err := supervisor.source.LoadConfiguration(ctx) + if err != nil || loaded.Value == nil { + return nil, errors.Join(errProviderConfigurationPending, err) + } + checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey) + if err != nil || checksum != snapshot.Config.Checksum { + return nil, errors.Join(errProviderConfigurationPending, err) + } + if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil { + return nil, errors.Join(errProviderConfigurationPending, err) + } + if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) { + return loaded.Value, nil + } + if supervisor.configuration.Revision() > snapshot.Config.Revision { + return nil, errProviderManagementSnapshotStale + } + return nil, errProviderConfigurationPending +} + +func startProvider( + ctx context.Context, + name string, + configuration config.Upstream, + runner lifecycle.Runner, + failures chan<- error, +) *runningProvider { + runCtx, cancel := context.WithCancel(ctx) + running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})} + go func() { + defer close(running.done) + err := runner.Run(runCtx) + if runCtx.Err() != nil { + return + } + if err == nil { + err = lifecycle.ErrRunnerStopped + } + select { + case failures <- fmt.Errorf("Provider %s runtime: %w", name, err): + default: + } + }() + return running +} + +func stopProvider(running *runningProvider) { + if running == nil { + return + } + running.cancel() + <-running.done +} + +func stopAllProviders(active map[string]*runningProvider) { + var wait sync.WaitGroup + for _, running := range active { + wait.Add(1) + go func(current *runningProvider) { + defer wait.Done() + stopProvider(current) + }(running) + } + wait.Wait() +} diff --git a/internal/controller/bootstrap/provider_supervisor_test.go b/internal/controller/bootstrap/provider_supervisor_test.go new file mode 100644 index 0000000..0c30a2a --- /dev/null +++ b/internal/controller/bootstrap/provider_supervisor_test.go @@ -0,0 +1,376 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "proxy-pool/internal/config" + "proxy-pool/internal/controller/admin" + "proxy-pool/internal/domain/adminstate" + "proxy-pool/internal/platform/lifecycle" +) + +func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + store, err := config.NewStore(configuration) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}} + started := make(chan providerRuntimeEvent, 4) + stopped := make(chan providerRuntimeEvent, 4) + supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) { + event := providerRuntimeEvent{name: name, url: upstream.API.URL} + return supervisorRunnerFunc(func(ctx context.Context) error { + started <- event + <-ctx.Done() + stopped <- event + return ctx.Err() + }), nil + }, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour) + if err != nil { + t.Fatalf("newProviderSupervisor(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- supervisor.Run(ctx) }() + waitForRuntimeEvents(t, started, 2) + + state.set("provider-a", false) + supervisor.Notify() + if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" { + t.Fatalf("stopped Provider = %s, want provider-a", event.name) + } + + updated := store.Current() + providerB := updated.Upstreams["provider-b"] + providerB.API.URL = "https://replacement.invalid/proxies" + updated.Upstreams["provider-b"] = providerB + if !store.PublishRevision(updated, 1) { + t.Fatal("PublishRevision() rejected updated configuration") + } + supervisor.Notify() + if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" { + t.Fatalf("replaced Provider = %s, want provider-b", event.name) + } + if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL { + t.Fatalf("replacement Provider = %+v", event) + } + + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context cancellation", err) + } +} + +func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + store, err := config.NewStore(configuration) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + wantErr := errors.New("coordination stopped") + supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) { + return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil + }, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour) + if err != nil { + t.Fatalf("newProviderSupervisor(): %v", err) + } + if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) { + t.Fatalf("Run() error = %v, want %v", err, wantErr) + } +} + +func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + store, err := config.NewStore(configuration) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}} + started := make(chan providerRuntimeEvent, 2) + stopped := make(chan providerRuntimeEvent, 2) + supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) { + event := providerRuntimeEvent{name: name, url: upstream.API.URL} + return supervisorRunnerFunc(func(ctx context.Context) error { + started <- event + <-ctx.Done() + stopped <- event + return ctx.Err() + }), nil + }, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour) + if err != nil { + t.Fatalf("newProviderSupervisor(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- supervisor.Run(ctx) }() + waitForRuntimeEvents(t, started, 2) + + state.setError(errors.New("PostgreSQL temporarily unavailable")) + supervisor.Notify() + select { + case err := <-done: + t.Fatalf("Supervisor stopped during transient state failure: %v", err) + case event := <-stopped: + t.Fatalf("Provider stopped during transient state failure: %+v", event) + case <-time.After(50 * time.Millisecond): + } + + state.setError(nil) + state.set("provider-a", false) + supervisor.Notify() + if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" { + t.Fatalf("stopped Provider = %s, want provider-a", event.name) + } + + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context cancellation", err) + } +} + +func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) { + initial, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + store, err := config.NewStore(initial) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey) + if err != nil { + t.Fatalf("config.Fingerprint(initial): %v", err) + } + state := &mutableProviderState{ + enabled: map[string]bool{"provider-a": true, "provider-b": true}, + checksum: initialChecksum, + revision: 1, + } + source := &mutableProviderConfigurationSource{configuration: initial} + started := make(chan providerRuntimeEvent, 4) + stopped := make(chan providerRuntimeEvent, 4) + supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) { + event := providerRuntimeEvent{name: name, url: upstream.API.URL} + return supervisorRunnerFunc(func(ctx context.Context) error { + started <- event + <-ctx.Done() + stopped <- event + return ctx.Err() + }), nil + }, nil, nil, source, bootstrapTestFingerprintKey, time.Hour) + if err != nil { + t.Fatalf("newProviderSupervisor(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- supervisor.Run(ctx) }() + waitForRuntimeEvents(t, started, 2) + + updated := store.Current() + providerA := updated.Upstreams["provider-a"] + providerA.API.URL = "https://replacement.invalid/proxies" + updated.Upstreams["provider-a"] = providerA + updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey) + if err != nil { + t.Fatalf("config.Fingerprint(updated): %v", err) + } + state.setChecksum(updatedChecksum) + supervisor.Notify() + waitForRuntimeEvents(t, stopped, 2) + select { + case err := <-done: + t.Fatalf("Supervisor stopped for stale local configuration: %v", err) + case <-time.After(50 * time.Millisecond): + } + + source.set(updated) + supervisor.Notify() + events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)} + foundReplacement := false + for _, event := range events { + if event.name == "provider-a" && event.url == providerA.API.URL { + foundReplacement = true + } + } + if !foundReplacement { + t.Fatalf("started Provider runtimes = %+v, replacement missing", events) + } + if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum { + t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum) + } + + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context cancellation", err) + } +} + +func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) { + initial, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + store, err := config.NewStore(initial) + if err != nil { + t.Fatalf("config.NewStore(): %v", err) + } + store.PublishRevision(initial, 1) + + candidate := store.Current() + providerA := candidate.Upstreams["provider-a"] + providerA.API.URL = "https://candidate.invalid/proxies" + candidate.Upstreams["provider-a"] = providerA + candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey) + if err != nil { + t.Fatalf("config.Fingerprint(candidate): %v", err) + } + newest := store.Current() + providerA = newest.Upstreams["provider-a"] + providerA.API.URL = "https://newest.invalid/proxies" + newest.Upstreams["provider-a"] = providerA + + source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) { + if !store.PublishRevision(newest, 3) { + t.Fatal("failed to publish simulated concurrent revision") + } + return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil + }) + supervisor, err := newProviderSupervisor( + store, + &mutableProviderState{}, + func(string, config.Upstream) (lifecycle.Runner, error) { + return supervisorRunnerFunc(func(context.Context) error { return nil }), nil + }, + nil, + nil, + source, + bootstrapTestFingerprintKey, + time.Hour, + ) + if err != nil { + t.Fatalf("newProviderSupervisor(): %v", err) + } + + _, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{ + Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum}, + }) + if !errors.Is(err, errProviderManagementSnapshotStale) { + t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err) + } + if got := store.Revision(); got != 3 { + t.Fatalf("configuration revision = %d, want 3", got) + } +} + +type providerRuntimeEvent struct { + name string + url string +} + +type supervisorRunnerFunc func(context.Context) error + +func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) } + +type mutableProviderState struct { + mu sync.Mutex + enabled map[string]bool + err error + checksum string + revision uint64 +} + +func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) { + state.mu.Lock() + defer state.mu.Unlock() + if state.err != nil { + return adminstate.Snapshot{}, state.err + } + snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))} + if state.checksum != "" { + snapshot.Config = &adminstate.ConfigRevision{ + Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum, + } + } + for name, enabled := range state.enabled { + snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled}) + } + return snapshot, nil +} + +func (state *mutableProviderState) setError(err error) { + state.mu.Lock() + defer state.mu.Unlock() + state.err = err +} + +func (state *mutableProviderState) setChecksum(checksum string) { + state.mu.Lock() + defer state.mu.Unlock() + state.checksum = checksum + state.revision++ +} + +type mutableProviderConfigurationSource struct { + mu sync.Mutex + configuration *config.Config +} + +type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error) + +func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) { + return source(ctx) +} + +func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) { + source.mu.Lock() + defer source.mu.Unlock() + return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil +} + +func (source *mutableProviderConfigurationSource) set(configuration *config.Config) { + source.mu.Lock() + defer source.mu.Unlock() + source.configuration = configuration +} + +func (state *mutableProviderState) set(name string, enabled bool) { + state.mu.Lock() + defer state.mu.Unlock() + state.enabled[name] = enabled +} + +func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) { + t.Helper() + for range count { + _ = waitForRuntimeEvent(t, events) + } +} + +func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent { + t.Helper() + select { + case event := <-events: + return event + case <-time.After(time.Second): + t.Fatal("timed out waiting for Provider runtime event") + return providerRuntimeEvent{} + } +} diff --git a/internal/controller/bootstrap/provider_test.go b/internal/controller/bootstrap/provider_test.go new file mode 100644 index 0000000..09ca17a --- /dev/null +++ b/internal/controller/bootstrap/provider_test.go @@ -0,0 +1,104 @@ +package bootstrap + +import ( + "strconv" + "strings" + "testing" + "time" + + "proxy-pool/internal/config" + "proxy-pool/internal/controller/provider" + "proxy-pool/internal/platform/credentials" +) + +func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + upstream := configuration.Upstreams["provider-a"] + mapped, err := providerRuntimeConfig("provider-a", upstream) + if err != nil { + t.Fatalf("providerRuntimeConfig(): %v", err) + } + + if mapped.Provider.UpstreamID != "provider-a" || + mapped.Provider.RequestInterval != time.Second || + mapped.Provider.Timeout != 3*time.Second || + mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 || + mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 || + mapped.Provider.TTL != 2*time.Minute || + mapped.Provider.AllocationSafetyMargin != 10*time.Second { + t.Fatalf("Provider config = %+v", mapped.Provider) + } + if mapped.ReconcileInterval != time.Second || + mapped.ReconcilePolicy.MinimumAvailableSlots != 100 || + mapped.ReconcilePolicy.TargetAvailableSlots != 200 || + mapped.ReconcilePolicy.ExpectedPerFetch != 10 || + mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 || + mapped.ReconcilePolicy.SafetyMargin != 10*time.Second { + t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval) + } +} + +func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) { + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + configuration.Upstreams["disabled"] = config.Upstream{} + credentialStore, err := credentials.NewMemoryStore(10) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + fleet, err := newProviderFleet(configuration, ports{ + activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore, + }) + if err != nil { + t.Fatalf("newProviderFleet(): %v", err) + } + if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" { + t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got) + } + + configuration.Upstreams["provider-a"] = config.Upstream{} + configuration.Upstreams["provider-b"] = config.Upstream{} + fleet, err = newProviderFleet(configuration, ports{}) + if err != nil || fleet != nil { + t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err) + } +} + +func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) { + if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil { + t.Fatal("providerRuntimeConfig(disabled) error = nil") + } + overflowing := config.Upstream{ + Enabled: true, + Pool: config.Pool{MaxSize: int(^uint(0) >> 1)}, + Capacity: config.Capacity{ + MaxConcurrencyPerProxy: int(^uint(0) >> 1), + }, + Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)}, + } + if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil { + t.Fatal("providerRuntimeConfig(overflow) error = nil") + } + configuration, err := config.Load(strings.NewReader(bootstrapTestConfig)) + if err != nil { + t.Fatalf("config.Load(): %v", err) + } + tooLarge := configuration.Upstreams["provider-a"] + tooLarge.Pool.MaxSize = redisMaximumScan + 1 + if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil { + t.Fatal("providerRuntimeConfig(oversized inventory) error = nil") + } + tooLarge = configuration.Upstreams["provider-a"] + tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger) + if strconv.IntSize == 64 { + tooLarge.Fetch.MaxTotal++ + if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil { + t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil") + } + } +} diff --git a/internal/controller/operations/reader.go b/internal/controller/operations/reader.go index 3be59fc..28e5005 100644 --- a/internal/controller/operations/reader.go +++ b/internal/controller/operations/reader.go @@ -9,6 +9,7 @@ import ( "proxy-pool/internal/config" "proxy-pool/internal/controller/admin" + "proxy-pool/internal/controller/provider" "proxy-pool/internal/domain/activitypool" ) @@ -24,6 +25,7 @@ type ConfigurationReader interface { type Reader struct { configuration ConfigurationReader inventory activitypool.StateInventoryReader + providerStats provider.StatsReader now func() time.Time } @@ -33,11 +35,17 @@ func NewReader( configuration ConfigurationReader, inventory activitypool.StateInventoryReader, now func() time.Time, + stats ...provider.StatsReader, ) (*Reader, error) { - if nilInterface(configuration) || nilInterface(inventory) || now == nil { + if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 || + (len(stats) == 1 && nilInterface(stats[0])) { return nil, ErrInvalidReader } - return &Reader{configuration: configuration, inventory: inventory, now: now}, nil + reader := &Reader{configuration: configuration, inventory: inventory, now: now} + if len(stats) == 1 { + reader.providerStats = stats[0] + } + return reader, nil } func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) { @@ -69,17 +77,27 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio } status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))} + providerStats := make([]provider.Stats, len(upstreamIDs)) + if reader.providerStats != nil { + providerStats = reader.providerStats.ReadProviderStats(upstreamIDs) + if len(providerStats) != len(upstreamIDs) { + return admin.OperationalStatus{}, ErrUnavailable + } + } for index, inventory := range inventories { - if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) { + if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) || + providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] { return admin.OperationalStatus{}, ErrUnavailable } status.Upstreams[index] = admin.UpstreamActivity{ - Name: inventory.UpstreamID, - Available: inventory.Available, - Checking: inventory.Checking, - Suspect: inventory.Suspect, - Draining: inventory.Draining, - Extracted: inventory.Extracted, + Name: inventory.UpstreamID, + Available: inventory.Available, + Checking: inventory.Checking, + Suspect: inventory.Suspect, + Draining: inventory.Draining, + Extracted: inventory.Extracted, + ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch, + FetchErrorCount: providerStats[index].FetchErrorCount, } } return status, nil diff --git a/internal/controller/operations/reader_test.go b/internal/controller/operations/reader_test.go index 493461d..eed2e91 100644 --- a/internal/controller/operations/reader_test.go +++ b/internal/controller/operations/reader_test.go @@ -7,6 +7,7 @@ import ( "time" "proxy-pool/internal/config" + "proxy-pool/internal/controller/provider" "proxy-pool/internal/domain/activitypool" ) @@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) { }} reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{ Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}}, - }}, inventory, func() time.Time { return now }) + }}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{ + {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4}, + {UpstreamID: "provider-b", FetchErrorCount: 1}, + }}) if err != nil { t.Fatalf("NewReader() error = %v", err) } @@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) { } first := status.Upstreams[0] if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 || - first.Draining != 4 || first.Extracted != 8 { + first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 { t.Fatalf("first upstream = %+v", first) } if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 { @@ -107,6 +111,14 @@ type recordingStateInventoryReader struct { now time.Time } +type staticProviderStatsReader struct { + result []provider.Stats +} + +func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats { + return append([]provider.Stats(nil), reader.result...) +} + func (reader *recordingStateInventoryReader) ReadStateInventory( _ context.Context, upstreamIDs []string, diff --git a/internal/controller/provider/coordination.go b/internal/controller/provider/coordination.go index b40067f..9f77d51 100644 --- a/internal/controller/provider/coordination.go +++ b/internal/controller/provider/coordination.go @@ -13,6 +13,8 @@ var ( ErrLeaderWorkStopped = errors.New("provider leader work stopped") ) +const MaximumCoordinationInteger = int64(1<<53 - 1) + type CoordinationLimits struct { RequestInterval time.Duration MaxInFlight int diff --git a/internal/controller/provider/fleet.go b/internal/controller/provider/fleet.go new file mode 100644 index 0000000..c575efe --- /dev/null +++ b/internal/controller/provider/fleet.go @@ -0,0 +1,64 @@ +package provider + +import ( + "context" + "errors" + "sort" + + "proxy-pool/internal/platform/lifecycle" +) + +var ErrInvalidFleet = errors.New("invalid Provider fleet") + +type Fleet struct { + runtimes []*UpstreamRuntime + group *lifecycle.Group +} + +func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) { + if len(runtimes) == 0 { + return nil, ErrInvalidFleet + } + seen := make(map[string]struct{}, len(runtimes)) + owned := make([]*UpstreamRuntime, len(runtimes)) + for index, runtime := range runtimes { + if runtime == nil || runtime.config.Provider.UpstreamID == "" { + return nil, ErrInvalidFleet + } + if _, exists := seen[runtime.config.Provider.UpstreamID]; exists { + return nil, ErrInvalidFleet + } + seen[runtime.config.Provider.UpstreamID] = struct{}{} + owned[index] = runtime + } + sort.Slice(owned, func(left, right int) bool { + return owned[left].ID() < owned[right].ID() + }) + runners := make([]lifecycle.Runner, len(owned)) + for index, runtime := range owned { + runners[index] = runtime + } + group, err := lifecycle.NewGroup(runners...) + if err != nil { + return nil, errors.Join(ErrInvalidFleet, err) + } + return &Fleet{runtimes: owned, group: group}, nil +} + +func (fleet *Fleet) Run(ctx context.Context) error { + if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil { + return ErrInvalidFleet + } + return fleet.group.Run(ctx) +} + +func (fleet *Fleet) IDs() []string { + if fleet == nil { + return nil + } + ids := make([]string, len(fleet.runtimes)) + for index, runtime := range fleet.runtimes { + ids[index] = runtime.ID() + } + return ids +} diff --git a/internal/controller/provider/fleet_test.go b/internal/controller/provider/fleet_test.go new file mode 100644 index 0000000..da32545 --- /dev/null +++ b/internal/controller/provider/fleet_test.go @@ -0,0 +1,101 @@ +package provider + +import ( + "context" + "errors" + "testing" + "time" + + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/domain/activitypool" + proxyDomain "proxy-pool/internal/domain/proxy" +) + +func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) { + started := make(chan string, 2) + siblingCancelled := make(chan struct{}, 1) + first := newFleetRuntime(t, "provider-a", coordinatorFunc(func( + context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error, + ) error { + started <- "provider-a" + return errors.New("provider-a stopped") + })) + second := newFleetRuntime(t, "provider-b", coordinatorFunc(func( + ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error, + ) error { + started <- "provider-b" + <-ctx.Done() + siblingCancelled <- struct{}{} + return ctx.Err() + })) + fleet, err := NewFleet(first, second) + if err != nil { + t.Fatalf("NewFleet(): %v", err) + } + + err = fleet.Run(context.Background()) + if err == nil || err.Error() != "provider-a stopped" { + t.Fatalf("Run() error = %v, want provider-a failure", err) + } + seen := map[string]bool{<-started: true, <-started: true} + if !seen["provider-a"] || !seen["provider-b"] { + t.Fatalf("started upstreams = %v", seen) + } + select { + case <-siblingCancelled: + case <-time.After(time.Second): + t.Fatal("sibling runtime was not cancelled") + } +} + +func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) { + valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func( + ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error, + ) error { + return ctx.Err() + })) + tests := []struct { + name string + runtimes []*UpstreamRuntime + }{ + {name: "empty"}, + {name: "nil", runtimes: []*UpstreamRuntime{nil}}, + {name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fleet, err := NewFleet(test.runtimes...) + if err == nil || fleet != nil { + t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err) + } + }) + } +} + +func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime { + t.Helper() + runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{ + Provider: runtimeProviderConfig(upstreamID), + ReconcilePolicy: runtimeReconcilePolicy(), + ReconcileInterval: time.Second, + }, UpstreamRuntimeDependencies{ + Coordinator: coordinator, + Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) { + return pool.InventorySnapshot{}, nil + }), + Adapter: adapterFunc(func(context.Context) (FetchResponse, error) { + return FetchResponse{Body: []byte("fixture")}, nil + }), + Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { + return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil + }), + Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) { + return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil + }), + Results: resultRecorderFunc(func(Result) {}), + }) + if err != nil { + t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err) + } + return runtime +} diff --git a/internal/controller/provider/ports.go b/internal/controller/provider/ports.go index c88d936..a4921b6 100644 --- a/internal/controller/provider/ports.go +++ b/internal/controller/provider/ports.go @@ -27,9 +27,11 @@ type RetryableError interface { Retryable() bool } -// Parser must be safe for concurrent calls and must honor context cancellation. +// Parser owns any transient resources attached to parsed candidates. It must +// be safe for concurrent calls and must honor context cancellation. type Parser interface { Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) + ReleaseCandidates([]proxyDomain.Proxy) } type Result struct { @@ -42,7 +44,8 @@ type Result struct { } type ResultRecorder interface { - // Record may be called concurrently and must not retain mutable result data. + // Record may be called concurrently, must return promptly, and must not + // retain mutable result data. Record(Result) } diff --git a/internal/controller/provider/reconciler.go b/internal/controller/provider/reconciler.go index 11b243f..c61051b 100644 --- a/internal/controller/provider/reconciler.go +++ b/internal/controller/provider/reconciler.go @@ -19,6 +19,7 @@ type Config struct { Timeout time.Duration MaxAttempts int MaxInFlight int + MaxTotal int64 MaxSize int TTL time.Duration AllocationSafetyMargin time.Duration @@ -31,6 +32,8 @@ type RetryConfig struct { Jitter int } +const requestPermitSettlementTimeout = 5 * time.Second + type Reconciler struct { config Config ports Ports @@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler return nil, fmt.Errorf("new provider reconciler: upstream ID is required") } if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 || - config.MaxInFlight <= 0 || config.MaxSize <= 0 { + config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 { return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive") } if config.TTL < 0 || config.AllocationSafetyMargin < 0 || @@ -140,11 +143,51 @@ func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error } func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) { + permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID) + if err != nil { + r.ports.Results.Record(Result{ + UpstreamID: r.config.UpstreamID, + Class: upstream.FetchError, + Err: fmt.Errorf("reserve fetch capacity: %w", err), + Attempt: 1, + }) + return + } + if !available { + return + } + if permit == nil || permit.Expected() <= 0 { + if permit != nil { + _ = permit.Cancel() + } + r.ports.Results.Record(Result{ + UpstreamID: r.config.UpstreamID, + Class: upstream.FetchError, + Err: fmt.Errorf("reserve fetch capacity: invalid permit"), + Attempt: 1, + }) + return + } + permitFinished := false + defer func() { + if !permitFinished { + _ = permit.Cancel() + } + }() + for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ { - response, result, retryable, ok := r.fetchAttempt(ctx, session, attempt) + response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt) if !ok { return } + if result.Class != upstream.FetchError { + if capacityErr := permit.Complete(result.NewCount); capacityErr != nil { + result.Class = upstream.FetchError + result.Err = errors.Join(result.Err, capacityErr) + } else { + permitFinished = true + } + } r.ports.Results.Record(result) if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts { return @@ -163,41 +206,12 @@ func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) { func (r *Reconciler) fetchAttempt( ctx context.Context, session LeaderSession, + permit upstream.FetchPermit, attempt int, ) (FetchResponse, Result, bool, bool) { if err := r.waitForRequestSlot(ctx); err != nil { return FetchResponse{}, Result{}, false, false } - permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID) - if err != nil { - resultErr := fmt.Errorf("reserve fetch capacity: %w", err) - return FetchResponse{}, Result{ - UpstreamID: r.config.UpstreamID, - Class: upstream.FetchError, - Err: resultErr, - Attempt: attempt, - }, false, true - } - if !available { - return FetchResponse{}, Result{}, false, false - } - if permit == nil || permit.Expected() <= 0 { - if permit != nil { - _ = permit.Cancel() - } - return FetchResponse{}, Result{ - UpstreamID: r.config.UpstreamID, - Class: upstream.FetchError, - Err: fmt.Errorf("reserve fetch capacity: invalid permit"), - Attempt: attempt, - }, false, true - } - permitFinished := false - defer func() { - if !permitFinished { - _ = permit.Cancel() - } - }() requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected()) if err != nil { if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) { @@ -236,13 +250,14 @@ func (r *Reconciler) fetchAttempt( defer cancel() response, callErr := r.ports.Adapter.Fetch(callCtx) - var parseErr, candidateErr, coordinationErr, capacityErr error + var parseErr, candidateErr, coordinationErr error var validCount, newCount int if callErr != nil { requestSettlementAttempted = true - coordinationErr = r.settleRequestPermit(requestPermit, false, 0) + coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected()) } else { candidates, err := r.ports.Parser.Parse(callCtx, response.Body) + defer r.ports.Parser.ReleaseCandidates(candidates) parseErr = err validCount = len(candidates) charged := validCount @@ -267,14 +282,10 @@ func (r *Reconciler) fetchAttempt( newCount = upserted.Inserted } } - if callErr == nil && parseErr == nil && candidateErr == nil && coordinationErr == nil { - capacityErr = permit.Complete(newCount) - permitFinished = capacityErr == nil - } - resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr, capacityErr) + resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr) class := upstream.ClassifyFetchResult( callErr, - errors.Join(parseErr, candidateErr, coordinationErr, capacityErr), + errors.Join(parseErr, candidateErr, coordinationErr), validCount, newCount, ) @@ -289,8 +300,7 @@ func (r *Reconciler) fetchAttempt( } func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error { - timeout := min(r.config.Timeout, 5*time.Second) - ctx, cancel := context.WithTimeout(context.Background(), timeout) + ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout) defer cancel() if complete { return permit.Complete(ctx, fetched) diff --git a/internal/controller/provider/reconciler_test.go b/internal/controller/provider/reconciler_test.go index 4c7c143..0756fba 100644 --- a/internal/controller/provider/reconciler_test.go +++ b/internal/controller/provider/reconciler_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + controllerPool "proxy-pool/internal/controller/pool" "proxy-pool/internal/domain/activitypool" proxyDomain "proxy-pool/internal/domain/proxy" "proxy-pool/internal/domain/upstream" @@ -722,9 +723,9 @@ func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testin } } -func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) { +func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) { results := make(chan Result, 1) - globalCancelled := make(chan struct{}, 1) + globalCompleted := make(chan int, 1) ports := successfulPorts(func() {}, results) ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) { return FetchResponse{}, errors.New("provider connection failed") @@ -736,17 +737,15 @@ func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) { t.Fatalf("NewReconciler(): %v", err) } session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) { - return &recordingRequestPermit{cancelled: globalCancelled}, true, nil + return &recordingRequestPermit{completed: globalCompleted}, true, nil }) result := runSingleReconcile(t, reconciler, results, session) if result.Class != upstream.FetchError { t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError) } - select { - case <-globalCancelled: - default: - t.Fatal("distributed request reservation was not cancelled") + if got := <-globalCompleted; got <= 0 { + t.Fatalf("global charged count = %d, want conservative expected count", got) } } @@ -754,10 +753,12 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing. results := make(chan Result, 1) localCompleted := make(chan fetchCompletion, 1) globalCompleted := make(chan int, 1) + released := make(chan []proxyDomain.Proxy, 1) ports := successfulPorts(func() {}, results) - ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { - return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil - }) + ports.Parser = &recordingCandidateParser{ + candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, + released: released, + } ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) { if len(batch.Proxies) != 2 { t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies)) @@ -791,6 +792,14 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing. if got := <-localCompleted; got.retained != 1 { t.Fatalf("local fetch completion = %+v, want retained=1", got) } + select { + case got := <-released: + if len(got) != 3 { + t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got)) + } + case <-time.After(time.Second): + t.Fatal("parsed candidates were not released") + } } func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) { @@ -838,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) { } } +func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) { + results := make(chan Result, 2) + budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{ + UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1, + }) + if err != nil { + t.Fatalf("NewFetchBudget(): %v", err) + } + var parses atomic.Int64 + ports := successfulPorts(func() {}, results) + ports.Capacity = budget + ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { + if parses.Add(1) == 1 { + return nil, errors.New("temporary parser failure") + } + return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil + }) + reconciler, err := NewReconciler(Config{ + UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2, + MaxInFlight: 1, MaxSize: 10, + Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond}, + }, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error { + if got := budget.Snapshot().PendingExpected; got != 1 { + t.Errorf("PendingExpected during retry backoff = %d, want 1", got) + } + return nil + })}) + if err != nil { + t.Fatalf("NewReconciler(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }() + reconciler.Notify() + <-results + <-results + cancel() + if err := <-done; err != nil { + t.Fatalf("RunLeader(): %v", err) + } + usage := budget.Snapshot() + if usage.PendingExpected != 0 || usage.Managed != 1 { + t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage) + } +} + func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) { results := make(chan Result, 1) ports := successfulPorts(func() {}, results) @@ -954,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error { return errors.New("unexpected sleep") } +type sleeperFunc func(context.Context, time.Duration) error + +func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error { + return f(ctx, duration) +} + func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error { if err := ctx.Err(); err != nil { return err @@ -981,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy return f(ctx, body) } +func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {} + +type recordingCandidateParser struct { + candidates []proxyDomain.Proxy + released chan<- []proxyDomain.Proxy +} + +func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) { + return append([]proxyDomain.Proxy(nil), parser.candidates...), nil +} + +func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) { + parser.released <- append([]proxyDomain.Proxy(nil), candidates...) +} + type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) { diff --git a/internal/controller/provider/stats.go b/internal/controller/provider/stats.go new file mode 100644 index 0000000..eaf4687 --- /dev/null +++ b/internal/controller/provider/stats.go @@ -0,0 +1,103 @@ +package provider + +import ( + "errors" + "math" + "sync" + + "proxy-pool/internal/domain/upstream" +) + +var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder") + +type Stats struct { + UpstreamID string + ConsecutiveEmptyFetch int64 + FetchErrorCount int64 +} + +type StatsReader interface { + ReadProviderStats([]string) []Stats +} + +type StatsRetainer interface { + RetainProviderStats([]string) +} + +type StatsRecorder struct { + mu sync.Mutex + maximum int + byID map[string]Stats +} + +func NewStatsRecorder(maximum int) (*StatsRecorder, error) { + if maximum <= 0 { + return nil, ErrInvalidStatsRecorder + } + return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil +} + +func (recorder *StatsRecorder) Record(result Result) { + if recorder == nil || result.UpstreamID == "" { + return + } + recorder.mu.Lock() + defer recorder.mu.Unlock() + stats, exists := recorder.byID[result.UpstreamID] + if !exists { + if len(recorder.byID) >= recorder.maximum { + return + } + stats.UpstreamID = result.UpstreamID + } + switch result.Class { + case upstream.FetchEmpty: + if stats.ConsecutiveEmptyFetch < math.MaxInt64 { + stats.ConsecutiveEmptyFetch++ + } + case upstream.FetchValid, upstream.FetchDuplicateOnly: + stats.ConsecutiveEmptyFetch = 0 + case upstream.FetchError: + if stats.FetchErrorCount < math.MaxInt64 { + stats.FetchErrorCount++ + } + default: + return + } + recorder.byID[result.UpstreamID] = stats +} + +func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats { + result := make([]Stats, len(upstreamIDs)) + if recorder == nil { + return result + } + recorder.mu.Lock() + defer recorder.mu.Unlock() + for index, upstreamID := range upstreamIDs { + result[index] = recorder.byID[upstreamID] + result[index].UpstreamID = upstreamID + } + return result +} + +// RetainProviderStats removes observations for upstreams no longer present in +// the complete configuration. Disabled but configured upstreams must be kept. +func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) { + if recorder == nil { + return + } + retained := make(map[string]struct{}, len(upstreamIDs)) + for _, upstreamID := range upstreamIDs { + if upstreamID != "" { + retained[upstreamID] = struct{}{} + } + } + recorder.mu.Lock() + defer recorder.mu.Unlock() + for upstreamID := range recorder.byID { + if _, keep := retained[upstreamID]; !keep { + delete(recorder.byID, upstreamID) + } + } +} diff --git a/internal/controller/provider/stats_test.go b/internal/controller/provider/stats_test.go new file mode 100644 index 0000000..ff1abc2 --- /dev/null +++ b/internal/controller/provider/stats_test.go @@ -0,0 +1,83 @@ +package provider + +import ( + "sync" + "testing" + + "proxy-pool/internal/domain/upstream" +) + +func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) { + recorder, err := NewStatsRecorder(2) + if err != nil { + t.Fatalf("NewStatsRecorder(): %v", err) + } + for _, class := range []upstream.FetchClass{ + upstream.FetchEmpty, + upstream.FetchEmpty, + upstream.FetchError, + } { + recorder.Record(Result{UpstreamID: "provider-a", Class: class}) + } + stats := recorder.ReadProviderStats([]string{"provider-a"})[0] + if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 { + t.Fatalf("stats = %+v, want empty=2 errors=1", stats) + } + recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly}) + if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 { + t.Fatalf("consecutive empty after duplicate = %d, want 0", got) + } +} + +func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) { + recorder, err := NewStatsRecorder(1) + if err != nil { + t.Fatalf("NewStatsRecorder(): %v", err) + } + const workers = 100 + var wait sync.WaitGroup + for range workers { + wait.Add(1) + go func() { + defer wait.Done() + recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError}) + }() + } + wait.Wait() + recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError}) + stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"}) + if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 { + t.Fatalf("stats = %+v, want bounded provider-a errors", stats) + } +} + +func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) { + recorder, err := NewStatsRecorder(2) + if err != nil { + t.Fatalf("NewStatsRecorder(): %v", err) + } + recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError}) + recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError}) + + const workers = 100 + var wait sync.WaitGroup + for range workers { + wait.Add(2) + go func() { + defer wait.Done() + recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError}) + }() + go func() { + defer wait.Done() + recorder.RetainProviderStats([]string{"provider-a", "provider-b"}) + }() + } + wait.Wait() + recorder.RetainProviderStats([]string{"provider-a", "provider-b"}) + recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError}) + + stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"}) + if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 { + t.Fatalf("stats after retention = %+v", stats) + } +} diff --git a/internal/controller/provider/upstream_runtime.go b/internal/controller/provider/upstream_runtime.go new file mode 100644 index 0000000..be30d21 --- /dev/null +++ b/internal/controller/provider/upstream_runtime.go @@ -0,0 +1,219 @@ +package provider + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "reflect" + "time" + + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/domain/activitypool" +) + +var ( + ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime") + ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped") +) + +type UpstreamRuntimeConfig struct { + Provider Config + ReconcilePolicy pool.ReconcilePolicy + ReconcileInterval time.Duration +} + +type UpstreamRuntimeDependencies struct { + Coordinator Coordinator + Inventory pool.InventoryReader + Adapter ProviderAdapter + Parser Parser + Activity activitypool.Upserter + Results ResultRecorder +} + +// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh +// local budget and coalescing signal are created for each leadership term. +type UpstreamRuntime struct { + config UpstreamRuntimeConfig + dependencies UpstreamRuntimeDependencies + sleeper Sleeper +} + +func NewUpstreamRuntime( + config UpstreamRuntimeConfig, + dependencies UpstreamRuntimeDependencies, +) (*UpstreamRuntime, error) { + if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) || + nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) || + nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) || + nilRuntimeDependency(dependencies.Results) { + return nil, ErrInvalidUpstreamRuntime + } + runtime := &UpstreamRuntime{ + config: config, dependencies: dependencies, sleeper: timerSleeper{}, + } + if _, err := runtime.newLeaderTerm(); err != nil { + return nil, errors.Join(ErrInvalidUpstreamRuntime, err) + } + return runtime, nil +} + +func (runtime *UpstreamRuntime) Run(ctx context.Context) error { + if runtime == nil || ctx == nil { + return ErrInvalidUpstreamRuntime + } + limits := CoordinationLimits{ + RequestInterval: runtime.config.Provider.RequestInterval, + MaxInFlight: runtime.config.Provider.MaxInFlight, + MaxAttemptDuration: runtime.config.Provider.Timeout, + MaxTotal: runtime.config.Provider.MaxTotal, + } + err := runtime.dependencies.Coordinator.RunLeader( + ctx, + runtime.config.Provider.UpstreamID, + limits, + func(leaderCtx context.Context, session LeaderSession) error { + term, buildErr := runtime.newLeaderTerm() + if buildErr != nil { + return buildErr + } + return term.run(leaderCtx, session) + }, + ) + if ctx.Err() != nil { + return ctx.Err() + } + if err == nil { + return ErrUpstreamRuntimeStopped + } + return err +} + +func (runtime *UpstreamRuntime) ID() string { + if runtime == nil { + return "" + } + return runtime.config.Provider.UpstreamID +} + +func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) { + budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{ + UpstreamID: runtime.config.Provider.UpstreamID, + MaxSize: runtime.config.Provider.MaxSize, + ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch, + }) + if err != nil { + return nil, err + } + providerConfig := runtime.config.Provider + providerConfig.RequestInterval = 0 + providerReconciler, err := NewReconciler(providerConfig, Ports{ + Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser, + Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results, + Capacity: budget, + }) + if err != nil { + return nil, err + } + poolReconciler, err := pool.NewReconciler( + runtime.config.ReconcilePolicy, + budget, + providerReconciler, + ) + if err != nil { + return nil, err + } + return &upstreamLeaderTerm{ + upstreamID: runtime.config.Provider.UpstreamID, + interval: runtime.config.ReconcileInterval, + safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin, + inventory: runtime.dependencies.Inventory, + pool: poolReconciler, + provider: providerReconciler, + sleeper: runtime.sleeper, + }, nil +} + +type upstreamLeaderTerm struct { + upstreamID string + interval time.Duration + safetyMargin time.Duration + inventory pool.InventoryReader + pool *pool.Reconciler + provider *Reconciler + sleeper Sleeper +} + +func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error { + termCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 2) + go func() { done <- term.provider.RunLeader(termCtx, session) }() + go func() { done <- term.reconcileInventory(termCtx) }() + + first := <-done + cancel() + second := <-done + if ctx.Err() != nil { + return nil + } + if first == nil { + first = ErrUpstreamRuntimeStopped + } + if second != nil && !errors.Is(second, context.Canceled) { + return errors.Join(first, second) + } + return first +} + +func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error { + if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 { + if err := term.sleeper.Sleep(ctx, delay); err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err) + } + } + for ctx.Err() == nil { + inventory, err := term.inventory.ReadInventory( + ctx, + term.upstreamID, + term.safetyMargin, + ) + if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 { + term.pool.ReconcileSnapshot(inventory) + } + if err := term.sleeper.Sleep(ctx, term.interval); err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("wait for Provider inventory reconciliation: %w", err) + } + } + return nil +} + +func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration { + window := min(interval/4, 250*time.Millisecond) + if window <= 1 { + return 0 + } + digest := fnv.New64a() + _, _ = digest.Write([]byte(upstreamID)) + return time.Duration(digest.Sum64() % uint64(window)) +} + +func nilRuntimeDependency(value any) bool { + if value == nil { + return true + } + reflected := reflect.ValueOf(value) + switch reflected.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return reflected.IsNil() + default: + return false + } +} diff --git a/internal/controller/provider/upstream_runtime_test.go b/internal/controller/provider/upstream_runtime_test.go new file mode 100644 index 0000000..495a14d --- /dev/null +++ b/internal/controller/provider/upstream_runtime_test.go @@ -0,0 +1,247 @@ +package provider + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "proxy-pool/internal/controller/pool" + "proxy-pool/internal/domain/activitypool" + proxyDomain "proxy-pool/internal/domain/proxy" +) + +func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) { + var inventoryReads atomic.Int64 + fetched := make(chan struct{}, 1) + runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{ + Provider: runtimeProviderConfig("provider-a"), + ReconcilePolicy: runtimeReconcilePolicy(), + ReconcileInterval: 10 * time.Millisecond, + }, UpstreamRuntimeDependencies{ + Coordinator: coordinatorFunc(func( + ctx context.Context, + upstreamID string, + limits CoordinationLimits, + work func(context.Context, LeaderSession) error, + ) error { + if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 { + t.Errorf("coordination = (%q, %+v)", upstreamID, limits) + } + return work(ctx, unlimitedLeaderSession{}) + }), + Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) { + inventoryReads.Add(1) + return pool.InventorySnapshot{}, nil + }), + Adapter: adapterFunc(func(context.Context) (FetchResponse, error) { + fetched <- struct{}{} + return FetchResponse{Body: []byte("fixture")}, nil + }), + Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { + return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil + }), + Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) { + return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil + }), + Results: resultRecorderFunc(func(Result) {}), + }) + if err != nil { + t.Fatalf("NewUpstreamRuntime(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runtime.Run(ctx) }() + select { + case <-fetched: + case <-time.After(time.Second): + t.Fatal("timed out waiting for leader fetch") + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context cancellation", err) + } + if got := inventoryReads.Load(); got == 0 { + t.Fatal("inventory was not read inside leader term") + } +} + +func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) { + var inventoryReads atomic.Int64 + var providerCalls atomic.Int64 + fetched := make(chan struct{}, 1) + runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{ + Provider: runtimeProviderConfig("provider-a"), + ReconcilePolicy: runtimeReconcilePolicy(), + ReconcileInterval: 10 * time.Millisecond, + }, UpstreamRuntimeDependencies{ + Coordinator: coordinatorFunc(func( + ctx context.Context, + _ string, + _ CoordinationLimits, + work func(context.Context, LeaderSession) error, + ) error { + return work(ctx, unlimitedLeaderSession{}) + }), + Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) { + if inventoryReads.Add(1) == 1 { + return pool.InventorySnapshot{}, errors.New("inventory unavailable") + } + return pool.InventorySnapshot{}, nil + }), + Adapter: adapterFunc(func(context.Context) (FetchResponse, error) { + providerCalls.Add(1) + fetched <- struct{}{} + return FetchResponse{Body: []byte("fixture")}, nil + }), + Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { + return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil + }), + Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) { + return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil + }), + Results: resultRecorderFunc(func(Result) {}), + }) + if err != nil { + t.Fatalf("NewUpstreamRuntime(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runtime.Run(ctx) }() + select { + case <-fetched: + case <-time.After(time.Second): + t.Fatal("timed out waiting for recovered inventory fetch") + } + cancel() + <-done + if got := inventoryReads.Load(); got < 2 { + t.Fatalf("inventory reads = %d, want recovery retry", got) + } + if got := providerCalls.Load(); got != 1 { + t.Fatalf("provider calls = %d, want one call after recovery", got) + } +} + +func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) { + providerConfig := runtimeProviderConfig("provider-a") + providerConfig.RequestInterval = 500 * time.Millisecond + fetched := make(chan struct{}, 2) + runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{ + Provider: providerConfig, + ReconcilePolicy: runtimeReconcilePolicy(), + ReconcileInterval: 5 * time.Millisecond, + }, UpstreamRuntimeDependencies{ + Coordinator: coordinatorFunc(func( + ctx context.Context, + _ string, + limits CoordinationLimits, + work func(context.Context, LeaderSession) error, + ) error { + if limits.RequestInterval != 500*time.Millisecond { + t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval) + } + return work(ctx, unlimitedLeaderSession{}) + }), + Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) { + return pool.InventorySnapshot{}, nil + }), + Adapter: adapterFunc(func(context.Context) (FetchResponse, error) { + fetched <- struct{}{} + return FetchResponse{Body: []byte("fixture")}, nil + }), + Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) { + return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil + }), + Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) { + return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil + }), + Results: resultRecorderFunc(func(Result) {}), + }) + if err != nil { + t.Fatalf("NewUpstreamRuntime(): %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runtime.Run(ctx) }() + for range 2 { + select { + case <-fetched: + case <-time.After(150 * time.Millisecond): + cancel() + <-done + t.Fatal("local Provider reconciler duplicated the distributed request interval") + } + } + cancel() + <-done +} + +func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) { + validConfig := UpstreamRuntimeConfig{ + Provider: runtimeProviderConfig("provider-a"), + ReconcilePolicy: runtimeReconcilePolicy(), + ReconcileInterval: time.Second, + } + if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil { + t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err) + } +} + +func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) { + const interval = time.Second + first := initialReconcileDelay("provider-a", interval) + if first != initialReconcileDelay("provider-a", interval) { + t.Fatal("initial reconcile delay is not stable") + } + if first < 0 || first >= 250*time.Millisecond { + t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first) + } + if other := initialReconcileDelay("provider-b", interval); other == first { + t.Fatalf("different upstreams have the same initial delay %s", first) + } +} + +func runtimeProviderConfig(upstreamID string) Config { + return Config{ + UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1, + MaxInFlight: 1, MaxTotal: 10, MaxSize: 10, + } +} + +func runtimeReconcilePolicy() pool.ReconcilePolicy { + return pool.ReconcilePolicy{ + MinimumAvailableSlots: 1, TargetAvailableSlots: 2, + ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1, + } +} + +type coordinatorFunc func( + context.Context, + string, + CoordinationLimits, + func(context.Context, LeaderSession) error, +) error + +func (f coordinatorFunc) RunLeader( + ctx context.Context, + upstreamID string, + limits CoordinationLimits, + work func(context.Context, LeaderSession) error, +) error { + return f(ctx, upstreamID, limits, work) +} + +type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) + +func (f inventoryReaderFunc) ReadInventory( + ctx context.Context, + upstreamID string, + safetyMargin time.Duration, +) (pool.InventorySnapshot, error) { + return f(ctx, upstreamID, safetyMargin) +} diff --git a/internal/platform/credentials/store.go b/internal/platform/credentials/store.go index 6d7f5fd..35ffcc1 100644 --- a/internal/platform/credentials/store.go +++ b/internal/platform/credentials/store.go @@ -48,18 +48,35 @@ type Store interface { Resolve(context.Context, Reference) (Value, error) } +// Releaser removes transient credential material after the consumer has copied +// it into its authoritative storage. Release is idempotent and version fenced. +type Releaser interface { + Release(context.Context, Reference) error +} + +type CapacityEnsurer interface { + EnsureCapacity(context.Context, int) error +} + type entry struct { + scope *scopeState value Value - version uint64 reference Reference } +type scopeState struct { + name string + value Value + version uint64 + leases int +} + // MemoryStore keeps credentials in process memory and serializes access with a // context-aware lock. type MemoryStore struct { lock chan struct{} capacity int - byScope map[string]*entry + byScope map[string]*scopeState byRef map[string]*entry } @@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) { return &MemoryStore{ lock: lock, capacity: capacity, - byScope: make(map[string]*entry), + byScope: make(map[string]*scopeState), byRef: make(map[string]*entry), }, nil } @@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer if err := s.acquire(ctx); err != nil { return Reference{}, err } - defer s.release() + defer s.unlock() if err := ctx.Err(); err != nil { return Reference{}, err } - - if current, ok := s.byScope[scope]; ok { - if current.value == value { - return current.reference, nil - } - current.value = value - current.version++ - current.reference.CredentialVersion = versionString(current.version) - return current.reference, nil + if len(s.byRef) >= s.capacity { + return Reference{}, ErrCapacityExceeded } - if len(s.byScope) >= s.capacity { + + current, exists := s.byScope[scope] + if !exists && len(s.byScope) >= s.capacity { return Reference{}, ErrCapacityExceeded } secretRef, err := s.newUniqueSecretRef() if err != nil { return Reference{}, err } + if !exists { + current = &scopeState{name: scope, value: value, version: 1} + s.byScope[scope] = current + } else if current.value != value { + current.value = value + current.version++ + } created := &entry{ - value: value, - version: 1, + scope: current, + value: value, reference: Reference{ SecretRef: secretRef, - CredentialVersion: versionString(1), + CredentialVersion: versionString(current.version), }, } - s.byScope[scope] = created + current.leases++ s.byRef[secretRef] = created return created.reference, nil } @@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value, if err := s.acquire(ctx); err != nil { return Value{}, err } - defer s.release() + defer s.unlock() if err := ctx.Err(); err != nil { return Value{}, err } @@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value, return current.value, nil } +func (s *MemoryStore) Release(ctx context.Context, reference Reference) error { + if err := contextError(ctx); err != nil { + return err + } + if !s.valid() { + return ErrInvalidStore + } + if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) { + return ErrInvalidReference + } + if err := s.acquire(ctx); err != nil { + return err + } + defer s.unlock() + if err := ctx.Err(); err != nil { + return err + } + current, ok := s.byRef[reference.SecretRef] + if !ok || current.reference != reference { + return nil + } + delete(s.byRef, reference.SecretRef) + current.scope.leases-- + if current.scope.leases == 0 { + delete(s.byScope, current.scope.name) + current.scope.value = Value{} + } + current.value = Value{} + return nil +} + +func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error { + if err := contextError(ctx); err != nil { + return err + } + if !s.valid() { + return ErrInvalidStore + } + if minimum <= 0 { + return ErrInvalidCapacity + } + if err := s.acquire(ctx); err != nil { + return err + } + defer s.unlock() + if minimum > s.capacity { + s.capacity = minimum + } + return nil +} + func (s *MemoryStore) acquire(ctx context.Context) error { if err := contextError(ctx); err != nil { return err @@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error { } } -func (s *MemoryStore) release() { +func (s *MemoryStore) unlock() { s.lock <- struct{}{} } diff --git a/internal/platform/credentials/store_test.go b/internal/platform/credentials/store_test.go index 48b0a09..4a3ef71 100644 --- a/internal/platform/credentials/store_test.go +++ b/internal/platform/credentials/store_test.go @@ -11,7 +11,7 @@ import ( "time" ) -func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) { +func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) { store, err := NewMemoryStore(2) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) @@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) { if err != nil { t.Fatalf("Put(second): %v", err) } - if first != second { - t.Fatalf("second reference = %#v, want %#v", second, first) + if first == second || first.SecretRef == second.SecretRef { + t.Fatalf("references = %#v and %#v, want independent leases", first, second) } - if first.SecretRef == "" || first.CredentialVersion != "v1" { + if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" { t.Fatalf("first reference = %#v, want opaque ref at v1", first) } for _, plaintext := range []string{"provider-a", value.Username, value.Password} { @@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) { if got != value { t.Fatalf("Resolve() = %#v, want %#v", got, value) } + if err := store.Release(context.Background(), first); err != nil { + t.Fatalf("Release(first lease): %v", err) + } + if got, err := store.Resolve(context.Background(), second); err != nil || got != value { + t.Fatalf("Resolve(second lease) = %#v, %v", got, err) + } } func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) { @@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) { } } -func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) { - store, err := NewMemoryStore(1) +func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) { + store, err := NewMemoryStore(2) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } @@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) { if err != nil { t.Fatalf("Put(new): %v", err) } - if newReference.SecretRef != oldReference.SecretRef { - t.Fatalf("new SecretRef changed across versions") + if newReference.SecretRef == oldReference.SecretRef { + t.Fatalf("new SecretRef reused an active lease") } if newReference.CredentialVersion != "v2" { t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion) } - if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) { - t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err) + old, err := store.Resolve(context.Background(), oldReference) + if err != nil || old.Password != "old-password" { + t.Fatalf("Resolve(active old lease) = %#v, %v", old, err) } got, err := store.Resolve(context.Background(), newReference) if err != nil { @@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin if got != want { t.Fatalf("Resolve(existing) returned changed credentials") } + if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) { + t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err) + } + if err := store.Release(context.Background(), reference); err != nil { + t.Fatalf("Release(first lease): %v", err) + } if _, err := store.Put(context.Background(), "provider-a", want); err != nil { - t.Fatalf("Put(idempotent at capacity): %v", err) + t.Fatalf("Put(after lease release): %v", err) + } +} + +func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) { + store, err := NewMemoryStore(1) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"}) + if err != nil { + t.Fatalf("Put(first): %v", err) + } + if err := store.Release(context.Background(), first); err != nil { + t.Fatalf("Release(first): %v", err) + } + if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) { + t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err) + } + if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil { + t.Fatalf("Put(after release): %v", err) + } + if err := store.Release(context.Background(), first); err != nil { + t.Fatalf("Release(idempotent): %v", err) + } +} + +func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) { + store, err := NewMemoryStore(2) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"}) + if err != nil { + t.Fatalf("Put(old): %v", err) + } + current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"}) + if err != nil { + t.Fatalf("Put(new): %v", err) + } + if err := store.Release(context.Background(), stale); err != nil { + t.Fatalf("Release(stale): %v", err) + } + if _, err := store.Resolve(context.Background(), current); err != nil { + t.Fatalf("Resolve(current): %v", err) + } +} + +func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) { + store, err := NewMemoryStore(1) + if err != nil { + t.Fatalf("NewMemoryStore(): %v", err) + } + if err := store.EnsureCapacity(context.Background(), 2); err != nil { + t.Fatalf("EnsureCapacity(2): %v", err) + } + if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil { + t.Fatalf("Put(provider-a): %v", err) + } + if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil { + t.Fatalf("Put(provider-b): %v", err) + } + if err := store.EnsureCapacity(context.Background(), 1); err != nil { + t.Fatalf("EnsureCapacity(shrink request): %v", err) + } + if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) { + t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err) } } @@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) { } } -func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) { - store, err := NewMemoryStore(1) +func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) { + const workers = 100 + store, err := NewMemoryStore(workers) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } want := Value{Username: "alice", Password: "shared-password"} - const workers = 100 references := make(chan Reference, workers) errorsSeen := make(chan error, workers) var wait sync.WaitGroup @@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) { for err := range errorsSeen { t.Errorf("concurrent Put(): %v", err) } - var first Reference + secretRefs := make(map[string]struct{}, workers) for reference := range references { - if first == (Reference{}) { - first = reference + if reference.CredentialVersion != "v1" { + t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion) } - if reference != first { - t.Errorf("concurrent Put() reference differs from first") + secretRefs[reference.SecretRef] = struct{}{} + } + if len(secretRefs) != workers { + t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers) + } + for secretRef := range secretRefs { + got, err := store.Resolve(context.Background(), Reference{ + SecretRef: secretRef, CredentialVersion: "v1", + }) + if err != nil || got != want { + t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err) } } - if first.CredentialVersion != "v1" { - t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion) - } - got, err := store.Resolve(context.Background(), first) - if err != nil { - t.Fatalf("Resolve(): %v", err) - } - if got != want { - t.Fatalf("Resolve() returned unexpected credentials") - } } func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) { - store, err := NewMemoryStore(1) + const workers = 100 + store, err := NewMemoryStore(workers) if err != nil { t.Fatalf("NewMemoryStore(): %v", err) } @@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) { reference Reference err error } - const workers = 100 results := make(chan result, workers) var wait sync.WaitGroup for index := range workers { @@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) { close(results) versions := make(map[string]struct{}, workers) - var secretRef string + secretRefs := make(map[string]struct{}, workers) var latest result for current := range results { if current.err != nil { t.Fatalf("concurrent Put(): %v", current.err) } - if secretRef == "" { - secretRef = current.reference.SecretRef - } - if current.reference.SecretRef != secretRef { - t.Fatal("SecretRef changed across concurrent updates") - } + secretRefs[current.reference.SecretRef] = struct{}{} versions[current.reference.CredentialVersion] = struct{}{} if current.reference.CredentialVersion == "v100" { latest = current @@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) { if len(versions) != workers { t.Fatalf("unique versions = %d, want %d", len(versions), workers) } + if len(secretRefs) != workers { + t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers) + } if latest.reference == (Reference{}) { t.Fatal("highest version v100 was not returned") } diff --git a/internal/platform/lifecycle/group.go b/internal/platform/lifecycle/group.go new file mode 100644 index 0000000..75c4145 --- /dev/null +++ b/internal/platform/lifecycle/group.go @@ -0,0 +1,72 @@ +package lifecycle + +import ( + "context" + "errors" + "reflect" +) + +var ( + ErrInvalidGroup = errors.New("invalid lifecycle group") + ErrRunnerStopped = errors.New("lifecycle runner stopped") +) + +type Runner interface { + Run(context.Context) error +} + +type Group struct { + runners []Runner +} + +func NewGroup(runners ...Runner) (*Group, error) { + if len(runners) == 0 { + return nil, ErrInvalidGroup + } + owned := make([]Runner, len(runners)) + for index, runner := range runners { + if isNilRunner(runner) { + return nil, ErrInvalidGroup + } + owned[index] = runner + } + return &Group{runners: owned}, nil +} + +func (group *Group) Run(ctx context.Context) error { + if group == nil || ctx == nil || len(group.runners) == 0 { + return ErrInvalidGroup + } + groupCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, len(group.runners)) + for _, runner := range group.runners { + go func() { done <- runner.Run(groupCtx) }() + } + + first := <-done + cancel() + for range len(group.runners) - 1 { + <-done + } + if ctx.Err() != nil { + return ctx.Err() + } + if first == nil { + return ErrRunnerStopped + } + return first +} + +func isNilRunner(runner Runner) bool { + if runner == nil { + return true + } + reflected := reflect.ValueOf(runner) + switch reflected.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return reflected.IsNil() + default: + return false + } +} diff --git a/internal/platform/lifecycle/group_test.go b/internal/platform/lifecycle/group_test.go new file mode 100644 index 0000000..76008c0 --- /dev/null +++ b/internal/platform/lifecycle/group_test.go @@ -0,0 +1,87 @@ +package lifecycle + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) { + started := make(chan struct{}, 2) + cancelled := make(chan struct{}, 1) + wantErr := errors.New("runner failed") + group, err := NewGroup( + runnerFunc(func(context.Context) error { + started <- struct{}{} + return wantErr + }), + runnerFunc(func(ctx context.Context) error { + started <- struct{}{} + <-ctx.Done() + cancelled <- struct{}{} + return ctx.Err() + }), + ) + if err != nil { + t.Fatalf("NewGroup(): %v", err) + } + if err := group.Run(context.Background()); !errors.Is(err, wantErr) { + t.Fatalf("Run() error = %v, want %v", err, wantErr) + } + <-started + <-started + select { + case <-cancelled: + case <-time.After(time.Second): + t.Fatal("sibling runner was not cancelled") + } +} + +func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) { + release := make(chan struct{}) + var exited atomic.Bool + group, err := NewGroup( + runnerFunc(func(context.Context) error { return errors.New("failed") }), + runnerFunc(func(ctx context.Context) error { + <-ctx.Done() + <-release + exited.Store(true) + return nil + }), + ) + if err != nil { + t.Fatalf("NewGroup(): %v", err) + } + done := make(chan error, 1) + go func() { done <- group.Run(context.Background()) }() + select { + case <-done: + t.Fatal("Run() returned before sibling exited") + case <-time.After(20 * time.Millisecond): + } + close(release) + <-done + if !exited.Load() { + t.Fatal("sibling exit was not observed") + } +} + +func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) { + var typedNil *nilRunner + for _, runners := range [][]Runner{nil, {typedNil}} { + group, err := NewGroup(runners...) + if err == nil || group != nil { + t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err) + } + } +} + +type runnerFunc func(context.Context) error + +func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) } + +type nilRunner struct{} + +func (*nilRunner) Run(context.Context) error { return nil }