youfak/proxy-pool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: run provider lifecycle in controller

Browse Source
This commit is contained in:
youfak 2026-07-30 19:47:02 +08:00
parent 40f4b3ffab
commit 8997e3880e
45 changed files with 3678 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/proxy-controller/main.go
View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/controller/bootstrap"
)
const configEnvironment = "PROXY_POOL_CONFIG"
const (
configEnvironment = "PROXY_POOL_CONFIG"
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
)
type environmentLookup func(string) string
type controllerRun func(context.Context, bootstrap.Options) error
@ -54,7 +57,13 @@ func execute(
return 2
}
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
var fingerprintKey []byte
if getenv != nil {
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
}
err := run(ctx, bootstrap.Options{
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}

12
cmd/proxy-controller/main_test.go
View File

@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
if name == configEnvironment {
return "environment.yaml"
}
if name == fingerprintKeyEnvironment {
return "0123456789abcdef0123456789abcdef"
}
return ""
}, func(_ context.Context, options bootstrap.Options) error {
received = options
return nil
}, &bytes.Buffer{})
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
t.Fatalf("execute() = %d, options = %+v", code, received)
}
}
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
func TestExecuteFallsBackToEnvironment(t *testing.T) {
t.Parallel()
var received bootstrap.Options
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
code := execute(context.Background(), nil, func(name string) string {
if name == configEnvironment {
return "environment.yaml"
}
return "0123456789abcdef0123456789abcdef"
}, func(
_ context.Context,
options bootstrap.Options,
) error {

19
deploy/docker-compose.yml
View File

@ -1,5 +1,13 @@
name: proxy-pool
x-app-environment: &app-environment
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
x-app: &app
build:
context: ..
@ -9,13 +17,7 @@ x-app: &app
networks: [frontend, backend]
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
environment:
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
environment: *app-environment
stop_grace_period: 45s
services:
@ -44,6 +46,9 @@ services:
controller:
<<: *app
command: ["proxy-controller"]
environment:
<<: *app-environment
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
depends_on:
postgres:
condition: service_healthy

2
deploy/kubernetes/base/secret.example.yaml
View File

@ -9,8 +9,8 @@ stringData:
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN

79
internal/adapters/providerapi/template_parser.go
View File

@ -17,11 +17,12 @@ import (
)
const (
defaultTemplateTimeout = 100 * time.Millisecond
defaultTemplateMaxBytes = int64(1 << 20)
defaultMaxCandidates = 10_000
maxRegexPatterns = 64
maxRegexPatternBytes = 1024
defaultTemplateTimeout = 100 * time.Millisecond
defaultTemplateMaxBytes = int64(1 << 20)
defaultMaxCandidates = 10_000
credentialReleaseTimeout = time.Second
maxRegexPatterns = 64
maxRegexPatternBytes = 1024
)
type TemplateParser struct {
@ -137,7 +138,14 @@ func NewTemplateParser(
return parser, nil
}
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
storedCredentials := make([]credentials.Reference, 0)
defer func() {
if resultErr == nil {
return
}
p.releaseCredentials(storedCredentials)
}()
if err := ctx.Err(); err != nil {
return nil, err
}
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > p.maxCandidates {
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
}
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
credentialIndexes := make(map[string]int)
for _, token := range tokens {
candidate, credential, ok := p.parseCandidate(token)
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
}
candidate.SecretRef = reference.SecretRef
candidate.CredentialVersion = reference.CredentialVersion
storedCredentials = append(storedCredentials, reference)
credentialKey = candidateCredentialKey(candidate)
if index, exists := credentialIndexes[credentialKey]; exists {
proxies[index] = candidate
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > 0 && len(proxies) == 0 {
return nil, ErrInvalidProxyOutput
}
p.releaseUnusedCredentials(storedCredentials, proxies)
return proxies, nil
}
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
references := make([]credentials.Reference, 0, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
references = append(references, credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
})
}
p.releaseCredentials(references)
}
func (p *TemplateParser) releaseUnusedCredentials(
stored []credentials.Reference,
candidates []proxyDomain.Proxy,
) {
retained := make(map[credentials.Reference]int, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
retained[credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
}]++
}
unused := make([]credentials.Reference, 0, len(stored))
for _, reference := range stored {
if retained[reference] > 0 {
retained[reference]--
continue
}
unused = append(unused, reference)
}
p.releaseCredentials(unused)
}
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
if p == nil {
return
}
releaser, ok := p.credentialStore.(credentials.Releaser)
if !ok {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
defer cancel()
for _, reference := range references {
if releaseCtx.Err() != nil {
return
}
_ = releaser.Release(releaseCtx, reference)
}
}
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
compiled, err := p.compileRegex(pattern)
if err != nil {

68
internal/adapters/providerapi/template_parser_test.go
View File

@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
}
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
if len(proxies) != 2 {
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
}
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
}
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
}
}
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
}
}
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: strings.Join([]string{
"http://alice:first-password@192.0.2.10:8080",
"http://bob:second-password@192.0.2.11:8080",
}, "\n")},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 2},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("newTemplateParser(): %v", err)
}
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
}
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after failed parse): %v", err)
}
}
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
store, err := credentials.NewMemoryStore(2)
if err != nil {
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
}
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
if value.Password != "new-secret" {
t.Fatalf("resolved latest password mismatch")
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after releasing latest candidates): %v", err)
}
}
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 1},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("NewTemplateParser(): %v", err)
}
proxies, err := parser.Parse(context.Background(), nil)
if err != nil {
t.Fatalf("Parse(): %v", err)
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after ReleaseCandidates): %v", err)
}
}
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {

15
internal/adapters/redisactivity/adapter.go
View File

@ -29,10 +29,11 @@ type Options struct {
}
type Adapter struct {
client redis.Scripter
credentials credentials.Store
keys keyspace
options Options
client redis.Scripter
credentials credentials.Store
credentialReleaser credentials.Releaser
keys keyspace
options Options
}
func New(client redis.Scripter, options Options) (*Adapter, error) {
@ -49,12 +50,14 @@ func New(client redis.Scripter, options Options) (*Adapter, error) {
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
return nil, ErrInvalidOptions
}
return &Adapter{
adapter := &Adapter{
client: client,
credentials: options.Credentials,
keys: newKeyspace(options.Namespace),
options: options,
}, nil
}
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
return adapter, nil
}
func (a *Adapter) Format(state fmt.State, _ rune) {

6
internal/adapters/redisactivity/testredis_test.go
View File

@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
}
func newRedisTestFixture(t *testing.T) redisTestFixture {
return newRedisFixtureWithCredentialCapacity(t, 10_000)
}
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
t.Helper()
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
if redisURL == "" {
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
_ = client.Close()
t.Fatalf("ping test Redis: %v", err)
}
credentialStore, err := credentials.NewMemoryStore(10_000)
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
if err != nil {
_ = client.Close()
t.Fatalf("NewMemoryStore(): %v", err)

28
internal/adapters/redisactivity/upsert.go
View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/platform/credentials"
)
const maxUpsertScriptBatch = 256
const (
maxUpsertScriptBatch = 256
transientCredentialReleaseTimeout = time.Second
)
type upsertCandidate struct {
ProxyID string `json:"proxyId"`
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
var result activitypool.UpsertResult
if a != nil {
defer a.releaseTransientCredentials(batch.Proxies)
}
if ctx == nil {
return result, activitypool.ErrInvalidBatch
}
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
return result, nil
}
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
if a.credentialReleaser == nil {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
defer cancel()
for _, candidate := range proxies {
if releaseCtx.Err() != nil {
return
}
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
SecretRef: candidate.SecretRef,
CredentialVersion: candidate.CredentialVersion,
})
}
}
func (a *Adapter) prepareUpsertCandidate(
ctx context.Context,
upstreamID string,

25
internal/adapters/redisactivity/upsert_integration_test.go
View File

@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
}
}
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
first := testProxy("proxy-a", "192.0.2.10")
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
Username: "first", Password: "first-password",
})
if err != nil {
t.Fatalf("Put(first credential): %v", err)
}
first.SecretRef = firstReference.SecretRef
first.CredentialVersion = firstReference.CredentialVersion
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
Proxies: []proxyDomain.Proxy{first},
}); err != nil {
t.Fatalf("UpsertFetched(first): %v", err)
}
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
Username: "second", Password: "second-password",
}); err != nil {
t.Fatalf("Put(second credential after upsert): %v", err)
}
}
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()

9
internal/adapters/redisprovider/adapter.go
View File

@ -19,8 +19,6 @@ import (
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
const maximumLuaInteger = int64(1<<53 - 1)
type Options struct {
Namespace string
HolderID string
@ -57,7 +55,8 @@ func (adapter *Adapter) RunLeader(
) error {
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
limits.MaxTotal < 0 || limits.MaxTotal > maximumLuaInteger ||
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
return controllerProvider.ErrInvalidCoordination
}
@ -222,7 +221,7 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
expected <= 0 || int64(expected) > maximumLuaInteger {
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
return nil, false, controllerProvider.ErrInvalidCoordination
}
operationCtx, cancel := context.WithCancel(ctx)
@ -295,7 +294,7 @@ type requestPermit struct {
}
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
if fetched < 0 || int64(fetched) > maximumLuaInteger {
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
return controllerProvider.ErrInvalidCoordination
}
return permit.finish(ctx, "complete_fetch", fetched)

6
internal/config/config.go
View File

@ -5,6 +5,12 @@ import (
"time"
)
const (
MaximumPoolSize = 1_000_000
MaximumExactCounter = int64(1<<53 - 1)
MaximumUpstreams = 4_096
)
type Duration time.Duration
func (d *Duration) UnmarshalText(text []byte) error {

60
internal/config/config_test.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
},
want: "estimatedIPsPerCall",
},
{
name: "pool exceeds runtime scan bound",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
},
want: "pool.maxSize",
},
{
name: "zero refill interval",
mutate: func(cfg *Config) {
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
}
}
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
cfg := mustLoadValidConfig(t)
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
t.Fatalf("Validate(too many upstreams) error = %v", err)
}
}
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
if strconv.IntSize < 64 {
t.Skip("64-bit int is required for values above the Redis exact range")
}
tests := []struct {
name string
mutate func(*Upstream)
}{
{name: "fetch total", mutate: func(upstream *Upstream) {
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
}},
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
}},
{name: "minimum slots", mutate: func(upstream *Upstream) {
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
}},
{name: "target slots", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
upstream.Pool.MaxSize = 1
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
}},
{name: "theoretical slots", mutate: func(upstream *Upstream) {
upstream.Pool.MaxSize = 2
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
upstream.Refill.MinimumAvailableSlots = 1
upstream.Refill.TargetAvailableSlots = 2
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := mustLoadValidConfig(t)
updateUpstream(cfg, test.mutate)
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
t.Fatalf("Validate(inexact counter) error = %v", err)
}
})
}
}
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
configured := strings.Replace(validConfig, ` auth:
mode: none`, ` auth:

30
internal/config/fingerprint.go Normal file
View File

@ -0,0 +1,30 @@
package config
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
)
const MinimumFingerprintKeyBytes = 32
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
// Fingerprint returns a keyed digest of one fully resolved configuration.
// Only the digest is persisted; the resolved configuration and key remain local.
func Fingerprint(configuration *Config, key []byte) (string, error) {
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
return "", ErrInvalidFingerprint
}
encoded, err := json.Marshal(configuration)
if err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
digest := hmac.New(sha256.New, key)
if _, err := digest.Write(encoded); err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
return hex.EncodeToString(digest.Sum(nil)), nil
}

68
internal/config/fingerprint_test.go Normal file
View File

@ -0,0 +1,68 @@
package config
import "testing"
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
first := storeTestConfig("provider-a")
upstream := first.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-a"
first.Upstreams["provider-a"] = upstream
stable, err := Fingerprint(first, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first): %v", err)
}
again, err := Fingerprint(first, testFingerprintKey)
if err != nil || again != stable {
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
}
rotated := storeTestConfig("provider-a")
upstream = rotated.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-b"
rotated.Upstreams["provider-a"] = upstream
changed, err := Fingerprint(rotated, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(rotated): %v", err)
}
if changed == stable {
t.Fatal("Fingerprint did not change after secret rotation")
}
if len(changed) != 64 {
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
}
}
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
configuration := storeTestConfig("provider-a")
first, err := Fingerprint(configuration, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first key): %v", err)
}
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
if err != nil {
t.Fatalf("Fingerprint(second key): %v", err)
}
if first == second {
t.Fatal("Fingerprint did not change with independent key")
}
}
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
for name, test := range map[string]struct {
configuration *Config
key []byte
}{
"nil configuration": {key: testFingerprintKey},
"missing key": {configuration: storeTestConfig("provider-a")},
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
} {
t.Run(name, func(t *testing.T) {
if _, err := Fingerprint(test.configuration, test.key); err == nil {
t.Fatal("Fingerprint() succeeded")
}
})
}
}

46
internal/config/store.go
View File

@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
// Store publishes complete validated configurations with one atomic pointer swap.
type Store struct {
current atomic.Pointer[Config]
current atomic.Pointer[publishedConfiguration]
}
type publishedConfiguration struct {
value Config
revision uint64
}
func NewStore(initial *Config) (*Store, error) {
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
return nil, errors.Join(ErrInvalidStore, err)
}
store := &Store{}
store.Publish(initial)
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
return store, nil
}
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
if store == nil {
return nil
}
current := store.current.Load()
if current == nil {
published := store.current.Load()
if published == nil {
return nil
}
cloned := cloneConfig(*current)
cloned := cloneConfig(published.value)
return &cloned
}
// Publish accepts a non-nil configuration already validated by the caller.
func (store *Store) Publish(configuration *Config) {
if store == nil || configuration == nil {
return
func (store *Store) Revision() uint64 {
if store == nil {
return 0
}
published := store.current.Load()
if published == nil {
return 0
}
return published.revision
}
// PublishRevision publishes only a strictly newer authoritative revision.
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
if store == nil || configuration == nil || revision == 0 {
return false
}
for {
current := store.current.Load()
if current != nil && revision <= current.revision {
return false
}
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
if store.current.CompareAndSwap(current, next) {
return true
}
}
cloned := cloneConfig(*configuration)
store.current.Store(&cloned)
}

29
internal/config/store_test.go
View File

@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
}
next := storeTestConfig("provider-b")
store.Publish(next)
if !store.PublishRevision(next, 1) {
t.Fatal("PublishRevision() rejected newer configuration")
}
next.Routing[0].Upstreams[0] = "mutated"
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Publish() retained caller state: %q", got)
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
if index%2 == 1 {
name = "provider-b"
}
store.Publish(storeTestConfig(name))
store.PublishRevision(storeTestConfig(name), uint64(index+1))
}(index)
go func() {
defer wait.Done()
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
wait.Wait()
}
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
t.Parallel()
store, err := NewStore(storeTestConfig("provider-a"))
if err != nil {
t.Fatalf("NewStore() error = %v", err)
}
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
t.Fatal("PublishRevision(newer) rejected")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
t.Fatal("PublishRevision(stale) succeeded")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
t.Fatal("PublishRevision(equal) succeeded")
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Current() upstream = %q, want provider-b", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("Revision() = %d, want 2", got)
}
}
func storeTestConfig(upstreamName string) *Config {
return &Config{
Version: 1,

24
internal/config/validate.go
View File

@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
return err
}
if len(cfg.Upstreams) > MaximumUpstreams {
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
}
enabledUpstreams := 0
for name, upstream := range cfg.Upstreams {
if upstream.Enabled {
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
return err
}
if upstream.Pool.MaxSize > MaximumPoolSize {
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
}
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
return err
}
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
}
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
}
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
return err
}
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
}
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
return err
}
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
return err
}
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
}
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
}
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
}
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
if theoreticalSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
}
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
return err
}
if int64(fetch.MaxInFlight) > MaximumExactCounter {
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
}
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
return err
}

70
internal/controller/admin/service.go
View File

@ -2,9 +2,6 @@ package admin
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"sort"
"strings"
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
LoadConfiguration(context.Context) (LoadedConfiguration, error)
}
// ConfigurationPublisher must atomically publish an already validated configuration.
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
type ConfigurationPublisher interface {
Publish(*config.Config)
PublishRevision(*config.Config, uint64) bool
}
type RuntimeController interface {
Notify()
ValidateConfiguration(context.Context, *config.Config) error
ValidateUpstream(context.Context, string) error
}
var _ ConfigurationPublisher = (*config.Store)(nil)
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
Operations OperationalStatusReader
Configuration ConfigurationLoader
Publisher ConfigurationPublisher
Runtime RuntimeController
}
type ApplicationOptions struct {
Now func() time.Time
Now func() time.Time
FingerprintKey []byte
}
type OperationalStatus struct {
@ -70,30 +75,40 @@ type LoadedConfiguration struct {
}
type ApplicationService struct {
state StateRepository
operations OperationalStatusReader
configuration ConfigurationLoader
publisher ConfigurationPublisher
now func() time.Time
state StateRepository
operations OperationalStatusReader
configuration ConfigurationLoader
publisher ConfigurationPublisher
runtime RuntimeController
now func() time.Time
fingerprintKey []byte
}
var _ Service = (*ApplicationService)(nil)
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
nilInterface(dependencies.Publisher) || options.Now == nil {
nilInterface(dependencies.Publisher) || options.Now == nil ||
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return nil, ErrInvalidApplicationService
}
return &ApplicationService{
state: dependencies.State,
operations: dependencies.Operations,
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
now: options.Now,
state: dependencies.State,
operations: dependencies.Operations,
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
runtime: dependencies.Runtime,
now: options.Now,
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
}, nil
}
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
if command.Enabled && service.runtime != nil {
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
RequestID: command.RequestID,
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
Name: command.Name,
Enabled: command.Enabled,
})
if err == nil && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), mapAdminStateError(err)
}
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
if err := config.Validate(loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
if service.runtime != nil {
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
managementView := loaded.Value.Redacted()
encoded, err := json.Marshal(managementView)
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
if err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
digest := sha256.Sum256(encoded)
checksum := hex.EncodeToString(digest[:])
current, err := service.state.Snapshot(ctx)
if err != nil {
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
if err != nil {
return mutationResult(result), mapAdminStateError(err)
}
service.publisher.Publish(loaded.Value)
if result.Revision == 0 {
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
}
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
if published && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), nil
}

215
internal/controller/admin/service_test.go
View File

@ -3,6 +3,7 @@ package admin
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
@ -10,6 +11,12 @@ import (
"proxy-pool/internal/domain/adminstate"
)
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func applicationTestOptions(now func() time.Time) ApplicationOptions {
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
}
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
Message: "enabled",
},
}
runtime := &recordingRuntimeNotifier{}
service, err := NewApplicationService(ApplicationDependencies{
State: state,
Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: func() time.Time { return now }})
Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
}) {
t.Fatalf("admin state command = %+v", state.lastUpstream)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
}
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
t.Parallel()
wantErr := errors.New("invalid Provider template")
state := &recordingAdminState{}
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
Runtime: runtime,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
RequestID: "req-enable", Name: "provider-a", Enabled: true,
})
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
t.Fatalf("SetUpstreamEnabled() error = %v", err)
}
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
}
}
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
dependencies ApplicationDependencies
options ApplicationOptions
}{
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "clock", dependencies: valid},
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
}
for _, test := range tests {
test := test
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
} {
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
}
}
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
configuration := validReloadConfiguration()
publisher := &recordingConfigurationPublisher{}
runtime := &recordingRuntimeNotifier{}
state := &recordingAdminState{
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "configs/proxy-pool.yaml",
}},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
Publisher: publisher, Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
if len(publisher.published) != 1 || publisher.published[0] != configuration {
t.Fatalf("published configurations = %+v", publisher.published)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
command := state.lastConfig
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
Operations: staticOperationalStatusReader{},
Configuration: forbiddenConfigurationLoader{},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
}
}
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
publisher := &recordingConfigurationPublisher{}
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
if !errors.Is(err, ErrUnavailable) {
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
}
if len(publisher.published) != 0 {
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
}
}
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &orderedCommitState{
firstCommitted: make(chan struct{}),
releaseFirst: make(chan struct{}),
}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: store,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
firstDone := make(chan error, 1)
go func() {
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
})
firstDone <- applyErr
}()
select {
case <-state.firstCommitted:
case <-time.After(time.Second):
t.Fatal("first commit did not reach delayed return")
}
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
}); err != nil {
t.Fatalf("ApplyConfiguration(new): %v", err)
}
close(state.releaseFirst)
if err := <-firstDone; err != nil {
t.Fatalf("ApplyConfiguration(old): %v", err)
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
t.Fatalf("published upstream = %q, want provider-c", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("published revision = %d, want 2", got)
}
}
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
t.Parallel()
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{err: context.Canceled},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
}
}
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
t.Parallel()
state := &recordingAdminState{}
publisher := &recordingConfigurationPublisher{}
var commands []adminstate.CommitConfigCommand
state.onCommit = func(command adminstate.CommitConfigCommand) {
commands = append(commands, command)
state.mutation = adminstate.MutationResult{
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
}
}
for _, secret := range []string{"secret-a", "secret-b"} {
configuration := validReloadConfiguration()
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
t.Fatalf("ReloadConfiguration() error = %v", err)
}
}
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
t.Fatalf("secret rotation changed public management digest: %+v", commands)
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
}
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
t.Fatalf("secret rotation was not published: %+v", publisher.published)
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
}
}
func configWithOnlyUpstream(name string) *config.Config {
configuration := validReloadConfiguration()
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
configuration.Routing = []config.Routing{{
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
}}
return configuration
}
func validReloadUpstream(secret string) config.Upstream {
return config.Upstream{
Enabled: true, Exposure: []string{"gateway"},
@ -505,6 +629,33 @@ type recordingAdminState struct {
onCommit func(adminstate.CommitConfigCommand)
}
type orderedCommitState struct {
next atomic.Uint64
firstCommitted chan struct{}
releaseFirst chan struct{}
}
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
revision := state.next.Add(1)
if revision == 1 {
close(state.firstCommitted)
<-state.releaseFirst
}
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
}
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
return adminstate.Snapshot{}, nil
}
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
state.lastUpstream = command
return state.mutation, state.err
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
type recordingConfigurationPublisher struct {
published []*config.Config
revisions []uint64
}
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
publisher.published = append(publisher.published, configuration)
type recordingRuntimeNotifier struct {
notifications int
validationErr error
}
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
return notifier.validationErr
}
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
return notifier.validationErr
}
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
publisher.published = append(publisher.published, configuration)
publisher.revisions = append(publisher.revisions, revision)
return true
}

131
internal/controller/bootstrap/bootstrap.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"reflect"
"sort"
"strings"
"time"
@ -15,11 +16,15 @@ import (
"proxy-pool/internal/controller/distribution"
"proxy-pool/internal/controller/extraction"
"proxy-pool/internal/controller/operations"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/admission"
"proxy-pool/internal/platform/credentials"
"proxy-pool/internal/platform/httpserver"
"proxy-pool/internal/platform/lifecycle"
platformMetrics "proxy-pool/internal/platform/metrics"
)
@ -29,14 +34,19 @@ var (
)
type Options struct {
ConfigPath string
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
ConfigPath string
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
HolderID string
RedisNamespace string
FingerprintKey []byte
}
type activityStore interface {
extractionDomain.Store
activitypool.Upserter
pool.InventoryReader
activitypool.StateInventoryReader
}
@ -45,6 +55,9 @@ type ports struct {
activity activityStore
readiness distribution.ReadinessChecker
metricsReadiness platformMetrics.ReadinessChecker
coordinator provider.Coordinator
credentials credentials.Store
providerResults provider.ResultRecorder
close func() error
}
@ -61,7 +74,9 @@ type runtimeFactory interface {
}
func Run(ctx context.Context, options Options) error {
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
return run(ctx, options, &productionInfrastructure{
holderID: options.HolderID, namespace: options.RedisNamespace,
}, productionRuntimeFactory{})
}
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if err != nil {
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
}
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
}
configurationStore, err := config.NewStore(loaded.Value)
if err != nil {
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
defer func() {
resultErr = errors.Join(resultErr, opened.close())
}()
var providerState providerStateReader
if loaded.Value.Admin.Enabled {
providerState = opened.state
}
supervisor, err := newProviderSupervisor(
configurationStore,
providerState,
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
buildRuntime, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
return buildRuntime(name, upstream)
},
func(ctx context.Context, configuration *config.Config) error {
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
},
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
loader,
options.FingerprintKey,
providerSupervisorInterval,
)
if err != nil {
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
}
dependencies := controllerRuntime.Dependencies{}
if loaded.Value.Distribution.Enabled {
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if nilInterface(opened.state) || nilInterface(opened.activity) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
var providerStats []provider.StatsReader
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
providerStats = append(providerStats, stats)
}
statusReader, statusErr := operations.NewReader(
configurationStore,
opened.activity,
options.Now,
providerStats...,
)
if statusErr != nil {
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
}
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
}, admin.ApplicationOptions{Now: options.Now})
Runtime: supervisor,
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
if serviceErr != nil {
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
}
@ -146,14 +199,68 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
dependencies.MetricsHandler = handler
}
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
runners := make([]lifecycle.Runner, 0, 2)
if hasHTTPRuntime(loaded.Value) {
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
}
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
runners = append(runners, runner)
}
runners = append(runners, supervisor)
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
}
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
return group.Run(ctx)
}
func prepareProviderConfiguration(
ctx context.Context,
configuration *config.Config,
credentialStore credentials.Store,
) error {
if ctx == nil || configuration == nil {
return ErrProviderRuntime
}
return runner.Run(ctx)
if err := ctx.Err(); err != nil {
return err
}
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
return nil
}
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
if !ok || nilInterface(credentialStore) {
return ErrProviderRuntime
}
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
return err
}
return nil
}
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
if configuration == nil {
return
}
retainer, ok := results.(provider.StatsRetainer)
if !ok || nilInterface(retainer) {
return
}
names := make([]string, 0, len(configuration.Upstreams))
for name := range configuration.Upstreams {
names = append(names, name)
}
sort.Strings(names)
retainer.RetainProviderStats(names)
}
func hasHTTPRuntime(configuration *config.Config) bool {
return configuration != nil &&
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
}
func extractionPolicy(configuration *config.Config) extraction.Policy {

144
internal/controller/bootstrap/bootstrap_integration_test.go
View File

@ -4,16 +4,25 @@ package bootstrap
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/platform/credentials"
)
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
if postgresURL == "" || redisURL == "" {
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
}
var providerCalls atomic.Int64
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
providerCalls.Add(1)
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
}))
defer providerServer.Close()
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
factory := &integrationRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
providerResults := make(chan provider.Result, 8)
factory := &integrationRuntimeFactory{
cancel: cancel, inventory: inventory, providerResults: providerResults,
}
infrastructure := &integrationInfrastructure{
productionInfrastructure: productionInfrastructure{namespace: namespace},
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
}
err := run(context.Background(), Options{
err := run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
},
}, &productionInfrastructure{}, factory)
if err != nil {
t.Fatalf("run() error = %v", err)
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run() error = %v, want context cancellation", err)
}
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
t.Fatalf("Admin Status = %+v", factory.status)
@ -46,13 +75,19 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
!strings.Contains(factory.metricsBody, "go_") {
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
}
if providerCalls.Load() == 0 {
t.Fatal("production Provider HTTP adapter was not called")
}
}
type integrationRuntimeFactory struct {
status admin.Status
readyStatus int
metricsStatus int
metricsBody string
status admin.Status
readyStatus int
metricsStatus int
metricsBody string
cancel context.CancelFunc
inventory pool.InventoryReader
providerResults <-chan provider.Result
}
func (factory *integrationRuntimeFactory) New(
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
factory.metricsBody = metrics.Body.String()
status, err := dependencies.AdminService.Status(ctx)
factory.status = status
return err
if err != nil {
return err
}
if err := waitForProviderInventory(
ctx,
factory.inventory,
factory.providerResults,
[]string{"provider-a", "provider-b"},
); err != nil {
return err
}
factory.cancel()
<-ctx.Done()
return ctx.Err()
}}, nil
}
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
t.Helper()
options, err := redis.ParseURL(redisURL)
if err != nil {
t.Fatalf("redis.ParseURL(): %v", err)
}
client := redis.NewClient(options)
t.Cleanup(func() { _ = client.Close() })
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("credentials.NewMemoryStore(): %v", err)
}
reader, err := redisactivity.New(client, redisactivity.Options{
Namespace: namespace, Credentials: credentialStore,
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
})
if err != nil {
t.Fatalf("redisactivity.New(): %v", err)
}
return reader
}
func waitForProviderInventory(
ctx context.Context,
inventory pool.InventoryReader,
results <-chan provider.Result,
upstreamIDs []string,
) error {
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
var latestResult provider.Result
for {
var inventoryErr error
for _, upstreamID := range upstreamIDs {
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
inventoryErr = errors.Join(inventoryErr, err)
if err == nil && snapshot.Managed > 0 {
return nil
}
}
select {
case <-deadline.Done():
return errors.Join(
errors.New("wait for Provider Redis inventory"),
deadline.Err(),
inventoryErr,
latestResult.Err,
)
case result := <-results:
latestResult = result
case <-ticker.C:
}
}
}
type integrationInfrastructure struct {
productionInfrastructure
results provider.ResultRecorder
}
func (infrastructure *integrationInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (ports, error) {
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
opened.providerResults = infrastructure.results
return opened, err
}
type providerResultRecorder func(provider.Result)
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
type integrationRunner struct {
run func(context.Context) error
}

216
internal/controller/bootstrap/bootstrap_test.go
View File

@ -3,32 +3,48 @@ package bootstrap
import (
"context"
"errors"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/domain/upstream"
"proxy-pool/internal/platform/credentials"
)
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
t.Parallel()
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
state := adminstate.NewMemoryStore()
activity := &stubActivityStore{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
closeErr := errors.New("close failed")
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return closeErr },
}}
runErr := errors.New("runtime failed")
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
err := run(context.Background(), Options{
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
t.Fatalf("run() error = %v, want runtime and close errors", err)
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
}
}
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
for name, key := range map[string][]byte{
"missing": nil,
"short": []byte("too-short"),
} {
t.Run(name, func(t *testing.T) {
infrastructure := &stubInfrastructure{}
err := run(context.Background(), Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: key,
}, infrastructure, &recordingRuntimeFactory{})
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
t.Fatalf("run() error = %v", err)
}
if infrastructure.opens != 0 {
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
}
})
}
}
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
infrastructure := &stubInfrastructure{ports: ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return nil },
}}
factory := &recordingRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(20*time.Millisecond, cancel)
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
Now: time.Now,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
}
if factory.configuration != nil {
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
}
}
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
state := adminstate.NewMemoryStore()
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
started := make(chan string, 2)
stopped := make(chan string, 2)
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
started <- upstreamID
<-ctx.Done()
stopped <- upstreamID
return ctx.Err()
}),
credentials: credentialStore,
close: func() error { return nil },
}}
wantErr := errors.New("test HTTP runtime stopped")
factory := runtimeFactoryFunc(func(
_ *config.Config,
dependencies controllerRuntime.Dependencies,
_ controllerRuntime.Options,
) (controllerRunner, error) {
return runnerFunc(func(ctx context.Context) error {
for {
select {
case upstreamID := <-started:
if upstreamID != "provider-a" {
continue
}
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
}); err != nil {
return err
}
for {
select {
case stoppedID := <-stopped:
if stoppedID == "provider-a" {
return wantErr
}
case <-ctx.Done():
return ctx.Err()
}
}
case <-ctx.Done():
return ctx.Err()
}
}
}), nil
})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, wantErr) {
t.Fatalf("run() error = %v, want %v", err, wantErr)
}
}
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
disabled := configuration.Upstreams["provider-b"]
disabled.Enabled = false
configuration.Upstreams["provider-b"] = disabled
stats, err := provider.NewStatsRecorder(2)
if err != nil {
t.Fatalf("provider.NewStatsRecorder(): %v", err)
}
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
retainProviderStats(configuration, stats)
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
t.Fatalf("Provider stats after retention = %+v", got)
}
}
type memoryResolver struct {
files map[string][]byte
reads int
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
func (runner runnerStub) Run(context.Context) error { return runner.err }
type runnerFunc func(context.Context) error
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
type runtimeFactoryFunc func(
*config.Config,
controllerRuntime.Dependencies,
controllerRuntime.Options,
) (controllerRunner, error)
func (factory runtimeFactoryFunc) New(
configuration *config.Config,
dependencies controllerRuntime.Dependencies,
options controllerRuntime.Options,
) (controllerRunner, error) {
return factory(configuration, dependencies, options)
}
type readyStub struct{}
func (readyStub) Ready(context.Context) error { return nil }
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
return result, nil
}
func (*stubActivityStore) UpsertFetched(
_ context.Context,
_ string,
batch activitypool.FetchedBatch,
) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
}
func (*stubActivityStore) ReadInventory(
context.Context,
string,
time.Duration,
) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
}
type coordinatorStub struct{}
func (coordinatorStub) RunLeader(
ctx context.Context,
_ string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
<-ctx.Done()
return ctx.Err()
}
type coordinatorFunc func(context.Context, string) error
func (run coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
return run(ctx, upstreamID)
}
const bootstrapTestConfig = `
version: 1
security:

127
internal/controller/bootstrap/infrastructure.go
View File

@ -2,6 +2,8 @@ package bootstrap
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"strings"
"time"
@ -11,16 +13,23 @@ import (
"proxy-pool/internal/adapters/postgresadmin"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/adapters/redisprovider"
"proxy-pool/internal/config"
controllerProvider "proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
platformMetrics "proxy-pool/internal/platform/metrics"
)
const (
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisCleanupLimit = 1_024
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisMaximumScan = config.MaximumPoolSize
redisCleanupLimit = 1_024
providerLeaseTTL = 15 * time.Second
providerRenewEvery = 3 * time.Second
providerRetryInterval = 100 * time.Millisecond
providerPermitGrace = 5 * time.Second
)
var (
@ -30,15 +39,22 @@ var (
ErrRedisUnavailable = errors.New("Redis unavailable")
)
type productionInfrastructure struct{}
type productionInfrastructure struct {
holderID string
namespace string
}
func (*productionInfrastructure) Open(
func (infrastructure *productionInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (_ ports, resultErr error) {
if ctx == nil || configuration == nil {
return ports{}, ErrInvalidOptions
}
namespace, err := resolveRedisNamespace(infrastructure.namespace)
if err != nil {
return ports{}, err
}
var postgresPool *pgxpool.Pool
var redisClient *redis.Client
closeResources := func() error {
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
}
}
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
providersEnabled := hasEnabledUpstream(configuration)
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
return ports{}, ErrRedisConfiguration
}
@ -95,17 +112,17 @@ func (*productionInfrastructure) Open(
if err = redisClient.Ping(ctx).Err(); err != nil {
return ports{}, contextOr(ctx, ErrRedisUnavailable)
}
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
if err != nil {
return ports{}, err
}
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
Namespace: redisNamespace,
Namespace: namespace,
Credentials: credentialStore,
OperationTTL: redisOperationTTL,
MaxCandidateScan: candidateScan(configuration),
MaxRuntimeCounters: credentialCapacity(configuration),
MaxInventoryScan: credentialCapacity(configuration),
MaxInventoryScan: maxInventoryScan(configuration),
CleanupLimit: redisCleanupLimit,
})
if err != nil {
@ -113,6 +130,26 @@ func (*productionInfrastructure) Open(
}
opened.activity = adapter
opened.readiness = redisReadiness{client: redisClient}
opened.credentials = credentialStore
if providersEnabled {
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
if statsErr != nil {
return ports{}, statsErr
}
opened.providerResults = stats
holderID, holderErr := resolveHolderID(infrastructure.holderID)
if holderErr != nil {
return ports{}, holderErr
}
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
Namespace: namespace, HolderID: holderID,
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
})
if err != nil {
return ports{}, err
}
}
}
if configuration.Metrics.Enabled {
opened.metricsReadiness = selectMetricsReadiness(
@ -124,11 +161,21 @@ func (*productionInfrastructure) Open(
return opened, nil
}
func resolveRedisNamespace(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured == "" {
return redisNamespace, nil
}
return configured, nil
}
func selectMetricsReadiness(
configuration *config.Config,
admin, activity platformMetrics.ReadinessChecker,
) platformMetrics.ReadinessChecker {
if configuration.Distribution.Enabled {
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
return activity
}
if configuration.Admin.Enabled {
@ -186,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
continue
}
if capacity > maximum-upstream.Pool.MaxSize {
@ -200,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
return capacity
}
func providerCredentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
continue
}
maxInFlight := upstream.Fetch.MaxInFlight
if maxInFlight <= 0 {
maxInFlight = 1
}
if upstream.Pool.MaxSize > maximum/maxInFlight {
return maximum
}
leases := upstream.Pool.MaxSize * maxInFlight
if capacity > maximum-leases {
return maximum
}
capacity += leases
}
if capacity == 0 {
return 1
}
return capacity
}
func maxInventoryScan(_ *config.Config) int {
return config.MaximumPoolSize
}
func hasEnabledUpstream(configuration *config.Config) bool {
if configuration == nil {
return false
}
for _, upstream := range configuration.Upstreams {
if upstream.Enabled {
return true
}
}
return false
}
func resolveHolderID(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured != "" {
return configured, nil
}
var entropy [16]byte
if _, err := rand.Read(entropy[:]); err != nil {
return "", errors.Join(ErrStartup, err)
}
return "controller-" + hex.EncodeToString(entropy[:]), nil
}
func candidateScan(configuration *config.Config) int {
configured := configuration.Distribution.Extraction
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {

19
internal/controller/bootstrap/infrastructure_test.go
View File

@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
},
}
if got := credentialCapacity(configuration); got != 5_000 {
t.Fatalf("credentialCapacity() = %d, want 5000", got)
}
if got := providerCredentialCapacity(configuration); got != 158_000 {
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
}
if got := candidateScan(configuration); got != 5_100 {
t.Fatalf("candidateScan() = %d, want 5100", got)
}
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
}
}
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
t.Parallel()
configuration := &config.Config{Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
}}
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
}
}

126
internal/controller/bootstrap/provider.go Normal file
View File

@ -0,0 +1,126 @@
package bootstrap
import (
"errors"
"math"
"sort"
"strings"
"time"
"proxy-pool/internal/adapters/providerapi"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/pool"
controllerProvider "proxy-pool/internal/controller/provider"
)
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
if configuration == nil {
return nil, ErrProviderRuntime
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
if len(names) == 0 {
return nil, nil
}
builder, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
sort.Strings(names)
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
for _, name := range names {
runtime, err := builder(name, configuration.Upstreams[name])
if err != nil {
return nil, err
}
runtimes = append(runtimes, runtime)
}
return controllerProvider.NewFleet(runtimes...)
}
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
return nil, ErrProviderRuntime
}
results := opened.providerResults
if nilInterface(results) {
results = discardProviderResults{}
}
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
upstream.Enabled = true
mapped, err := providerRuntimeConfig(name, upstream)
if err != nil {
return nil, err
}
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
if err != nil {
return nil, err
}
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
if err != nil {
return nil, err
}
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
Coordinator: opened.coordinator,
Inventory: opened.activity,
Adapter: adapter,
Parser: parser,
Activity: opened.activity,
Results: results,
})
}, nil
}
func providerRuntimeConfig(
upstreamID string,
upstream config.Upstream,
) (controllerProvider.UpstreamRuntimeConfig, error) {
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
upstream.Pool.MaxSize > config.MaximumPoolSize ||
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
}
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
return controllerProvider.UpstreamRuntimeConfig{
Provider: controllerProvider.Config{
UpstreamID: upstreamID,
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
Timeout: time.Duration(upstream.Fetch.Timeout),
MaxAttempts: upstream.Fetch.MaxAttempts,
MaxInFlight: upstream.Fetch.MaxInFlight,
MaxTotal: int64(upstream.Fetch.MaxTotal),
MaxSize: upstream.Pool.MaxSize,
TTL: time.Duration(upstream.Lifecycle.TTL),
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
Retry: controllerProvider.RetryConfig{
Initial: time.Duration(upstream.Fetch.Retry.Initial),
Max: time.Duration(upstream.Fetch.Retry.Max),
Jitter: upstream.Fetch.Retry.Jitter,
},
},
ReconcilePolicy: pool.ReconcilePolicy{
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
ExpectedSlotsPerFetch: expectedSlots,
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
},
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
}, nil
}
type discardProviderResults struct{}
func (discardProviderResults) Record(controllerProvider.Result) {}

375
internal/controller/bootstrap/provider_supervisor.go Normal file
View File

@ -0,0 +1,375 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"sync"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
const providerSupervisorInterval = time.Second
var (
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
)
type providerConfigurationStore interface {
Current() *config.Config
Revision() uint64
PublishRevision(*config.Config, uint64) bool
}
type providerConfigurationSource interface {
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
}
type providerStateReader interface {
Snapshot(context.Context) (adminstate.Snapshot, error)
}
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
type providerConfigurationPreparer func(context.Context, *config.Config) error
type providerConfigurationObserver func(*config.Config)
type providerSupervisor struct {
configuration providerConfigurationStore
state providerStateReader
build providerRunnerBuilder
prepare providerConfigurationPreparer
observe providerConfigurationObserver
source providerConfigurationSource
fingerprintKey []byte
interval time.Duration
notify chan struct{}
}
type runningProvider struct {
configuration config.Upstream
cancel context.CancelFunc
done chan struct{}
}
func newProviderSupervisor(
configuration providerConfigurationStore,
state providerStateReader,
build providerRunnerBuilder,
prepare providerConfigurationPreparer,
observe providerConfigurationObserver,
source providerConfigurationSource,
fingerprintKey []byte,
interval time.Duration,
) (*providerSupervisor, error) {
if nilInterface(configuration) || build == nil || interval <= 0 ||
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
return nil, ErrProviderSupervisor
}
return &providerSupervisor{
configuration: configuration,
state: state,
build: build,
prepare: prepare,
observe: observe,
source: source,
fingerprintKey: append([]byte(nil), fingerprintKey...),
interval: interval,
notify: make(chan struct{}, 1),
}, nil
}
func (supervisor *providerSupervisor) Notify() {
if supervisor == nil || supervisor.notify == nil {
return
}
select {
case supervisor.notify <- struct{}{}:
default:
}
}
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
if err := ctx.Err(); err != nil {
return err
}
if err := config.Validate(configuration); err != nil {
return err
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return err
}
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
sort.Strings(names)
for _, name := range names {
if err := ctx.Err(); err != nil {
return err
}
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
return err
}
}
return nil
}
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := ctx.Err(); err != nil {
return err
}
configuration := supervisor.configuration.Current()
if configuration == nil {
return ErrProviderSupervisor
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
}
upstream, exists := configuration.Upstreams[name]
if !exists {
return ErrProviderSupervisor
}
upstream.Enabled = true
if _, err := supervisor.build(name, upstream); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
return ErrProviderSupervisor
}
active := make(map[string]*runningProvider)
failures := make(chan error, 1)
defer stopAllProviders(active)
ticker := time.NewTicker(supervisor.interval)
defer ticker.Stop()
for {
if err := supervisor.reconcile(ctx, active, failures); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-failures:
return err
case <-supervisor.notify:
case <-ticker.C:
}
}
}
func (supervisor *providerSupervisor) reconcile(
ctx context.Context,
active map[string]*runningProvider,
failures chan<- error,
) error {
desired, err := supervisor.desired(ctx)
if err != nil {
if errors.Is(err, errProviderManagementStateUnavailable) ||
errors.Is(err, errProviderManagementSnapshotStale) {
return nil
}
if errors.Is(err, errProviderConfigurationPending) {
stopAllProviders(active)
clear(active)
return nil
}
return err
}
names := make([]string, 0, len(desired))
prepared := make(map[string]lifecycle.Runner)
for name, upstream := range desired {
names = append(names, name)
current := active[name]
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
continue
}
runner, buildErr := supervisor.build(name, upstream)
if buildErr != nil || nilInterface(runner) {
return errors.Join(ErrProviderSupervisor, buildErr)
}
prepared[name] = runner
}
sort.Strings(names)
for name, current := range active {
if _, keep := desired[name]; !keep {
stopProvider(current)
delete(active, name)
}
}
for _, name := range names {
runner := prepared[name]
if runner == nil {
continue
}
if current := active[name]; current != nil {
stopProvider(current)
}
active[name] = startProvider(ctx, name, desired[name], runner, failures)
}
return nil
}
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
configuration := supervisor.configuration.Current()
if configuration == nil {
return nil, ErrProviderSupervisor
}
enabled := make(map[string]bool, len(configuration.Upstreams))
var snapshot adminstate.Snapshot
if !nilInterface(supervisor.state) {
var err error
snapshot, err = supervisor.state.Snapshot(ctx)
if err != nil {
return nil, errors.Join(errProviderManagementStateUnavailable, err)
}
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
if err != nil {
return nil, err
}
for _, upstream := range snapshot.Upstreams {
enabled[upstream.Name] = upstream.Enabled
}
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
}
if supervisor.observe != nil {
supervisor.observe(configuration)
}
desired := make(map[string]config.Upstream)
for name, upstream := range configuration.Upstreams {
isEnabled := upstream.Enabled
if !nilInterface(supervisor.state) {
isEnabled = enabled[name]
}
if isEnabled {
upstream.Enabled = true
desired[name] = upstream
}
}
return desired, nil
}
func (supervisor *providerSupervisor) synchronizeConfiguration(
ctx context.Context,
current *config.Config,
snapshot adminstate.Snapshot,
) (*config.Config, error) {
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
return current, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
if err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
if checksum == snapshot.Config.Checksum {
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
return current, nil
}
if nilInterface(supervisor.source) {
return nil, errProviderConfigurationPending
}
loaded, err := supervisor.source.LoadConfiguration(ctx)
if err != nil || loaded.Value == nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
if err != nil || checksum != snapshot.Config.Checksum {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
return loaded.Value, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
return nil, errProviderConfigurationPending
}
func startProvider(
ctx context.Context,
name string,
configuration config.Upstream,
runner lifecycle.Runner,
failures chan<- error,
) *runningProvider {
runCtx, cancel := context.WithCancel(ctx)
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
go func() {
defer close(running.done)
err := runner.Run(runCtx)
if runCtx.Err() != nil {
return
}
if err == nil {
err = lifecycle.ErrRunnerStopped
}
select {
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
default:
}
}()
return running
}
func stopProvider(running *runningProvider) {
if running == nil {
return
}
running.cancel()
<-running.done
}
func stopAllProviders(active map[string]*runningProvider) {
var wait sync.WaitGroup
for _, running := range active {
wait.Add(1)
go func(current *runningProvider) {
defer wait.Done()
stopProvider(current)
}(running)
}
wait.Wait()
}

376
internal/controller/bootstrap/provider_supervisor_test.go Normal file
View File

@ -0,0 +1,376 @@
package bootstrap
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
updated := store.Current()
providerB := updated.Upstreams["provider-b"]
providerB.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-b"] = providerB
if !store.PublishRevision(updated, 1) {
t.Fatal("PublishRevision() rejected updated configuration")
}
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
}
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
t.Fatalf("replacement Provider = %+v", event)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
wantErr := errors.New("coordination stopped")
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
}
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 2)
stopped := make(chan providerRuntimeEvent, 2)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.setError(errors.New("PostgreSQL temporarily unavailable"))
supervisor.Notify()
select {
case err := <-done:
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
case event := <-stopped:
t.Fatalf("Provider stopped during transient state failure: %+v", event)
case <-time.After(50 * time.Millisecond):
}
state.setError(nil)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(initial): %v", err)
}
state := &mutableProviderState{
enabled: map[string]bool{"provider-a": true, "provider-b": true},
checksum: initialChecksum,
revision: 1,
}
source := &mutableProviderConfigurationSource{configuration: initial}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
updated := store.Current()
providerA := updated.Upstreams["provider-a"]
providerA.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-a"] = providerA
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(updated): %v", err)
}
state.setChecksum(updatedChecksum)
supervisor.Notify()
waitForRuntimeEvents(t, stopped, 2)
select {
case err := <-done:
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
case <-time.After(50 * time.Millisecond):
}
source.set(updated)
supervisor.Notify()
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
foundReplacement := false
for _, event := range events {
if event.name == "provider-a" && event.url == providerA.API.URL {
foundReplacement = true
}
}
if !foundReplacement {
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
}
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
store.PublishRevision(initial, 1)
candidate := store.Current()
providerA := candidate.Upstreams["provider-a"]
providerA.API.URL = "https://candidate.invalid/proxies"
candidate.Upstreams["provider-a"] = providerA
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(candidate): %v", err)
}
newest := store.Current()
providerA = newest.Upstreams["provider-a"]
providerA.API.URL = "https://newest.invalid/proxies"
newest.Upstreams["provider-a"] = providerA
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
if !store.PublishRevision(newest, 3) {
t.Fatal("failed to publish simulated concurrent revision")
}
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
})
supervisor, err := newProviderSupervisor(
store,
&mutableProviderState{},
func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
},
nil,
nil,
source,
bootstrapTestFingerprintKey,
time.Hour,
)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
})
if !errors.Is(err, errProviderManagementSnapshotStale) {
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
}
if got := store.Revision(); got != 3 {
t.Fatalf("configuration revision = %d, want 3", got)
}
}
type providerRuntimeEvent struct {
name string
url string
}
type supervisorRunnerFunc func(context.Context) error
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
type mutableProviderState struct {
mu sync.Mutex
enabled map[string]bool
err error
checksum string
revision uint64
}
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
state.mu.Lock()
defer state.mu.Unlock()
if state.err != nil {
return adminstate.Snapshot{}, state.err
}
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
if state.checksum != "" {
snapshot.Config = &adminstate.ConfigRevision{
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
}
}
for name, enabled := range state.enabled {
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
}
return snapshot, nil
}
func (state *mutableProviderState) setError(err error) {
state.mu.Lock()
defer state.mu.Unlock()
state.err = err
}
func (state *mutableProviderState) setChecksum(checksum string) {
state.mu.Lock()
defer state.mu.Unlock()
state.checksum = checksum
state.revision++
}
type mutableProviderConfigurationSource struct {
mu sync.Mutex
configuration *config.Config
}
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
return source(ctx)
}
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
source.mu.Lock()
defer source.mu.Unlock()
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
}
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
source.mu.Lock()
defer source.mu.Unlock()
source.configuration = configuration
}
func (state *mutableProviderState) set(name string, enabled bool) {
state.mu.Lock()
defer state.mu.Unlock()
state.enabled[name] = enabled
}
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
t.Helper()
for range count {
_ = waitForRuntimeEvent(t, events)
}
}
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
t.Helper()
select {
case event := <-events:
return event
case <-time.After(time.Second):
t.Fatal("timed out waiting for Provider runtime event")
return providerRuntimeEvent{}
}
}

104
internal/controller/bootstrap/provider_test.go Normal file
View File

@ -0,0 +1,104 @@
package bootstrap
import (
"strconv"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
)
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
upstream := configuration.Upstreams["provider-a"]
mapped, err := providerRuntimeConfig("provider-a", upstream)
if err != nil {
t.Fatalf("providerRuntimeConfig(): %v", err)
}
if mapped.Provider.UpstreamID != "provider-a" ||
mapped.Provider.RequestInterval != time.Second ||
mapped.Provider.Timeout != 3*time.Second ||
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
mapped.Provider.TTL != 2*time.Minute ||
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
t.Fatalf("Provider config = %+v", mapped.Provider)
}
if mapped.ReconcileInterval != time.Second ||
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
}
}
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
configuration.Upstreams["disabled"] = config.Upstream{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
fleet, err := newProviderFleet(configuration, ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
})
if err != nil {
t.Fatalf("newProviderFleet(): %v", err)
}
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
}
configuration.Upstreams["provider-a"] = config.Upstream{}
configuration.Upstreams["provider-b"] = config.Upstream{}
fleet, err = newProviderFleet(configuration, ports{})
if err != nil || fleet != nil {
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
}
}
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
t.Fatal("providerRuntimeConfig(disabled) error = nil")
}
overflowing := config.Upstream{
Enabled: true,
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
Capacity: config.Capacity{
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
},
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
}
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
t.Fatal("providerRuntimeConfig(overflow) error = nil")
}
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
tooLarge := configuration.Upstreams["provider-a"]
tooLarge.Pool.MaxSize = redisMaximumScan + 1
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
}
tooLarge = configuration.Upstreams["provider-a"]
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
if strconv.IntSize == 64 {
tooLarge.Fetch.MaxTotal++
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
}
}
}

36
internal/controller/operations/reader.go
View File

@ -9,6 +9,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -24,6 +25,7 @@ type ConfigurationReader interface {
type Reader struct {
configuration ConfigurationReader
inventory activitypool.StateInventoryReader
providerStats provider.StatsReader
now func() time.Time
}
@ -33,11 +35,17 @@ func NewReader(
configuration ConfigurationReader,
inventory activitypool.StateInventoryReader,
now func() time.Time,
stats ...provider.StatsReader,
) (*Reader, error) {
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
(len(stats) == 1 && nilInterface(stats[0])) {
return nil, ErrInvalidReader
}
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
if len(stats) == 1 {
reader.providerStats = stats[0]
}
return reader, nil
}
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
@ -69,17 +77,27 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
}
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
providerStats := make([]provider.Stats, len(upstreamIDs))
if reader.providerStats != nil {
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
if len(providerStats) != len(upstreamIDs) {
return admin.OperationalStatus{}, ErrUnavailable
}
}
for index, inventory := range inventories {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
return admin.OperationalStatus{}, ErrUnavailable
}
status.Upstreams[index] = admin.UpstreamActivity{
Name: inventory.UpstreamID,
Available: inventory.Available,
Checking: inventory.Checking,
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
Name: inventory.UpstreamID,
Available: inventory.Available,
Checking: inventory.Checking,
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
FetchErrorCount: providerStats[index].FetchErrorCount,
}
}
return status, nil

16
internal/controller/operations/reader_test.go
View File

@ -7,6 +7,7 @@ import (
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}}
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
}}, inventory, func() time.Time { return now })
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
{UpstreamID: "provider-b", FetchErrorCount: 1},
}})
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}
first := status.Upstreams[0]
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
first.Draining != 4 || first.Extracted != 8 {
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
t.Fatalf("first upstream = %+v", first)
}
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
now time.Time
}
type staticProviderStatsReader struct {
result []provider.Stats
}
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
return append([]provider.Stats(nil), reader.result...)
}
func (reader *recordingStateInventoryReader) ReadStateInventory(
_ context.Context,
upstreamIDs []string,

2
internal/controller/provider/coordination.go
View File

@ -13,6 +13,8 @@ var (
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
)
const MaximumCoordinationInteger = int64(1<<53 - 1)
type CoordinationLimits struct {
RequestInterval time.Duration
MaxInFlight int

64
internal/controller/provider/fleet.go Normal file
View File

@ -0,0 +1,64 @@
package provider
import (
"context"
"errors"
"sort"
"proxy-pool/internal/platform/lifecycle"
)
var ErrInvalidFleet = errors.New("invalid Provider fleet")
type Fleet struct {
runtimes []*UpstreamRuntime
group *lifecycle.Group
}
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
if len(runtimes) == 0 {
return nil, ErrInvalidFleet
}
seen := make(map[string]struct{}, len(runtimes))
owned := make([]*UpstreamRuntime, len(runtimes))
for index, runtime := range runtimes {
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
return nil, ErrInvalidFleet
}
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
return nil, ErrInvalidFleet
}
seen[runtime.config.Provider.UpstreamID] = struct{}{}
owned[index] = runtime
}
sort.Slice(owned, func(left, right int) bool {
return owned[left].ID() < owned[right].ID()
})
runners := make([]lifecycle.Runner, len(owned))
for index, runtime := range owned {
runners[index] = runtime
}
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return nil, errors.Join(ErrInvalidFleet, err)
}
return &Fleet{runtimes: owned, group: group}, nil
}
func (fleet *Fleet) Run(ctx context.Context) error {
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
return ErrInvalidFleet
}
return fleet.group.Run(ctx)
}
func (fleet *Fleet) IDs() []string {
if fleet == nil {
return nil
}
ids := make([]string, len(fleet.runtimes))
for index, runtime := range fleet.runtimes {
ids[index] = runtime.ID()
}
return ids
}

101
internal/controller/provider/fleet_test.go Normal file
View File

@ -0,0 +1,101 @@
package provider
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
started := make(chan string, 2)
siblingCancelled := make(chan struct{}, 1)
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
) error {
started <- "provider-a"
return errors.New("provider-a stopped")
}))
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
started <- "provider-b"
<-ctx.Done()
siblingCancelled <- struct{}{}
return ctx.Err()
}))
fleet, err := NewFleet(first, second)
if err != nil {
t.Fatalf("NewFleet(): %v", err)
}
err = fleet.Run(context.Background())
if err == nil || err.Error() != "provider-a stopped" {
t.Fatalf("Run() error = %v, want provider-a failure", err)
}
seen := map[string]bool{<-started: true, <-started: true}
if !seen["provider-a"] || !seen["provider-b"] {
t.Fatalf("started upstreams = %v", seen)
}
select {
case <-siblingCancelled:
case <-time.After(time.Second):
t.Fatal("sibling runtime was not cancelled")
}
}
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
return ctx.Err()
}))
tests := []struct {
name string
runtimes []*UpstreamRuntime
}{
{name: "empty"},
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fleet, err := NewFleet(test.runtimes...)
if err == nil || fleet != nil {
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
}
})
}
}
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
t.Helper()
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig(upstreamID),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}, UpstreamRuntimeDependencies{
Coordinator: coordinator,
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
}
return runtime
}

7
internal/controller/provider/ports.go
View File

@ -27,9 +27,11 @@ type RetryableError interface {
Retryable() bool
}
// Parser must be safe for concurrent calls and must honor context cancellation.
// Parser owns any transient resources attached to parsed candidates. It must
// be safe for concurrent calls and must honor context cancellation.
type Parser interface {
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
ReleaseCandidates([]proxyDomain.Proxy)
}
type Result struct {
@ -42,7 +44,8 @@ type Result struct {
}
type ResultRecorder interface {
// Record may be called concurrently and must not retain mutable result data.
// Record may be called concurrently, must return promptly, and must not
// retain mutable result data.
Record(Result)
}

94
internal/controller/provider/reconciler.go
View File

@ -19,6 +19,7 @@ type Config struct {
Timeout time.Duration
MaxAttempts int
MaxInFlight int
MaxTotal int64
MaxSize int
TTL time.Duration
AllocationSafetyMargin time.Duration
@ -31,6 +32,8 @@ type RetryConfig struct {
Jitter int
}
const requestPermitSettlementTimeout = 5 * time.Second
type Reconciler struct {
config Config
ports Ports
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
}
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
}
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
@ -140,11 +143,51 @@ func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error
}
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
if err != nil {
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: %w", err),
Attempt: 1,
})
return
}
if !available {
return
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Attempt: 1,
})
return
}
permitFinished := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
}
}()
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
response, result, retryable, ok := r.fetchAttempt(ctx, session, attempt)
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
if !ok {
return
}
if result.Class != upstream.FetchError {
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
result.Class = upstream.FetchError
result.Err = errors.Join(result.Err, capacityErr)
} else {
permitFinished = true
}
}
r.ports.Results.Record(result)
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
return
@ -163,41 +206,12 @@ func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
func (r *Reconciler) fetchAttempt(
ctx context.Context,
session LeaderSession,
permit upstream.FetchPermit,
attempt int,
) (FetchResponse, Result, bool, bool) {
if err := r.waitForRequestSlot(ctx); err != nil {
return FetchResponse{}, Result{}, false, false
}
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
if err != nil {
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: resultErr,
Attempt: attempt,
}, false, true
}
if !available {
return FetchResponse{}, Result{}, false, false
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Attempt: attempt,
}, false, true
}
permitFinished := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
}
}()
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
if err != nil {
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
@ -236,13 +250,14 @@ func (r *Reconciler) fetchAttempt(
defer cancel()
response, callErr := r.ports.Adapter.Fetch(callCtx)
var parseErr, candidateErr, coordinationErr, capacityErr error
var parseErr, candidateErr, coordinationErr error
var validCount, newCount int
if callErr != nil {
requestSettlementAttempted = true
coordinationErr = r.settleRequestPermit(requestPermit, false, 0)
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
} else {
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
defer r.ports.Parser.ReleaseCandidates(candidates)
parseErr = err
validCount = len(candidates)
charged := validCount
@ -267,14 +282,10 @@ func (r *Reconciler) fetchAttempt(
newCount = upserted.Inserted
}
}
if callErr == nil && parseErr == nil && candidateErr == nil && coordinationErr == nil {
capacityErr = permit.Complete(newCount)
permitFinished = capacityErr == nil
}
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr, capacityErr)
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
class := upstream.ClassifyFetchResult(
callErr,
errors.Join(parseErr, candidateErr, coordinationErr, capacityErr),
errors.Join(parseErr, candidateErr, coordinationErr),
validCount,
newCount,
)
@ -289,8 +300,7 @@ func (r *Reconciler) fetchAttempt(
}
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
timeout := min(r.config.Timeout, 5*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
defer cancel()
if complete {
return permit.Complete(ctx, fetched)

97
internal/controller/provider/reconciler_test.go
View File

@ -8,6 +8,7 @@ import (
"testing"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/upstream"
@ -722,9 +723,9 @@ func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testin
}
}
func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
results := make(chan Result, 1)
globalCancelled := make(chan struct{}, 1)
globalCompleted := make(chan int, 1)
ports := successfulPorts(func() {}, results)
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{}, errors.New("provider connection failed")
@ -736,17 +737,15 @@ func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{cancelled: globalCancelled}, true, nil
return &recordingRequestPermit{completed: globalCompleted}, true, nil
})
result := runSingleReconcile(t, reconciler, results, session)
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
select {
case <-globalCancelled:
default:
t.Fatal("distributed request reservation was not cancelled")
if got := <-globalCompleted; got <= 0 {
t.Fatalf("global charged count = %d, want conservative expected count", got)
}
}
@ -754,10 +753,12 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.
results := make(chan Result, 1)
localCompleted := make(chan fetchCompletion, 1)
globalCompleted := make(chan int, 1)
released := make(chan []proxyDomain.Proxy, 1)
ports := successfulPorts(func() {}, results)
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
})
ports.Parser = &recordingCandidateParser{
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
released: released,
}
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
if len(batch.Proxies) != 2 {
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
@ -791,6 +792,14 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.
if got := <-localCompleted; got.retained != 1 {
t.Fatalf("local fetch completion = %+v, want retained=1", got)
}
select {
case got := <-released:
if len(got) != 3 {
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
}
case <-time.After(time.Second):
t.Fatal("parsed candidates were not released")
}
}
func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
@ -838,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
}
}
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
results := make(chan Result, 2)
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
var parses atomic.Int64
ports := successfulPorts(func() {}, results)
ports.Capacity = budget
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
if parses.Add(1) == 1 {
return nil, errors.New("temporary parser failure")
}
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
MaxInFlight: 1, MaxSize: 10,
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
if got := budget.Snapshot().PendingExpected; got != 1 {
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
}
return nil
})})
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
cancel()
if err := <-done; err != nil {
t.Fatalf("RunLeader(): %v", err)
}
usage := budget.Snapshot()
if usage.PendingExpected != 0 || usage.Managed != 1 {
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
}
}
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
results := make(chan Result, 1)
ports := successfulPorts(func() {}, results)
@ -954,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
return errors.New("unexpected sleep")
}
type sleeperFunc func(context.Context, time.Duration) error
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
return f(ctx, duration)
}
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
if err := ctx.Err(); err != nil {
return err
@ -981,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
return f(ctx, body)
}
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
type recordingCandidateParser struct {
candidates []proxyDomain.Proxy
released chan<- []proxyDomain.Proxy
}
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
}
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
}
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {

103
internal/controller/provider/stats.go Normal file
View File

@ -0,0 +1,103 @@
package provider
import (
"errors"
"math"
"sync"
"proxy-pool/internal/domain/upstream"
)
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
type Stats struct {
UpstreamID string
ConsecutiveEmptyFetch int64
FetchErrorCount int64
}
type StatsReader interface {
ReadProviderStats([]string) []Stats
}
type StatsRetainer interface {
RetainProviderStats([]string)
}
type StatsRecorder struct {
mu sync.Mutex
maximum int
byID map[string]Stats
}
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
if maximum <= 0 {
return nil, ErrInvalidStatsRecorder
}
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
}
func (recorder *StatsRecorder) Record(result Result) {
if recorder == nil || result.UpstreamID == "" {
return
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
stats, exists := recorder.byID[result.UpstreamID]
if !exists {
if len(recorder.byID) >= recorder.maximum {
return
}
stats.UpstreamID = result.UpstreamID
}
switch result.Class {
case upstream.FetchEmpty:
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
stats.ConsecutiveEmptyFetch++
}
case upstream.FetchValid, upstream.FetchDuplicateOnly:
stats.ConsecutiveEmptyFetch = 0
case upstream.FetchError:
if stats.FetchErrorCount < math.MaxInt64 {
stats.FetchErrorCount++
}
default:
return
}
recorder.byID[result.UpstreamID] = stats
}
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
result := make([]Stats, len(upstreamIDs))
if recorder == nil {
return result
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for index, upstreamID := range upstreamIDs {
result[index] = recorder.byID[upstreamID]
result[index].UpstreamID = upstreamID
}
return result
}
// RetainProviderStats removes observations for upstreams no longer present in
// the complete configuration. Disabled but configured upstreams must be kept.
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
if recorder == nil {
return
}
retained := make(map[string]struct{}, len(upstreamIDs))
for _, upstreamID := range upstreamIDs {
if upstreamID != "" {
retained[upstreamID] = struct{}{}
}
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for upstreamID := range recorder.byID {
if _, keep := retained[upstreamID]; !keep {
delete(recorder.byID, upstreamID)
}
}
}

83
internal/controller/provider/stats_test.go Normal file
View File

@ -0,0 +1,83 @@
package provider
import (
"sync"
"testing"
"proxy-pool/internal/domain/upstream"
)
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
for _, class := range []upstream.FetchClass{
upstream.FetchEmpty,
upstream.FetchEmpty,
upstream.FetchError,
} {
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
}
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
}
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
}
}
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
recorder, err := NewStatsRecorder(1)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(1)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
}()
}
wait.Wait()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
}
}
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(2)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
}()
go func() {
defer wait.Done()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
}()
}
wait.Wait()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
t.Fatalf("stats after retention = %+v", stats)
}
}

219
internal/controller/provider/upstream_runtime.go Normal file
View File

@ -0,0 +1,219 @@
package provider
import (
"context"
"errors"
"fmt"
"hash/fnv"
"reflect"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
)
var (
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
)
type UpstreamRuntimeConfig struct {
Provider Config
ReconcilePolicy pool.ReconcilePolicy
ReconcileInterval time.Duration
}
type UpstreamRuntimeDependencies struct {
Coordinator Coordinator
Inventory pool.InventoryReader
Adapter ProviderAdapter
Parser Parser
Activity activitypool.Upserter
Results ResultRecorder
}
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
// local budget and coalescing signal are created for each leadership term.
type UpstreamRuntime struct {
config UpstreamRuntimeConfig
dependencies UpstreamRuntimeDependencies
sleeper Sleeper
}
func NewUpstreamRuntime(
config UpstreamRuntimeConfig,
dependencies UpstreamRuntimeDependencies,
) (*UpstreamRuntime, error) {
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
nilRuntimeDependency(dependencies.Results) {
return nil, ErrInvalidUpstreamRuntime
}
runtime := &UpstreamRuntime{
config: config, dependencies: dependencies, sleeper: timerSleeper{},
}
if _, err := runtime.newLeaderTerm(); err != nil {
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
}
return runtime, nil
}
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
if runtime == nil || ctx == nil {
return ErrInvalidUpstreamRuntime
}
limits := CoordinationLimits{
RequestInterval: runtime.config.Provider.RequestInterval,
MaxInFlight: runtime.config.Provider.MaxInFlight,
MaxAttemptDuration: runtime.config.Provider.Timeout,
MaxTotal: runtime.config.Provider.MaxTotal,
}
err := runtime.dependencies.Coordinator.RunLeader(
ctx,
runtime.config.Provider.UpstreamID,
limits,
func(leaderCtx context.Context, session LeaderSession) error {
term, buildErr := runtime.newLeaderTerm()
if buildErr != nil {
return buildErr
}
return term.run(leaderCtx, session)
},
)
if ctx.Err() != nil {
return ctx.Err()
}
if err == nil {
return ErrUpstreamRuntimeStopped
}
return err
}
func (runtime *UpstreamRuntime) ID() string {
if runtime == nil {
return ""
}
return runtime.config.Provider.UpstreamID
}
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
UpstreamID: runtime.config.Provider.UpstreamID,
MaxSize: runtime.config.Provider.MaxSize,
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
})
if err != nil {
return nil, err
}
providerConfig := runtime.config.Provider
providerConfig.RequestInterval = 0
providerReconciler, err := NewReconciler(providerConfig, Ports{
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
Capacity: budget,
})
if err != nil {
return nil, err
}
poolReconciler, err := pool.NewReconciler(
runtime.config.ReconcilePolicy,
budget,
providerReconciler,
)
if err != nil {
return nil, err
}
return &upstreamLeaderTerm{
upstreamID: runtime.config.Provider.UpstreamID,
interval: runtime.config.ReconcileInterval,
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
inventory: runtime.dependencies.Inventory,
pool: poolReconciler,
provider: providerReconciler,
sleeper: runtime.sleeper,
}, nil
}
type upstreamLeaderTerm struct {
upstreamID string
interval time.Duration
safetyMargin time.Duration
inventory pool.InventoryReader
pool *pool.Reconciler
provider *Reconciler
sleeper Sleeper
}
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
termCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 2)
go func() { done <- term.provider.RunLeader(termCtx, session) }()
go func() { done <- term.reconcileInventory(termCtx) }()
first := <-done
cancel()
second := <-done
if ctx.Err() != nil {
return nil
}
if first == nil {
first = ErrUpstreamRuntimeStopped
}
if second != nil && !errors.Is(second, context.Canceled) {
return errors.Join(first, second)
}
return first
}
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
if err := term.sleeper.Sleep(ctx, delay); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
}
}
for ctx.Err() == nil {
inventory, err := term.inventory.ReadInventory(
ctx,
term.upstreamID,
term.safetyMargin,
)
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
term.pool.ReconcileSnapshot(inventory)
}
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
}
}
return nil
}
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
window := min(interval/4, 250*time.Millisecond)
if window <= 1 {
return 0
}
digest := fnv.New64a()
_, _ = digest.Write([]byte(upstreamID))
return time.Duration(digest.Sum64() % uint64(window))
}
func nilRuntimeDependency(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

247
internal/controller/provider/upstream_runtime_test.go Normal file
View File

@ -0,0 +1,247 @@
package provider
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
var inventoryReads atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
inventoryReads.Add(1)
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for leader fetch")
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
if got := inventoryReads.Load(); got == 0 {
t.Fatal("inventory was not read inside leader term")
}
}
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
var inventoryReads atomic.Int64
var providerCalls atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
_ CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
if inventoryReads.Add(1) == 1 {
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
}
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
providerCalls.Add(1)
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for recovered inventory fetch")
}
cancel()
<-done
if got := inventoryReads.Load(); got < 2 {
t.Fatalf("inventory reads = %d, want recovery retry", got)
}
if got := providerCalls.Load(); got != 1 {
t.Fatalf("provider calls = %d, want one call after recovery", got)
}
}
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
providerConfig := runtimeProviderConfig("provider-a")
providerConfig.RequestInterval = 500 * time.Millisecond
fetched := make(chan struct{}, 2)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: providerConfig,
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 5 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if limits.RequestInterval != 500*time.Millisecond {
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
for range 2 {
select {
case <-fetched:
case <-time.After(150 * time.Millisecond):
cancel()
<-done
t.Fatal("local Provider reconciler duplicated the distributed request interval")
}
}
cancel()
<-done
}
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
validConfig := UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
}
}
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
const interval = time.Second
first := initialReconcileDelay("provider-a", interval)
if first != initialReconcileDelay("provider-a", interval) {
t.Fatal("initial reconcile delay is not stable")
}
if first < 0 || first >= 250*time.Millisecond {
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
}
if other := initialReconcileDelay("provider-b", interval); other == first {
t.Fatalf("different upstreams have the same initial delay %s", first)
}
}
func runtimeProviderConfig(upstreamID string) Config {
return Config{
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
}
}
func runtimeReconcilePolicy() pool.ReconcilePolicy {
return pool.ReconcilePolicy{
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
}
}
type coordinatorFunc func(
context.Context,
string,
CoordinationLimits,
func(context.Context, LeaderSession) error,
) error
func (f coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return f(ctx, upstreamID, limits, work)
}
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
func (f inventoryReaderFunc) ReadInventory(
ctx context.Context,
upstreamID string,
safetyMargin time.Duration,
) (pool.InventorySnapshot, error) {
return f(ctx, upstreamID, safetyMargin)
}

110
internal/platform/credentials/store.go
View File

@ -48,18 +48,35 @@ type Store interface {
Resolve(context.Context, Reference) (Value, error)
}
// Releaser removes transient credential material after the consumer has copied
// it into its authoritative storage. Release is idempotent and version fenced.
type Releaser interface {
Release(context.Context, Reference) error
}
type CapacityEnsurer interface {
EnsureCapacity(context.Context, int) error
}
type entry struct {
scope *scopeState
value Value
version uint64
reference Reference
}
type scopeState struct {
name string
value Value
version uint64
leases int
}
// MemoryStore keeps credentials in process memory and serializes access with a
// context-aware lock.
type MemoryStore struct {
lock chan struct{}
capacity int
byScope map[string]*entry
byScope map[string]*scopeState
byRef map[string]*entry
}
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
return &MemoryStore{
lock: lock,
capacity: capacity,
byScope: make(map[string]*entry),
byScope: make(map[string]*scopeState),
byRef: make(map[string]*entry),
}, nil
}
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
if err := s.acquire(ctx); err != nil {
return Reference{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Reference{}, err
}
if current, ok := s.byScope[scope]; ok {
if current.value == value {
return current.reference, nil
}
current.value = value
current.version++
current.reference.CredentialVersion = versionString(current.version)
return current.reference, nil
if len(s.byRef) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
if len(s.byScope) >= s.capacity {
current, exists := s.byScope[scope]
if !exists && len(s.byScope) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
secretRef, err := s.newUniqueSecretRef()
if err != nil {
return Reference{}, err
}
if !exists {
current = &scopeState{name: scope, value: value, version: 1}
s.byScope[scope] = current
} else if current.value != value {
current.value = value
current.version++
}
created := &entry{
value: value,
version: 1,
scope: current,
value: value,
reference: Reference{
SecretRef: secretRef,
CredentialVersion: versionString(1),
CredentialVersion: versionString(current.version),
},
}
s.byScope[scope] = created
current.leases++
s.byRef[secretRef] = created
return created.reference, nil
}
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
if err := s.acquire(ctx); err != nil {
return Value{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Value{}, err
}
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
return current.value, nil
}
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
return ErrInvalidReference
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if err := ctx.Err(); err != nil {
return err
}
current, ok := s.byRef[reference.SecretRef]
if !ok || current.reference != reference {
return nil
}
delete(s.byRef, reference.SecretRef)
current.scope.leases--
if current.scope.leases == 0 {
delete(s.byScope, current.scope.name)
current.scope.value = Value{}
}
current.value = Value{}
return nil
}
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if minimum <= 0 {
return ErrInvalidCapacity
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if minimum > s.capacity {
s.capacity = minimum
}
return nil
}
func (s *MemoryStore) acquire(ctx context.Context) error {
if err := contextError(ctx); err != nil {
return err
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
}
}
func (s *MemoryStore) release() {
func (s *MemoryStore) unlock() {
s.lock <- struct{}{}
}

152
internal/platform/credentials/store_test.go
View File

@ -11,7 +11,7 @@ import (
"time"
)
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if err != nil {
t.Fatalf("Put(second): %v", err)
}
if first != second {
t.Fatalf("second reference = %#v, want %#v", second, first)
if first == second || first.SecretRef == second.SecretRef {
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
}
if first.SecretRef == "" || first.CredentialVersion != "v1" {
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
}
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if got != value {
t.Fatalf("Resolve() = %#v, want %#v", got, value)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
}
}
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
}
}
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if newReference.SecretRef != oldReference.SecretRef {
t.Fatalf("new SecretRef changed across versions")
if newReference.SecretRef == oldReference.SecretRef {
t.Fatalf("new SecretRef reused an active lease")
}
if newReference.CredentialVersion != "v2" {
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
}
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
old, err := store.Resolve(context.Background(), oldReference)
if err != nil || old.Password != "old-password" {
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
}
got, err := store.Resolve(context.Background(), newReference)
if err != nil {
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
if got != want {
t.Fatalf("Resolve(existing) returned changed credentials")
}
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
}
if err := store.Release(context.Background(), reference); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
t.Fatalf("Put(idempotent at capacity): %v", err)
t.Fatalf("Put(after lease release): %v", err)
}
}
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
if err != nil {
t.Fatalf("Put(first): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first): %v", err)
}
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
t.Fatalf("Put(after release): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(idempotent): %v", err)
}
}
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
if err != nil {
t.Fatalf("Put(old): %v", err)
}
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if err := store.Release(context.Background(), stale); err != nil {
t.Fatalf("Release(stale): %v", err)
}
if _, err := store.Resolve(context.Background(), current); err != nil {
t.Fatalf("Resolve(current): %v", err)
}
}
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
t.Fatalf("EnsureCapacity(2): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
t.Fatalf("Put(provider-a): %v", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
t.Fatalf("Put(provider-b): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
t.Fatalf("EnsureCapacity(shrink request): %v", err)
}
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
}
}
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
}
}
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
want := Value{Username: "alice", Password: "shared-password"}
const workers = 100
references := make(chan Reference, workers)
errorsSeen := make(chan error, workers)
var wait sync.WaitGroup
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
for err := range errorsSeen {
t.Errorf("concurrent Put(): %v", err)
}
var first Reference
secretRefs := make(map[string]struct{}, workers)
for reference := range references {
if first == (Reference{}) {
first = reference
if reference.CredentialVersion != "v1" {
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
}
if reference != first {
t.Errorf("concurrent Put() reference differs from first")
secretRefs[reference.SecretRef] = struct{}{}
}
if len(secretRefs) != workers {
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
}
for secretRef := range secretRefs {
got, err := store.Resolve(context.Background(), Reference{
SecretRef: secretRef, CredentialVersion: "v1",
})
if err != nil || got != want {
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
}
}
if first.CredentialVersion != "v1" {
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
}
got, err := store.Resolve(context.Background(), first)
if err != nil {
t.Fatalf("Resolve(): %v", err)
}
if got != want {
t.Fatalf("Resolve() returned unexpected credentials")
}
}
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
store, err := NewMemoryStore(1)
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
reference Reference
err error
}
const workers = 100
results := make(chan result, workers)
var wait sync.WaitGroup
for index := range workers {
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
close(results)
versions := make(map[string]struct{}, workers)
var secretRef string
secretRefs := make(map[string]struct{}, workers)
var latest result
for current := range results {
if current.err != nil {
t.Fatalf("concurrent Put(): %v", current.err)
}
if secretRef == "" {
secretRef = current.reference.SecretRef
}
if current.reference.SecretRef != secretRef {
t.Fatal("SecretRef changed across concurrent updates")
}
secretRefs[current.reference.SecretRef] = struct{}{}
versions[current.reference.CredentialVersion] = struct{}{}
if current.reference.CredentialVersion == "v100" {
latest = current
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
if len(versions) != workers {
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
}
if len(secretRefs) != workers {
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
}
if latest.reference == (Reference{}) {
t.Fatal("highest version v100 was not returned")
}

72
internal/platform/lifecycle/group.go Normal file
View File

@ -0,0 +1,72 @@
package lifecycle
import (
"context"
"errors"
"reflect"
)
var (
ErrInvalidGroup = errors.New("invalid lifecycle group")
ErrRunnerStopped = errors.New("lifecycle runner stopped")
)
type Runner interface {
Run(context.Context) error
}
type Group struct {
runners []Runner
}
func NewGroup(runners ...Runner) (*Group, error) {
if len(runners) == 0 {
return nil, ErrInvalidGroup
}
owned := make([]Runner, len(runners))
for index, runner := range runners {
if isNilRunner(runner) {
return nil, ErrInvalidGroup
}
owned[index] = runner
}
return &Group{runners: owned}, nil
}
func (group *Group) Run(ctx context.Context) error {
if group == nil || ctx == nil || len(group.runners) == 0 {
return ErrInvalidGroup
}
groupCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, len(group.runners))
for _, runner := range group.runners {
go func() { done <- runner.Run(groupCtx) }()
}
first := <-done
cancel()
for range len(group.runners) - 1 {
<-done
}
if ctx.Err() != nil {
return ctx.Err()
}
if first == nil {
return ErrRunnerStopped
}
return first
}
func isNilRunner(runner Runner) bool {
if runner == nil {
return true
}
reflected := reflect.ValueOf(runner)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

87
internal/platform/lifecycle/group_test.go Normal file
View File

@ -0,0 +1,87 @@
package lifecycle
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
started := make(chan struct{}, 2)
cancelled := make(chan struct{}, 1)
wantErr := errors.New("runner failed")
group, err := NewGroup(
runnerFunc(func(context.Context) error {
started <- struct{}{}
return wantErr
}),
runnerFunc(func(ctx context.Context) error {
started <- struct{}{}
<-ctx.Done()
cancelled <- struct{}{}
return ctx.Err()
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
<-started
<-started
select {
case <-cancelled:
case <-time.After(time.Second):
t.Fatal("sibling runner was not cancelled")
}
}
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
release := make(chan struct{})
var exited atomic.Bool
group, err := NewGroup(
runnerFunc(func(context.Context) error { return errors.New("failed") }),
runnerFunc(func(ctx context.Context) error {
<-ctx.Done()
<-release
exited.Store(true)
return nil
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
done := make(chan error, 1)
go func() { done <- group.Run(context.Background()) }()
select {
case <-done:
t.Fatal("Run() returned before sibling exited")
case <-time.After(20 * time.Millisecond):
}
close(release)
<-done
if !exited.Load() {
t.Fatal("sibling exit was not observed")
}
}
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
var typedNil *nilRunner
for _, runners := range [][]Runner{nil, {typedNil}} {
group, err := NewGroup(runners...)
if err == nil || group != nil {
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
}
}
}
type runnerFunc func(context.Context) error
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
type nilRunner struct{}
func (*nilRunner) Run(context.Context) error { return nil }