feat: run provider lifecycle in controller
This commit is contained in:
parent
40f4b3ffab
commit
8997e3880e
@ -15,7 +15,10 @@ import (
|
|||||||
"proxy-pool/internal/controller/bootstrap"
|
"proxy-pool/internal/controller/bootstrap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const configEnvironment = "PROXY_POOL_CONFIG"
|
const (
|
||||||
|
configEnvironment = "PROXY_POOL_CONFIG"
|
||||||
|
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
|
||||||
|
)
|
||||||
|
|
||||||
type environmentLookup func(string) string
|
type environmentLookup func(string) string
|
||||||
type controllerRun func(context.Context, bootstrap.Options) error
|
type controllerRun func(context.Context, bootstrap.Options) error
|
||||||
@ -54,7 +57,13 @@ func execute(
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
|
var fingerprintKey []byte
|
||||||
|
if getenv != nil {
|
||||||
|
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
|
||||||
|
}
|
||||||
|
err := run(ctx, bootstrap.Options{
|
||||||
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
|
||||||
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
|||||||
if name == configEnvironment {
|
if name == configEnvironment {
|
||||||
return "environment.yaml"
|
return "environment.yaml"
|
||||||
}
|
}
|
||||||
|
if name == fingerprintKeyEnvironment {
|
||||||
|
return "0123456789abcdef0123456789abcdef"
|
||||||
|
}
|
||||||
return ""
|
return ""
|
||||||
}, func(_ context.Context, options bootstrap.Options) error {
|
}, func(_ context.Context, options bootstrap.Options) error {
|
||||||
received = options
|
received = options
|
||||||
return nil
|
return nil
|
||||||
}, &bytes.Buffer{})
|
}, &bytes.Buffer{})
|
||||||
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
|
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
|
||||||
t.Fatalf("execute() = %d, options = %+v", code, received)
|
t.Fatalf("execute() = %d, options = %+v", code, received)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
|||||||
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
var received bootstrap.Options
|
var received bootstrap.Options
|
||||||
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
|
code := execute(context.Background(), nil, func(name string) string {
|
||||||
|
if name == configEnvironment {
|
||||||
|
return "environment.yaml"
|
||||||
|
}
|
||||||
|
return "0123456789abcdef0123456789abcdef"
|
||||||
|
}, func(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
options bootstrap.Options,
|
options bootstrap.Options,
|
||||||
) error {
|
) error {
|
||||||
|
|||||||
@ -1,5 +1,13 @@
|
|||||||
name: proxy-pool
|
name: proxy-pool
|
||||||
|
|
||||||
|
x-app-environment: &app-environment
|
||||||
|
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
||||||
|
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
||||||
|
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
||||||
|
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
||||||
|
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||||
|
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||||
|
|
||||||
x-app: &app
|
x-app: &app
|
||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
@ -9,13 +17,7 @@ x-app: &app
|
|||||||
networks: [frontend, backend]
|
networks: [frontend, backend]
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
environment:
|
environment: *app-environment
|
||||||
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
|
||||||
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
|
||||||
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
|
||||||
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
|
||||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
|
||||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
|
||||||
stop_grace_period: 45s
|
stop_grace_period: 45s
|
||||||
|
|
||||||
services:
|
services:
|
||||||
@ -44,6 +46,9 @@ services:
|
|||||||
controller:
|
controller:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-controller"]
|
command: ["proxy-controller"]
|
||||||
|
environment:
|
||||||
|
<<: *app-environment
|
||||||
|
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
@ -9,8 +9,8 @@ stringData:
|
|||||||
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
||||||
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
||||||
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
||||||
|
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
|
||||||
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
||||||
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
||||||
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
||||||
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
||||||
|
|
||||||
|
|||||||
@ -20,6 +20,7 @@ const (
|
|||||||
defaultTemplateTimeout = 100 * time.Millisecond
|
defaultTemplateTimeout = 100 * time.Millisecond
|
||||||
defaultTemplateMaxBytes = int64(1 << 20)
|
defaultTemplateMaxBytes = int64(1 << 20)
|
||||||
defaultMaxCandidates = 10_000
|
defaultMaxCandidates = 10_000
|
||||||
|
credentialReleaseTimeout = time.Second
|
||||||
maxRegexPatterns = 64
|
maxRegexPatterns = 64
|
||||||
maxRegexPatternBytes = 1024
|
maxRegexPatternBytes = 1024
|
||||||
)
|
)
|
||||||
@ -137,7 +138,14 @@ func NewTemplateParser(
|
|||||||
return parser, nil
|
return parser, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
|
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
|
||||||
|
storedCredentials := make([]credentials.Reference, 0)
|
||||||
|
defer func() {
|
||||||
|
if resultErr == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.releaseCredentials(storedCredentials)
|
||||||
|
}()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
if len(tokens) > p.maxCandidates {
|
if len(tokens) > p.maxCandidates {
|
||||||
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
|
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
|
||||||
}
|
}
|
||||||
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
|
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
|
||||||
credentialIndexes := make(map[string]int)
|
credentialIndexes := make(map[string]int)
|
||||||
for _, token := range tokens {
|
for _, token := range tokens {
|
||||||
candidate, credential, ok := p.parseCandidate(token)
|
candidate, credential, ok := p.parseCandidate(token)
|
||||||
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
}
|
}
|
||||||
candidate.SecretRef = reference.SecretRef
|
candidate.SecretRef = reference.SecretRef
|
||||||
candidate.CredentialVersion = reference.CredentialVersion
|
candidate.CredentialVersion = reference.CredentialVersion
|
||||||
|
storedCredentials = append(storedCredentials, reference)
|
||||||
credentialKey = candidateCredentialKey(candidate)
|
credentialKey = candidateCredentialKey(candidate)
|
||||||
if index, exists := credentialIndexes[credentialKey]; exists {
|
if index, exists := credentialIndexes[credentialKey]; exists {
|
||||||
proxies[index] = candidate
|
proxies[index] = candidate
|
||||||
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
|||||||
if len(tokens) > 0 && len(proxies) == 0 {
|
if len(tokens) > 0 && len(proxies) == 0 {
|
||||||
return nil, ErrInvalidProxyOutput
|
return nil, ErrInvalidProxyOutput
|
||||||
}
|
}
|
||||||
|
p.releaseUnusedCredentials(storedCredentials, proxies)
|
||||||
return proxies, nil
|
return proxies, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||||
|
references := make([]credentials.Reference, 0, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
references = append(references, credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
p.releaseCredentials(references)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) releaseUnusedCredentials(
|
||||||
|
stored []credentials.Reference,
|
||||||
|
candidates []proxyDomain.Proxy,
|
||||||
|
) {
|
||||||
|
retained := make(map[credentials.Reference]int, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
retained[credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||||
|
}]++
|
||||||
|
}
|
||||||
|
unused := make([]credentials.Reference, 0, len(stored))
|
||||||
|
for _, reference := range stored {
|
||||||
|
if retained[reference] > 0 {
|
||||||
|
retained[reference]--
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
unused = append(unused, reference)
|
||||||
|
}
|
||||||
|
p.releaseCredentials(unused)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaser, ok := p.credentialStore.(credentials.Releaser)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
|
||||||
|
defer cancel()
|
||||||
|
for _, reference := range references {
|
||||||
|
if releaseCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = releaser.Release(releaseCtx, reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
|
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
|
||||||
compiled, err := p.compileRegex(pattern)
|
compiled, err := p.compileRegex(pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
|
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(1)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
|
|||||||
if len(proxies) != 2 {
|
if len(proxies) != 2 {
|
||||||
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
|
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
|
||||||
}
|
}
|
||||||
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
|
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
|
||||||
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
|
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
|
||||||
|
}
|
||||||
|
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
|
||||||
|
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
|
||||||
|
store, err := credentials.NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||||
|
Provider: config.Provider{Protocols: []string{"http"}},
|
||||||
|
API: config.ProviderAPI{Template: strings.Join([]string{
|
||||||
|
"http://alice:first-password@192.0.2.10:8080",
|
||||||
|
"http://bob:second-password@192.0.2.11:8080",
|
||||||
|
}, "\n")},
|
||||||
|
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||||
|
Pool: config.Pool{MaxSize: 2},
|
||||||
|
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||||
|
}, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newTemplateParser(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after failed parse): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
|
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(2)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
|
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
|
||||||
store, err := credentials.NewMemoryStore(1)
|
store, err := credentials.NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
|
|||||||
if value.Password != "new-secret" {
|
if value.Password != "new-secret" {
|
||||||
t.Fatalf("resolved latest password mismatch")
|
t.Fatalf("resolved latest password mismatch")
|
||||||
}
|
}
|
||||||
|
parser.ReleaseCandidates(proxies)
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after releasing latest candidates): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
|
||||||
|
store, err := credentials.NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||||
|
Provider: config.Provider{Protocols: []string{"http"}},
|
||||||
|
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
|
||||||
|
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||||
|
Pool: config.Pool{MaxSize: 1},
|
||||||
|
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||||
|
}, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewTemplateParser(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
proxies, err := parser.Parse(context.Background(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse(): %v", err)
|
||||||
|
}
|
||||||
|
parser.ReleaseCandidates(proxies)
|
||||||
|
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||||
|
t.Fatalf("Put(after ReleaseCandidates): %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {
|
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {
|
||||||
|
|||||||
@ -31,6 +31,7 @@ type Options struct {
|
|||||||
type Adapter struct {
|
type Adapter struct {
|
||||||
client redis.Scripter
|
client redis.Scripter
|
||||||
credentials credentials.Store
|
credentials credentials.Store
|
||||||
|
credentialReleaser credentials.Releaser
|
||||||
keys keyspace
|
keys keyspace
|
||||||
options Options
|
options Options
|
||||||
}
|
}
|
||||||
@ -49,12 +50,14 @@ func New(client redis.Scripter, options Options) (*Adapter, error) {
|
|||||||
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
|
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
|
||||||
return nil, ErrInvalidOptions
|
return nil, ErrInvalidOptions
|
||||||
}
|
}
|
||||||
return &Adapter{
|
adapter := &Adapter{
|
||||||
client: client,
|
client: client,
|
||||||
credentials: options.Credentials,
|
credentials: options.Credentials,
|
||||||
keys: newKeyspace(options.Namespace),
|
keys: newKeyspace(options.Namespace),
|
||||||
options: options,
|
options: options,
|
||||||
}, nil
|
}
|
||||||
|
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
|
||||||
|
return adapter, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) Format(state fmt.State, _ rune) {
|
func (a *Adapter) Format(state fmt.State, _ rune) {
|
||||||
|
|||||||
@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
||||||
|
return newRedisFixtureWithCredentialCapacity(t, 10_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
||||||
if redisURL == "" {
|
if redisURL == "" {
|
||||||
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
|
|||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
t.Fatalf("ping test Redis: %v", err)
|
t.Fatalf("ping test Redis: %v", err)
|
||||||
}
|
}
|
||||||
credentialStore, err := credentials.NewMemoryStore(10_000)
|
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
|||||||
@ -15,7 +15,10 @@ import (
|
|||||||
"proxy-pool/internal/platform/credentials"
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxUpsertScriptBatch = 256
|
const (
|
||||||
|
maxUpsertScriptBatch = 256
|
||||||
|
transientCredentialReleaseTimeout = time.Second
|
||||||
|
)
|
||||||
|
|
||||||
type upsertCandidate struct {
|
type upsertCandidate struct {
|
||||||
ProxyID string `json:"proxyId"`
|
ProxyID string `json:"proxyId"`
|
||||||
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
|
|||||||
|
|
||||||
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
var result activitypool.UpsertResult
|
var result activitypool.UpsertResult
|
||||||
|
if a != nil {
|
||||||
|
defer a.releaseTransientCredentials(batch.Proxies)
|
||||||
|
}
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return result, activitypool.ErrInvalidBatch
|
return result, activitypool.ErrInvalidBatch
|
||||||
}
|
}
|
||||||
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
|
||||||
|
if a.credentialReleaser == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
|
||||||
|
defer cancel()
|
||||||
|
for _, candidate := range proxies {
|
||||||
|
if releaseCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
|
||||||
|
SecretRef: candidate.SecretRef,
|
||||||
|
CredentialVersion: candidate.CredentialVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (a *Adapter) prepareUpsertCandidate(
|
func (a *Adapter) prepareUpsertCandidate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
upstreamID string,
|
upstreamID string,
|
||||||
|
|||||||
@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
|
||||||
|
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
|
||||||
|
first := testProxy("proxy-a", "192.0.2.10")
|
||||||
|
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
|
||||||
|
Username: "first", Password: "first-password",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(first credential): %v", err)
|
||||||
|
}
|
||||||
|
first.SecretRef = firstReference.SecretRef
|
||||||
|
first.CredentialVersion = firstReference.CredentialVersion
|
||||||
|
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||||
|
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
|
||||||
|
Proxies: []proxyDomain.Proxy{first},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("UpsertFetched(first): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
|
||||||
|
Username: "second", Password: "second-password",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Put(second credential after upsert): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
||||||
fixture := newRedisTestFixture(t)
|
fixture := newRedisTestFixture(t)
|
||||||
now := redisTestNow()
|
now := redisTestNow()
|
||||||
|
|||||||
@ -19,8 +19,6 @@ import (
|
|||||||
|
|
||||||
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||||
|
|
||||||
const maximumLuaInteger = int64(1<<53 - 1)
|
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
HolderID string
|
HolderID string
|
||||||
@ -57,7 +55,8 @@ func (adapter *Adapter) RunLeader(
|
|||||||
) error {
|
) error {
|
||||||
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
||||||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
||||||
limits.MaxTotal < 0 || limits.MaxTotal > maximumLuaInteger ||
|
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
@ -222,7 +221,7 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
|
|||||||
|
|
||||||
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
|
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
|
||||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
|
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
|
||||||
expected <= 0 || int64(expected) > maximumLuaInteger {
|
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
|
||||||
return nil, false, controllerProvider.ErrInvalidCoordination
|
return nil, false, controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
operationCtx, cancel := context.WithCancel(ctx)
|
operationCtx, cancel := context.WithCancel(ctx)
|
||||||
@ -295,7 +294,7 @@ type requestPermit struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
|
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
|
||||||
if fetched < 0 || int64(fetched) > maximumLuaInteger {
|
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
return permit.finish(ctx, "complete_fetch", fetched)
|
return permit.finish(ctx, "complete_fetch", fetched)
|
||||||
|
|||||||
@ -5,6 +5,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
MaximumPoolSize = 1_000_000
|
||||||
|
MaximumExactCounter = int64(1<<53 - 1)
|
||||||
|
MaximumUpstreams = 4_096
|
||||||
|
)
|
||||||
|
|
||||||
type Duration time.Duration
|
type Duration time.Duration
|
||||||
|
|
||||||
func (d *Duration) UnmarshalText(text []byte) error {
|
func (d *Duration) UnmarshalText(text []byte) error {
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
|||||||
},
|
},
|
||||||
want: "estimatedIPsPerCall",
|
want: "estimatedIPsPerCall",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "pool exceeds runtime scan bound",
|
||||||
|
mutate: func(cfg *Config) {
|
||||||
|
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
|
||||||
|
},
|
||||||
|
want: "pool.maxSize",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "zero refill interval",
|
name: "zero refill interval",
|
||||||
mutate: func(cfg *Config) {
|
mutate: func(cfg *Config) {
|
||||||
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
|
||||||
|
cfg := mustLoadValidConfig(t)
|
||||||
|
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
|
||||||
|
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
|
||||||
|
t.Fatalf("Validate(too many upstreams) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
|
||||||
|
if strconv.IntSize < 64 {
|
||||||
|
t.Skip("64-bit int is required for values above the Redis exact range")
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*Upstream)
|
||||||
|
}{
|
||||||
|
{name: "fetch total", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
|
||||||
|
}},
|
||||||
|
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
|
||||||
|
}},
|
||||||
|
{name: "minimum slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
|
||||||
|
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
|
||||||
|
}},
|
||||||
|
{name: "target slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
|
||||||
|
upstream.Pool.MaxSize = 1
|
||||||
|
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
|
||||||
|
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
|
||||||
|
}},
|
||||||
|
{name: "theoretical slots", mutate: func(upstream *Upstream) {
|
||||||
|
upstream.Pool.MaxSize = 2
|
||||||
|
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
|
||||||
|
upstream.Refill.MinimumAvailableSlots = 1
|
||||||
|
upstream.Refill.TargetAvailableSlots = 2
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg := mustLoadValidConfig(t)
|
||||||
|
updateUpstream(cfg, test.mutate)
|
||||||
|
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
|
||||||
|
t.Fatalf("Validate(inexact counter) error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
|
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
|
||||||
configured := strings.Replace(validConfig, ` auth:
|
configured := strings.Replace(validConfig, ` auth:
|
||||||
mode: none`, ` auth:
|
mode: none`, ` auth:
|
||||||
|
|||||||
30
internal/config/fingerprint.go
Normal file
30
internal/config/fingerprint.go
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
const MinimumFingerprintKeyBytes = 32
|
||||||
|
|
||||||
|
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
|
||||||
|
|
||||||
|
// Fingerprint returns a keyed digest of one fully resolved configuration.
|
||||||
|
// Only the digest is persisted; the resolved configuration and key remain local.
|
||||||
|
func Fingerprint(configuration *Config, key []byte) (string, error) {
|
||||||
|
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
|
||||||
|
return "", ErrInvalidFingerprint
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(configuration)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||||
|
}
|
||||||
|
digest := hmac.New(sha256.New, key)
|
||||||
|
if _, err := digest.Write(encoded); err != nil {
|
||||||
|
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(digest.Sum(nil)), nil
|
||||||
|
}
|
||||||
68
internal/config/fingerprint_test.go
Normal file
68
internal/config/fingerprint_test.go
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
|
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
|
||||||
|
first := storeTestConfig("provider-a")
|
||||||
|
upstream := first.Upstreams["provider-a"]
|
||||||
|
upstream.ProxyAuth.Password = "secret-a"
|
||||||
|
first.Upstreams["provider-a"] = upstream
|
||||||
|
|
||||||
|
stable, err := Fingerprint(first, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(first): %v", err)
|
||||||
|
}
|
||||||
|
again, err := Fingerprint(first, testFingerprintKey)
|
||||||
|
if err != nil || again != stable {
|
||||||
|
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
|
||||||
|
}
|
||||||
|
|
||||||
|
rotated := storeTestConfig("provider-a")
|
||||||
|
upstream = rotated.Upstreams["provider-a"]
|
||||||
|
upstream.ProxyAuth.Password = "secret-b"
|
||||||
|
rotated.Upstreams["provider-a"] = upstream
|
||||||
|
changed, err := Fingerprint(rotated, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(rotated): %v", err)
|
||||||
|
}
|
||||||
|
if changed == stable {
|
||||||
|
t.Fatal("Fingerprint did not change after secret rotation")
|
||||||
|
}
|
||||||
|
if len(changed) != 64 {
|
||||||
|
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
|
||||||
|
configuration := storeTestConfig("provider-a")
|
||||||
|
first, err := Fingerprint(configuration, testFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(first key): %v", err)
|
||||||
|
}
|
||||||
|
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fingerprint(second key): %v", err)
|
||||||
|
}
|
||||||
|
if first == second {
|
||||||
|
t.Fatal("Fingerprint did not change with independent key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
|
||||||
|
for name, test := range map[string]struct {
|
||||||
|
configuration *Config
|
||||||
|
key []byte
|
||||||
|
}{
|
||||||
|
"nil configuration": {key: testFingerprintKey},
|
||||||
|
"missing key": {configuration: storeTestConfig("provider-a")},
|
||||||
|
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := Fingerprint(test.configuration, test.key); err == nil {
|
||||||
|
t.Fatal("Fingerprint() succeeded")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
|
|||||||
|
|
||||||
// Store publishes complete validated configurations with one atomic pointer swap.
|
// Store publishes complete validated configurations with one atomic pointer swap.
|
||||||
type Store struct {
|
type Store struct {
|
||||||
current atomic.Pointer[Config]
|
current atomic.Pointer[publishedConfiguration]
|
||||||
|
}
|
||||||
|
|
||||||
|
type publishedConfiguration struct {
|
||||||
|
value Config
|
||||||
|
revision uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStore(initial *Config) (*Store, error) {
|
func NewStore(initial *Config) (*Store, error) {
|
||||||
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
|
|||||||
return nil, errors.Join(ErrInvalidStore, err)
|
return nil, errors.Join(ErrInvalidStore, err)
|
||||||
}
|
}
|
||||||
store := &Store{}
|
store := &Store{}
|
||||||
store.Publish(initial)
|
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
|
||||||
return store, nil
|
return store, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
|
|||||||
if store == nil {
|
if store == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
current := store.current.Load()
|
published := store.current.Load()
|
||||||
if current == nil {
|
if published == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
cloned := cloneConfig(*current)
|
cloned := cloneConfig(published.value)
|
||||||
return &cloned
|
return &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// Publish accepts a non-nil configuration already validated by the caller.
|
func (store *Store) Revision() uint64 {
|
||||||
func (store *Store) Publish(configuration *Config) {
|
if store == nil {
|
||||||
if store == nil || configuration == nil {
|
return 0
|
||||||
return
|
}
|
||||||
|
published := store.current.Load()
|
||||||
|
if published == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return published.revision
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishRevision publishes only a strictly newer authoritative revision.
|
||||||
|
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
|
||||||
|
if store == nil || configuration == nil || revision == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
current := store.current.Load()
|
||||||
|
if current != nil && revision <= current.revision {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
|
||||||
|
if store.current.CompareAndSwap(current, next) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
cloned := cloneConfig(*configuration)
|
|
||||||
store.current.Store(&cloned)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
next := storeTestConfig("provider-b")
|
next := storeTestConfig("provider-b")
|
||||||
store.Publish(next)
|
if !store.PublishRevision(next, 1) {
|
||||||
|
t.Fatal("PublishRevision() rejected newer configuration")
|
||||||
|
}
|
||||||
next.Routing[0].Upstreams[0] = "mutated"
|
next.Routing[0].Upstreams[0] = "mutated"
|
||||||
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||||
t.Fatalf("Publish() retained caller state: %q", got)
|
t.Fatalf("Publish() retained caller state: %q", got)
|
||||||
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
|||||||
if index%2 == 1 {
|
if index%2 == 1 {
|
||||||
name = "provider-b"
|
name = "provider-b"
|
||||||
}
|
}
|
||||||
store.Publish(storeTestConfig(name))
|
store.PublishRevision(storeTestConfig(name), uint64(index+1))
|
||||||
}(index)
|
}(index)
|
||||||
go func() {
|
go func() {
|
||||||
defer wait.Done()
|
defer wait.Done()
|
||||||
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
|||||||
wait.Wait()
|
wait.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
store, err := NewStore(storeTestConfig("provider-a"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStore() error = %v", err)
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
|
||||||
|
t.Fatal("PublishRevision(newer) rejected")
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
|
||||||
|
t.Fatal("PublishRevision(stale) succeeded")
|
||||||
|
}
|
||||||
|
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
|
||||||
|
t.Fatal("PublishRevision(equal) succeeded")
|
||||||
|
}
|
||||||
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||||
|
t.Fatalf("Current() upstream = %q, want provider-b", got)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 2 {
|
||||||
|
t.Fatalf("Revision() = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func storeTestConfig(upstreamName string) *Config {
|
func storeTestConfig(upstreamName string) *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
Version: 1,
|
Version: 1,
|
||||||
|
|||||||
@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
|
|||||||
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
|
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(cfg.Upstreams) > MaximumUpstreams {
|
||||||
|
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
|
||||||
|
}
|
||||||
enabledUpstreams := 0
|
enabledUpstreams := 0
|
||||||
for name, upstream := range cfg.Upstreams {
|
for name, upstream := range cfg.Upstreams {
|
||||||
if upstream.Enabled {
|
if upstream.Enabled {
|
||||||
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
|
|||||||
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
|
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if upstream.Pool.MaxSize > MaximumPoolSize {
|
||||||
|
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
|
||||||
|
}
|
||||||
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
|
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
||||||
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
|
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
|
||||||
}
|
}
|
||||||
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
||||||
}
|
}
|
||||||
|
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
||||||
}
|
}
|
||||||
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||||
|
if theoreticalSlots > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
||||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
||||||
}
|
}
|
||||||
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
|
|||||||
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
|
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if int64(fetch.MaxInFlight) > MaximumExactCounter {
|
||||||
|
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
|
||||||
|
}
|
||||||
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
|
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,9 +2,6 @@ package admin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
|
|||||||
LoadConfiguration(context.Context) (LoadedConfiguration, error)
|
LoadConfiguration(context.Context) (LoadedConfiguration, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigurationPublisher must atomically publish an already validated configuration.
|
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
|
||||||
type ConfigurationPublisher interface {
|
type ConfigurationPublisher interface {
|
||||||
Publish(*config.Config)
|
PublishRevision(*config.Config, uint64) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type RuntimeController interface {
|
||||||
|
Notify()
|
||||||
|
ValidateConfiguration(context.Context, *config.Config) error
|
||||||
|
ValidateUpstream(context.Context, string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ ConfigurationPublisher = (*config.Store)(nil)
|
var _ ConfigurationPublisher = (*config.Store)(nil)
|
||||||
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
|
|||||||
Operations OperationalStatusReader
|
Operations OperationalStatusReader
|
||||||
Configuration ConfigurationLoader
|
Configuration ConfigurationLoader
|
||||||
Publisher ConfigurationPublisher
|
Publisher ConfigurationPublisher
|
||||||
|
Runtime RuntimeController
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApplicationOptions struct {
|
type ApplicationOptions struct {
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
|
FingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
type OperationalStatus struct {
|
type OperationalStatus struct {
|
||||||
@ -74,14 +79,17 @@ type ApplicationService struct {
|
|||||||
operations OperationalStatusReader
|
operations OperationalStatusReader
|
||||||
configuration ConfigurationLoader
|
configuration ConfigurationLoader
|
||||||
publisher ConfigurationPublisher
|
publisher ConfigurationPublisher
|
||||||
|
runtime RuntimeController
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
|
fingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Service = (*ApplicationService)(nil)
|
var _ Service = (*ApplicationService)(nil)
|
||||||
|
|
||||||
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
|
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
|
||||||
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
|
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
|
||||||
nilInterface(dependencies.Publisher) || options.Now == nil {
|
nilInterface(dependencies.Publisher) || options.Now == nil ||
|
||||||
|
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||||
return nil, ErrInvalidApplicationService
|
return nil, ErrInvalidApplicationService
|
||||||
}
|
}
|
||||||
return &ApplicationService{
|
return &ApplicationService{
|
||||||
@ -89,11 +97,18 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
|
|||||||
operations: dependencies.Operations,
|
operations: dependencies.Operations,
|
||||||
configuration: dependencies.Configuration,
|
configuration: dependencies.Configuration,
|
||||||
publisher: dependencies.Publisher,
|
publisher: dependencies.Publisher,
|
||||||
|
runtime: dependencies.Runtime,
|
||||||
now: options.Now,
|
now: options.Now,
|
||||||
|
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
||||||
|
if command.Enabled && service.runtime != nil {
|
||||||
|
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
|
||||||
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
|
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
|
||||||
RequestID: command.RequestID,
|
RequestID: command.RequestID,
|
||||||
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
|
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
|
||||||
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
|
|||||||
Name: command.Name,
|
Name: command.Name,
|
||||||
Enabled: command.Enabled,
|
Enabled: command.Enabled,
|
||||||
})
|
})
|
||||||
|
if err == nil && service.runtime != nil {
|
||||||
|
service.runtime.Notify()
|
||||||
|
}
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
|
|||||||
if err := config.Validate(loaded.Value); err != nil {
|
if err := config.Validate(loaded.Value); err != nil {
|
||||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
}
|
}
|
||||||
|
if service.runtime != nil {
|
||||||
|
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
|
||||||
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
managementView := loaded.Value.Redacted()
|
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
|
||||||
encoded, err := json.Marshal(managementView)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||||
}
|
}
|
||||||
digest := sha256.Sum256(encoded)
|
|
||||||
checksum := hex.EncodeToString(digest[:])
|
|
||||||
|
|
||||||
current, err := service.state.Snapshot(ctx)
|
current, err := service.state.Snapshot(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
service.publisher.Publish(loaded.Value)
|
if result.Revision == 0 {
|
||||||
|
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
|
||||||
|
}
|
||||||
|
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
|
||||||
|
if published && service.runtime != nil {
|
||||||
|
service.runtime.Notify()
|
||||||
|
}
|
||||||
return mutationResult(result), nil
|
return mutationResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package admin
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -10,6 +11,12 @@ import (
|
|||||||
"proxy-pool/internal/domain/adminstate"
|
"proxy-pool/internal/domain/adminstate"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
|
func applicationTestOptions(now func() time.Time) ApplicationOptions {
|
||||||
|
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
||||||
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
Message: "enabled",
|
Message: "enabled",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
runtime := &recordingRuntimeNotifier{}
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state,
|
State: state,
|
||||||
Operations: staticOperationalStatusReader{},
|
Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{},
|
Configuration: staticConfigurationLoader{},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
Runtime: runtime,
|
||||||
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
}) {
|
}) {
|
||||||
t.Fatalf("admin state command = %+v", state.lastUpstream)
|
t.Fatalf("admin state command = %+v", state.lastUpstream)
|
||||||
}
|
}
|
||||||
|
if runtime.notifications != 1 {
|
||||||
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
wantErr := errors.New("invalid Provider template")
|
||||||
|
state := &recordingAdminState{}
|
||||||
|
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
|
Runtime: runtime,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
|
||||||
|
RequestID: "req-enable", Name: "provider-a", Enabled: true,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("SetUpstreamEnabled() error = %v", err)
|
||||||
|
}
|
||||||
|
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
|
||||||
|
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
||||||
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
|||||||
dependencies ApplicationDependencies
|
dependencies ApplicationDependencies
|
||||||
options ApplicationOptions
|
options ApplicationOptions
|
||||||
}{
|
}{
|
||||||
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||||
{name: "clock", dependencies: valid},
|
{name: "clock", dependencies: valid},
|
||||||
|
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
test := test
|
test := test
|
||||||
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
|
|||||||
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
|
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
|
||||||
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
|
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
|
||||||
} {
|
} {
|
||||||
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
|
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
|
||||||
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
|
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||||
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
||||||
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
|
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
|
||||||
|
|
||||||
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
||||||
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
||||||
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
|
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
|
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err = NewApplicationService(ApplicationDependencies{
|
service, err = NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
|||||||
service, err = NewApplicationService(ApplicationDependencies{
|
service, err = NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
|
||||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||||
configuration := validReloadConfiguration()
|
configuration := validReloadConfiguration()
|
||||||
publisher := &recordingConfigurationPublisher{}
|
publisher := &recordingConfigurationPublisher{}
|
||||||
|
runtime := &recordingRuntimeNotifier{}
|
||||||
state := &recordingAdminState{
|
state := &recordingAdminState{
|
||||||
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
||||||
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
||||||
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: configuration, Source: "configs/proxy-pool.yaml",
|
Value: configuration, Source: "configs/proxy-pool.yaml",
|
||||||
}},
|
}},
|
||||||
Publisher: publisher,
|
Publisher: publisher, Runtime: runtime,
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
||||||
t.Fatalf("published configurations = %+v", publisher.published)
|
t.Fatalf("published configurations = %+v", publisher.published)
|
||||||
}
|
}
|
||||||
|
if runtime.notifications != 1 {
|
||||||
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
|
}
|
||||||
command := state.lastConfig
|
command := state.lastConfig
|
||||||
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
||||||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
||||||
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
|
|||||||
Operations: staticOperationalStatusReader{},
|
Operations: staticOperationalStatusReader{},
|
||||||
Configuration: forbiddenConfigurationLoader{},
|
Configuration: forbiddenConfigurationLoader{},
|
||||||
Publisher: publisher,
|
Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
|
|||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state, Operations: staticOperationalStatusReader{},
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
|
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: validReloadConfiguration(), Source: "config.yaml",
|
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||||
}}, Publisher: publisher,
|
}}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
|
||||||
|
publisher := &recordingConfigurationPublisher{}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
|
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||||
|
}}, Publisher: publisher,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
|
||||||
|
if !errors.Is(err, ErrUnavailable) {
|
||||||
|
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
|
||||||
|
}
|
||||||
|
if len(publisher.published) != 0 {
|
||||||
|
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
|
||||||
|
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &orderedCommitState{
|
||||||
|
firstCommitted: make(chan struct{}),
|
||||||
|
releaseFirst: make(chan struct{}),
|
||||||
|
}
|
||||||
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
|
State: state, Operations: staticOperationalStatusReader{},
|
||||||
|
Configuration: staticConfigurationLoader{}, Publisher: store,
|
||||||
|
}, applicationTestOptions(time.Now))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewApplicationService(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
|
||||||
|
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
|
||||||
|
})
|
||||||
|
firstDone <- applyErr
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-state.firstCommitted:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("first commit did not reach delayed return")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
|
||||||
|
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("ApplyConfiguration(new): %v", err)
|
||||||
|
}
|
||||||
|
close(state.releaseFirst)
|
||||||
|
if err := <-firstDone; err != nil {
|
||||||
|
t.Fatalf("ApplyConfiguration(old): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
|
||||||
|
t.Fatalf("published upstream = %q, want provider-c", got)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 2 {
|
||||||
|
t.Fatalf("published revision = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{err: context.Canceled},
|
Configuration: staticConfigurationLoader{err: context.Canceled},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
state := &recordingAdminState{}
|
state := &recordingAdminState{}
|
||||||
publisher := &recordingConfigurationPublisher{}
|
publisher := &recordingConfigurationPublisher{}
|
||||||
var commands []adminstate.CommitConfigCommand
|
var commands []adminstate.CommitConfigCommand
|
||||||
state.onCommit = func(command adminstate.CommitConfigCommand) {
|
state.onCommit = func(command adminstate.CommitConfigCommand) {
|
||||||
commands = append(commands, command)
|
commands = append(commands, command)
|
||||||
|
state.mutation = adminstate.MutationResult{
|
||||||
|
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, secret := range []string{"secret-a", "secret-b"} {
|
for _, secret := range []string{"secret-a", "secret-b"} {
|
||||||
configuration := validReloadConfiguration()
|
configuration := validReloadConfiguration()
|
||||||
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: configuration, Source: "config.yaml",
|
Value: configuration, Source: "config.yaml",
|
||||||
}}, Publisher: publisher,
|
}}, Publisher: publisher,
|
||||||
}, ApplicationOptions{Now: time.Now})
|
}, applicationTestOptions(time.Now))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
|||||||
t.Fatalf("ReloadConfiguration() error = %v", err)
|
t.Fatalf("ReloadConfiguration() error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
|
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
|
||||||
t.Fatalf("secret rotation changed public management digest: %+v", commands)
|
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
|
||||||
}
|
}
|
||||||
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
|
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
|
||||||
t.Fatalf("secret rotation was not published: %+v", publisher.published)
|
t.Fatalf("secret rotation was not published: %+v", publisher.published)
|
||||||
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func configWithOnlyUpstream(name string) *config.Config {
|
||||||
|
configuration := validReloadConfiguration()
|
||||||
|
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
|
||||||
|
configuration.Routing = []config.Routing{{
|
||||||
|
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
|
||||||
|
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
|
||||||
|
}}
|
||||||
|
return configuration
|
||||||
|
}
|
||||||
|
|
||||||
func validReloadUpstream(secret string) config.Upstream {
|
func validReloadUpstream(secret string) config.Upstream {
|
||||||
return config.Upstream{
|
return config.Upstream{
|
||||||
Enabled: true, Exposure: []string{"gateway"},
|
Enabled: true, Exposure: []string{"gateway"},
|
||||||
@ -505,6 +629,33 @@ type recordingAdminState struct {
|
|||||||
onCommit func(adminstate.CommitConfigCommand)
|
onCommit func(adminstate.CommitConfigCommand)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type orderedCommitState struct {
|
||||||
|
next atomic.Uint64
|
||||||
|
firstCommitted chan struct{}
|
||||||
|
releaseFirst chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||||
|
return adminstate.MutationResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
|
||||||
|
return adminstate.MutationResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||||
|
revision := state.next.Add(1)
|
||||||
|
if revision == 1 {
|
||||||
|
close(state.firstCommitted)
|
||||||
|
<-state.releaseFirst
|
||||||
|
}
|
||||||
|
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||||
|
return adminstate.Snapshot{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||||
state.lastUpstream = command
|
state.lastUpstream = command
|
||||||
return state.mutation, state.err
|
return state.mutation, state.err
|
||||||
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
|
|||||||
|
|
||||||
type recordingConfigurationPublisher struct {
|
type recordingConfigurationPublisher struct {
|
||||||
published []*config.Config
|
published []*config.Config
|
||||||
|
revisions []uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
|
type recordingRuntimeNotifier struct {
|
||||||
publisher.published = append(publisher.published, configuration)
|
notifications int
|
||||||
|
validationErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
||||||
|
return notifier.validationErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
|
||||||
|
return notifier.validationErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
|
||||||
|
publisher.published = append(publisher.published, configuration)
|
||||||
|
publisher.revisions = append(publisher.revisions, revision)
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -15,11 +16,15 @@ import (
|
|||||||
"proxy-pool/internal/controller/distribution"
|
"proxy-pool/internal/controller/distribution"
|
||||||
"proxy-pool/internal/controller/extraction"
|
"proxy-pool/internal/controller/extraction"
|
||||||
"proxy-pool/internal/controller/operations"
|
"proxy-pool/internal/controller/operations"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
"proxy-pool/internal/platform/admission"
|
"proxy-pool/internal/platform/admission"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
"proxy-pool/internal/platform/httpserver"
|
"proxy-pool/internal/platform/httpserver"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -33,10 +38,15 @@ type Options struct {
|
|||||||
Resolver config.Resolver
|
Resolver config.Resolver
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
HTTP httpserver.Options
|
HTTP httpserver.Options
|
||||||
|
HolderID string
|
||||||
|
RedisNamespace string
|
||||||
|
FingerprintKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
type activityStore interface {
|
type activityStore interface {
|
||||||
extractionDomain.Store
|
extractionDomain.Store
|
||||||
|
activitypool.Upserter
|
||||||
|
pool.InventoryReader
|
||||||
activitypool.StateInventoryReader
|
activitypool.StateInventoryReader
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -45,6 +55,9 @@ type ports struct {
|
|||||||
activity activityStore
|
activity activityStore
|
||||||
readiness distribution.ReadinessChecker
|
readiness distribution.ReadinessChecker
|
||||||
metricsReadiness platformMetrics.ReadinessChecker
|
metricsReadiness platformMetrics.ReadinessChecker
|
||||||
|
coordinator provider.Coordinator
|
||||||
|
credentials credentials.Store
|
||||||
|
providerResults provider.ResultRecorder
|
||||||
close func() error
|
close func() error
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -61,7 +74,9 @@ type runtimeFactory interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Run(ctx context.Context, options Options) error {
|
func Run(ctx context.Context, options Options) error {
|
||||||
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
|
return run(ctx, options, &productionInfrastructure{
|
||||||
|
holderID: options.HolderID, namespace: options.RedisNamespace,
|
||||||
|
}, productionRuntimeFactory{})
|
||||||
}
|
}
|
||||||
|
|
||||||
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
||||||
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
|
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||||
|
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
|
||||||
|
}
|
||||||
configurationStore, err := config.NewStore(loaded.Value)
|
configurationStore, err := config.NewStore(loaded.Value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
||||||
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
defer func() {
|
defer func() {
|
||||||
resultErr = errors.Join(resultErr, opened.close())
|
resultErr = errors.Join(resultErr, opened.close())
|
||||||
}()
|
}()
|
||||||
|
var providerState providerStateReader
|
||||||
|
if loaded.Value.Admin.Enabled {
|
||||||
|
providerState = opened.state
|
||||||
|
}
|
||||||
|
supervisor, err := newProviderSupervisor(
|
||||||
|
configurationStore,
|
||||||
|
providerState,
|
||||||
|
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
buildRuntime, err := providerRuntimeBuilder(opened)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return buildRuntime(name, upstream)
|
||||||
|
},
|
||||||
|
func(ctx context.Context, configuration *config.Config) error {
|
||||||
|
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
|
||||||
|
},
|
||||||
|
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
|
||||||
|
loader,
|
||||||
|
options.FingerprintKey,
|
||||||
|
providerSupervisorInterval,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
|
|
||||||
dependencies := controllerRuntime.Dependencies{}
|
dependencies := controllerRuntime.Dependencies{}
|
||||||
if loaded.Value.Distribution.Enabled {
|
if loaded.Value.Distribution.Enabled {
|
||||||
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
}
|
}
|
||||||
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
|
var providerStats []provider.StatsReader
|
||||||
|
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
||||||
|
providerStats = append(providerStats, stats)
|
||||||
|
}
|
||||||
|
statusReader, statusErr := operations.NewReader(
|
||||||
|
configurationStore,
|
||||||
|
opened.activity,
|
||||||
|
options.Now,
|
||||||
|
providerStats...,
|
||||||
|
)
|
||||||
if statusErr != nil {
|
if statusErr != nil {
|
||||||
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
||||||
}
|
}
|
||||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
||||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
||||||
}, admin.ApplicationOptions{Now: options.Now})
|
Runtime: supervisor,
|
||||||
|
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
||||||
if serviceErr != nil {
|
if serviceErr != nil {
|
||||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
||||||
}
|
}
|
||||||
@ -146,6 +199,8 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
dependencies.MetricsHandler = handler
|
dependencies.MetricsHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runners := make([]lifecycle.Runner, 0, 2)
|
||||||
|
if hasHTTPRuntime(loaded.Value) {
|
||||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
||||||
@ -153,7 +208,59 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
|||||||
if nilInterface(runner) {
|
if nilInterface(runner) {
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
}
|
}
|
||||||
return runner.Run(ctx)
|
runners = append(runners, runner)
|
||||||
|
}
|
||||||
|
runners = append(runners, supervisor)
|
||||||
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
|
return group.Run(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareProviderConfiguration(
|
||||||
|
ctx context.Context,
|
||||||
|
configuration *config.Config,
|
||||||
|
credentialStore credentials.Store,
|
||||||
|
) error {
|
||||||
|
if ctx == nil || configuration == nil {
|
||||||
|
return ErrProviderRuntime
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
|
||||||
|
if !ok || nilInterface(credentialStore) {
|
||||||
|
return ErrProviderRuntime
|
||||||
|
}
|
||||||
|
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
|
||||||
|
if configuration == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
retainer, ok := results.(provider.StatsRetainer)
|
||||||
|
if !ok || nilInterface(retainer) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name := range configuration.Upstreams {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
retainer.RetainProviderStats(names)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasHTTPRuntime(configuration *config.Config) bool {
|
||||||
|
return configuration != nil &&
|
||||||
|
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
||||||
|
|||||||
@ -4,16 +4,25 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"proxy-pool/internal/adapters/redisactivity"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controller/admin"
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
||||||
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
|||||||
if postgresURL == "" || redisURL == "" {
|
if postgresURL == "" || redisURL == "" {
|
||||||
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
||||||
}
|
}
|
||||||
|
var providerCalls atomic.Int64
|
||||||
|
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||||
|
providerCalls.Add(1)
|
||||||
|
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
|
||||||
|
}))
|
||||||
|
defer providerServer.Close()
|
||||||
|
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||||
|
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
|
||||||
|
|
||||||
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
||||||
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
||||||
|
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
||||||
factory := &integrationRuntimeFactory{}
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
providerResults := make(chan provider.Result, 8)
|
||||||
|
factory := &integrationRuntimeFactory{
|
||||||
|
cancel: cancel, inventory: inventory, providerResults: providerResults,
|
||||||
|
}
|
||||||
|
infrastructure := &integrationInfrastructure{
|
||||||
|
productionInfrastructure: productionInfrastructure{namespace: namespace},
|
||||||
|
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
|
||||||
|
}
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
err := run(ctx, Options{
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
||||||
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
||||||
},
|
},
|
||||||
}, &productionInfrastructure{}, factory)
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
if err != nil {
|
}, infrastructure, factory)
|
||||||
t.Fatalf("run() error = %v", err)
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("run() error = %v, want context cancellation", err)
|
||||||
}
|
}
|
||||||
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
||||||
t.Fatalf("Admin Status = %+v", factory.status)
|
t.Fatalf("Admin Status = %+v", factory.status)
|
||||||
@ -46,6 +75,9 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
|||||||
!strings.Contains(factory.metricsBody, "go_") {
|
!strings.Contains(factory.metricsBody, "go_") {
|
||||||
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
||||||
}
|
}
|
||||||
|
if providerCalls.Load() == 0 {
|
||||||
|
t.Fatal("production Provider HTTP adapter was not called")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type integrationRuntimeFactory struct {
|
type integrationRuntimeFactory struct {
|
||||||
@ -53,6 +85,9 @@ type integrationRuntimeFactory struct {
|
|||||||
readyStatus int
|
readyStatus int
|
||||||
metricsStatus int
|
metricsStatus int
|
||||||
metricsBody string
|
metricsBody string
|
||||||
|
cancel context.CancelFunc
|
||||||
|
inventory pool.InventoryReader
|
||||||
|
providerResults <-chan provider.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (factory *integrationRuntimeFactory) New(
|
func (factory *integrationRuntimeFactory) New(
|
||||||
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
|
|||||||
factory.metricsBody = metrics.Body.String()
|
factory.metricsBody = metrics.Body.String()
|
||||||
status, err := dependencies.AdminService.Status(ctx)
|
status, err := dependencies.AdminService.Status(ctx)
|
||||||
factory.status = status
|
factory.status = status
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
|
if err := waitForProviderInventory(
|
||||||
|
ctx,
|
||||||
|
factory.inventory,
|
||||||
|
factory.providerResults,
|
||||||
|
[]string{"provider-a", "provider-b"},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
factory.cancel()
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
}}, nil
|
}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
|
||||||
|
t.Helper()
|
||||||
|
options, err := redis.ParseURL(redisURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("redis.ParseURL(): %v", err)
|
||||||
|
}
|
||||||
|
client := redis.NewClient(options)
|
||||||
|
t.Cleanup(func() { _ = client.Close() })
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("credentials.NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
reader, err := redisactivity.New(client, redisactivity.Options{
|
||||||
|
Namespace: namespace, Credentials: credentialStore,
|
||||||
|
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
|
||||||
|
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("redisactivity.New(): %v", err)
|
||||||
|
}
|
||||||
|
return reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForProviderInventory(
|
||||||
|
ctx context.Context,
|
||||||
|
inventory pool.InventoryReader,
|
||||||
|
results <-chan provider.Result,
|
||||||
|
upstreamIDs []string,
|
||||||
|
) error {
|
||||||
|
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
ticker := time.NewTicker(10 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
var latestResult provider.Result
|
||||||
|
for {
|
||||||
|
var inventoryErr error
|
||||||
|
for _, upstreamID := range upstreamIDs {
|
||||||
|
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
|
||||||
|
inventoryErr = errors.Join(inventoryErr, err)
|
||||||
|
if err == nil && snapshot.Managed > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline.Done():
|
||||||
|
return errors.Join(
|
||||||
|
errors.New("wait for Provider Redis inventory"),
|
||||||
|
deadline.Err(),
|
||||||
|
inventoryErr,
|
||||||
|
latestResult.Err,
|
||||||
|
)
|
||||||
|
case result := <-results:
|
||||||
|
latestResult = result
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type integrationInfrastructure struct {
|
||||||
|
productionInfrastructure
|
||||||
|
results provider.ResultRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (infrastructure *integrationInfrastructure) Open(
|
||||||
|
ctx context.Context,
|
||||||
|
configuration *config.Config,
|
||||||
|
) (ports, error) {
|
||||||
|
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
|
||||||
|
opened.providerResults = infrastructure.results
|
||||||
|
return opened, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerResultRecorder func(provider.Result)
|
||||||
|
|
||||||
|
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
|
||||||
|
|
||||||
type integrationRunner struct {
|
type integrationRunner struct {
|
||||||
run func(context.Context) error
|
run func(context.Context) error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,32 +3,48 @@ package bootstrap
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
"proxy-pool/internal/domain/adminstate"
|
"proxy-pool/internal/domain/adminstate"
|
||||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
|
||||||
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
||||||
state := adminstate.NewMemoryStore()
|
state := adminstate.NewMemoryStore()
|
||||||
activity := &stubActivityStore{}
|
activity := &stubActivityStore{}
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
closeErr := errors.New("close failed")
|
closeErr := errors.New("close failed")
|
||||||
infrastructure := &stubInfrastructure{ports: ports{
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||||
|
coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
close: func() error { return closeErr },
|
close: func() error { return closeErr },
|
||||||
}}
|
}}
|
||||||
runErr := errors.New("runtime failed")
|
runErr := errors.New("runtime failed")
|
||||||
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
||||||
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
err := run(context.Background(), Options{
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
err = run(ctx, Options{
|
||||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
||||||
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
}, infrastructure, factory)
|
}, infrastructure, factory)
|
||||||
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
||||||
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
||||||
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
|
||||||
|
for name, key := range map[string][]byte{
|
||||||
|
"missing": nil,
|
||||||
|
"short": []byte("too-short"),
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
infrastructure := &stubInfrastructure{}
|
||||||
|
err := run(context.Background(), Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||||
|
Now: time.Now,
|
||||||
|
FingerprintKey: key,
|
||||||
|
}, infrastructure, &recordingRuntimeFactory{})
|
||||||
|
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
|
||||||
|
t.Fatalf("run() error = %v", err)
|
||||||
|
}
|
||||||
|
if infrastructure.opens != 0 {
|
||||||
|
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
|
||||||
|
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
|
||||||
|
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
|
||||||
|
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
|
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
|
close: func() error { return nil },
|
||||||
|
}}
|
||||||
|
factory := &recordingRuntimeFactory{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
time.AfterFunc(20*time.Millisecond, cancel)
|
||||||
|
err = run(ctx, Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
|
||||||
|
Now: time.Now,
|
||||||
|
}, infrastructure, factory)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if factory.configuration != nil {
|
||||||
|
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
|
||||||
|
state := adminstate.NewMemoryStore()
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
started := make(chan string, 2)
|
||||||
|
stopped := make(chan string, 2)
|
||||||
|
infrastructure := &stubInfrastructure{ports: ports{
|
||||||
|
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||||
|
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
|
||||||
|
started <- upstreamID
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- upstreamID
|
||||||
|
return ctx.Err()
|
||||||
|
}),
|
||||||
|
credentials: credentialStore,
|
||||||
|
close: func() error { return nil },
|
||||||
|
}}
|
||||||
|
wantErr := errors.New("test HTTP runtime stopped")
|
||||||
|
factory := runtimeFactoryFunc(func(
|
||||||
|
_ *config.Config,
|
||||||
|
dependencies controllerRuntime.Dependencies,
|
||||||
|
_ controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error) {
|
||||||
|
return runnerFunc(func(ctx context.Context) error {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case upstreamID := <-started:
|
||||||
|
if upstreamID != "provider-a" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
|
||||||
|
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case stoppedID := <-stopped:
|
||||||
|
if stoppedID == "provider-a" {
|
||||||
|
return wantErr
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}), nil
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
err = run(ctx, Options{
|
||||||
|
ConfigPath: "controller.yaml",
|
||||||
|
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||||
|
Now: time.Now,
|
||||||
|
FingerprintKey: bootstrapTestFingerprintKey,
|
||||||
|
}, infrastructure, factory)
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
disabled := configuration.Upstreams["provider-b"]
|
||||||
|
disabled.Enabled = false
|
||||||
|
configuration.Upstreams["provider-b"] = disabled
|
||||||
|
stats, err := provider.NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("provider.NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
|
||||||
|
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
retainProviderStats(configuration, stats)
|
||||||
|
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
|
||||||
|
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
|
||||||
|
t.Fatalf("Provider stats after retention = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type memoryResolver struct {
|
type memoryResolver struct {
|
||||||
files map[string][]byte
|
files map[string][]byte
|
||||||
reads int
|
reads int
|
||||||
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
|
|||||||
|
|
||||||
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
||||||
|
|
||||||
|
type runnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||||
|
|
||||||
|
type runtimeFactoryFunc func(
|
||||||
|
*config.Config,
|
||||||
|
controllerRuntime.Dependencies,
|
||||||
|
controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error)
|
||||||
|
|
||||||
|
func (factory runtimeFactoryFunc) New(
|
||||||
|
configuration *config.Config,
|
||||||
|
dependencies controllerRuntime.Dependencies,
|
||||||
|
options controllerRuntime.Options,
|
||||||
|
) (controllerRunner, error) {
|
||||||
|
return factory(configuration, dependencies, options)
|
||||||
|
}
|
||||||
|
|
||||||
type readyStub struct{}
|
type readyStub struct{}
|
||||||
|
|
||||||
func (readyStub) Ready(context.Context) error { return nil }
|
func (readyStub) Ready(context.Context) error { return nil }
|
||||||
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*stubActivityStore) UpsertFetched(
|
||||||
|
_ context.Context,
|
||||||
|
_ string,
|
||||||
|
batch activitypool.FetchedBatch,
|
||||||
|
) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*stubActivityStore) ReadInventory(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
time.Duration,
|
||||||
|
) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorStub struct{}
|
||||||
|
|
||||||
|
func (coordinatorStub) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
_ provider.CoordinationLimits,
|
||||||
|
_ func(context.Context, provider.LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorFunc func(context.Context, string) error
|
||||||
|
|
||||||
|
func (run coordinatorFunc) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
_ provider.CoordinationLimits,
|
||||||
|
_ func(context.Context, provider.LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return run(ctx, upstreamID)
|
||||||
|
}
|
||||||
|
|
||||||
const bootstrapTestConfig = `
|
const bootstrapTestConfig = `
|
||||||
version: 1
|
version: 1
|
||||||
security:
|
security:
|
||||||
|
|||||||
@ -2,6 +2,8 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@ -11,7 +13,9 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/adapters/postgresadmin"
|
"proxy-pool/internal/adapters/postgresadmin"
|
||||||
"proxy-pool/internal/adapters/redisactivity"
|
"proxy-pool/internal/adapters/redisactivity"
|
||||||
|
"proxy-pool/internal/adapters/redisprovider"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
controllerProvider "proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/platform/credentials"
|
"proxy-pool/internal/platform/credentials"
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||||
)
|
)
|
||||||
@ -20,7 +24,12 @@ const (
|
|||||||
redisNamespace = "controller"
|
redisNamespace = "controller"
|
||||||
redisOperationTTL = 30 * time.Second
|
redisOperationTTL = 30 * time.Second
|
||||||
redisMinimumScan = 4_096
|
redisMinimumScan = 4_096
|
||||||
|
redisMaximumScan = config.MaximumPoolSize
|
||||||
redisCleanupLimit = 1_024
|
redisCleanupLimit = 1_024
|
||||||
|
providerLeaseTTL = 15 * time.Second
|
||||||
|
providerRenewEvery = 3 * time.Second
|
||||||
|
providerRetryInterval = 100 * time.Millisecond
|
||||||
|
providerPermitGrace = 5 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -30,15 +39,22 @@ var (
|
|||||||
ErrRedisUnavailable = errors.New("Redis unavailable")
|
ErrRedisUnavailable = errors.New("Redis unavailable")
|
||||||
)
|
)
|
||||||
|
|
||||||
type productionInfrastructure struct{}
|
type productionInfrastructure struct {
|
||||||
|
holderID string
|
||||||
|
namespace string
|
||||||
|
}
|
||||||
|
|
||||||
func (*productionInfrastructure) Open(
|
func (infrastructure *productionInfrastructure) Open(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
configuration *config.Config,
|
configuration *config.Config,
|
||||||
) (_ ports, resultErr error) {
|
) (_ ports, resultErr error) {
|
||||||
if ctx == nil || configuration == nil {
|
if ctx == nil || configuration == nil {
|
||||||
return ports{}, ErrInvalidOptions
|
return ports{}, ErrInvalidOptions
|
||||||
}
|
}
|
||||||
|
namespace, err := resolveRedisNamespace(infrastructure.namespace)
|
||||||
|
if err != nil {
|
||||||
|
return ports{}, err
|
||||||
|
}
|
||||||
var postgresPool *pgxpool.Pool
|
var postgresPool *pgxpool.Pool
|
||||||
var redisClient *redis.Client
|
var redisClient *redis.Client
|
||||||
closeResources := func() error {
|
closeResources := func() error {
|
||||||
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
|
providersEnabled := hasEnabledUpstream(configuration)
|
||||||
|
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
|
||||||
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
||||||
return ports{}, ErrRedisConfiguration
|
return ports{}, ErrRedisConfiguration
|
||||||
}
|
}
|
||||||
@ -95,17 +112,17 @@ func (*productionInfrastructure) Open(
|
|||||||
if err = redisClient.Ping(ctx).Err(); err != nil {
|
if err = redisClient.Ping(ctx).Err(); err != nil {
|
||||||
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
||||||
}
|
}
|
||||||
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
|
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ports{}, err
|
return ports{}, err
|
||||||
}
|
}
|
||||||
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
||||||
Namespace: redisNamespace,
|
Namespace: namespace,
|
||||||
Credentials: credentialStore,
|
Credentials: credentialStore,
|
||||||
OperationTTL: redisOperationTTL,
|
OperationTTL: redisOperationTTL,
|
||||||
MaxCandidateScan: candidateScan(configuration),
|
MaxCandidateScan: candidateScan(configuration),
|
||||||
MaxRuntimeCounters: credentialCapacity(configuration),
|
MaxRuntimeCounters: credentialCapacity(configuration),
|
||||||
MaxInventoryScan: credentialCapacity(configuration),
|
MaxInventoryScan: maxInventoryScan(configuration),
|
||||||
CleanupLimit: redisCleanupLimit,
|
CleanupLimit: redisCleanupLimit,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -113,6 +130,26 @@ func (*productionInfrastructure) Open(
|
|||||||
}
|
}
|
||||||
opened.activity = adapter
|
opened.activity = adapter
|
||||||
opened.readiness = redisReadiness{client: redisClient}
|
opened.readiness = redisReadiness{client: redisClient}
|
||||||
|
opened.credentials = credentialStore
|
||||||
|
if providersEnabled {
|
||||||
|
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
|
||||||
|
if statsErr != nil {
|
||||||
|
return ports{}, statsErr
|
||||||
|
}
|
||||||
|
opened.providerResults = stats
|
||||||
|
holderID, holderErr := resolveHolderID(infrastructure.holderID)
|
||||||
|
if holderErr != nil {
|
||||||
|
return ports{}, holderErr
|
||||||
|
}
|
||||||
|
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
|
||||||
|
Namespace: namespace, HolderID: holderID,
|
||||||
|
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
|
||||||
|
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ports{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if configuration.Metrics.Enabled {
|
if configuration.Metrics.Enabled {
|
||||||
opened.metricsReadiness = selectMetricsReadiness(
|
opened.metricsReadiness = selectMetricsReadiness(
|
||||||
@ -124,11 +161,21 @@ func (*productionInfrastructure) Open(
|
|||||||
return opened, nil
|
return opened, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveRedisNamespace(configured string) (string, error) {
|
||||||
|
if strings.TrimSpace(configured) != configured {
|
||||||
|
return "", ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if configured == "" {
|
||||||
|
return redisNamespace, nil
|
||||||
|
}
|
||||||
|
return configured, nil
|
||||||
|
}
|
||||||
|
|
||||||
func selectMetricsReadiness(
|
func selectMetricsReadiness(
|
||||||
configuration *config.Config,
|
configuration *config.Config,
|
||||||
admin, activity platformMetrics.ReadinessChecker,
|
admin, activity platformMetrics.ReadinessChecker,
|
||||||
) platformMetrics.ReadinessChecker {
|
) platformMetrics.ReadinessChecker {
|
||||||
if configuration.Distribution.Enabled {
|
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
|
||||||
return activity
|
return activity
|
||||||
}
|
}
|
||||||
if configuration.Admin.Enabled {
|
if configuration.Admin.Enabled {
|
||||||
@ -186,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
|
|||||||
capacity := 0
|
capacity := 0
|
||||||
maximum := int(^uint(0) >> 1)
|
maximum := int(^uint(0) >> 1)
|
||||||
for _, upstream := range configuration.Upstreams {
|
for _, upstream := range configuration.Upstreams {
|
||||||
if upstream.Pool.MaxSize <= 0 {
|
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if capacity > maximum-upstream.Pool.MaxSize {
|
if capacity > maximum-upstream.Pool.MaxSize {
|
||||||
@ -200,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
|
|||||||
return capacity
|
return capacity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func providerCredentialCapacity(configuration *config.Config) int {
|
||||||
|
capacity := 0
|
||||||
|
maximum := int(^uint(0) >> 1)
|
||||||
|
for _, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Pool.MaxSize <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
maxInFlight := upstream.Fetch.MaxInFlight
|
||||||
|
if maxInFlight <= 0 {
|
||||||
|
maxInFlight = 1
|
||||||
|
}
|
||||||
|
if upstream.Pool.MaxSize > maximum/maxInFlight {
|
||||||
|
return maximum
|
||||||
|
}
|
||||||
|
leases := upstream.Pool.MaxSize * maxInFlight
|
||||||
|
if capacity > maximum-leases {
|
||||||
|
return maximum
|
||||||
|
}
|
||||||
|
capacity += leases
|
||||||
|
}
|
||||||
|
if capacity == 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return capacity
|
||||||
|
}
|
||||||
|
|
||||||
|
func maxInventoryScan(_ *config.Config) int {
|
||||||
|
return config.MaximumPoolSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasEnabledUpstream(configuration *config.Config) bool {
|
||||||
|
if configuration == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveHolderID(configured string) (string, error) {
|
||||||
|
if strings.TrimSpace(configured) != configured {
|
||||||
|
return "", ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if configured != "" {
|
||||||
|
return configured, nil
|
||||||
|
}
|
||||||
|
var entropy [16]byte
|
||||||
|
if _, err := rand.Read(entropy[:]); err != nil {
|
||||||
|
return "", errors.Join(ErrStartup, err)
|
||||||
|
}
|
||||||
|
return "controller-" + hex.EncodeToString(entropy[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
func candidateScan(configuration *config.Config) int {
|
func candidateScan(configuration *config.Config) int {
|
||||||
configured := configuration.Distribution.Extraction
|
configured := configuration.Distribution.Extraction
|
||||||
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
||||||
|
|||||||
@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
|||||||
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
||||||
}},
|
}},
|
||||||
Upstreams: map[string]config.Upstream{
|
Upstreams: map[string]config.Upstream{
|
||||||
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
|
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
|
||||||
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
|
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
|
||||||
|
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if got := credentialCapacity(configuration); got != 5_000 {
|
if got := credentialCapacity(configuration); got != 5_000 {
|
||||||
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
||||||
}
|
}
|
||||||
|
if got := providerCredentialCapacity(configuration); got != 158_000 {
|
||||||
|
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
|
||||||
|
}
|
||||||
if got := candidateScan(configuration); got != 5_100 {
|
if got := candidateScan(configuration); got != 5_100 {
|
||||||
t.Fatalf("candidateScan() = %d, want 5100", got)
|
t.Fatalf("candidateScan() = %d, want 5100", got)
|
||||||
}
|
}
|
||||||
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
|||||||
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
||||||
|
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
|
||||||
|
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
126
internal/controller/bootstrap/provider.go
Normal file
126
internal/controller/bootstrap/provider.go
Normal file
@ -0,0 +1,126 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/adapters/providerapi"
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
controllerProvider "proxy-pool/internal/controller/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
|
||||||
|
|
||||||
|
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
|
||||||
|
if configuration == nil {
|
||||||
|
return nil, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
builder, err := providerRuntimeBuilder(opened)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
runtime, err := builder(name, configuration.Upstreams[name])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runtimes = append(runtimes, runtime)
|
||||||
|
}
|
||||||
|
return controllerProvider.NewFleet(runtimes...)
|
||||||
|
}
|
||||||
|
|
||||||
|
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
|
||||||
|
|
||||||
|
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
|
||||||
|
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
|
||||||
|
return nil, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
results := opened.providerResults
|
||||||
|
if nilInterface(results) {
|
||||||
|
results = discardProviderResults{}
|
||||||
|
}
|
||||||
|
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
|
||||||
|
upstream.Enabled = true
|
||||||
|
mapped, err := providerRuntimeConfig(name, upstream)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: opened.coordinator,
|
||||||
|
Inventory: opened.activity,
|
||||||
|
Adapter: adapter,
|
||||||
|
Parser: parser,
|
||||||
|
Activity: opened.activity,
|
||||||
|
Results: results,
|
||||||
|
})
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func providerRuntimeConfig(
|
||||||
|
upstreamID string,
|
||||||
|
upstream config.Upstream,
|
||||||
|
) (controllerProvider.UpstreamRuntimeConfig, error) {
|
||||||
|
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
|
||||||
|
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
|
||||||
|
upstream.Pool.MaxSize > config.MaximumPoolSize ||
|
||||||
|
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
|
||||||
|
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||||
|
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
|
||||||
|
}
|
||||||
|
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||||
|
return controllerProvider.UpstreamRuntimeConfig{
|
||||||
|
Provider: controllerProvider.Config{
|
||||||
|
UpstreamID: upstreamID,
|
||||||
|
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
|
||||||
|
Timeout: time.Duration(upstream.Fetch.Timeout),
|
||||||
|
MaxAttempts: upstream.Fetch.MaxAttempts,
|
||||||
|
MaxInFlight: upstream.Fetch.MaxInFlight,
|
||||||
|
MaxTotal: int64(upstream.Fetch.MaxTotal),
|
||||||
|
MaxSize: upstream.Pool.MaxSize,
|
||||||
|
TTL: time.Duration(upstream.Lifecycle.TTL),
|
||||||
|
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||||
|
Retry: controllerProvider.RetryConfig{
|
||||||
|
Initial: time.Duration(upstream.Fetch.Retry.Initial),
|
||||||
|
Max: time.Duration(upstream.Fetch.Retry.Max),
|
||||||
|
Jitter: upstream.Fetch.Retry.Jitter,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ReconcilePolicy: pool.ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
|
||||||
|
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
|
||||||
|
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
|
||||||
|
ExpectedSlotsPerFetch: expectedSlots,
|
||||||
|
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||||
|
},
|
||||||
|
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type discardProviderResults struct{}
|
||||||
|
|
||||||
|
func (discardProviderResults) Record(controllerProvider.Result) {}
|
||||||
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
@ -0,0 +1,375 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
const providerSupervisorInterval = time.Second
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
|
||||||
|
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
|
||||||
|
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
|
||||||
|
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
|
||||||
|
)
|
||||||
|
|
||||||
|
type providerConfigurationStore interface {
|
||||||
|
Current() *config.Config
|
||||||
|
Revision() uint64
|
||||||
|
PublishRevision(*config.Config, uint64) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerConfigurationSource interface {
|
||||||
|
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerStateReader interface {
|
||||||
|
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
|
||||||
|
type providerConfigurationPreparer func(context.Context, *config.Config) error
|
||||||
|
type providerConfigurationObserver func(*config.Config)
|
||||||
|
|
||||||
|
type providerSupervisor struct {
|
||||||
|
configuration providerConfigurationStore
|
||||||
|
state providerStateReader
|
||||||
|
build providerRunnerBuilder
|
||||||
|
prepare providerConfigurationPreparer
|
||||||
|
observe providerConfigurationObserver
|
||||||
|
source providerConfigurationSource
|
||||||
|
fingerprintKey []byte
|
||||||
|
interval time.Duration
|
||||||
|
notify chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runningProvider struct {
|
||||||
|
configuration config.Upstream
|
||||||
|
cancel context.CancelFunc
|
||||||
|
done chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProviderSupervisor(
|
||||||
|
configuration providerConfigurationStore,
|
||||||
|
state providerStateReader,
|
||||||
|
build providerRunnerBuilder,
|
||||||
|
prepare providerConfigurationPreparer,
|
||||||
|
observe providerConfigurationObserver,
|
||||||
|
source providerConfigurationSource,
|
||||||
|
fingerprintKey []byte,
|
||||||
|
interval time.Duration,
|
||||||
|
) (*providerSupervisor, error) {
|
||||||
|
if nilInterface(configuration) || build == nil || interval <= 0 ||
|
||||||
|
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
|
||||||
|
return nil, ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
return &providerSupervisor{
|
||||||
|
configuration: configuration,
|
||||||
|
state: state,
|
||||||
|
build: build,
|
||||||
|
prepare: prepare,
|
||||||
|
observe: observe,
|
||||||
|
source: source,
|
||||||
|
fingerprintKey: append([]byte(nil), fingerprintKey...),
|
||||||
|
interval: interval,
|
||||||
|
notify: make(chan struct{}, 1),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) Notify() {
|
||||||
|
if supervisor == nil || supervisor.notify == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case supervisor.notify <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||||
|
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := config.Validate(configuration); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(configuration.Upstreams))
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
if upstream.Enabled {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
|
||||||
|
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
configuration := supervisor.configuration.Current()
|
||||||
|
if configuration == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
upstream, exists := configuration.Upstreams[name]
|
||||||
|
if !exists {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
upstream.Enabled = true
|
||||||
|
if _, err := supervisor.build(name, upstream); err != nil {
|
||||||
|
return errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
|
||||||
|
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
|
||||||
|
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
|
||||||
|
return ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
active := make(map[string]*runningProvider)
|
||||||
|
failures := make(chan error, 1)
|
||||||
|
defer stopAllProviders(active)
|
||||||
|
ticker := time.NewTicker(supervisor.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
if err := supervisor.reconcile(ctx, active, failures); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case err := <-failures:
|
||||||
|
return err
|
||||||
|
case <-supervisor.notify:
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) reconcile(
|
||||||
|
ctx context.Context,
|
||||||
|
active map[string]*runningProvider,
|
||||||
|
failures chan<- error,
|
||||||
|
) error {
|
||||||
|
desired, err := supervisor.desired(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, errProviderManagementStateUnavailable) ||
|
||||||
|
errors.Is(err, errProviderManagementSnapshotStale) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, errProviderConfigurationPending) {
|
||||||
|
stopAllProviders(active)
|
||||||
|
clear(active)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(desired))
|
||||||
|
prepared := make(map[string]lifecycle.Runner)
|
||||||
|
for name, upstream := range desired {
|
||||||
|
names = append(names, name)
|
||||||
|
current := active[name]
|
||||||
|
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
runner, buildErr := supervisor.build(name, upstream)
|
||||||
|
if buildErr != nil || nilInterface(runner) {
|
||||||
|
return errors.Join(ErrProviderSupervisor, buildErr)
|
||||||
|
}
|
||||||
|
prepared[name] = runner
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
for name, current := range active {
|
||||||
|
if _, keep := desired[name]; !keep {
|
||||||
|
stopProvider(current)
|
||||||
|
delete(active, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range names {
|
||||||
|
runner := prepared[name]
|
||||||
|
if runner == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if current := active[name]; current != nil {
|
||||||
|
stopProvider(current)
|
||||||
|
}
|
||||||
|
active[name] = startProvider(ctx, name, desired[name], runner, failures)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
|
||||||
|
configuration := supervisor.configuration.Current()
|
||||||
|
if configuration == nil {
|
||||||
|
return nil, ErrProviderSupervisor
|
||||||
|
}
|
||||||
|
enabled := make(map[string]bool, len(configuration.Upstreams))
|
||||||
|
var snapshot adminstate.Snapshot
|
||||||
|
if !nilInterface(supervisor.state) {
|
||||||
|
var err error
|
||||||
|
snapshot, err = supervisor.state.Snapshot(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(errProviderManagementStateUnavailable, err)
|
||||||
|
}
|
||||||
|
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, upstream := range snapshot.Upstreams {
|
||||||
|
enabled[upstream.Name] = upstream.Enabled
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if supervisor.prepare != nil {
|
||||||
|
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||||
|
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if supervisor.observe != nil {
|
||||||
|
supervisor.observe(configuration)
|
||||||
|
}
|
||||||
|
desired := make(map[string]config.Upstream)
|
||||||
|
for name, upstream := range configuration.Upstreams {
|
||||||
|
isEnabled := upstream.Enabled
|
||||||
|
if !nilInterface(supervisor.state) {
|
||||||
|
isEnabled = enabled[name]
|
||||||
|
}
|
||||||
|
if isEnabled {
|
||||||
|
upstream.Enabled = true
|
||||||
|
desired[name] = upstream
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return desired, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (supervisor *providerSupervisor) synchronizeConfiguration(
|
||||||
|
ctx context.Context,
|
||||||
|
current *config.Config,
|
||||||
|
snapshot adminstate.Snapshot,
|
||||||
|
) (*config.Config, error) {
|
||||||
|
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
|
||||||
|
return current, nil
|
||||||
|
}
|
||||||
|
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||||
|
return nil, errProviderManagementSnapshotStale
|
||||||
|
}
|
||||||
|
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||||
|
}
|
||||||
|
if checksum == snapshot.Config.Checksum {
|
||||||
|
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
|
||||||
|
return current, nil
|
||||||
|
}
|
||||||
|
if nilInterface(supervisor.source) {
|
||||||
|
return nil, errProviderConfigurationPending
|
||||||
|
}
|
||||||
|
loaded, err := supervisor.source.LoadConfiguration(ctx)
|
||||||
|
if err != nil || loaded.Value == nil {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
|
||||||
|
if err != nil || checksum != snapshot.Config.Checksum {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
|
||||||
|
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||||
|
}
|
||||||
|
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
|
||||||
|
return loaded.Value, nil
|
||||||
|
}
|
||||||
|
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||||
|
return nil, errProviderManagementSnapshotStale
|
||||||
|
}
|
||||||
|
return nil, errProviderConfigurationPending
|
||||||
|
}
|
||||||
|
|
||||||
|
func startProvider(
|
||||||
|
ctx context.Context,
|
||||||
|
name string,
|
||||||
|
configuration config.Upstream,
|
||||||
|
runner lifecycle.Runner,
|
||||||
|
failures chan<- error,
|
||||||
|
) *runningProvider {
|
||||||
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
|
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
|
||||||
|
go func() {
|
||||||
|
defer close(running.done)
|
||||||
|
err := runner.Run(runCtx)
|
||||||
|
if runCtx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = lifecycle.ErrRunnerStopped
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopProvider(running *runningProvider) {
|
||||||
|
if running == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running.cancel()
|
||||||
|
<-running.done
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopAllProviders(active map[string]*runningProvider) {
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for _, running := range active {
|
||||||
|
wait.Add(1)
|
||||||
|
go func(current *runningProvider) {
|
||||||
|
defer wait.Done()
|
||||||
|
stopProvider(current)
|
||||||
|
}(running)
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
}
|
||||||
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
@ -0,0 +1,376 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||||
|
started := make(chan providerRuntimeEvent, 4)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 4)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
state.set("provider-a", false)
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||||
|
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
updated := store.Current()
|
||||||
|
providerB := updated.Upstreams["provider-b"]
|
||||||
|
providerB.API.URL = "https://replacement.invalid/proxies"
|
||||||
|
updated.Upstreams["provider-b"] = providerB
|
||||||
|
if !store.PublishRevision(updated, 1) {
|
||||||
|
t.Fatal("PublishRevision() rejected updated configuration")
|
||||||
|
}
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
|
||||||
|
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
|
||||||
|
}
|
||||||
|
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
|
||||||
|
t.Fatalf("replacement Provider = %+v", event)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
wantErr := errors.New("coordination stopped")
|
||||||
|
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(configuration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||||
|
started := make(chan providerRuntimeEvent, 2)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 2)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
state.setError(errors.New("PostgreSQL temporarily unavailable"))
|
||||||
|
supervisor.Notify()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
|
||||||
|
case event := <-stopped:
|
||||||
|
t.Fatalf("Provider stopped during transient state failure: %+v", event)
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
|
||||||
|
state.setError(nil)
|
||||||
|
state.set("provider-a", false)
|
||||||
|
supervisor.Notify()
|
||||||
|
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||||
|
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
|
||||||
|
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(initial)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(initial): %v", err)
|
||||||
|
}
|
||||||
|
state := &mutableProviderState{
|
||||||
|
enabled: map[string]bool{"provider-a": true, "provider-b": true},
|
||||||
|
checksum: initialChecksum,
|
||||||
|
revision: 1,
|
||||||
|
}
|
||||||
|
source := &mutableProviderConfigurationSource{configuration: initial}
|
||||||
|
started := make(chan providerRuntimeEvent, 4)
|
||||||
|
stopped := make(chan providerRuntimeEvent, 4)
|
||||||
|
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||||
|
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- event
|
||||||
|
<-ctx.Done()
|
||||||
|
stopped <- event
|
||||||
|
return ctx.Err()
|
||||||
|
}), nil
|
||||||
|
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- supervisor.Run(ctx) }()
|
||||||
|
waitForRuntimeEvents(t, started, 2)
|
||||||
|
|
||||||
|
updated := store.Current()
|
||||||
|
providerA := updated.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://replacement.invalid/proxies"
|
||||||
|
updated.Upstreams["provider-a"] = providerA
|
||||||
|
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(updated): %v", err)
|
||||||
|
}
|
||||||
|
state.setChecksum(updatedChecksum)
|
||||||
|
supervisor.Notify()
|
||||||
|
waitForRuntimeEvents(t, stopped, 2)
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
|
||||||
|
source.set(updated)
|
||||||
|
supervisor.Notify()
|
||||||
|
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
|
||||||
|
foundReplacement := false
|
||||||
|
for _, event := range events {
|
||||||
|
if event.name == "provider-a" && event.url == providerA.API.URL {
|
||||||
|
foundReplacement = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundReplacement {
|
||||||
|
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
|
||||||
|
}
|
||||||
|
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
|
||||||
|
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
|
||||||
|
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
store, err := config.NewStore(initial)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.NewStore(): %v", err)
|
||||||
|
}
|
||||||
|
store.PublishRevision(initial, 1)
|
||||||
|
|
||||||
|
candidate := store.Current()
|
||||||
|
providerA := candidate.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://candidate.invalid/proxies"
|
||||||
|
candidate.Upstreams["provider-a"] = providerA
|
||||||
|
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Fingerprint(candidate): %v", err)
|
||||||
|
}
|
||||||
|
newest := store.Current()
|
||||||
|
providerA = newest.Upstreams["provider-a"]
|
||||||
|
providerA.API.URL = "https://newest.invalid/proxies"
|
||||||
|
newest.Upstreams["provider-a"] = providerA
|
||||||
|
|
||||||
|
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
if !store.PublishRevision(newest, 3) {
|
||||||
|
t.Fatal("failed to publish simulated concurrent revision")
|
||||||
|
}
|
||||||
|
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
|
||||||
|
})
|
||||||
|
supervisor, err := newProviderSupervisor(
|
||||||
|
store,
|
||||||
|
&mutableProviderState{},
|
||||||
|
func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||||
|
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
source,
|
||||||
|
bootstrapTestFingerprintKey,
|
||||||
|
time.Hour,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
|
||||||
|
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errProviderManagementSnapshotStale) {
|
||||||
|
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
|
||||||
|
}
|
||||||
|
if got := store.Revision(); got != 3 {
|
||||||
|
t.Fatalf("configuration revision = %d, want 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerRuntimeEvent struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
type supervisorRunnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||||
|
|
||||||
|
type mutableProviderState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
enabled map[string]bool
|
||||||
|
err error
|
||||||
|
checksum string
|
||||||
|
revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
if state.err != nil {
|
||||||
|
return adminstate.Snapshot{}, state.err
|
||||||
|
}
|
||||||
|
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
|
||||||
|
if state.checksum != "" {
|
||||||
|
snapshot.Config = &adminstate.ConfigRevision{
|
||||||
|
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, enabled := range state.enabled {
|
||||||
|
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
|
||||||
|
}
|
||||||
|
return snapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) setError(err error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.err = err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) setChecksum(checksum string) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.checksum = checksum
|
||||||
|
state.revision++
|
||||||
|
}
|
||||||
|
|
||||||
|
type mutableProviderConfigurationSource struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
configuration *config.Config
|
||||||
|
}
|
||||||
|
|
||||||
|
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
|
||||||
|
|
||||||
|
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
return source(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
source.configuration = configuration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *mutableProviderState) set(name string, enabled bool) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
state.enabled[name] = enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
|
||||||
|
t.Helper()
|
||||||
|
for range count {
|
||||||
|
_ = waitForRuntimeEvent(t, events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case event := <-events:
|
||||||
|
return event
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for Provider runtime event")
|
||||||
|
return providerRuntimeEvent{}
|
||||||
|
}
|
||||||
|
}
|
||||||
104
internal/controller/bootstrap/provider_test.go
Normal file
104
internal/controller/bootstrap/provider_test.go
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
|
"proxy-pool/internal/platform/credentials"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
upstream := configuration.Upstreams["provider-a"]
|
||||||
|
mapped, err := providerRuntimeConfig("provider-a", upstream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("providerRuntimeConfig(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if mapped.Provider.UpstreamID != "provider-a" ||
|
||||||
|
mapped.Provider.RequestInterval != time.Second ||
|
||||||
|
mapped.Provider.Timeout != 3*time.Second ||
|
||||||
|
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
|
||||||
|
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
|
||||||
|
mapped.Provider.TTL != 2*time.Minute ||
|
||||||
|
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
|
||||||
|
t.Fatalf("Provider config = %+v", mapped.Provider)
|
||||||
|
}
|
||||||
|
if mapped.ReconcileInterval != time.Second ||
|
||||||
|
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
|
||||||
|
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
|
||||||
|
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
|
||||||
|
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
|
||||||
|
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
|
||||||
|
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
configuration.Upstreams["disabled"] = config.Upstream{}
|
||||||
|
credentialStore, err := credentials.NewMemoryStore(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
fleet, err := newProviderFleet(configuration, ports{
|
||||||
|
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newProviderFleet(): %v", err)
|
||||||
|
}
|
||||||
|
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
|
||||||
|
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
configuration.Upstreams["provider-a"] = config.Upstream{}
|
||||||
|
configuration.Upstreams["provider-b"] = config.Upstream{}
|
||||||
|
fleet, err = newProviderFleet(configuration, ports{})
|
||||||
|
if err != nil || fleet != nil {
|
||||||
|
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(disabled) error = nil")
|
||||||
|
}
|
||||||
|
overflowing := config.Upstream{
|
||||||
|
Enabled: true,
|
||||||
|
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
|
||||||
|
Capacity: config.Capacity{
|
||||||
|
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
|
||||||
|
},
|
||||||
|
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
|
||||||
|
}
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(overflow) error = nil")
|
||||||
|
}
|
||||||
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config.Load(): %v", err)
|
||||||
|
}
|
||||||
|
tooLarge := configuration.Upstreams["provider-a"]
|
||||||
|
tooLarge.Pool.MaxSize = redisMaximumScan + 1
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
|
||||||
|
}
|
||||||
|
tooLarge = configuration.Upstreams["provider-a"]
|
||||||
|
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
|
||||||
|
if strconv.IntSize == 64 {
|
||||||
|
tooLarge.Fetch.MaxTotal++
|
||||||
|
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||||
|
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controller/admin"
|
"proxy-pool/internal/controller/admin"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -24,6 +25,7 @@ type ConfigurationReader interface {
|
|||||||
type Reader struct {
|
type Reader struct {
|
||||||
configuration ConfigurationReader
|
configuration ConfigurationReader
|
||||||
inventory activitypool.StateInventoryReader
|
inventory activitypool.StateInventoryReader
|
||||||
|
providerStats provider.StatsReader
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -33,11 +35,17 @@ func NewReader(
|
|||||||
configuration ConfigurationReader,
|
configuration ConfigurationReader,
|
||||||
inventory activitypool.StateInventoryReader,
|
inventory activitypool.StateInventoryReader,
|
||||||
now func() time.Time,
|
now func() time.Time,
|
||||||
|
stats ...provider.StatsReader,
|
||||||
) (*Reader, error) {
|
) (*Reader, error) {
|
||||||
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
|
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
|
||||||
|
(len(stats) == 1 && nilInterface(stats[0])) {
|
||||||
return nil, ErrInvalidReader
|
return nil, ErrInvalidReader
|
||||||
}
|
}
|
||||||
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
|
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
|
||||||
|
if len(stats) == 1 {
|
||||||
|
reader.providerStats = stats[0]
|
||||||
|
}
|
||||||
|
return reader, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
||||||
@ -69,8 +77,16 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
|
|||||||
}
|
}
|
||||||
|
|
||||||
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
||||||
|
providerStats := make([]provider.Stats, len(upstreamIDs))
|
||||||
|
if reader.providerStats != nil {
|
||||||
|
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
|
||||||
|
if len(providerStats) != len(upstreamIDs) {
|
||||||
|
return admin.OperationalStatus{}, ErrUnavailable
|
||||||
|
}
|
||||||
|
}
|
||||||
for index, inventory := range inventories {
|
for index, inventory := range inventories {
|
||||||
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
|
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
|
||||||
|
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
|
||||||
return admin.OperationalStatus{}, ErrUnavailable
|
return admin.OperationalStatus{}, ErrUnavailable
|
||||||
}
|
}
|
||||||
status.Upstreams[index] = admin.UpstreamActivity{
|
status.Upstreams[index] = admin.UpstreamActivity{
|
||||||
@ -80,6 +96,8 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
|
|||||||
Suspect: inventory.Suspect,
|
Suspect: inventory.Suspect,
|
||||||
Draining: inventory.Draining,
|
Draining: inventory.Draining,
|
||||||
Extracted: inventory.Extracted,
|
Extracted: inventory.Extracted,
|
||||||
|
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
|
||||||
|
FetchErrorCount: providerStats[index].FetchErrorCount,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return status, nil
|
return status, nil
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
|||||||
}}
|
}}
|
||||||
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
||||||
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
||||||
}}, inventory, func() time.Time { return now })
|
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
|
||||||
|
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
|
||||||
|
{UpstreamID: "provider-b", FetchErrorCount: 1},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewReader() error = %v", err)
|
t.Fatalf("NewReader() error = %v", err)
|
||||||
}
|
}
|
||||||
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
first := status.Upstreams[0]
|
first := status.Upstreams[0]
|
||||||
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
||||||
first.Draining != 4 || first.Extracted != 8 {
|
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
|
||||||
t.Fatalf("first upstream = %+v", first)
|
t.Fatalf("first upstream = %+v", first)
|
||||||
}
|
}
|
||||||
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
||||||
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
|
|||||||
now time.Time
|
now time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type staticProviderStatsReader struct {
|
||||||
|
result []provider.Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
|
||||||
|
return append([]provider.Stats(nil), reader.result...)
|
||||||
|
}
|
||||||
|
|
||||||
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
upstreamIDs []string,
|
upstreamIDs []string,
|
||||||
|
|||||||
@ -13,6 +13,8 @@ var (
|
|||||||
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const MaximumCoordinationInteger = int64(1<<53 - 1)
|
||||||
|
|
||||||
type CoordinationLimits struct {
|
type CoordinationLimits struct {
|
||||||
RequestInterval time.Duration
|
RequestInterval time.Duration
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
|
|||||||
64
internal/controller/provider/fleet.go
Normal file
64
internal/controller/provider/fleet.go
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidFleet = errors.New("invalid Provider fleet")
|
||||||
|
|
||||||
|
type Fleet struct {
|
||||||
|
runtimes []*UpstreamRuntime
|
||||||
|
group *lifecycle.Group
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
|
||||||
|
if len(runtimes) == 0 {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(runtimes))
|
||||||
|
owned := make([]*UpstreamRuntime, len(runtimes))
|
||||||
|
for index, runtime := range runtimes {
|
||||||
|
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
|
||||||
|
return nil, ErrInvalidFleet
|
||||||
|
}
|
||||||
|
seen[runtime.config.Provider.UpstreamID] = struct{}{}
|
||||||
|
owned[index] = runtime
|
||||||
|
}
|
||||||
|
sort.Slice(owned, func(left, right int) bool {
|
||||||
|
return owned[left].ID() < owned[right].ID()
|
||||||
|
})
|
||||||
|
runners := make([]lifecycle.Runner, len(owned))
|
||||||
|
for index, runtime := range owned {
|
||||||
|
runners[index] = runtime
|
||||||
|
}
|
||||||
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Join(ErrInvalidFleet, err)
|
||||||
|
}
|
||||||
|
return &Fleet{runtimes: owned, group: group}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fleet *Fleet) Run(ctx context.Context) error {
|
||||||
|
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
|
||||||
|
return ErrInvalidFleet
|
||||||
|
}
|
||||||
|
return fleet.group.Run(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fleet *Fleet) IDs() []string {
|
||||||
|
if fleet == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ids := make([]string, len(fleet.runtimes))
|
||||||
|
for index, runtime := range fleet.runtimes {
|
||||||
|
ids[index] = runtime.ID()
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
101
internal/controller/provider/fleet_test.go
Normal file
101
internal/controller/provider/fleet_test.go
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
|
||||||
|
started := make(chan string, 2)
|
||||||
|
siblingCancelled := make(chan struct{}, 1)
|
||||||
|
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||||
|
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
started <- "provider-a"
|
||||||
|
return errors.New("provider-a stopped")
|
||||||
|
}))
|
||||||
|
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
|
||||||
|
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
started <- "provider-b"
|
||||||
|
<-ctx.Done()
|
||||||
|
siblingCancelled <- struct{}{}
|
||||||
|
return ctx.Err()
|
||||||
|
}))
|
||||||
|
fleet, err := NewFleet(first, second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFleet(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = fleet.Run(context.Background())
|
||||||
|
if err == nil || err.Error() != "provider-a stopped" {
|
||||||
|
t.Fatalf("Run() error = %v, want provider-a failure", err)
|
||||||
|
}
|
||||||
|
seen := map[string]bool{<-started: true, <-started: true}
|
||||||
|
if !seen["provider-a"] || !seen["provider-b"] {
|
||||||
|
t.Fatalf("started upstreams = %v", seen)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-siblingCancelled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("sibling runtime was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
|
||||||
|
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||||
|
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return ctx.Err()
|
||||||
|
}))
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
runtimes []*UpstreamRuntime
|
||||||
|
}{
|
||||||
|
{name: "empty"},
|
||||||
|
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
|
||||||
|
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
fleet, err := NewFleet(test.runtimes...)
|
||||||
|
if err == nil || fleet != nil {
|
||||||
|
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
|
||||||
|
t.Helper()
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig(upstreamID),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: time.Second,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinator,
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
|
||||||
|
}
|
||||||
|
return runtime
|
||||||
|
}
|
||||||
@ -27,9 +27,11 @@ type RetryableError interface {
|
|||||||
Retryable() bool
|
Retryable() bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parser must be safe for concurrent calls and must honor context cancellation.
|
// Parser owns any transient resources attached to parsed candidates. It must
|
||||||
|
// be safe for concurrent calls and must honor context cancellation.
|
||||||
type Parser interface {
|
type Parser interface {
|
||||||
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
||||||
|
ReleaseCandidates([]proxyDomain.Proxy)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Result struct {
|
type Result struct {
|
||||||
@ -42,7 +44,8 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResultRecorder interface {
|
type ResultRecorder interface {
|
||||||
// Record may be called concurrently and must not retain mutable result data.
|
// Record may be called concurrently, must return promptly, and must not
|
||||||
|
// retain mutable result data.
|
||||||
Record(Result)
|
Record(Result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,7 @@ type Config struct {
|
|||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
MaxAttempts int
|
MaxAttempts int
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
|
MaxTotal int64
|
||||||
MaxSize int
|
MaxSize int
|
||||||
TTL time.Duration
|
TTL time.Duration
|
||||||
AllocationSafetyMargin time.Duration
|
AllocationSafetyMargin time.Duration
|
||||||
@ -31,6 +32,8 @@ type RetryConfig struct {
|
|||||||
Jitter int
|
Jitter int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const requestPermitSettlementTimeout = 5 * time.Second
|
||||||
|
|
||||||
type Reconciler struct {
|
type Reconciler struct {
|
||||||
config Config
|
config Config
|
||||||
ports Ports
|
ports Ports
|
||||||
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
|
|||||||
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
||||||
}
|
}
|
||||||
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
||||||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
|
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
|
||||||
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
||||||
}
|
}
|
||||||
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
||||||
@ -140,11 +143,51 @@ func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
||||||
|
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
||||||
|
if err != nil {
|
||||||
|
r.ports.Results.Record(Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("reserve fetch capacity: %w", err),
|
||||||
|
Attempt: 1,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !available {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if permit == nil || permit.Expected() <= 0 {
|
||||||
|
if permit != nil {
|
||||||
|
_ = permit.Cancel()
|
||||||
|
}
|
||||||
|
r.ports.Results.Record(Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
||||||
|
Attempt: 1,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
permitFinished := false
|
||||||
|
defer func() {
|
||||||
|
if !permitFinished {
|
||||||
|
_ = permit.Cancel()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
||||||
response, result, retryable, ok := r.fetchAttempt(ctx, session, attempt)
|
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
|
||||||
|
result.Class = upstream.FetchError
|
||||||
|
result.Err = errors.Join(result.Err, capacityErr)
|
||||||
|
} else {
|
||||||
|
permitFinished = true
|
||||||
|
}
|
||||||
|
}
|
||||||
r.ports.Results.Record(result)
|
r.ports.Results.Record(result)
|
||||||
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
|
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
|
||||||
return
|
return
|
||||||
@ -163,41 +206,12 @@ func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
|||||||
func (r *Reconciler) fetchAttempt(
|
func (r *Reconciler) fetchAttempt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
session LeaderSession,
|
session LeaderSession,
|
||||||
|
permit upstream.FetchPermit,
|
||||||
attempt int,
|
attempt int,
|
||||||
) (FetchResponse, Result, bool, bool) {
|
) (FetchResponse, Result, bool, bool) {
|
||||||
if err := r.waitForRequestSlot(ctx); err != nil {
|
if err := r.waitForRequestSlot(ctx); err != nil {
|
||||||
return FetchResponse{}, Result{}, false, false
|
return FetchResponse{}, Result{}, false, false
|
||||||
}
|
}
|
||||||
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
|
||||||
if err != nil {
|
|
||||||
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
|
|
||||||
return FetchResponse{}, Result{
|
|
||||||
UpstreamID: r.config.UpstreamID,
|
|
||||||
Class: upstream.FetchError,
|
|
||||||
Err: resultErr,
|
|
||||||
Attempt: attempt,
|
|
||||||
}, false, true
|
|
||||||
}
|
|
||||||
if !available {
|
|
||||||
return FetchResponse{}, Result{}, false, false
|
|
||||||
}
|
|
||||||
if permit == nil || permit.Expected() <= 0 {
|
|
||||||
if permit != nil {
|
|
||||||
_ = permit.Cancel()
|
|
||||||
}
|
|
||||||
return FetchResponse{}, Result{
|
|
||||||
UpstreamID: r.config.UpstreamID,
|
|
||||||
Class: upstream.FetchError,
|
|
||||||
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
|
||||||
Attempt: attempt,
|
|
||||||
}, false, true
|
|
||||||
}
|
|
||||||
permitFinished := false
|
|
||||||
defer func() {
|
|
||||||
if !permitFinished {
|
|
||||||
_ = permit.Cancel()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
|
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
|
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
|
||||||
@ -236,13 +250,14 @@ func (r *Reconciler) fetchAttempt(
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
||||||
|
|
||||||
var parseErr, candidateErr, coordinationErr, capacityErr error
|
var parseErr, candidateErr, coordinationErr error
|
||||||
var validCount, newCount int
|
var validCount, newCount int
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
requestSettlementAttempted = true
|
requestSettlementAttempted = true
|
||||||
coordinationErr = r.settleRequestPermit(requestPermit, false, 0)
|
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
|
||||||
} else {
|
} else {
|
||||||
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
||||||
|
defer r.ports.Parser.ReleaseCandidates(candidates)
|
||||||
parseErr = err
|
parseErr = err
|
||||||
validCount = len(candidates)
|
validCount = len(candidates)
|
||||||
charged := validCount
|
charged := validCount
|
||||||
@ -267,14 +282,10 @@ func (r *Reconciler) fetchAttempt(
|
|||||||
newCount = upserted.Inserted
|
newCount = upserted.Inserted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if callErr == nil && parseErr == nil && candidateErr == nil && coordinationErr == nil {
|
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
|
||||||
capacityErr = permit.Complete(newCount)
|
|
||||||
permitFinished = capacityErr == nil
|
|
||||||
}
|
|
||||||
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr, capacityErr)
|
|
||||||
class := upstream.ClassifyFetchResult(
|
class := upstream.ClassifyFetchResult(
|
||||||
callErr,
|
callErr,
|
||||||
errors.Join(parseErr, candidateErr, coordinationErr, capacityErr),
|
errors.Join(parseErr, candidateErr, coordinationErr),
|
||||||
validCount,
|
validCount,
|
||||||
newCount,
|
newCount,
|
||||||
)
|
)
|
||||||
@ -289,8 +300,7 @@ func (r *Reconciler) fetchAttempt(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
|
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
|
||||||
timeout := min(r.config.Timeout, 5*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if complete {
|
if complete {
|
||||||
return permit.Complete(ctx, fetched)
|
return permit.Complete(ctx, fetched)
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
controllerPool "proxy-pool/internal/controller/pool"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
"proxy-pool/internal/domain/upstream"
|
"proxy-pool/internal/domain/upstream"
|
||||||
@ -722,9 +723,9 @@ func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
|
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
|
||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
globalCancelled := make(chan struct{}, 1)
|
globalCompleted := make(chan int, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
return FetchResponse{}, errors.New("provider connection failed")
|
return FetchResponse{}, errors.New("provider connection failed")
|
||||||
@ -736,17 +737,15 @@ func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
return &recordingRequestPermit{cancelled: globalCancelled}, true, nil
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
result := runSingleReconcile(t, reconciler, results, session)
|
result := runSingleReconcile(t, reconciler, results, session)
|
||||||
if result.Class != upstream.FetchError {
|
if result.Class != upstream.FetchError {
|
||||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
}
|
}
|
||||||
select {
|
if got := <-globalCompleted; got <= 0 {
|
||||||
case <-globalCancelled:
|
t.Fatalf("global charged count = %d, want conservative expected count", got)
|
||||||
default:
|
|
||||||
t.Fatal("distributed request reservation was not cancelled")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -754,10 +753,12 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.
|
|||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
localCompleted := make(chan fetchCompletion, 1)
|
localCompleted := make(chan fetchCompletion, 1)
|
||||||
globalCompleted := make(chan int, 1)
|
globalCompleted := make(chan int, 1)
|
||||||
|
released := make(chan []proxyDomain.Proxy, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
ports.Parser = &recordingCandidateParser{
|
||||||
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
|
||||||
})
|
released: released,
|
||||||
|
}
|
||||||
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
if len(batch.Proxies) != 2 {
|
if len(batch.Proxies) != 2 {
|
||||||
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
||||||
@ -791,6 +792,14 @@ func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.
|
|||||||
if got := <-localCompleted; got.retained != 1 {
|
if got := <-localCompleted; got.retained != 1 {
|
||||||
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case got := <-released:
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("parsed candidates were not released")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
||||||
@ -838,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
|
||||||
|
results := make(chan Result, 2)
|
||||||
|
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
|
||||||
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
|
}
|
||||||
|
var parses atomic.Int64
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Capacity = budget
|
||||||
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
if parses.Add(1) == 1 {
|
||||||
|
return nil, errors.New("temporary parser failure")
|
||||||
|
}
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
|
||||||
|
MaxInFlight: 1, MaxSize: 10,
|
||||||
|
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
|
||||||
|
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
|
||||||
|
if got := budget.Snapshot().PendingExpected; got != 1 {
|
||||||
|
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
reconciler.Notify()
|
||||||
|
<-results
|
||||||
|
<-results
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("RunLeader(): %v", err)
|
||||||
|
}
|
||||||
|
usage := budget.Snapshot()
|
||||||
|
if usage.PendingExpected != 0 || usage.Managed != 1 {
|
||||||
|
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
@ -954,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
|
|||||||
return errors.New("unexpected sleep")
|
return errors.New("unexpected sleep")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type sleeperFunc func(context.Context, time.Duration) error
|
||||||
|
|
||||||
|
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
|
||||||
|
return f(ctx, duration)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -981,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
|
|||||||
return f(ctx, body)
|
return f(ctx, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
|
||||||
|
|
||||||
|
type recordingCandidateParser struct {
|
||||||
|
candidates []proxyDomain.Proxy
|
||||||
|
released chan<- []proxyDomain.Proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||||
|
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
|
||||||
|
}
|
||||||
|
|
||||||
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
||||||
|
|
||||||
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
|||||||
103
internal/controller/provider/stats.go
Normal file
103
internal/controller/provider/stats.go
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
|
||||||
|
|
||||||
|
type Stats struct {
|
||||||
|
UpstreamID string
|
||||||
|
ConsecutiveEmptyFetch int64
|
||||||
|
FetchErrorCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsReader interface {
|
||||||
|
ReadProviderStats([]string) []Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsRetainer interface {
|
||||||
|
RetainProviderStats([]string)
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatsRecorder struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
maximum int
|
||||||
|
byID map[string]Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
|
||||||
|
if maximum <= 0 {
|
||||||
|
return nil, ErrInvalidStatsRecorder
|
||||||
|
}
|
||||||
|
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (recorder *StatsRecorder) Record(result Result) {
|
||||||
|
if recorder == nil || result.UpstreamID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
stats, exists := recorder.byID[result.UpstreamID]
|
||||||
|
if !exists {
|
||||||
|
if len(recorder.byID) >= recorder.maximum {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stats.UpstreamID = result.UpstreamID
|
||||||
|
}
|
||||||
|
switch result.Class {
|
||||||
|
case upstream.FetchEmpty:
|
||||||
|
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
|
||||||
|
stats.ConsecutiveEmptyFetch++
|
||||||
|
}
|
||||||
|
case upstream.FetchValid, upstream.FetchDuplicateOnly:
|
||||||
|
stats.ConsecutiveEmptyFetch = 0
|
||||||
|
case upstream.FetchError:
|
||||||
|
if stats.FetchErrorCount < math.MaxInt64 {
|
||||||
|
stats.FetchErrorCount++
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
recorder.byID[result.UpstreamID] = stats
|
||||||
|
}
|
||||||
|
|
||||||
|
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
|
||||||
|
result := make([]Stats, len(upstreamIDs))
|
||||||
|
if recorder == nil {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
for index, upstreamID := range upstreamIDs {
|
||||||
|
result[index] = recorder.byID[upstreamID]
|
||||||
|
result[index].UpstreamID = upstreamID
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetainProviderStats removes observations for upstreams no longer present in
|
||||||
|
// the complete configuration. Disabled but configured upstreams must be kept.
|
||||||
|
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
|
||||||
|
if recorder == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
retained := make(map[string]struct{}, len(upstreamIDs))
|
||||||
|
for _, upstreamID := range upstreamIDs {
|
||||||
|
if upstreamID != "" {
|
||||||
|
retained[upstreamID] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
recorder.mu.Lock()
|
||||||
|
defer recorder.mu.Unlock()
|
||||||
|
for upstreamID := range recorder.byID {
|
||||||
|
if _, keep := retained[upstreamID]; !keep {
|
||||||
|
delete(recorder.byID, upstreamID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
83
internal/controller/provider/stats_test.go
Normal file
83
internal/controller/provider/stats_test.go
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
for _, class := range []upstream.FetchClass{
|
||||||
|
upstream.FetchEmpty,
|
||||||
|
upstream.FetchEmpty,
|
||||||
|
upstream.FetchError,
|
||||||
|
} {
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
|
||||||
|
}
|
||||||
|
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
|
||||||
|
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
|
||||||
|
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
|
||||||
|
}
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
|
||||||
|
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
|
||||||
|
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
const workers = 100
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for range workers {
|
||||||
|
wait.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
|
||||||
|
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||||
|
}
|
||||||
|
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
const workers = 100
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for range workers {
|
||||||
|
wait.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||||
|
|
||||||
|
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
|
||||||
|
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
|
||||||
|
t.Fatalf("stats after retention = %+v", stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
219
internal/controller/provider/upstream_runtime.go
Normal file
219
internal/controller/provider/upstream_runtime.go
Normal file
@ -0,0 +1,219 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"hash/fnv"
|
||||||
|
"reflect"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
|
||||||
|
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
|
||||||
|
)
|
||||||
|
|
||||||
|
type UpstreamRuntimeConfig struct {
|
||||||
|
Provider Config
|
||||||
|
ReconcilePolicy pool.ReconcilePolicy
|
||||||
|
ReconcileInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type UpstreamRuntimeDependencies struct {
|
||||||
|
Coordinator Coordinator
|
||||||
|
Inventory pool.InventoryReader
|
||||||
|
Adapter ProviderAdapter
|
||||||
|
Parser Parser
|
||||||
|
Activity activitypool.Upserter
|
||||||
|
Results ResultRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
|
||||||
|
// local budget and coalescing signal are created for each leadership term.
|
||||||
|
type UpstreamRuntime struct {
|
||||||
|
config UpstreamRuntimeConfig
|
||||||
|
dependencies UpstreamRuntimeDependencies
|
||||||
|
sleeper Sleeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewUpstreamRuntime(
|
||||||
|
config UpstreamRuntimeConfig,
|
||||||
|
dependencies UpstreamRuntimeDependencies,
|
||||||
|
) (*UpstreamRuntime, error) {
|
||||||
|
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
|
||||||
|
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
|
||||||
|
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
|
||||||
|
nilRuntimeDependency(dependencies.Results) {
|
||||||
|
return nil, ErrInvalidUpstreamRuntime
|
||||||
|
}
|
||||||
|
runtime := &UpstreamRuntime{
|
||||||
|
config: config, dependencies: dependencies, sleeper: timerSleeper{},
|
||||||
|
}
|
||||||
|
if _, err := runtime.newLeaderTerm(); err != nil {
|
||||||
|
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
|
||||||
|
}
|
||||||
|
return runtime, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
|
||||||
|
if runtime == nil || ctx == nil {
|
||||||
|
return ErrInvalidUpstreamRuntime
|
||||||
|
}
|
||||||
|
limits := CoordinationLimits{
|
||||||
|
RequestInterval: runtime.config.Provider.RequestInterval,
|
||||||
|
MaxInFlight: runtime.config.Provider.MaxInFlight,
|
||||||
|
MaxAttemptDuration: runtime.config.Provider.Timeout,
|
||||||
|
MaxTotal: runtime.config.Provider.MaxTotal,
|
||||||
|
}
|
||||||
|
err := runtime.dependencies.Coordinator.RunLeader(
|
||||||
|
ctx,
|
||||||
|
runtime.config.Provider.UpstreamID,
|
||||||
|
limits,
|
||||||
|
func(leaderCtx context.Context, session LeaderSession) error {
|
||||||
|
term, buildErr := runtime.newLeaderTerm()
|
||||||
|
if buildErr != nil {
|
||||||
|
return buildErr
|
||||||
|
}
|
||||||
|
return term.run(leaderCtx, session)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
return ErrUpstreamRuntimeStopped
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) ID() string {
|
||||||
|
if runtime == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return runtime.config.Provider.UpstreamID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
|
||||||
|
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
|
||||||
|
UpstreamID: runtime.config.Provider.UpstreamID,
|
||||||
|
MaxSize: runtime.config.Provider.MaxSize,
|
||||||
|
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
providerConfig := runtime.config.Provider
|
||||||
|
providerConfig.RequestInterval = 0
|
||||||
|
providerReconciler, err := NewReconciler(providerConfig, Ports{
|
||||||
|
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
|
||||||
|
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
|
||||||
|
Capacity: budget,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
poolReconciler, err := pool.NewReconciler(
|
||||||
|
runtime.config.ReconcilePolicy,
|
||||||
|
budget,
|
||||||
|
providerReconciler,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &upstreamLeaderTerm{
|
||||||
|
upstreamID: runtime.config.Provider.UpstreamID,
|
||||||
|
interval: runtime.config.ReconcileInterval,
|
||||||
|
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
|
||||||
|
inventory: runtime.dependencies.Inventory,
|
||||||
|
pool: poolReconciler,
|
||||||
|
provider: providerReconciler,
|
||||||
|
sleeper: runtime.sleeper,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type upstreamLeaderTerm struct {
|
||||||
|
upstreamID string
|
||||||
|
interval time.Duration
|
||||||
|
safetyMargin time.Duration
|
||||||
|
inventory pool.InventoryReader
|
||||||
|
pool *pool.Reconciler
|
||||||
|
provider *Reconciler
|
||||||
|
sleeper Sleeper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
|
||||||
|
termCtx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 2)
|
||||||
|
go func() { done <- term.provider.RunLeader(termCtx, session) }()
|
||||||
|
go func() { done <- term.reconcileInventory(termCtx) }()
|
||||||
|
|
||||||
|
first := <-done
|
||||||
|
cancel()
|
||||||
|
second := <-done
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
first = ErrUpstreamRuntimeStopped
|
||||||
|
}
|
||||||
|
if second != nil && !errors.Is(second, context.Canceled) {
|
||||||
|
return errors.Join(first, second)
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
|
||||||
|
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
|
||||||
|
if err := term.sleeper.Sleep(ctx, delay); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
inventory, err := term.inventory.ReadInventory(
|
||||||
|
ctx,
|
||||||
|
term.upstreamID,
|
||||||
|
term.safetyMargin,
|
||||||
|
)
|
||||||
|
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
|
||||||
|
term.pool.ReconcileSnapshot(inventory)
|
||||||
|
}
|
||||||
|
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
|
||||||
|
window := min(interval/4, 250*time.Millisecond)
|
||||||
|
if window <= 1 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
digest := fnv.New64a()
|
||||||
|
_, _ = digest.Write([]byte(upstreamID))
|
||||||
|
return time.Duration(digest.Sum64() % uint64(window))
|
||||||
|
}
|
||||||
|
|
||||||
|
func nilRuntimeDependency(value any) bool {
|
||||||
|
if value == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
reflected := reflect.ValueOf(value)
|
||||||
|
switch reflected.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return reflected.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
247
internal/controller/provider/upstream_runtime_test.go
Normal file
247
internal/controller/provider/upstream_runtime_test.go
Normal file
@ -0,0 +1,247 @@
|
|||||||
|
package provider
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/controller/pool"
|
||||||
|
"proxy-pool/internal/domain/activitypool"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
|
||||||
|
var inventoryReads atomic.Int64
|
||||||
|
fetched := make(chan struct{}, 1)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 10 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
|
||||||
|
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
|
||||||
|
}
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
inventoryReads.Add(1)
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for leader fetch")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if got := inventoryReads.Load(); got == 0 {
|
||||||
|
t.Fatal("inventory was not read inside leader term")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
|
||||||
|
var inventoryReads atomic.Int64
|
||||||
|
var providerCalls atomic.Int64
|
||||||
|
fetched := make(chan struct{}, 1)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 10 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
_ CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
if inventoryReads.Add(1) == 1 {
|
||||||
|
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
|
||||||
|
}
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
providerCalls.Add(1)
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("timed out waiting for recovered inventory fetch")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
if got := inventoryReads.Load(); got < 2 {
|
||||||
|
t.Fatalf("inventory reads = %d, want recovery retry", got)
|
||||||
|
}
|
||||||
|
if got := providerCalls.Load(); got != 1 {
|
||||||
|
t.Fatalf("provider calls = %d, want one call after recovery", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
|
||||||
|
providerConfig := runtimeProviderConfig("provider-a")
|
||||||
|
providerConfig.RequestInterval = 500 * time.Millisecond
|
||||||
|
fetched := make(chan struct{}, 2)
|
||||||
|
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||||
|
Provider: providerConfig,
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: 5 * time.Millisecond,
|
||||||
|
}, UpstreamRuntimeDependencies{
|
||||||
|
Coordinator: coordinatorFunc(func(
|
||||||
|
ctx context.Context,
|
||||||
|
_ string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
if limits.RequestInterval != 500*time.Millisecond {
|
||||||
|
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
|
||||||
|
}
|
||||||
|
return work(ctx, unlimitedLeaderSession{})
|
||||||
|
}),
|
||||||
|
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||||
|
return pool.InventorySnapshot{}, nil
|
||||||
|
}),
|
||||||
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
fetched <- struct{}{}
|
||||||
|
return FetchResponse{Body: []byte("fixture")}, nil
|
||||||
|
}),
|
||||||
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||||
|
}),
|
||||||
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||||
|
}),
|
||||||
|
Results: resultRecorderFunc(func(Result) {}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- runtime.Run(ctx) }()
|
||||||
|
for range 2 {
|
||||||
|
select {
|
||||||
|
case <-fetched:
|
||||||
|
case <-time.After(150 * time.Millisecond):
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
t.Fatal("local Provider reconciler duplicated the distributed request interval")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
|
||||||
|
validConfig := UpstreamRuntimeConfig{
|
||||||
|
Provider: runtimeProviderConfig("provider-a"),
|
||||||
|
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||||
|
ReconcileInterval: time.Second,
|
||||||
|
}
|
||||||
|
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
|
||||||
|
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
|
||||||
|
const interval = time.Second
|
||||||
|
first := initialReconcileDelay("provider-a", interval)
|
||||||
|
if first != initialReconcileDelay("provider-a", interval) {
|
||||||
|
t.Fatal("initial reconcile delay is not stable")
|
||||||
|
}
|
||||||
|
if first < 0 || first >= 250*time.Millisecond {
|
||||||
|
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
|
||||||
|
}
|
||||||
|
if other := initialReconcileDelay("provider-b", interval); other == first {
|
||||||
|
t.Fatalf("different upstreams have the same initial delay %s", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runtimeProviderConfig(upstreamID string) Config {
|
||||||
|
return Config{
|
||||||
|
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
|
||||||
|
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runtimeReconcilePolicy() pool.ReconcilePolicy {
|
||||||
|
return pool.ReconcilePolicy{
|
||||||
|
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
||||||
|
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type coordinatorFunc func(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
CoordinationLimits,
|
||||||
|
func(context.Context, LeaderSession) error,
|
||||||
|
) error
|
||||||
|
|
||||||
|
func (f coordinatorFunc) RunLeader(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
limits CoordinationLimits,
|
||||||
|
work func(context.Context, LeaderSession) error,
|
||||||
|
) error {
|
||||||
|
return f(ctx, upstreamID, limits, work)
|
||||||
|
}
|
||||||
|
|
||||||
|
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
|
||||||
|
|
||||||
|
func (f inventoryReaderFunc) ReadInventory(
|
||||||
|
ctx context.Context,
|
||||||
|
upstreamID string,
|
||||||
|
safetyMargin time.Duration,
|
||||||
|
) (pool.InventorySnapshot, error) {
|
||||||
|
return f(ctx, upstreamID, safetyMargin)
|
||||||
|
}
|
||||||
@ -48,10 +48,27 @@ type Store interface {
|
|||||||
Resolve(context.Context, Reference) (Value, error)
|
Resolve(context.Context, Reference) (Value, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Releaser removes transient credential material after the consumer has copied
|
||||||
|
// it into its authoritative storage. Release is idempotent and version fenced.
|
||||||
|
type Releaser interface {
|
||||||
|
Release(context.Context, Reference) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type CapacityEnsurer interface {
|
||||||
|
EnsureCapacity(context.Context, int) error
|
||||||
|
}
|
||||||
|
|
||||||
type entry struct {
|
type entry struct {
|
||||||
|
scope *scopeState
|
||||||
|
value Value
|
||||||
|
reference Reference
|
||||||
|
}
|
||||||
|
|
||||||
|
type scopeState struct {
|
||||||
|
name string
|
||||||
value Value
|
value Value
|
||||||
version uint64
|
version uint64
|
||||||
reference Reference
|
leases int
|
||||||
}
|
}
|
||||||
|
|
||||||
// MemoryStore keeps credentials in process memory and serializes access with a
|
// MemoryStore keeps credentials in process memory and serializes access with a
|
||||||
@ -59,7 +76,7 @@ type entry struct {
|
|||||||
type MemoryStore struct {
|
type MemoryStore struct {
|
||||||
lock chan struct{}
|
lock chan struct{}
|
||||||
capacity int
|
capacity int
|
||||||
byScope map[string]*entry
|
byScope map[string]*scopeState
|
||||||
byRef map[string]*entry
|
byRef map[string]*entry
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
|
|||||||
return &MemoryStore{
|
return &MemoryStore{
|
||||||
lock: lock,
|
lock: lock,
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
byScope: make(map[string]*entry),
|
byScope: make(map[string]*scopeState),
|
||||||
byRef: make(map[string]*entry),
|
byRef: make(map[string]*entry),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
|
|||||||
if err := s.acquire(ctx); err != nil {
|
if err := s.acquire(ctx); err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
defer s.release()
|
defer s.unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
|
if len(s.byRef) >= s.capacity {
|
||||||
|
return Reference{}, ErrCapacityExceeded
|
||||||
|
}
|
||||||
|
|
||||||
if current, ok := s.byScope[scope]; ok {
|
current, exists := s.byScope[scope]
|
||||||
if current.value == value {
|
if !exists && len(s.byScope) >= s.capacity {
|
||||||
return current.reference, nil
|
|
||||||
}
|
|
||||||
current.value = value
|
|
||||||
current.version++
|
|
||||||
current.reference.CredentialVersion = versionString(current.version)
|
|
||||||
return current.reference, nil
|
|
||||||
}
|
|
||||||
if len(s.byScope) >= s.capacity {
|
|
||||||
return Reference{}, ErrCapacityExceeded
|
return Reference{}, ErrCapacityExceeded
|
||||||
}
|
}
|
||||||
secretRef, err := s.newUniqueSecretRef()
|
secretRef, err := s.newUniqueSecretRef()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Reference{}, err
|
return Reference{}, err
|
||||||
}
|
}
|
||||||
|
if !exists {
|
||||||
|
current = &scopeState{name: scope, value: value, version: 1}
|
||||||
|
s.byScope[scope] = current
|
||||||
|
} else if current.value != value {
|
||||||
|
current.value = value
|
||||||
|
current.version++
|
||||||
|
}
|
||||||
created := &entry{
|
created := &entry{
|
||||||
|
scope: current,
|
||||||
value: value,
|
value: value,
|
||||||
version: 1,
|
|
||||||
reference: Reference{
|
reference: Reference{
|
||||||
SecretRef: secretRef,
|
SecretRef: secretRef,
|
||||||
CredentialVersion: versionString(1),
|
CredentialVersion: versionString(current.version),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
s.byScope[scope] = created
|
current.leases++
|
||||||
s.byRef[secretRef] = created
|
s.byRef[secretRef] = created
|
||||||
return created.reference, nil
|
return created.reference, nil
|
||||||
}
|
}
|
||||||
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
|||||||
if err := s.acquire(ctx); err != nil {
|
if err := s.acquire(ctx); err != nil {
|
||||||
return Value{}, err
|
return Value{}, err
|
||||||
}
|
}
|
||||||
defer s.release()
|
defer s.unlock()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return Value{}, err
|
return Value{}, err
|
||||||
}
|
}
|
||||||
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
|||||||
return current.value, nil
|
return current.value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
|
||||||
|
if err := contextError(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !s.valid() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
|
||||||
|
return ErrInvalidReference
|
||||||
|
}
|
||||||
|
if err := s.acquire(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.unlock()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current, ok := s.byRef[reference.SecretRef]
|
||||||
|
if !ok || current.reference != reference {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
delete(s.byRef, reference.SecretRef)
|
||||||
|
current.scope.leases--
|
||||||
|
if current.scope.leases == 0 {
|
||||||
|
delete(s.byScope, current.scope.name)
|
||||||
|
current.scope.value = Value{}
|
||||||
|
}
|
||||||
|
current.value = Value{}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
|
||||||
|
if err := contextError(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !s.valid() {
|
||||||
|
return ErrInvalidStore
|
||||||
|
}
|
||||||
|
if minimum <= 0 {
|
||||||
|
return ErrInvalidCapacity
|
||||||
|
}
|
||||||
|
if err := s.acquire(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.unlock()
|
||||||
|
if minimum > s.capacity {
|
||||||
|
s.capacity = minimum
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MemoryStore) acquire(ctx context.Context) error {
|
func (s *MemoryStore) acquire(ctx context.Context) error {
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MemoryStore) release() {
|
func (s *MemoryStore) unlock() {
|
||||||
s.lock <- struct{}{}
|
s.lock <- struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
|
||||||
store, err := NewMemoryStore(2)
|
store, err := NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Put(second): %v", err)
|
t.Fatalf("Put(second): %v", err)
|
||||||
}
|
}
|
||||||
if first != second {
|
if first == second || first.SecretRef == second.SecretRef {
|
||||||
t.Fatalf("second reference = %#v, want %#v", second, first)
|
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
|
||||||
}
|
}
|
||||||
if first.SecretRef == "" || first.CredentialVersion != "v1" {
|
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
|
||||||
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
|
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
|
||||||
}
|
}
|
||||||
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
|
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
|
||||||
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
|||||||
if got != value {
|
if got != value {
|
||||||
t.Fatalf("Resolve() = %#v, want %#v", got, value)
|
t.Fatalf("Resolve() = %#v, want %#v", got, value)
|
||||||
}
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(first lease): %v", err)
|
||||||
|
}
|
||||||
|
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
|
||||||
|
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
||||||
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
store, err := NewMemoryStore(2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Put(new): %v", err)
|
t.Fatalf("Put(new): %v", err)
|
||||||
}
|
}
|
||||||
if newReference.SecretRef != oldReference.SecretRef {
|
if newReference.SecretRef == oldReference.SecretRef {
|
||||||
t.Fatalf("new SecretRef changed across versions")
|
t.Fatalf("new SecretRef reused an active lease")
|
||||||
}
|
}
|
||||||
if newReference.CredentialVersion != "v2" {
|
if newReference.CredentialVersion != "v2" {
|
||||||
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
|
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
|
||||||
}
|
}
|
||||||
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
|
old, err := store.Resolve(context.Background(), oldReference)
|
||||||
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
|
if err != nil || old.Password != "old-password" {
|
||||||
|
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
|
||||||
}
|
}
|
||||||
got, err := store.Resolve(context.Background(), newReference)
|
got, err := store.Resolve(context.Background(), newReference)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
|
|||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("Resolve(existing) returned changed credentials")
|
t.Fatalf("Resolve(existing) returned changed credentials")
|
||||||
}
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), reference); err != nil {
|
||||||
|
t.Fatalf("Release(first lease): %v", err)
|
||||||
|
}
|
||||||
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
|
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
|
||||||
t.Fatalf("Put(idempotent at capacity): %v", err)
|
t.Fatalf("Put(after lease release): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(first): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(first): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
|
||||||
|
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
|
||||||
|
t.Fatalf("Put(after release): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), first); err != nil {
|
||||||
|
t.Fatalf("Release(idempotent): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(old): %v", err)
|
||||||
|
}
|
||||||
|
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put(new): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Release(context.Background(), stale); err != nil {
|
||||||
|
t.Fatalf("Release(stale): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Resolve(context.Background(), current); err != nil {
|
||||||
|
t.Fatalf("Resolve(current): %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
|
||||||
|
store, err := NewMemoryStore(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
|
||||||
|
t.Fatalf("EnsureCapacity(2): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
|
||||||
|
t.Fatalf("Put(provider-a): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
|
||||||
|
t.Fatalf("Put(provider-b): %v", err)
|
||||||
|
}
|
||||||
|
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("EnsureCapacity(shrink request): %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
const workers = 100
|
||||||
|
store, err := NewMemoryStore(workers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
want := Value{Username: "alice", Password: "shared-password"}
|
want := Value{Username: "alice", Password: "shared-password"}
|
||||||
const workers = 100
|
|
||||||
references := make(chan Reference, workers)
|
references := make(chan Reference, workers)
|
||||||
errorsSeen := make(chan error, workers)
|
errorsSeen := make(chan error, workers)
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
|||||||
for err := range errorsSeen {
|
for err := range errorsSeen {
|
||||||
t.Errorf("concurrent Put(): %v", err)
|
t.Errorf("concurrent Put(): %v", err)
|
||||||
}
|
}
|
||||||
var first Reference
|
secretRefs := make(map[string]struct{}, workers)
|
||||||
for reference := range references {
|
for reference := range references {
|
||||||
if first == (Reference{}) {
|
if reference.CredentialVersion != "v1" {
|
||||||
first = reference
|
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
|
||||||
}
|
}
|
||||||
if reference != first {
|
secretRefs[reference.SecretRef] = struct{}{}
|
||||||
t.Errorf("concurrent Put() reference differs from first")
|
|
||||||
}
|
}
|
||||||
|
if len(secretRefs) != workers {
|
||||||
|
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
|
||||||
}
|
}
|
||||||
if first.CredentialVersion != "v1" {
|
for secretRef := range secretRefs {
|
||||||
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
|
got, err := store.Resolve(context.Background(), Reference{
|
||||||
|
SecretRef: secretRef, CredentialVersion: "v1",
|
||||||
|
})
|
||||||
|
if err != nil || got != want {
|
||||||
|
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
|
||||||
}
|
}
|
||||||
got, err := store.Resolve(context.Background(), first)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve(): %v", err)
|
|
||||||
}
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("Resolve() returned unexpected credentials")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||||
store, err := NewMemoryStore(1)
|
const workers = 100
|
||||||
|
store, err := NewMemoryStore(workers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewMemoryStore(): %v", err)
|
t.Fatalf("NewMemoryStore(): %v", err)
|
||||||
}
|
}
|
||||||
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
reference Reference
|
reference Reference
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
const workers = 100
|
|
||||||
results := make(chan result, workers)
|
results := make(chan result, workers)
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
for index := range workers {
|
for index := range workers {
|
||||||
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
close(results)
|
close(results)
|
||||||
|
|
||||||
versions := make(map[string]struct{}, workers)
|
versions := make(map[string]struct{}, workers)
|
||||||
var secretRef string
|
secretRefs := make(map[string]struct{}, workers)
|
||||||
var latest result
|
var latest result
|
||||||
for current := range results {
|
for current := range results {
|
||||||
if current.err != nil {
|
if current.err != nil {
|
||||||
t.Fatalf("concurrent Put(): %v", current.err)
|
t.Fatalf("concurrent Put(): %v", current.err)
|
||||||
}
|
}
|
||||||
if secretRef == "" {
|
secretRefs[current.reference.SecretRef] = struct{}{}
|
||||||
secretRef = current.reference.SecretRef
|
|
||||||
}
|
|
||||||
if current.reference.SecretRef != secretRef {
|
|
||||||
t.Fatal("SecretRef changed across concurrent updates")
|
|
||||||
}
|
|
||||||
versions[current.reference.CredentialVersion] = struct{}{}
|
versions[current.reference.CredentialVersion] = struct{}{}
|
||||||
if current.reference.CredentialVersion == "v100" {
|
if current.reference.CredentialVersion == "v100" {
|
||||||
latest = current
|
latest = current
|
||||||
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
|||||||
if len(versions) != workers {
|
if len(versions) != workers {
|
||||||
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
|
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
|
||||||
}
|
}
|
||||||
|
if len(secretRefs) != workers {
|
||||||
|
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
|
||||||
|
}
|
||||||
if latest.reference == (Reference{}) {
|
if latest.reference == (Reference{}) {
|
||||||
t.Fatal("highest version v100 was not returned")
|
t.Fatal("highest version v100 was not returned")
|
||||||
}
|
}
|
||||||
|
|||||||
72
internal/platform/lifecycle/group.go
Normal file
72
internal/platform/lifecycle/group.go
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidGroup = errors.New("invalid lifecycle group")
|
||||||
|
ErrRunnerStopped = errors.New("lifecycle runner stopped")
|
||||||
|
)
|
||||||
|
|
||||||
|
type Runner interface {
|
||||||
|
Run(context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type Group struct {
|
||||||
|
runners []Runner
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewGroup(runners ...Runner) (*Group, error) {
|
||||||
|
if len(runners) == 0 {
|
||||||
|
return nil, ErrInvalidGroup
|
||||||
|
}
|
||||||
|
owned := make([]Runner, len(runners))
|
||||||
|
for index, runner := range runners {
|
||||||
|
if isNilRunner(runner) {
|
||||||
|
return nil, ErrInvalidGroup
|
||||||
|
}
|
||||||
|
owned[index] = runner
|
||||||
|
}
|
||||||
|
return &Group{runners: owned}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (group *Group) Run(ctx context.Context) error {
|
||||||
|
if group == nil || ctx == nil || len(group.runners) == 0 {
|
||||||
|
return ErrInvalidGroup
|
||||||
|
}
|
||||||
|
groupCtx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, len(group.runners))
|
||||||
|
for _, runner := range group.runners {
|
||||||
|
go func() { done <- runner.Run(groupCtx) }()
|
||||||
|
}
|
||||||
|
|
||||||
|
first := <-done
|
||||||
|
cancel()
|
||||||
|
for range len(group.runners) - 1 {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if first == nil {
|
||||||
|
return ErrRunnerStopped
|
||||||
|
}
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
|
||||||
|
func isNilRunner(runner Runner) bool {
|
||||||
|
if runner == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
reflected := reflect.ValueOf(runner)
|
||||||
|
switch reflected.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return reflected.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
87
internal/platform/lifecycle/group_test.go
Normal file
87
internal/platform/lifecycle/group_test.go
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
|
||||||
|
started := make(chan struct{}, 2)
|
||||||
|
cancelled := make(chan struct{}, 1)
|
||||||
|
wantErr := errors.New("runner failed")
|
||||||
|
group, err := NewGroup(
|
||||||
|
runnerFunc(func(context.Context) error {
|
||||||
|
started <- struct{}{}
|
||||||
|
return wantErr
|
||||||
|
}),
|
||||||
|
runnerFunc(func(ctx context.Context) error {
|
||||||
|
started <- struct{}{}
|
||||||
|
<-ctx.Done()
|
||||||
|
cancelled <- struct{}{}
|
||||||
|
return ctx.Err()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGroup(): %v", err)
|
||||||
|
}
|
||||||
|
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
<-started
|
||||||
|
<-started
|
||||||
|
select {
|
||||||
|
case <-cancelled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("sibling runner was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
var exited atomic.Bool
|
||||||
|
group, err := NewGroup(
|
||||||
|
runnerFunc(func(context.Context) error { return errors.New("failed") }),
|
||||||
|
runnerFunc(func(ctx context.Context) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
<-release
|
||||||
|
exited.Store(true)
|
||||||
|
return nil
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewGroup(): %v", err)
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- group.Run(context.Background()) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatal("Run() returned before sibling exited")
|
||||||
|
case <-time.After(20 * time.Millisecond):
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
<-done
|
||||||
|
if !exited.Load() {
|
||||||
|
t.Fatal("sibling exit was not observed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
|
||||||
|
var typedNil *nilRunner
|
||||||
|
for _, runners := range [][]Runner{nil, {typedNil}} {
|
||||||
|
group, err := NewGroup(runners...)
|
||||||
|
if err == nil || group != nil {
|
||||||
|
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runnerFunc func(context.Context) error
|
||||||
|
|
||||||
|
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
|
||||||
|
|
||||||
|
type nilRunner struct{}
|
||||||
|
|
||||||
|
func (*nilRunner) Run(context.Context) error { return nil }
|
||||||
Loading…
Reference in New Issue
Block a user