feat: enforce distributed provider fetch quota
This commit is contained in:
parent
d648ba37e0
commit
40f4b3ffab
@ -19,6 +19,8 @@ import (
|
|||||||
|
|
||||||
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
var namespacePattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||||
|
|
||||||
|
const maximumLuaInteger = int64(1<<53 - 1)
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
HolderID string
|
HolderID string
|
||||||
@ -55,6 +57,7 @@ func (adapter *Adapter) RunLeader(
|
|||||||
) error {
|
) error {
|
||||||
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
||||||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
||||||
|
limits.MaxTotal < 0 || limits.MaxTotal > maximumLuaInteger ||
|
||||||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
@ -217,9 +220,10 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
|
|||||||
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
|
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
|
||||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
|
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
|
||||||
return nil, controllerProvider.ErrInvalidCoordination
|
expected <= 0 || int64(expected) > maximumLuaInteger {
|
||||||
|
return nil, false, controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
operationCtx, cancel := context.WithCancel(ctx)
|
operationCtx, cancel := context.WithCancel(ctx)
|
||||||
stop := context.AfterFunc(session.ctx, cancel)
|
stop := context.AfterFunc(session.ctx, cancel)
|
||||||
@ -229,32 +233,37 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
|||||||
}()
|
}()
|
||||||
permitToken, err := randomToken()
|
permitToken, err := randomToken()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
return nil, false, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
||||||
}
|
}
|
||||||
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
||||||
for operationCtx.Err() == nil {
|
for operationCtx.Err() == nil {
|
||||||
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
||||||
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
||||||
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
||||||
durationMillis(permitTTL),
|
durationMillis(permitTTL), expected, session.limits.MaxTotal,
|
||||||
)
|
)
|
||||||
if scriptErr != nil {
|
if scriptErr != nil {
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
if operationCtx.Err() != nil {
|
if operationCtx.Err() != nil {
|
||||||
return nil, operationCtx.Err()
|
return nil, false, operationCtx.Err()
|
||||||
}
|
}
|
||||||
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
switch reply.Status {
|
switch reply.Status {
|
||||||
case "ok":
|
case "ok":
|
||||||
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
|
return &requestPermit{
|
||||||
|
adapter: session.adapter, keys: session.keys, token: permitToken,
|
||||||
|
settlementTTL: permitTTL,
|
||||||
|
}, true, nil
|
||||||
|
case "quota_exhausted":
|
||||||
|
return nil, false, nil
|
||||||
case "stale":
|
case "stale":
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
case "rate_limited", "at_capacity":
|
case "rate_limited", "at_capacity":
|
||||||
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
||||||
if delay <= 0 {
|
if delay <= 0 {
|
||||||
@ -262,30 +271,42 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
|||||||
}
|
}
|
||||||
if err := wait(operationCtx, delay); err != nil {
|
if err := wait(operationCtx, delay); err != nil {
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return nil, controllerProvider.ErrCoordinationUnavailable
|
return nil, false, controllerProvider.ErrCoordinationUnavailable
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if session.ctx.Err() != nil {
|
if session.ctx.Err() != nil {
|
||||||
return nil, controllerProvider.ErrLeadershipLost
|
return nil, false, controllerProvider.ErrLeadershipLost
|
||||||
}
|
}
|
||||||
return nil, operationCtx.Err()
|
return nil, false, operationCtx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
type requestPermit struct {
|
type requestPermit struct {
|
||||||
adapter *Adapter
|
adapter *Adapter
|
||||||
keys upstreamKeys
|
keys upstreamKeys
|
||||||
token string
|
token string
|
||||||
|
settlementTTL time.Duration
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
done bool
|
done bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (permit *requestPermit) Release(ctx context.Context) error {
|
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
|
||||||
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
|
if fetched < 0 || int64(fetched) > maximumLuaInteger {
|
||||||
|
return controllerProvider.ErrInvalidCoordination
|
||||||
|
}
|
||||||
|
return permit.finish(ctx, "complete_fetch", fetched)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (permit *requestPermit) Cancel(ctx context.Context) error {
|
||||||
|
return permit.finish(ctx, "cancel_fetch", 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (permit *requestPermit) finish(ctx context.Context, operation string, fetched int) error {
|
||||||
|
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" || permit.settlementTTL <= 0 {
|
||||||
return controllerProvider.ErrInvalidCoordination
|
return controllerProvider.ErrInvalidCoordination
|
||||||
}
|
}
|
||||||
permit.mu.Lock()
|
permit.mu.Lock()
|
||||||
@ -293,9 +314,14 @@ func (permit *requestPermit) Release(ctx context.Context) error {
|
|||||||
if permit.done {
|
if permit.done {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
|
for ctx.Err() == nil {
|
||||||
|
reply, err := runScript(ctx, permit.adapter.client, permit.keys,
|
||||||
|
operation, permit.token, fetched, durationMillis(permit.settlementTTL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
if waitErr := wait(ctx, permit.adapter.options.RetryInterval); waitErr != nil {
|
||||||
|
return waitErr
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if reply.Status != "ok" {
|
if reply.Status != "ok" {
|
||||||
return controllerProvider.ErrCoordinationUnavailable
|
return controllerProvider.ErrCoordinationUnavailable
|
||||||
@ -303,6 +329,8 @@ func (permit *requestPermit) Release(ctx context.Context) error {
|
|||||||
permit.done = true
|
permit.done = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
func randomToken() (string, error) {
|
func randomToken() (string, error) {
|
||||||
var token [16]byte
|
var token [16]byte
|
||||||
|
|||||||
@ -89,6 +89,27 @@ func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunLeaderRejectsNegativeFetchQuota(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
||||||
|
t.Cleanup(func() { _ = client.Close() })
|
||||||
|
adapter, err := New(client, Options{
|
||||||
|
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
|
||||||
|
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(): %v", err)
|
||||||
|
}
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: -1,
|
||||||
|
}
|
||||||
|
err = adapter.RunLeader(context.Background(), "provider-a", limits,
|
||||||
|
func(context.Context, controllerProvider.LeaderSession) error { return nil })
|
||||||
|
if !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
|
||||||
|
t.Fatalf("RunLeader() error = %v, want ErrInvalidCoordination", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|||||||
@ -103,36 +103,201 @@ func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T)
|
|||||||
})
|
})
|
||||||
}()
|
}()
|
||||||
session := receiveSession(t, sessions)
|
session := receiveSession(t, sessions)
|
||||||
first, err := session.AcquireFetch(context.Background())
|
first, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
if err != nil {
|
if err != nil || !available {
|
||||||
t.Fatalf("first AcquireFetch(): %v", err)
|
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", first, available, err)
|
||||||
}
|
}
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
secondResult := make(chan permitResultFixture, 1)
|
secondResult := make(chan permitResultFixture, 1)
|
||||||
go func() {
|
go func() {
|
||||||
permit, acquireErr := session.AcquireFetch(context.Background())
|
permit, permitAvailable, acquireErr := session.AcquireFetch(context.Background(), 1)
|
||||||
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
|
secondResult <- permitResultFixture{permit: permit, available: permitAvailable, err: acquireErr}
|
||||||
}()
|
}()
|
||||||
select {
|
select {
|
||||||
case result := <-secondResult:
|
case result := <-secondResult:
|
||||||
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
||||||
case <-time.After(100 * time.Millisecond):
|
case <-time.After(100 * time.Millisecond):
|
||||||
}
|
}
|
||||||
if err := first.Release(context.Background()); err != nil {
|
if err := first.Complete(context.Background(), 1); err != nil {
|
||||||
t.Fatalf("first Release(): %v", err)
|
t.Fatalf("first Complete(): %v", err)
|
||||||
}
|
}
|
||||||
result := receivePermit(t, secondResult)
|
result := receivePermit(t, secondResult)
|
||||||
if result.err != nil || result.permit == nil {
|
if result.err != nil || !result.available || result.permit == nil {
|
||||||
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
|
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", result.permit, result.available, result.err)
|
||||||
}
|
}
|
||||||
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
||||||
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
||||||
}
|
}
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||||
t.Fatalf("second Release(): %v", err)
|
t.Fatalf("second Cancel(): %v", err)
|
||||||
}
|
}
|
||||||
if err := result.permit.Release(context.Background()); err != nil {
|
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||||
t.Fatalf("idempotent second Release(): %v", err)
|
t.Fatalf("idempotent second Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
waitRunner(t, done)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisLeaderSessionPreservesFetchQuotaAcrossFailover(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
firstCtx, cancelFirst := context.WithCancel(context.Background())
|
||||||
|
firstSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
firstDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
firstDone <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||||
|
firstCtx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
firstSessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
firstSession := receiveSession(t, firstSessions)
|
||||||
|
firstPermit, available, err := firstSession.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || firstPermit == nil {
|
||||||
|
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", firstPermit, available, err)
|
||||||
|
}
|
||||||
|
firstFence := firstSession.Fence()
|
||||||
|
cancelFirst()
|
||||||
|
waitRunner(t, firstDone)
|
||||||
|
if err := firstPermit.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("Complete() after leadership loss: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
secondCtx, cancelSecond := context.WithCancel(context.Background())
|
||||||
|
defer cancelSecond()
|
||||||
|
secondSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
secondDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
secondDone <- fixture.coordinator(t, "controller-b").RunLeader(
|
||||||
|
secondCtx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
secondSessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
secondSession := receiveSession(t, secondSessions)
|
||||||
|
secondFence := secondSession.Fence()
|
||||||
|
if secondFence.Generation != firstFence.Generation || secondFence.Epoch <= firstFence.Epoch {
|
||||||
|
t.Fatalf("second fence = %+v, first = %+v", secondFence, firstFence)
|
||||||
|
}
|
||||||
|
|
||||||
|
secondPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || secondPermit == nil {
|
||||||
|
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", secondPermit, available, err)
|
||||||
|
}
|
||||||
|
if err := secondPermit.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("second Complete(): %v", err)
|
||||||
|
}
|
||||||
|
exhaustedPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhaustedPermit != nil {
|
||||||
|
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v), want unavailable", exhaustedPermit, available, err)
|
||||||
|
}
|
||||||
|
cancelSecond()
|
||||||
|
waitRunner(t, secondDone)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisFetchQuotaSettlementIsIdempotentAndCancellationRefundsReservation(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 2, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||||
|
ctx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
sessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}()
|
||||||
|
session := receiveSession(t, sessions)
|
||||||
|
|
||||||
|
cancelled, available, err := session.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || cancelled == nil {
|
||||||
|
t.Fatalf("cancelled AcquireFetch() = (%v, %t, %v)", cancelled, available, err)
|
||||||
|
}
|
||||||
|
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("idempotent Cancel(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
completed, available, err := session.AcquireFetch(context.Background(), 2)
|
||||||
|
if err != nil || !available || completed == nil {
|
||||||
|
t.Fatalf("completed AcquireFetch() = (%v, %t, %v)", completed, available, err)
|
||||||
|
}
|
||||||
|
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("Complete(): %v", err)
|
||||||
|
}
|
||||||
|
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("idempotent Complete(): %v", err)
|
||||||
|
}
|
||||||
|
if err := completed.Cancel(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Cancel() after Complete(): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
last, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || last == nil {
|
||||||
|
t.Fatalf("last AcquireFetch() = (%v, %t, %v)", last, available, err)
|
||||||
|
}
|
||||||
|
if err := last.Complete(context.Background(), 1); err != nil {
|
||||||
|
t.Fatalf("last Complete(): %v", err)
|
||||||
|
}
|
||||||
|
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhausted != nil {
|
||||||
|
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v)", exhausted, available, err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
waitRunner(t, done)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRedisExpiredFetchReservationIsConservativelyCharged(t *testing.T) {
|
||||||
|
fixture := newRedisFixture(t)
|
||||||
|
coordinator, err := New(fixture.client, Options{
|
||||||
|
Namespace: fixture.namespace, HolderID: "controller-a", LeaseTTL: 600 * time.Millisecond,
|
||||||
|
RenewEvery: 150 * time.Millisecond, RetryInterval: 10 * time.Millisecond, PermitGrace: 10 * time.Millisecond,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New(): %v", err)
|
||||||
|
}
|
||||||
|
limits := controllerProvider.CoordinationLimits{
|
||||||
|
MaxInFlight: 1, MaxAttemptDuration: 40 * time.Millisecond, MaxTotal: 1,
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- coordinator.RunLeader(ctx, "provider-a", limits,
|
||||||
|
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||||
|
sessions <- session
|
||||||
|
<-workCtx.Done()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
session := receiveSession(t, sessions)
|
||||||
|
abandoned, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || !available || abandoned == nil {
|
||||||
|
t.Fatalf("abandoned AcquireFetch() = (%v, %t, %v)", abandoned, available, err)
|
||||||
|
}
|
||||||
|
time.Sleep(80 * time.Millisecond)
|
||||||
|
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||||
|
if err != nil || available || exhausted != nil {
|
||||||
|
t.Fatalf("post-expiry AcquireFetch() = (%v, %t, %v), want charged quota", exhausted, available, err)
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
waitRunner(t, done)
|
waitRunner(t, done)
|
||||||
@ -281,6 +446,7 @@ func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResult
|
|||||||
|
|
||||||
type permitResultFixture struct {
|
type permitResultFixture struct {
|
||||||
permit controllerProvider.RequestPermit
|
permit controllerProvider.RequestPermit
|
||||||
|
available bool
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,10 @@ type upstreamKeys struct {
|
|||||||
leader string
|
leader string
|
||||||
next string
|
next string
|
||||||
inflight string
|
inflight string
|
||||||
|
fetchedTotal string
|
||||||
|
pendingTotal string
|
||||||
|
permits string
|
||||||
|
permitExpiry string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
||||||
@ -33,11 +37,18 @@ func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
|||||||
leader: prefix + ":leader",
|
leader: prefix + ":leader",
|
||||||
next: prefix + ":next-request",
|
next: prefix + ":next-request",
|
||||||
inflight: prefix + ":inflight",
|
inflight: prefix + ":inflight",
|
||||||
|
fetchedTotal: prefix + ":fetched-total",
|
||||||
|
pendingTotal: prefix + ":pending-total",
|
||||||
|
permits: prefix + ":permits",
|
||||||
|
permitExpiry: prefix + ":permit-expiry",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (keys upstreamKeys) all() []string {
|
func (keys upstreamKeys) all() []string {
|
||||||
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
|
return []string{
|
||||||
|
keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight,
|
||||||
|
keys.fetchedTotal, keys.pendingTotal, keys.permits, keys.permitExpiry,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func digestParts(values ...string) string {
|
func digestParts(values ...string) string {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
|
local max_safe_integer = 9007199254740991
|
||||||
|
|
||||||
local function now_ms()
|
local function now_ms()
|
||||||
local value = redis.call('TIME')
|
local value = redis.call('TIME')
|
||||||
@ -33,6 +34,63 @@ local function same_leader(value, generation, holder_id, token, epoch)
|
|||||||
value.token == token and value.epoch == tostring(epoch)
|
value.token == token and value.epoch == tostring(epoch)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function read_counter(key)
|
||||||
|
local encoded = redis.call('GET', key)
|
||||||
|
if not encoded then
|
||||||
|
return 0, nil
|
||||||
|
end
|
||||||
|
local value = tonumber(encoded)
|
||||||
|
if not value or value < 0 or value ~= math.floor(value) then
|
||||||
|
return nil, 'invalid'
|
||||||
|
end
|
||||||
|
return value, nil
|
||||||
|
end
|
||||||
|
|
||||||
|
local function read_permit(token)
|
||||||
|
local encoded = redis.call('HGET', KEYS[8], token)
|
||||||
|
if not encoded then
|
||||||
|
return nil, nil
|
||||||
|
end
|
||||||
|
local ok, value = pcall(cjson.decode, encoded)
|
||||||
|
if not ok or type(value) ~= 'table' or type(value.state) ~= 'string' or
|
||||||
|
type(value.expected) ~= 'number' or value.expected <= 0 or
|
||||||
|
value.expected ~= math.floor(value.expected) then
|
||||||
|
return nil, 'invalid'
|
||||||
|
end
|
||||||
|
return value, nil
|
||||||
|
end
|
||||||
|
|
||||||
|
local function cleanup_expired(now)
|
||||||
|
local expired = redis.call('ZRANGEBYSCORE', KEYS[9], '-inf', now, 'LIMIT', 0, 256)
|
||||||
|
if #expired == 0 then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
for _, permit_token in ipairs(expired) do
|
||||||
|
local permit, permit_error = read_permit(permit_token)
|
||||||
|
if permit_error then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
if permit and permit.state == 'reserved' then
|
||||||
|
if pending < permit.expected or fetched > max_safe_integer - permit.expected then
|
||||||
|
return 'invalid'
|
||||||
|
end
|
||||||
|
pending = pending - permit.expected
|
||||||
|
fetched = fetched + permit.expected
|
||||||
|
end
|
||||||
|
redis.call('HDEL', KEYS[8], permit_token)
|
||||||
|
redis.call('ZREM', KEYS[5], permit_token)
|
||||||
|
redis.call('ZREM', KEYS[9], permit_token)
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[6], fetched)
|
||||||
|
redis.call('SET', KEYS[7], pending)
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
|
||||||
if operation == 'acquire_leader' then
|
if operation == 'acquire_leader' then
|
||||||
local generation_candidate = ARGV[2]
|
local generation_candidate = ARGV[2]
|
||||||
local holder_id = ARGV[3]
|
local holder_id = ARGV[3]
|
||||||
@ -41,7 +99,10 @@ if operation == 'acquire_leader' then
|
|||||||
if not lease_ttl or lease_ttl <= 0 then
|
if not lease_ttl or lease_ttl <= 0 then
|
||||||
return reply('invalid', '', 0, 0)
|
return reply('invalid', '', 0, 0)
|
||||||
end
|
end
|
||||||
redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
local created = redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
||||||
|
if created then
|
||||||
|
redis.call('DEL', KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9])
|
||||||
|
end
|
||||||
local generation = redis.call('GET', KEYS[1])
|
local generation = redis.call('GET', KEYS[1])
|
||||||
local current, current_error = read_leader()
|
local current, current_error = read_leader()
|
||||||
if current_error then
|
if current_error then
|
||||||
@ -112,6 +173,13 @@ if operation == 'acquire_fetch' then
|
|||||||
local request_interval = tonumber(ARGV[7])
|
local request_interval = tonumber(ARGV[7])
|
||||||
local max_in_flight = tonumber(ARGV[8])
|
local max_in_flight = tonumber(ARGV[8])
|
||||||
local permit_ttl = tonumber(ARGV[9])
|
local permit_ttl = tonumber(ARGV[9])
|
||||||
|
local expected = tonumber(ARGV[10])
|
||||||
|
local max_total = tonumber(ARGV[11])
|
||||||
|
if not expected or expected <= 0 or expected ~= math.floor(expected) or
|
||||||
|
expected > max_safe_integer or not max_total or max_total < 0 or
|
||||||
|
max_total > max_safe_integer or max_total ~= math.floor(max_total) then
|
||||||
|
return reply('invalid', generation, epoch or 0, 0)
|
||||||
|
end
|
||||||
local current, current_error = read_leader()
|
local current, current_error = read_leader()
|
||||||
if current_error then
|
if current_error then
|
||||||
return reply('unavailable', generation, epoch or 0, 0)
|
return reply('unavailable', generation, epoch or 0, 0)
|
||||||
@ -120,11 +188,27 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('stale', generation, epoch or 0, 0)
|
return reply('stale', generation, epoch or 0, 0)
|
||||||
end
|
end
|
||||||
local now = now_ms()
|
local now = now_ms()
|
||||||
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
|
if cleanup_expired(now) then
|
||||||
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
|
return reply('unavailable', generation, epoch, 0)
|
||||||
if existing then
|
end
|
||||||
|
local existing, existing_error = read_permit(permit_token)
|
||||||
|
if existing_error then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
if existing and existing.state == 'reserved' then
|
||||||
return reply('ok', generation, epoch, 0)
|
return reply('ok', generation, epoch, 0)
|
||||||
end
|
end
|
||||||
|
if existing then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error then
|
||||||
|
return reply('unavailable', generation, epoch, 0)
|
||||||
|
end
|
||||||
|
if max_total > 0 and fetched + pending + expected > max_total then
|
||||||
|
return reply('quota_exhausted', generation, epoch, 0)
|
||||||
|
end
|
||||||
local next_request = redis.call('GET', KEYS[4])
|
local next_request = redis.call('GET', KEYS[4])
|
||||||
if next_request and not tonumber(next_request) then
|
if next_request and not tonumber(next_request) then
|
||||||
return reply('unavailable', generation, epoch, 0)
|
return reply('unavailable', generation, epoch, 0)
|
||||||
@ -141,7 +225,13 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('at_capacity', generation, epoch, wait_ms)
|
return reply('at_capacity', generation, epoch, wait_ms)
|
||||||
end
|
end
|
||||||
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
||||||
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
|
redis.call('ZADD', KEYS[9], now + permit_ttl, permit_token)
|
||||||
|
redis.call('HSET', KEYS[8], permit_token, cjson.encode({
|
||||||
|
version = 1,
|
||||||
|
state = 'reserved',
|
||||||
|
expected = expected
|
||||||
|
}))
|
||||||
|
redis.call('SET', KEYS[7], pending + expected)
|
||||||
if request_interval > 0 then
|
if request_interval > 0 then
|
||||||
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
||||||
else
|
else
|
||||||
@ -150,8 +240,43 @@ if operation == 'acquire_fetch' then
|
|||||||
return reply('ok', generation, epoch, 0)
|
return reply('ok', generation, epoch, 0)
|
||||||
end
|
end
|
||||||
|
|
||||||
if operation == 'release_fetch' then
|
if operation == 'complete_fetch' or operation == 'cancel_fetch' then
|
||||||
redis.call('ZREM', KEYS[5], ARGV[2])
|
local permit_token = ARGV[2]
|
||||||
|
local fetched_count = tonumber(ARGV[3])
|
||||||
|
local settlement_ttl = tonumber(ARGV[4])
|
||||||
|
if not fetched_count or fetched_count < 0 or fetched_count ~= math.floor(fetched_count) or
|
||||||
|
fetched_count > max_safe_integer or not settlement_ttl or settlement_ttl <= 0 then
|
||||||
|
return reply('invalid', '', 0, 0)
|
||||||
|
end
|
||||||
|
local now = now_ms()
|
||||||
|
if cleanup_expired(now) then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
local permit, permit_error = read_permit(permit_token)
|
||||||
|
if permit_error then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
if not permit or permit.state ~= 'reserved' then
|
||||||
|
return reply('ok', '', 0, 0)
|
||||||
|
end
|
||||||
|
local fetched, fetched_error = read_counter(KEYS[6])
|
||||||
|
local pending, pending_error = read_counter(KEYS[7])
|
||||||
|
if fetched_error or pending_error or pending < permit.expected or
|
||||||
|
fetched > max_safe_integer - fetched_count then
|
||||||
|
return reply('unavailable', '', 0, 0)
|
||||||
|
end
|
||||||
|
pending = pending - permit.expected
|
||||||
|
if operation == 'complete_fetch' then
|
||||||
|
fetched = fetched + fetched_count
|
||||||
|
permit.state = 'completed'
|
||||||
|
else
|
||||||
|
permit.state = 'cancelled'
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[6], fetched)
|
||||||
|
redis.call('SET', KEYS[7], pending)
|
||||||
|
redis.call('HSET', KEYS[8], permit_token, cjson.encode(permit))
|
||||||
|
redis.call('ZREM', KEYS[5], permit_token)
|
||||||
|
redis.call('ZADD', KEYS[9], now + settlement_ttl, permit_token)
|
||||||
return reply('ok', '', 0, 0)
|
return reply('ok', '', 0, 0)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@ -20,48 +20,38 @@ var (
|
|||||||
type FetchBudgetConfig struct {
|
type FetchBudgetConfig struct {
|
||||||
UpstreamID string
|
UpstreamID string
|
||||||
MaxSize int
|
MaxSize int
|
||||||
MaxTotal int64
|
|
||||||
ExpectedPerFetch int
|
ExpectedPerFetch int
|
||||||
Managed int
|
Managed int
|
||||||
FetchedTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type FetchBudgetSnapshot struct {
|
type FetchBudgetSnapshot struct {
|
||||||
Managed int
|
Managed int
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FetchBudget owns both current-inventory and cumulative-fetch accounting for
|
// FetchBudget owns current-inventory accounting for one upstream. Reserving
|
||||||
// one upstream. Reserving the expected response before I/O closes the race
|
// the expected response before I/O closes the race between concurrent calls;
|
||||||
// between concurrent provider calls.
|
// distributed cumulative quota belongs to the Provider coordination permit.
|
||||||
type FetchBudget struct {
|
type FetchBudget struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
|
||||||
upstreamID string
|
upstreamID string
|
||||||
maxSize int
|
maxSize int
|
||||||
maxTotal int64
|
|
||||||
expected int
|
expected int
|
||||||
usage FetchBudgetSnapshot
|
usage FetchBudgetSnapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
|
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
|
||||||
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
|
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
|
||||||
config.ExpectedPerFetch > config.MaxSize || config.MaxTotal < 0 ||
|
config.ExpectedPerFetch > config.MaxSize || config.Managed < 0 {
|
||||||
config.Managed < 0 || config.FetchedTotal < 0 {
|
|
||||||
return nil, ErrInvalidFetchBudget
|
|
||||||
}
|
|
||||||
if config.MaxTotal > 0 && int64(config.ExpectedPerFetch) > config.MaxTotal {
|
|
||||||
return nil, ErrInvalidFetchBudget
|
return nil, ErrInvalidFetchBudget
|
||||||
}
|
}
|
||||||
return &FetchBudget{
|
return &FetchBudget{
|
||||||
upstreamID: config.UpstreamID,
|
upstreamID: config.UpstreamID,
|
||||||
maxSize: config.MaxSize,
|
maxSize: config.MaxSize,
|
||||||
maxTotal: config.MaxTotal,
|
|
||||||
expected: config.ExpectedPerFetch,
|
expected: config.ExpectedPerFetch,
|
||||||
usage: FetchBudgetSnapshot{
|
usage: FetchBudgetSnapshot{
|
||||||
Managed: config.Managed,
|
Managed: config.Managed,
|
||||||
FetchedTotal: config.FetchedTotal,
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@ -95,14 +85,7 @@ func (b *FetchBudget) FetchAllowance() int {
|
|||||||
|
|
||||||
func (b *FetchBudget) canReserveLocked() bool {
|
func (b *FetchBudget) canReserveLocked() bool {
|
||||||
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
|
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
|
||||||
if poolRoom < b.expected {
|
return poolRoom >= b.expected
|
||||||
return false
|
|
||||||
}
|
|
||||||
if b.maxTotal > 0 {
|
|
||||||
totalRoom := b.maxTotal - b.usage.FetchedTotal - int64(b.usage.PendingExpected)
|
|
||||||
return totalRoom >= int64(b.expected)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
||||||
@ -115,8 +98,8 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SynchronizeManaged replaces the local current-inventory count with the
|
// SynchronizeManaged replaces the local current-inventory count with the
|
||||||
// authoritative activity-store observation. Pending requests and cumulative
|
// authoritative activity-store observation. Pending reservations remain owned
|
||||||
// fetch usage remain owned by this budget.
|
// by this budget.
|
||||||
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
||||||
if b == nil || managed < 0 {
|
if b == nil || managed < 0 {
|
||||||
return ErrInvalidManagedSynchronization
|
return ErrInvalidManagedSynchronization
|
||||||
@ -158,11 +141,11 @@ func (p *fetchPermit) Expected() int {
|
|||||||
return p.expected
|
return p.expected
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fetchPermit) Complete(fetched, retained int) error {
|
func (p *fetchPermit) Complete(retained int) error {
|
||||||
if p == nil || p.budget == nil {
|
if p == nil || p.budget == nil {
|
||||||
return ErrFetchPermitFinished
|
return ErrFetchPermitFinished
|
||||||
}
|
}
|
||||||
if fetched < 0 || retained < 0 || retained > fetched || retained > p.expected {
|
if retained < 0 || retained > p.expected {
|
||||||
return ErrInvalidFetchCompletion
|
return ErrInvalidFetchCompletion
|
||||||
}
|
}
|
||||||
p.budget.mu.Lock()
|
p.budget.mu.Lock()
|
||||||
@ -173,7 +156,6 @@ func (p *fetchPermit) Complete(fetched, retained int) error {
|
|||||||
p.finished = true
|
p.finished = true
|
||||||
p.budget.usage.PendingExpected -= p.expected
|
p.budget.usage.PendingExpected -= p.expected
|
||||||
p.budget.usage.Managed += retained
|
p.budget.usage.Managed += retained
|
||||||
p.budget.usage.FetchedTotal += int64(fetched)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -6,14 +6,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
func TestFetchBudgetReservesExpectedPoolCapacity(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a",
|
UpstreamID: "provider-a",
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 20,
|
|
||||||
ExpectedPerFetch: 4,
|
ExpectedPerFetch: 4,
|
||||||
Managed: 2,
|
Managed: 2,
|
||||||
FetchedTotal: 3,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -31,21 +29,21 @@ func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
|||||||
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := first.Complete(4, 3); err != nil {
|
if err := first.Complete(3); err != nil {
|
||||||
t.Fatalf("first.Complete(): %v", err)
|
t.Fatalf("first.Complete(): %v", err)
|
||||||
}
|
}
|
||||||
if err := second.Cancel(); err != nil {
|
if err := second.Cancel(); err != nil {
|
||||||
t.Fatalf("second.Cancel(): %v", err)
|
t.Fatalf("second.Cancel(): %v", err)
|
||||||
}
|
}
|
||||||
usage := budget.Snapshot()
|
usage := budget.Snapshot()
|
||||||
if usage.Managed != 5 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
if usage.Managed != 5 || usage.PendingExpected != 0 {
|
||||||
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0 fetched=7", usage)
|
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0", usage)
|
||||||
}
|
}
|
||||||
if err := budget.ReleaseManaged(2); err != nil {
|
if err := budget.ReleaseManaged(2); err != nil {
|
||||||
t.Fatalf("ReleaseManaged(): %v", err)
|
t.Fatalf("ReleaseManaged(): %v", err)
|
||||||
}
|
}
|
||||||
if usage := budget.Snapshot(); usage.Managed != 3 || usage.FetchedTotal != 7 {
|
if usage := budget.Snapshot(); usage.Managed != 3 {
|
||||||
t.Fatalf("Snapshot() after release = %+v, want managed=3 fetched=7", usage)
|
t.Fatalf("Snapshot() after release = %+v, want managed=3", usage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -63,7 +61,7 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
|
|||||||
|
|
||||||
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
|
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -77,8 +75,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
|||||||
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
||||||
}
|
}
|
||||||
usage := budget.Snapshot()
|
usage := budget.Snapshot()
|
||||||
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
|
if usage.Managed != 3 || usage.PendingExpected != 2 {
|
||||||
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
|
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2", usage)
|
||||||
}
|
}
|
||||||
if err := permit.Cancel(); err != nil {
|
if err := permit.Cancel(); err != nil {
|
||||||
t.Fatalf("Cancel(): %v", err)
|
t.Fatalf("Cancel(): %v", err)
|
||||||
@ -86,8 +84,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
|||||||
if err := budget.SynchronizeManaged(1); err != nil {
|
if err := budget.SynchronizeManaged(1); err != nil {
|
||||||
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
||||||
}
|
}
|
||||||
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 {
|
||||||
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
|
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0", usage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -103,36 +101,16 @@ func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
|
func TestFetchBudgetRequiresWholeExpectedBatchToFitPool(t *testing.T) {
|
||||||
tests := []struct {
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
name string
|
|
||||||
config FetchBudgetConfig
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "pool size",
|
|
||||||
config: FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
||||||
},
|
})
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cumulative total",
|
|
||||||
config: FetchBudgetConfig{
|
|
||||||
UpstreamID: "a", MaxSize: 10, MaxTotal: 5, ExpectedPerFetch: 4, FetchedTotal: 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
budget, err := NewFetchBudget(tt.config)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
}
|
}
|
||||||
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetchBudgetConcurrentReservationsNeverExceedMaxSize(t *testing.T) {
|
func TestFetchBudgetConcurrentReservationsNeverExceedMaxSize(t *testing.T) {
|
||||||
@ -185,7 +163,7 @@ func TestFetchPermitRejectsDoubleFinishAndInvalidCounts(t *testing.T) {
|
|||||||
if err != nil || !ok {
|
if err != nil || !ok {
|
||||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
||||||
}
|
}
|
||||||
if err := permit.Complete(1, 2); !errors.Is(err, ErrInvalidFetchCompletion) {
|
if err := permit.Complete(3); !errors.Is(err, ErrInvalidFetchCompletion) {
|
||||||
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
|
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
|
||||||
}
|
}
|
||||||
if err := permit.Cancel(); err != nil {
|
if err := permit.Cancel(); err != nil {
|
||||||
|
|||||||
@ -26,7 +26,6 @@ type FetchNotifier interface {
|
|||||||
type ReconcileDecision struct {
|
type ReconcileDecision struct {
|
||||||
AvailableSlots int64
|
AvailableSlots int64
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
FetchAllowance int
|
FetchAllowance int
|
||||||
EffectiveSlots int64
|
EffectiveSlots int64
|
||||||
Triggered bool
|
Triggered bool
|
||||||
@ -82,7 +81,6 @@ func (r *Reconciler) reconcileSnapshot(inventory InventorySnapshot, synchronizeM
|
|||||||
decision := ReconcileDecision{
|
decision := ReconcileDecision{
|
||||||
AvailableSlots: inventory.AvailableSlots,
|
AvailableSlots: inventory.AvailableSlots,
|
||||||
PendingExpected: usage.PendingExpected,
|
PendingExpected: usage.PendingExpected,
|
||||||
FetchedTotal: usage.FetchedTotal,
|
|
||||||
FetchAllowance: r.budget.FetchAllowance(),
|
FetchAllowance: r.budget.FetchAllowance(),
|
||||||
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
|
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
|
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a", MaxSize: 10, MaxTotal: 20, ExpectedPerFetch: 2,
|
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -32,7 +32,6 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
|||||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
|
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
|
||||||
}},
|
}},
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 20,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := reconciler.Reconcile(now, inventory)
|
decision := reconciler.Reconcile(now, inventory)
|
||||||
@ -44,9 +43,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
func TestPoolReconcilerUsesPendingPoolReservation(t *testing.T) {
|
||||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||||
UpstreamID: "provider-a", MaxSize: 2, MaxTotal: 2, ExpectedPerFetch: 2,
|
UpstreamID: "provider-a", MaxSize: 2, ExpectedPerFetch: 2,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewFetchBudget(): %v", err)
|
t.Fatalf("NewFetchBudget(): %v", err)
|
||||||
@ -65,7 +64,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2, MaxTotal: 2})
|
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2})
|
||||||
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
|
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
|
||||||
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
|
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
|
||||||
}
|
}
|
||||||
@ -133,7 +132,7 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
|
|||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
||||||
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
||||||
}
|
}
|
||||||
if err := permit.Complete(2, 1); err != nil {
|
if err := permit.Complete(1); err != nil {
|
||||||
t.Fatalf("Complete(): %v", err)
|
t.Fatalf("Complete(): %v", err)
|
||||||
}
|
}
|
||||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
||||||
|
|||||||
@ -17,6 +17,7 @@ type CoordinationLimits struct {
|
|||||||
RequestInterval time.Duration
|
RequestInterval time.Duration
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
MaxAttemptDuration time.Duration
|
MaxAttemptDuration time.Duration
|
||||||
|
MaxTotal int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type Fence struct {
|
type Fence struct {
|
||||||
@ -32,10 +33,14 @@ type Coordinator interface {
|
|||||||
|
|
||||||
type LeaderSession interface {
|
type LeaderSession interface {
|
||||||
Fence() Fence
|
Fence() Fence
|
||||||
AcquireFetch(context.Context) (RequestPermit, error)
|
AcquireFetch(context.Context, int) (RequestPermit, bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type RequestPermit interface {
|
type RequestPermit interface {
|
||||||
// Release is idempotent. A failed release expires automatically in storage.
|
// Complete atomically releases in-flight capacity and charges the actual
|
||||||
Release(context.Context) error
|
// successful candidate count. Repeated calls are idempotent.
|
||||||
|
Complete(context.Context, int) error
|
||||||
|
// Cancel releases a reservation that is known not to have consumed Provider
|
||||||
|
// quota. A crashed or abandoned reservation is conservatively charged.
|
||||||
|
Cancel(context.Context) error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -112,7 +112,10 @@ func (r *Reconciler) Notify() {
|
|||||||
r.signal.Notify()
|
r.signal.Notify()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) Run(ctx context.Context) error {
|
func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error {
|
||||||
|
if ctx == nil || session == nil {
|
||||||
|
return ErrInvalidCoordination
|
||||||
|
}
|
||||||
var workers sync.WaitGroup
|
var workers sync.WaitGroup
|
||||||
defer workers.Wait()
|
defer workers.Wait()
|
||||||
for {
|
for {
|
||||||
@ -131,14 +134,14 @@ func (r *Reconciler) Run(ctx context.Context) error {
|
|||||||
go func() {
|
go func() {
|
||||||
defer workers.Done()
|
defer workers.Done()
|
||||||
defer func() { <-r.inFlight }()
|
defer func() { <-r.inFlight }()
|
||||||
r.reconcile(ctx)
|
r.reconcile(ctx, session)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) reconcile(ctx context.Context) {
|
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
||||||
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
||||||
response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
|
response, result, retryable, ok := r.fetchAttempt(ctx, session, attempt)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -157,7 +160,11 @@ func (r *Reconciler) reconcile(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
|
func (r *Reconciler) fetchAttempt(
|
||||||
|
ctx context.Context,
|
||||||
|
session LeaderSession,
|
||||||
|
attempt int,
|
||||||
|
) (FetchResponse, Result, bool, bool) {
|
||||||
if err := r.waitForRequestSlot(ctx); err != nil {
|
if err := r.waitForRequestSlot(ctx); err != nil {
|
||||||
return FetchResponse{}, Result{}, false, false
|
return FetchResponse{}, Result{}, false, false
|
||||||
}
|
}
|
||||||
@ -191,6 +198,35 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
_ = permit.Cancel()
|
_ = permit.Cancel()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
|
||||||
|
return FetchResponse{}, Result{}, false, false
|
||||||
|
}
|
||||||
|
return FetchResponse{}, Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("acquire distributed fetch capacity: %w", err),
|
||||||
|
Attempt: attempt,
|
||||||
|
}, false, true
|
||||||
|
}
|
||||||
|
if !available {
|
||||||
|
return FetchResponse{}, Result{}, false, false
|
||||||
|
}
|
||||||
|
if requestPermit == nil {
|
||||||
|
return FetchResponse{}, Result{
|
||||||
|
UpstreamID: r.config.UpstreamID,
|
||||||
|
Class: upstream.FetchError,
|
||||||
|
Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"),
|
||||||
|
Attempt: attempt,
|
||||||
|
}, false, true
|
||||||
|
}
|
||||||
|
requestSettlementAttempted := false
|
||||||
|
defer func() {
|
||||||
|
if !requestSettlementAttempted {
|
||||||
|
_ = r.settleRequestPermit(requestPermit, false, 0)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
callCtx := ctx
|
callCtx := ctx
|
||||||
cancel := func() {}
|
cancel := func() {}
|
||||||
@ -200,13 +236,22 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
||||||
|
|
||||||
var parseErr, candidateErr, capacityErr error
|
var parseErr, candidateErr, coordinationErr, capacityErr error
|
||||||
var validCount, newCount int
|
var validCount, newCount int
|
||||||
if callErr == nil {
|
if callErr != nil {
|
||||||
|
requestSettlementAttempted = true
|
||||||
|
coordinationErr = r.settleRequestPermit(requestPermit, false, 0)
|
||||||
|
} else {
|
||||||
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
||||||
parseErr = err
|
parseErr = err
|
||||||
validCount = len(candidates)
|
validCount = len(candidates)
|
||||||
if parseErr == nil && validCount > 0 {
|
charged := validCount
|
||||||
|
if parseErr != nil {
|
||||||
|
charged = permit.Expected()
|
||||||
|
}
|
||||||
|
requestSettlementAttempted = true
|
||||||
|
coordinationErr = r.settleRequestPermit(requestPermit, true, charged)
|
||||||
|
if parseErr == nil && coordinationErr == nil && validCount > 0 {
|
||||||
retained := candidates
|
retained := candidates
|
||||||
if expected := permit.Expected(); expected < len(retained) {
|
if expected := permit.Expected(); expected < len(retained) {
|
||||||
retained = retained[:expected]
|
retained = retained[:expected]
|
||||||
@ -222,12 +267,17 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
newCount = upserted.Inserted
|
newCount = upserted.Inserted
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if callErr == nil && parseErr == nil && candidateErr == nil {
|
if callErr == nil && parseErr == nil && candidateErr == nil && coordinationErr == nil {
|
||||||
capacityErr = permit.Complete(validCount, newCount)
|
capacityErr = permit.Complete(newCount)
|
||||||
permitFinished = capacityErr == nil
|
permitFinished = capacityErr == nil
|
||||||
}
|
}
|
||||||
resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
|
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr, capacityErr)
|
||||||
class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
|
class := upstream.ClassifyFetchResult(
|
||||||
|
callErr,
|
||||||
|
errors.Join(parseErr, candidateErr, coordinationErr, capacityErr),
|
||||||
|
validCount,
|
||||||
|
newCount,
|
||||||
|
)
|
||||||
return response, Result{
|
return response, Result{
|
||||||
UpstreamID: r.config.UpstreamID,
|
UpstreamID: r.config.UpstreamID,
|
||||||
Class: class,
|
Class: class,
|
||||||
@ -238,6 +288,16 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
|||||||
}, isRetryable(callErr) || parseErr != nil, true
|
}, isRetryable(callErr) || parseErr != nil, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
|
||||||
|
timeout := min(r.config.Timeout, 5*time.Second)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
if complete {
|
||||||
|
return permit.Complete(ctx, fetched)
|
||||||
|
}
|
||||||
|
return permit.Cancel(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
func isRetryable(err error) bool {
|
func isRetryable(err error) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@ -89,7 +89,7 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case got := <-result:
|
case got := <-result:
|
||||||
@ -130,7 +130,7 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
@ -183,7 +183,7 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
|
|
||||||
wantClasses := []upstream.FetchClass{
|
wantClasses := []upstream.FetchClass{
|
||||||
@ -291,7 +291,7 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
<-results
|
<-results
|
||||||
@ -332,7 +332,7 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-results
|
<-results
|
||||||
<-results
|
<-results
|
||||||
@ -417,7 +417,7 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
|
|
||||||
@ -473,8 +473,8 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 2)
|
done := make(chan error, 2)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
|
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
@ -519,7 +519,7 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
<-started
|
<-started
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
@ -545,6 +545,7 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|||||||
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
||||||
sleeper := &fakeSleeper{clock: clock}
|
sleeper := &fakeSleeper{clock: clock}
|
||||||
results := make(chan Result, 3)
|
results := make(chan Result, 3)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
var calls atomic.Int64
|
var calls atomic.Int64
|
||||||
ports := successfulPorts(func() { calls.Add(1) }, results)
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||||
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||||
@ -562,13 +563,20 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := runSingleReconcile(t, reconciler, results)
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||||
|
func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
},
|
||||||
|
))
|
||||||
if result.Class != upstream.FetchError {
|
if result.Class != upstream.FetchError {
|
||||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
}
|
}
|
||||||
if got := calls.Load(); got != 1 {
|
if got := calls.Load(); got != 1 {
|
||||||
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
||||||
}
|
}
|
||||||
|
if got := <-globalCompleted; got != 1 {
|
||||||
|
t.Fatalf("global charged count = %d, want fetched=1", got)
|
||||||
|
}
|
||||||
if got := sleeper.Durations(); len(got) != 0 {
|
if got := sleeper.Durations(); len(got) != 0 {
|
||||||
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
||||||
}
|
}
|
||||||
@ -623,7 +631,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
time.Sleep(20 * time.Millisecond)
|
time.Sleep(20 * time.Millisecond)
|
||||||
cancel()
|
cancel()
|
||||||
@ -640,9 +648,112 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T) {
|
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
|
||||||
results := make(chan Result, 1)
|
results := make(chan Result, 1)
|
||||||
completed := make(chan fetchCompletion, 1)
|
var calls atomic.Int64
|
||||||
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||||
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
|
return &recordingFetchPermit{expected: 2}, true, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
if expected != 2 {
|
||||||
|
t.Errorf("distributed expected = %d, want 2", expected)
|
||||||
|
}
|
||||||
|
return nil, false, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||||
|
reconciler.Notify()
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("RunLeader(): %v", err)
|
||||||
|
}
|
||||||
|
if got := calls.Load(); got != 0 {
|
||||||
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case result := <-results:
|
||||||
|
t.Fatalf("unexpected fetch result: %+v", result)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
|
localCancelled := make(chan struct{}, 1)
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
|
return nil, errors.New("invalid provider payload")
|
||||||
|
})
|
||||||
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
|
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
result := runSingleReconcile(t, reconciler, results, session)
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
|
}
|
||||||
|
if got := <-globalCompleted; got != 2 {
|
||||||
|
t.Fatalf("global charged count = %d, want expected=2", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-localCancelled:
|
||||||
|
default:
|
||||||
|
t.Fatal("local pool reservation was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
globalCancelled := make(chan struct{}, 1)
|
||||||
|
ports := successfulPorts(func() {}, results)
|
||||||
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||||
|
return FetchResponse{}, errors.New("provider connection failed")
|
||||||
|
})
|
||||||
|
reconciler, err := NewReconciler(Config{
|
||||||
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
|
}, ports)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
|
}
|
||||||
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{cancelled: globalCancelled}, true, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
result := runSingleReconcile(t, reconciler, results, session)
|
||||||
|
if result.Class != upstream.FetchError {
|
||||||
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-globalCancelled:
|
||||||
|
default:
|
||||||
|
t.Fatal("distributed request reservation was not cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
|
||||||
|
results := make(chan Result, 1)
|
||||||
|
localCompleted := make(chan fetchCompletion, 1)
|
||||||
|
globalCompleted := make(chan int, 1)
|
||||||
ports := successfulPorts(func() {}, results)
|
ports := successfulPorts(func() {}, results)
|
||||||
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||||
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
||||||
@ -654,7 +765,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
|||||||
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
||||||
})
|
})
|
||||||
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||||
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
|
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
|
||||||
})
|
})
|
||||||
reconciler, err := NewReconciler(Config{
|
reconciler, err := NewReconciler(Config{
|
||||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||||
@ -663,12 +774,22 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
|||||||
t.Fatalf("NewReconciler(): %v", err)
|
t.Fatalf("NewReconciler(): %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := runSingleReconcile(t, reconciler, results)
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||||
|
func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
if expected != 2 {
|
||||||
|
t.Errorf("distributed expected = %d, want 2", expected)
|
||||||
|
}
|
||||||
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||||
|
},
|
||||||
|
))
|
||||||
if result.ValidCount != 3 || result.NewCount != 1 {
|
if result.ValidCount != 3 || result.NewCount != 1 {
|
||||||
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
||||||
}
|
}
|
||||||
if got := <-completed; got.fetched != 3 || got.retained != 1 {
|
if got := <-globalCompleted; got != 3 {
|
||||||
t.Fatalf("fetch completion = %+v, want fetched=3 retained=1", got)
|
t.Fatalf("global fetch completion = %d, want fetched=3", got)
|
||||||
|
}
|
||||||
|
if got := <-localCompleted; got.retained != 1 {
|
||||||
|
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -699,7 +820,7 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
first, second := <-results, <-results
|
first, second := <-results, <-results
|
||||||
cancel()
|
cancel()
|
||||||
@ -744,11 +865,23 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func runSingleReconcile(t *testing.T, reconciler *Reconciler, results <-chan Result) Result {
|
func runSingleReconcile(
|
||||||
|
t *testing.T,
|
||||||
|
reconciler *Reconciler,
|
||||||
|
results <-chan Result,
|
||||||
|
sessions ...LeaderSession,
|
||||||
|
) Result {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
session := LeaderSession(unlimitedLeaderSession{})
|
||||||
|
if len(sessions) > 1 {
|
||||||
|
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
|
||||||
|
}
|
||||||
|
if len(sessions) == 1 {
|
||||||
|
session = sessions[0]
|
||||||
|
}
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- reconciler.Run(ctx) }()
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||||
reconciler.Notify()
|
reconciler.Notify()
|
||||||
var result Result
|
var result Result
|
||||||
select {
|
select {
|
||||||
@ -867,26 +1000,66 @@ func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool,
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fetchCompletion struct {
|
type fetchCompletion struct {
|
||||||
fetched int
|
|
||||||
retained int
|
retained int
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingFetchPermit struct {
|
type recordingFetchPermit struct {
|
||||||
expected int
|
expected int
|
||||||
completed chan<- fetchCompletion
|
completed chan<- fetchCompletion
|
||||||
|
cancelled chan<- struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
||||||
|
|
||||||
func (p *recordingFetchPermit) Complete(fetched, retained int) error {
|
func (p *recordingFetchPermit) Complete(retained int) error {
|
||||||
if p.completed != nil {
|
if p.completed != nil {
|
||||||
p.completed <- fetchCompletion{fetched: fetched, retained: retained}
|
p.completed <- fetchCompletion{retained: retained}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*recordingFetchPermit) Cancel() error { return nil }
|
func (p *recordingFetchPermit) Cancel() error {
|
||||||
|
if p.cancelled != nil {
|
||||||
|
p.cancelled <- struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type resultRecorderFunc func(Result)
|
type resultRecorderFunc func(Result)
|
||||||
|
|
||||||
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
||||||
|
|
||||||
|
type unlimitedLeaderSession struct{}
|
||||||
|
|
||||||
|
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||||
|
|
||||||
|
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
|
||||||
|
return &recordingRequestPermit{}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
|
||||||
|
|
||||||
|
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||||
|
|
||||||
|
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
|
||||||
|
return f(ctx, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingRequestPermit struct {
|
||||||
|
completed chan<- int
|
||||||
|
cancelled chan<- struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
|
||||||
|
if p.completed != nil {
|
||||||
|
p.completed <- fetched
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingRequestPermit) Cancel(context.Context) error {
|
||||||
|
if p.cancelled != nil {
|
||||||
|
p.cancelled <- struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@ -8,6 +8,6 @@ type FetchCapacity interface {
|
|||||||
|
|
||||||
type FetchPermit interface {
|
type FetchPermit interface {
|
||||||
Expected() int
|
Expected() int
|
||||||
Complete(fetched, retained int) error
|
Complete(retained int) error
|
||||||
Cancel() error
|
Cancel() error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,9 +17,7 @@ type ProxyCapacity struct {
|
|||||||
type Inventory struct {
|
type Inventory struct {
|
||||||
Proxies []ProxyCapacity
|
Proxies []ProxyCapacity
|
||||||
PendingExpected int
|
PendingExpected int
|
||||||
FetchedTotal int64
|
|
||||||
MaxSize int
|
MaxSize int
|
||||||
MaxTotal int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
|
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
|
||||||
@ -55,15 +53,6 @@ func (i Inventory) FetchAllowance(requested int) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
|
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
|
||||||
if i.MaxTotal > 0 {
|
|
||||||
remaining := i.MaxTotal - i.FetchedTotal
|
|
||||||
if remaining <= 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
if int64(allowed) > remaining {
|
|
||||||
allowed = int(remaining)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return allowed
|
return allowed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -21,7 +21,7 @@ func TestInventoryAvailableSlotsUsesOnlyAllocatableCapacity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
func TestInventoryFetchAllowanceUsesManagedPoolCapacity(t *testing.T) {
|
||||||
inventory := Inventory{
|
inventory := Inventory{
|
||||||
Proxies: []ProxyCapacity{
|
Proxies: []ProxyCapacity{
|
||||||
{State: proxyDomain.StateFetched},
|
{State: proxyDomain.StateFetched},
|
||||||
@ -32,20 +32,13 @@ func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
|||||||
{State: proxyDomain.StateExtracted},
|
{State: proxyDomain.StateExtracted},
|
||||||
},
|
},
|
||||||
PendingExpected: 2,
|
PendingExpected: 2,
|
||||||
FetchedTotal: 98,
|
|
||||||
MaxSize: 10,
|
MaxSize: 10,
|
||||||
MaxTotal: 100,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := inventory.ManagedCount(); got != 7 {
|
if got := inventory.ManagedCount(); got != 7 {
|
||||||
t.Fatalf("ManagedCount() = %d, want 7", got)
|
t.Fatalf("ManagedCount() = %d, want 7", got)
|
||||||
}
|
}
|
||||||
if got := inventory.FetchAllowance(10); got != 2 {
|
|
||||||
t.Fatalf("FetchAllowance() = %d, want 2 from cumulative quota", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
inventory.MaxTotal = 0
|
|
||||||
if got := inventory.FetchAllowance(10); got != 3 {
|
if got := inventory.FetchAllowance(10); got != 3 {
|
||||||
t.Fatalf("FetchAllowance() with unlimited cumulative quota = %d, want 3 from pool size", got)
|
t.Fatalf("FetchAllowance() = %d, want 3 from pool size", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user