feat: fence checker facts by lease token
This commit is contained in:
youfak
parent
2bdc1ebda3
commit
9934c659ab
@ -235,6 +235,7 @@ message CheckTask {
|
||||
string password = 14;
|
||||
string credential_version = 15;
|
||||
uint32 max_attempts = 16;
|
||||
string lease_token = 17;
|
||||
}
|
||||
|
||||
enum CheckLevel {
|
||||
@ -260,6 +261,7 @@ message HealthObservation {
|
||||
google.protobuf.Duration latency = 8;
|
||||
string observed_egress_ip = 9;
|
||||
google.protobuf.Timestamp observed_at = 10;
|
||||
string lease_token = 11;
|
||||
}
|
||||
|
||||
message ReportObservationsResponse {
|
||||
|
||||
@ -171,6 +171,9 @@ Routing 决定 AVAILABLE、SUSPECT 或 UNHEALTHY，并更新 Redis 活动池，
|
||||
检查层级、目标 Profile 与领取者一致，Reducer 成功后才确认任务；相同领取者对已确认
|
||||
任务的同一事实可重放，由活动池摘要幂等处理。当前生产配置尚未装配 Redis 共享 broker，
|
||||
因此无共享 broker 的服务会以 `Unavailable` 拒绝任务流，而不下发无租约任务。
|
||||
每次 Claim 还会签发新的不可预测 `lease_token`；Observation 必须回传该值。任务被重新
|
||||
领取后，旧 token 即使拥有相同 `task_id` 和 `checker_id` 也会被拒绝，避免过期实例的
|
||||
迟到事实覆盖新租约结果。
|
||||
|
||||
`proxy-checker` 使用固定大小 worker-pool 执行每个 pull 批次，任务数不超过该请求的
|
||||
`max_in_flight`；每次尝试都受 `deadline` 和 `timeout` 的较小值约束，失败可在同一
|
||||
|
||||
@ -281,7 +281,7 @@ Lua 同一原子边界归并，覆盖首次 CHECKING 失败进入 UNHEALTHY、AV
|
||||
Checker Observation 上报 RPC 已复用既有控制面监听接入 Controller：SPIFFE `checker` 身份、
|
||||
每批上限、配置阈值解析、代理归属查询与 Reducer 均已闭环。`StreamCheckTasks` 现已实现为
|
||||
有界 pull，并通过通用任务 broker 契约完成能力协商、同 Checker 并发窗口、租约到期回收、
|
||||
领取者栅栏和完成后重放；任务凭据仅由认证流在执行期下发。Redis 共享 due-index/租约持久化、
|
||||
领取者加不可预测 lease token 的栅栏和完成后重放；任务凭据仅由认证流在执行期下发。Redis 共享 due-index/租约持久化、
|
||||
生产 broker 已有契约但 Redis 实现尚未完成；`proxy-checker` 独立进程、固定大小 worker-pool、
|
||||
任务期重试/微批上报和 HTTP/HTTPS BASIC/TARGET 探测器已完成并有测试。SOCKS5、EGRESS
|
||||
专用出口探测、Redis due-index/跨副本租约和部署运行态仍未实现，因此本任务保持未完成。
|
||||
|
||||
@ -1801,6 +1801,7 @@ type CheckTask struct {
|
||||
Password string `protobuf:"bytes,14,opt,name=password,proto3" json:"password,omitempty"`
|
||||
CredentialVersion string `protobuf:"bytes,15,opt,name=credential_version,json=credentialVersion,proto3" json:"credential_version,omitempty"`
|
||||
MaxAttempts uint32 `protobuf:"varint,16,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"`
|
||||
LeaseToken string `protobuf:"bytes,17,opt,name=lease_token,json=leaseToken,proto3" json:"lease_token,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@ -1947,6 +1948,13 @@ func (x *CheckTask) GetMaxAttempts() uint32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *CheckTask) GetLeaseToken() string {
|
||||
if x != nil {
|
||||
return x.LeaseToken
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type ObservationBatch struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
CheckerId string `protobuf:"bytes,1,opt,name=checker_id,json=checkerId,proto3" json:"checker_id,omitempty"`
|
||||
@ -2011,6 +2019,7 @@ type HealthObservation struct {
|
||||
Latency *durationpb.Duration `protobuf:"bytes,8,opt,name=latency,proto3" json:"latency,omitempty"`
|
||||
ObservedEgressIp string `protobuf:"bytes,9,opt,name=observed_egress_ip,json=observedEgressIp,proto3" json:"observed_egress_ip,omitempty"`
|
||||
ObservedAt *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=observed_at,json=observedAt,proto3" json:"observed_at,omitempty"`
|
||||
LeaseToken string `protobuf:"bytes,11,opt,name=lease_token,json=leaseToken,proto3" json:"lease_token,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@ -2115,6 +2124,13 @@ func (x *HealthObservation) GetObservedAt() *timestamppb.Timestamp {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *HealthObservation) GetLeaseToken() string {
|
||||
if x != nil {
|
||||
return x.LeaseToken
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type ReportObservationsResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Accepted uint32 `protobuf:"varint,1,opt,name=accepted,proto3" json:"accepted,omitempty"`
|
||||
@ -2324,7 +2340,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\vinstance_id\x18\x02 \x01(\tR\n" +
|
||||
"instanceId\x12\"\n" +
|
||||
"\rmax_in_flight\x18\x03 \x01(\rR\vmaxInFlight\x12Q\n" +
|
||||
"\x10supported_levels\x18\x04 \x03(\x0e2&.proxy_pool.controlplane.v1.CheckLevelR\x0fsupportedLevels\"\xde\x04\n" +
|
||||
"\x10supported_levels\x18\x04 \x03(\x0e2&.proxy_pool.controlplane.v1.CheckLevelR\x0fsupportedLevels\"\xff\x04\n" +
|
||||
"\tCheckTask\x12\x17\n" +
|
||||
"\atask_id\x18\x01 \x01(\tR\x06taskId\x12\x19\n" +
|
||||
"\bproxy_id\x18\x02 \x01(\tR\aproxyId\x12E\n" +
|
||||
@ -2344,11 +2360,13 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\busername\x18\r \x01(\tR\busername\x12\x1a\n" +
|
||||
"\bpassword\x18\x0e \x01(\tR\bpassword\x12-\n" +
|
||||
"\x12credential_version\x18\x0f \x01(\tR\x11credentialVersion\x12!\n" +
|
||||
"\fmax_attempts\x18\x10 \x01(\rR\vmaxAttempts\"\x84\x01\n" +
|
||||
"\fmax_attempts\x18\x10 \x01(\rR\vmaxAttempts\x12\x1f\n" +
|
||||
"\vlease_token\x18\x11 \x01(\tR\n" +
|
||||
"leaseToken\"\x84\x01\n" +
|
||||
"\x10ObservationBatch\x12\x1d\n" +
|
||||
"\n" +
|
||||
"checker_id\x18\x01 \x01(\tR\tcheckerId\x12Q\n" +
|
||||
"\fobservations\x18\x02 \x03(\v2-.proxy_pool.controlplane.v1.HealthObservationR\fobservations\"\xa6\x03\n" +
|
||||
"\fobservations\x18\x02 \x03(\v2-.proxy_pool.controlplane.v1.HealthObservationR\fobservations\"\xc7\x03\n" +
|
||||
"\x11HealthObservation\x12\x17\n" +
|
||||
"\atask_id\x18\x01 \x01(\tR\x06taskId\x12\x19\n" +
|
||||
"\bproxy_id\x18\x02 \x01(\tR\aproxyId\x12<\n" +
|
||||
@ -2362,7 +2380,9 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\x12observed_egress_ip\x18\t \x01(\tR\x10observedEgressIp\x12;\n" +
|
||||
"\vobserved_at\x18\n" +
|
||||
" \x01(\v2\x1a.google.protobuf.TimestampR\n" +
|
||||
"observedAt\"T\n" +
|
||||
"observedAt\x12\x1f\n" +
|
||||
"\vlease_token\x18\v \x01(\tR\n" +
|
||||
"leaseToken\"T\n" +
|
||||
"\x1aReportObservationsResponse\x12\x1a\n" +
|
||||
"\baccepted\x18\x01 \x01(\rR\baccepted\x12\x1a\n" +
|
||||
"\brejected\x18\x02 \x01(\rR\brejected*\xc5\x01\n" +
|
||||
|
||||
@ -212,7 +212,7 @@ func (runner *Runner) executeTask(ctx context.Context, task *controlplanev1.Chec
|
||||
result.FailureClass = probe.FailureProxyRequest
|
||||
}
|
||||
observation := &controlplanev1.HealthObservation{
|
||||
TaskId: task.GetTaskId(), ProxyId: task.GetProxyId(), Level: task.GetLevel(), Success: result.Success,
|
||||
TaskId: task.GetTaskId(), LeaseToken: task.GetLeaseToken(), ProxyId: task.GetProxyId(), Level: task.GetLevel(), Success: result.Success,
|
||||
FailureClass: result.FailureClass, Latency: durationpb.New(result.Latency),
|
||||
ObservedAt: timestamppb.New(runner.options.Now().UTC()),
|
||||
}
|
||||
|
||||
@ -146,7 +146,7 @@ func (handler *GRPCHandler) ReportObservations(
|
||||
if now.IsZero() {
|
||||
return nil, healthGRPCError(ErrInvalidGRPCHandler)
|
||||
}
|
||||
err = handler.options.TaskBroker.AuthorizeObservation(ctx, request.GetCheckerId(), observation, now)
|
||||
err = handler.options.TaskBroker.AuthorizeObservation(ctx, request.GetCheckerId(), item.GetLeaseToken(), observation, now)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = handler.reducer.Apply(ctx, observation)
|
||||
@ -156,7 +156,7 @@ func (handler *GRPCHandler) ReportObservations(
|
||||
if now.IsZero() {
|
||||
return nil, healthGRPCError(ErrInvalidGRPCHandler)
|
||||
}
|
||||
err = handler.options.TaskBroker.CompleteObservation(ctx, request.GetCheckerId(), observation, now)
|
||||
err = handler.options.TaskBroker.CompleteObservation(ctx, request.GetCheckerId(), item.GetLeaseToken(), observation, now)
|
||||
}
|
||||
if err == nil {
|
||||
response.Accepted++
|
||||
@ -226,7 +226,7 @@ func healthGRPCError(err error) error {
|
||||
}
|
||||
|
||||
func wireCheckTask(task LeasedTask, now time.Time) (*controlplanev1.CheckTask, error) {
|
||||
if !validTaskIdentifier(task.TaskID) || !validTaskIdentifier(task.ProxyID) || task.Host == "" ||
|
||||
if !validTaskIdentifier(task.TaskID) || !validTaskIdentifier(task.LeaseToken) || !validTaskIdentifier(task.ProxyID) || task.Host == "" ||
|
||||
strings.TrimSpace(task.Host) != task.Host || task.Port == 0 || task.Attempts <= 0 || task.Attempts > math.MaxUint32 ||
|
||||
!task.Deadline.After(now) || (task.SecretRef == "") != (task.CredentialVersion == "") {
|
||||
return nil, ErrInvalidLeasedTask
|
||||
@ -249,7 +249,7 @@ func wireCheckTask(task LeasedTask, now time.Time) (*controlplanev1.CheckTask, e
|
||||
return nil, ErrInvalidLeasedTask
|
||||
}
|
||||
return &controlplanev1.CheckTask{
|
||||
TaskId: task.TaskID, ProxyId: task.ProxyID, Protocol: protocol, Host: task.Host, Port: uint32(task.Port),
|
||||
TaskId: task.TaskID, LeaseToken: task.LeaseToken, ProxyId: task.ProxyID, Protocol: protocol, Host: task.Host, Port: uint32(task.Port),
|
||||
SecretRef: task.SecretRef, CredentialVersion: task.CredentialVersion, Username: task.Username, Password: task.Password,
|
||||
Level: level, RoutingName: task.RoutingName, TargetUrl: task.TargetURL,
|
||||
Timeout: durationpb.New(task.Deadline.Sub(now)), Attempt: 1, MaxAttempts: uint32(task.Attempts),
|
||||
|
||||
@ -151,17 +151,21 @@ func TestGRPCHandlerStreamsLeasedTasksAndFencesReportedFacts(t *testing.T) {
|
||||
if _, err := stream.Recv(); !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("second Recv() error = %v, want EOF", err)
|
||||
}
|
||||
acceptedObservation := grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second))
|
||||
acceptedObservation.LeaseToken = task.GetLeaseToken()
|
||||
response, err := client.ReportObservations(context.Background(), &controlplanev1.ObservationBatch{
|
||||
CheckerId: "checker-a", Observations: []*controlplanev1.HealthObservation{
|
||||
grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)),
|
||||
acceptedObservation,
|
||||
},
|
||||
})
|
||||
if err != nil || response.GetAccepted() != 1 || len(global.commands) != 1 {
|
||||
t.Fatalf("ReportObservations() = (%+v, %v), global=%d", response, err, len(global.commands))
|
||||
}
|
||||
rejectedObservation := grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second))
|
||||
rejectedObservation.LeaseToken = task.GetLeaseToken()
|
||||
rejected, err := client.ReportObservations(context.Background(), &controlplanev1.ObservationBatch{
|
||||
CheckerId: "checker-b", Observations: []*controlplanev1.HealthObservation{
|
||||
grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)),
|
||||
rejectedObservation,
|
||||
},
|
||||
})
|
||||
if err != nil || rejected.GetRejected() != 1 || len(global.commands) != 1 {
|
||||
|
||||
@ -2,6 +2,7 @@ package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
@ -73,6 +74,7 @@ type TaskClaim struct {
|
||||
// for the duration of a task lease and must not be logged.
|
||||
type LeasedTask struct {
|
||||
TaskID string
|
||||
LeaseToken string
|
||||
ProxyID string
|
||||
Protocol proxyDomain.Scheme
|
||||
Host string
|
||||
@ -98,8 +100,8 @@ func (LeasedTask) Format(state fmt.State, _ rune) {
|
||||
type TaskBroker interface {
|
||||
TaskSink
|
||||
Claim(context.Context, TaskClaim) ([]LeasedTask, error)
|
||||
AuthorizeObservation(context.Context, string, healthDomain.Observation, time.Time) error
|
||||
CompleteObservation(context.Context, string, healthDomain.Observation, time.Time) error
|
||||
AuthorizeObservation(context.Context, string, string, healthDomain.Observation, time.Time) error
|
||||
CompleteObservation(context.Context, string, string, healthDomain.Observation, time.Time) error
|
||||
}
|
||||
|
||||
type MemoryTaskBrokerOptions struct {
|
||||
@ -127,11 +129,13 @@ type brokerTask struct {
|
||||
type taskLease struct {
|
||||
checkerID string
|
||||
instanceID string
|
||||
token string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type taskCompletion struct {
|
||||
checkerID string
|
||||
token string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
@ -218,6 +222,7 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L
|
||||
return []LeasedTask{}, nil
|
||||
}
|
||||
materials := make([]TaskMaterial, len(taskIDs))
|
||||
tokens := make([]string, len(taskIDs))
|
||||
for index, taskID := range taskIDs {
|
||||
material, resolveErr := broker.options.Material.ResolveCheckTask(ctx, broker.tasks[taskID].plan.Candidate)
|
||||
if resolveErr != nil {
|
||||
@ -227,6 +232,11 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L
|
||||
return nil, ErrInvalidLeasedTask
|
||||
}
|
||||
materials[index] = material
|
||||
leaseToken, tokenErr := newLeaseToken()
|
||||
if tokenErr != nil {
|
||||
return nil, tokenErr
|
||||
}
|
||||
tokens[index] = leaseToken
|
||||
}
|
||||
result := make([]LeasedTask, len(taskIDs))
|
||||
for index, taskID := range taskIDs {
|
||||
@ -235,8 +245,9 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L
|
||||
if plan.Deadline.Before(leaseUntil) {
|
||||
leaseUntil = plan.Deadline
|
||||
}
|
||||
broker.leases[taskID] = taskLease{checkerID: claim.CheckerID, instanceID: claim.InstanceID, expiresAt: leaseUntil}
|
||||
result[index] = leasedTask(taskID, plan, materials[index])
|
||||
leaseToken := tokens[index]
|
||||
broker.leases[taskID] = taskLease{checkerID: claim.CheckerID, instanceID: claim.InstanceID, token: leaseToken, expiresAt: leaseUntil}
|
||||
result[index] = leasedTask(taskID, leaseToken, plan, materials[index])
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@ -244,10 +255,11 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L
|
||||
func (broker *MemoryTaskBroker) AuthorizeObservation(
|
||||
ctx context.Context,
|
||||
checkerID string,
|
||||
leaseToken string,
|
||||
observation healthDomain.Observation,
|
||||
now time.Time,
|
||||
) error {
|
||||
if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || now.IsZero() {
|
||||
if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || !validTaskIdentifier(leaseToken) || now.IsZero() {
|
||||
return ErrInvalidTaskBroker
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
@ -259,16 +271,17 @@ func (broker *MemoryTaskBroker) AuthorizeObservation(
|
||||
}
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
return broker.authorizeObservationLocked(checkerID, normalized, now.UTC())
|
||||
return broker.authorizeObservationLocked(checkerID, leaseToken, normalized, now.UTC())
|
||||
}
|
||||
|
||||
func (broker *MemoryTaskBroker) CompleteObservation(
|
||||
ctx context.Context,
|
||||
checkerID string,
|
||||
leaseToken string,
|
||||
observation healthDomain.Observation,
|
||||
now time.Time,
|
||||
) error {
|
||||
if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || now.IsZero() {
|
||||
if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || !validTaskIdentifier(leaseToken) || now.IsZero() {
|
||||
return ErrInvalidTaskBroker
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
@ -281,7 +294,7 @@ func (broker *MemoryTaskBroker) CompleteObservation(
|
||||
now = now.UTC()
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
if err := broker.authorizeObservationLocked(checkerID, normalized, now); err != nil {
|
||||
if err := broker.authorizeObservationLocked(checkerID, leaseToken, normalized, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, complete := broker.finished[normalized.TaskID]; complete {
|
||||
@ -289,12 +302,13 @@ func (broker *MemoryTaskBroker) CompleteObservation(
|
||||
}
|
||||
task := broker.tasks[normalized.TaskID]
|
||||
delete(broker.leases, normalized.TaskID)
|
||||
broker.finished[normalized.TaskID] = taskCompletion{checkerID: checkerID, expiresAt: task.plan.Deadline}
|
||||
broker.finished[normalized.TaskID] = taskCompletion{checkerID: checkerID, token: leaseToken, expiresAt: task.plan.Deadline}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (broker *MemoryTaskBroker) authorizeObservationLocked(
|
||||
checkerID string,
|
||||
leaseToken string,
|
||||
observation healthDomain.Observation,
|
||||
now time.Time,
|
||||
) error {
|
||||
@ -307,7 +321,7 @@ func (broker *MemoryTaskBroker) authorizeObservationLocked(
|
||||
return ErrTaskObservation
|
||||
}
|
||||
if finished, complete := broker.finished[observation.TaskID]; complete {
|
||||
if finished.checkerID != checkerID {
|
||||
if finished.checkerID != checkerID || finished.token != leaseToken {
|
||||
return ErrTaskLeaseNotOwned
|
||||
}
|
||||
return nil
|
||||
@ -316,7 +330,7 @@ func (broker *MemoryTaskBroker) authorizeObservationLocked(
|
||||
if !leased || !lease.expiresAt.After(now) {
|
||||
return ErrTaskLeaseExpired
|
||||
}
|
||||
if lease.checkerID != checkerID {
|
||||
if lease.checkerID != checkerID || lease.token != leaseToken {
|
||||
return ErrTaskLeaseNotOwned
|
||||
}
|
||||
return nil
|
||||
@ -445,15 +459,23 @@ func deterministicTaskID(task PlannedTask) string {
|
||||
return "check_" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func leasedTask(taskID string, plan PlannedTask, material TaskMaterial) LeasedTask {
|
||||
func leasedTask(taskID, leaseToken string, plan PlannedTask, material TaskMaterial) LeasedTask {
|
||||
return LeasedTask{
|
||||
TaskID: taskID, ProxyID: plan.Candidate.ProxyID, Protocol: material.Protocol, Host: material.Host, Port: material.Port,
|
||||
TaskID: taskID, LeaseToken: leaseToken, ProxyID: plan.Candidate.ProxyID, Protocol: material.Protocol, Host: material.Host, Port: material.Port,
|
||||
SecretRef: material.SecretRef, CredentialVersion: material.CredentialVersion, Username: material.Username, Password: material.Password,
|
||||
Level: plan.Candidate.Level, RoutingName: plan.Candidate.RoutingName, TargetURL: plan.Candidate.TargetURL,
|
||||
Deadline: plan.Deadline.UTC(), Attempts: plan.Attempts,
|
||||
}
|
||||
}
|
||||
|
||||
func newLeaseToken() (string, error) {
|
||||
var entropy [24]byte
|
||||
if _, err := rand.Read(entropy[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "lease_" + hex.EncodeToString(entropy[:]), nil
|
||||
}
|
||||
|
||||
func taskMatchesObservation(task PlannedTask, observation healthDomain.Observation) bool {
|
||||
return task.Candidate.ProxyID == observation.ProxyID && task.Candidate.Level == observation.Level &&
|
||||
task.Candidate.RoutingName == observation.RoutingName && task.Candidate.TargetURL == observation.TargetURL
|
||||
|
||||
@ -42,16 +42,16 @@ func TestMemoryTaskBrokerClaimsSupportedTasksAndFencesObservations(t *testing.T)
|
||||
TaskID: claimed[0].TaskID, ProxyID: "basic", Level: healthDomain.LevelBasic,
|
||||
Success: true, Latency: time.Millisecond, ObservedAt: now.Add(time.Second),
|
||||
}
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-b", observation, now.Add(time.Second)); !errors.Is(err, ErrTaskLeaseNotOwned) {
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-b", claimed[0].LeaseToken, observation, now.Add(time.Second)); !errors.Is(err, ErrTaskLeaseNotOwned) {
|
||||
t.Fatalf("AuthorizeObservation(other checker) error = %v, want ErrTaskLeaseNotOwned", err)
|
||||
}
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-a", observation, now.Add(time.Second)); err != nil {
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(time.Second)); err != nil {
|
||||
t.Fatalf("AuthorizeObservation(owner): %v", err)
|
||||
}
|
||||
if err := broker.CompleteObservation(context.Background(), "checker-a", observation, now.Add(time.Second)); err != nil {
|
||||
if err := broker.CompleteObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(time.Second)); err != nil {
|
||||
t.Fatalf("CompleteObservation(): %v", err)
|
||||
}
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-a", observation, now.Add(2*time.Second)); err != nil {
|
||||
if err := broker.AuthorizeObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(2*time.Second)); err != nil {
|
||||
t.Fatalf("AuthorizeObservation(replay): %v", err)
|
||||
}
|
||||
}
|
||||
@ -82,7 +82,7 @@ func TestMemoryTaskBrokerReclaimsExpiredLeaseWithoutExceedingCheckerLimit(t *tes
|
||||
}
|
||||
now = now.Add(2 * time.Second)
|
||||
reclaimed, err := broker.Claim(context.Background(), TaskClaim{CheckerID: "checker-b", InstanceID: "instance-b", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic}})
|
||||
if err != nil || len(reclaimed) != 1 || reclaimed[0].TaskID != first[0].TaskID {
|
||||
if err != nil || len(reclaimed) != 1 || reclaimed[0].TaskID != first[0].TaskID || reclaimed[0].LeaseToken == first[0].LeaseToken {
|
||||
t.Fatalf("reclaimed Claim() = (%+v, %v)", reclaimed, err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user