diff --git a/api/proto/controlplane/v1/controlplane.proto b/api/proto/controlplane/v1/controlplane.proto index 0ad8edd..1e8f963 100644 --- a/api/proto/controlplane/v1/controlplane.proto +++ b/api/proto/controlplane/v1/controlplane.proto @@ -235,6 +235,7 @@ message CheckTask { string password = 14; string credential_version = 15; uint32 max_attempts = 16; + string lease_token = 17; } enum CheckLevel { @@ -260,6 +261,7 @@ message HealthObservation { google.protobuf.Duration latency = 8; string observed_egress_ip = 9; google.protobuf.Timestamp observed_at = 10; + string lease_token = 11; } message ReportObservationsResponse { diff --git a/docs/api/control-plane.md b/docs/api/control-plane.md index a33c83e..7f8da76 100644 --- a/docs/api/control-plane.md +++ b/docs/api/control-plane.md @@ -171,6 +171,9 @@ Routing 决定 AVAILABLE、SUSPECT 或 UNHEALTHY,并更新 Redis 活动池, 检查层级、目标 Profile 与领取者一致,Reducer 成功后才确认任务;相同领取者对已确认 任务的同一事实可重放,由活动池摘要幂等处理。当前生产配置尚未装配 Redis 共享 broker, 因此无共享 broker 的服务会以 `Unavailable` 拒绝任务流,而不下发无租约任务。 +每次 Claim 还会签发新的不可预测 `lease_token`;Observation 必须回传该值。任务被重新 +领取后,旧 token 即使拥有相同 `task_id` 和 `checker_id` 也会被拒绝,避免过期实例的 +迟到事实覆盖新租约结果。 `proxy-checker` 使用固定大小 worker-pool 执行每个 pull 批次,任务数不超过该请求的 `max_in_flight`;每次尝试都受 `deadline` 和 `timeout` 的较小值约束,失败可在同一 diff --git a/docs/development/implementation-plan.md b/docs/development/implementation-plan.md index d10ca09..e365702 100644 --- a/docs/development/implementation-plan.md +++ b/docs/development/implementation-plan.md @@ -281,7 +281,7 @@ Lua 同一原子边界归并,覆盖首次 CHECKING 失败进入 UNHEALTHY、AV Checker Observation 上报 RPC 已复用既有控制面监听接入 Controller:SPIFFE `checker` 身份、 每批上限、配置阈值解析、代理归属查询与 Reducer 均已闭环。`StreamCheckTasks` 现已实现为 有界 pull,并通过通用任务 broker 契约完成能力协商、同 Checker 并发窗口、租约到期回收、 -领取者栅栏和完成后重放;任务凭据仅由认证流在执行期下发。Redis 共享 due-index/租约持久化、 +领取者加不可预测 lease token 的栅栏和完成后重放;任务凭据仅由认证流在执行期下发。Redis 共享 due-index/租约持久化、 生产 broker 已有契约但 Redis 实现尚未完成;`proxy-checker` 独立进程、固定大小 worker-pool、 任务期重试/微批上报和 HTTP/HTTPS BASIC/TARGET 探测器已完成并有测试。SOCKS5、EGRESS 专用出口探测、Redis due-index/跨副本租约和部署运行态仍未实现,因此本任务保持未完成。 diff --git a/gen/controlplane/v1/controlplane.pb.go b/gen/controlplane/v1/controlplane.pb.go index 1077128..ba4cf73 100644 --- a/gen/controlplane/v1/controlplane.pb.go +++ b/gen/controlplane/v1/controlplane.pb.go @@ -1801,6 +1801,7 @@ type CheckTask struct { Password string `protobuf:"bytes,14,opt,name=password,proto3" json:"password,omitempty"` CredentialVersion string `protobuf:"bytes,15,opt,name=credential_version,json=credentialVersion,proto3" json:"credential_version,omitempty"` MaxAttempts uint32 `protobuf:"varint,16,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"` + LeaseToken string `protobuf:"bytes,17,opt,name=lease_token,json=leaseToken,proto3" json:"lease_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1947,6 +1948,13 @@ func (x *CheckTask) GetMaxAttempts() uint32 { return 0 } +func (x *CheckTask) GetLeaseToken() string { + if x != nil { + return x.LeaseToken + } + return "" +} + type ObservationBatch struct { state protoimpl.MessageState `protogen:"open.v1"` CheckerId string `protobuf:"bytes,1,opt,name=checker_id,json=checkerId,proto3" json:"checker_id,omitempty"` @@ -2011,6 +2019,7 @@ type HealthObservation struct { Latency *durationpb.Duration `protobuf:"bytes,8,opt,name=latency,proto3" json:"latency,omitempty"` ObservedEgressIp string `protobuf:"bytes,9,opt,name=observed_egress_ip,json=observedEgressIp,proto3" json:"observed_egress_ip,omitempty"` ObservedAt *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=observed_at,json=observedAt,proto3" json:"observed_at,omitempty"` + LeaseToken string `protobuf:"bytes,11,opt,name=lease_token,json=leaseToken,proto3" json:"lease_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2115,6 +2124,13 @@ func (x *HealthObservation) GetObservedAt() *timestamppb.Timestamp { return nil } +func (x *HealthObservation) GetLeaseToken() string { + if x != nil { + return x.LeaseToken + } + return "" +} + type ReportObservationsResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Accepted uint32 `protobuf:"varint,1,opt,name=accepted,proto3" json:"accepted,omitempty"` @@ -2324,7 +2340,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" + "\vinstance_id\x18\x02 \x01(\tR\n" + "instanceId\x12\"\n" + "\rmax_in_flight\x18\x03 \x01(\rR\vmaxInFlight\x12Q\n" + - "\x10supported_levels\x18\x04 \x03(\x0e2&.proxy_pool.controlplane.v1.CheckLevelR\x0fsupportedLevels\"\xde\x04\n" + + "\x10supported_levels\x18\x04 \x03(\x0e2&.proxy_pool.controlplane.v1.CheckLevelR\x0fsupportedLevels\"\xff\x04\n" + "\tCheckTask\x12\x17\n" + "\atask_id\x18\x01 \x01(\tR\x06taskId\x12\x19\n" + "\bproxy_id\x18\x02 \x01(\tR\aproxyId\x12E\n" + @@ -2344,11 +2360,13 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" + "\busername\x18\r \x01(\tR\busername\x12\x1a\n" + "\bpassword\x18\x0e \x01(\tR\bpassword\x12-\n" + "\x12credential_version\x18\x0f \x01(\tR\x11credentialVersion\x12!\n" + - "\fmax_attempts\x18\x10 \x01(\rR\vmaxAttempts\"\x84\x01\n" + + "\fmax_attempts\x18\x10 \x01(\rR\vmaxAttempts\x12\x1f\n" + + "\vlease_token\x18\x11 \x01(\tR\n" + + "leaseToken\"\x84\x01\n" + "\x10ObservationBatch\x12\x1d\n" + "\n" + "checker_id\x18\x01 \x01(\tR\tcheckerId\x12Q\n" + - "\fobservations\x18\x02 \x03(\v2-.proxy_pool.controlplane.v1.HealthObservationR\fobservations\"\xa6\x03\n" + + "\fobservations\x18\x02 \x03(\v2-.proxy_pool.controlplane.v1.HealthObservationR\fobservations\"\xc7\x03\n" + "\x11HealthObservation\x12\x17\n" + "\atask_id\x18\x01 \x01(\tR\x06taskId\x12\x19\n" + "\bproxy_id\x18\x02 \x01(\tR\aproxyId\x12<\n" + @@ -2362,7 +2380,9 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" + "\x12observed_egress_ip\x18\t \x01(\tR\x10observedEgressIp\x12;\n" + "\vobserved_at\x18\n" + " \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "observedAt\"T\n" + + "observedAt\x12\x1f\n" + + "\vlease_token\x18\v \x01(\tR\n" + + "leaseToken\"T\n" + "\x1aReportObservationsResponse\x12\x1a\n" + "\baccepted\x18\x01 \x01(\rR\baccepted\x12\x1a\n" + "\brejected\x18\x02 \x01(\rR\brejected*\xc5\x01\n" + diff --git a/internal/checker/controlplane/runner.go b/internal/checker/controlplane/runner.go index eb35afe..6008c93 100644 --- a/internal/checker/controlplane/runner.go +++ b/internal/checker/controlplane/runner.go @@ -212,7 +212,7 @@ func (runner *Runner) executeTask(ctx context.Context, task *controlplanev1.Chec result.FailureClass = probe.FailureProxyRequest } observation := &controlplanev1.HealthObservation{ - TaskId: task.GetTaskId(), ProxyId: task.GetProxyId(), Level: task.GetLevel(), Success: result.Success, + TaskId: task.GetTaskId(), LeaseToken: task.GetLeaseToken(), ProxyId: task.GetProxyId(), Level: task.GetLevel(), Success: result.Success, FailureClass: result.FailureClass, Latency: durationpb.New(result.Latency), ObservedAt: timestamppb.New(runner.options.Now().UTC()), } diff --git a/internal/controller/health/grpc_handler.go b/internal/controller/health/grpc_handler.go index 6714a98..c3952f8 100644 --- a/internal/controller/health/grpc_handler.go +++ b/internal/controller/health/grpc_handler.go @@ -146,7 +146,7 @@ func (handler *GRPCHandler) ReportObservations( if now.IsZero() { return nil, healthGRPCError(ErrInvalidGRPCHandler) } - err = handler.options.TaskBroker.AuthorizeObservation(ctx, request.GetCheckerId(), observation, now) + err = handler.options.TaskBroker.AuthorizeObservation(ctx, request.GetCheckerId(), item.GetLeaseToken(), observation, now) } if err == nil { _, err = handler.reducer.Apply(ctx, observation) @@ -156,7 +156,7 @@ func (handler *GRPCHandler) ReportObservations( if now.IsZero() { return nil, healthGRPCError(ErrInvalidGRPCHandler) } - err = handler.options.TaskBroker.CompleteObservation(ctx, request.GetCheckerId(), observation, now) + err = handler.options.TaskBroker.CompleteObservation(ctx, request.GetCheckerId(), item.GetLeaseToken(), observation, now) } if err == nil { response.Accepted++ @@ -226,7 +226,7 @@ func healthGRPCError(err error) error { } func wireCheckTask(task LeasedTask, now time.Time) (*controlplanev1.CheckTask, error) { - if !validTaskIdentifier(task.TaskID) || !validTaskIdentifier(task.ProxyID) || task.Host == "" || + if !validTaskIdentifier(task.TaskID) || !validTaskIdentifier(task.LeaseToken) || !validTaskIdentifier(task.ProxyID) || task.Host == "" || strings.TrimSpace(task.Host) != task.Host || task.Port == 0 || task.Attempts <= 0 || task.Attempts > math.MaxUint32 || !task.Deadline.After(now) || (task.SecretRef == "") != (task.CredentialVersion == "") { return nil, ErrInvalidLeasedTask @@ -249,7 +249,7 @@ func wireCheckTask(task LeasedTask, now time.Time) (*controlplanev1.CheckTask, e return nil, ErrInvalidLeasedTask } return &controlplanev1.CheckTask{ - TaskId: task.TaskID, ProxyId: task.ProxyID, Protocol: protocol, Host: task.Host, Port: uint32(task.Port), + TaskId: task.TaskID, LeaseToken: task.LeaseToken, ProxyId: task.ProxyID, Protocol: protocol, Host: task.Host, Port: uint32(task.Port), SecretRef: task.SecretRef, CredentialVersion: task.CredentialVersion, Username: task.Username, Password: task.Password, Level: level, RoutingName: task.RoutingName, TargetUrl: task.TargetURL, Timeout: durationpb.New(task.Deadline.Sub(now)), Attempt: 1, MaxAttempts: uint32(task.Attempts), diff --git a/internal/controller/health/grpc_handler_test.go b/internal/controller/health/grpc_handler_test.go index 751ec61..5f2f6f4 100644 --- a/internal/controller/health/grpc_handler_test.go +++ b/internal/controller/health/grpc_handler_test.go @@ -151,17 +151,21 @@ func TestGRPCHandlerStreamsLeasedTasksAndFencesReportedFacts(t *testing.T) { if _, err := stream.Recv(); !errors.Is(err, io.EOF) { t.Fatalf("second Recv() error = %v, want EOF", err) } + acceptedObservation := grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)) + acceptedObservation.LeaseToken = task.GetLeaseToken() response, err := client.ReportObservations(context.Background(), &controlplanev1.ObservationBatch{ CheckerId: "checker-a", Observations: []*controlplanev1.HealthObservation{ - grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)), + acceptedObservation, }, }) if err != nil || response.GetAccepted() != 1 || len(global.commands) != 1 { t.Fatalf("ReportObservations() = (%+v, %v), global=%d", response, err, len(global.commands)) } + rejectedObservation := grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)) + rejectedObservation.LeaseToken = task.GetLeaseToken() rejected, err := client.ReportObservations(context.Background(), &controlplanev1.ObservationBatch{ CheckerId: "checker-b", Observations: []*controlplanev1.HealthObservation{ - grpcHealthObservation(task.GetTaskId(), controlplanev1.CheckLevel_CHECK_LEVEL_BASIC, now.Add(time.Second)), + rejectedObservation, }, }) if err != nil || rejected.GetRejected() != 1 || len(global.commands) != 1 { diff --git a/internal/controller/health/task_broker.go b/internal/controller/health/task_broker.go index 57c78fd..377edbb 100644 --- a/internal/controller/health/task_broker.go +++ b/internal/controller/health/task_broker.go @@ -2,6 +2,7 @@ package health import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "errors" @@ -73,6 +74,7 @@ type TaskClaim struct { // for the duration of a task lease and must not be logged. type LeasedTask struct { TaskID string + LeaseToken string ProxyID string Protocol proxyDomain.Scheme Host string @@ -98,8 +100,8 @@ func (LeasedTask) Format(state fmt.State, _ rune) { type TaskBroker interface { TaskSink Claim(context.Context, TaskClaim) ([]LeasedTask, error) - AuthorizeObservation(context.Context, string, healthDomain.Observation, time.Time) error - CompleteObservation(context.Context, string, healthDomain.Observation, time.Time) error + AuthorizeObservation(context.Context, string, string, healthDomain.Observation, time.Time) error + CompleteObservation(context.Context, string, string, healthDomain.Observation, time.Time) error } type MemoryTaskBrokerOptions struct { @@ -127,11 +129,13 @@ type brokerTask struct { type taskLease struct { checkerID string instanceID string + token string expiresAt time.Time } type taskCompletion struct { checkerID string + token string expiresAt time.Time } @@ -218,6 +222,7 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L return []LeasedTask{}, nil } materials := make([]TaskMaterial, len(taskIDs)) + tokens := make([]string, len(taskIDs)) for index, taskID := range taskIDs { material, resolveErr := broker.options.Material.ResolveCheckTask(ctx, broker.tasks[taskID].plan.Candidate) if resolveErr != nil { @@ -227,6 +232,11 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L return nil, ErrInvalidLeasedTask } materials[index] = material + leaseToken, tokenErr := newLeaseToken() + if tokenErr != nil { + return nil, tokenErr + } + tokens[index] = leaseToken } result := make([]LeasedTask, len(taskIDs)) for index, taskID := range taskIDs { @@ -235,8 +245,9 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L if plan.Deadline.Before(leaseUntil) { leaseUntil = plan.Deadline } - broker.leases[taskID] = taskLease{checkerID: claim.CheckerID, instanceID: claim.InstanceID, expiresAt: leaseUntil} - result[index] = leasedTask(taskID, plan, materials[index]) + leaseToken := tokens[index] + broker.leases[taskID] = taskLease{checkerID: claim.CheckerID, instanceID: claim.InstanceID, token: leaseToken, expiresAt: leaseUntil} + result[index] = leasedTask(taskID, leaseToken, plan, materials[index]) } return result, nil } @@ -244,10 +255,11 @@ func (broker *MemoryTaskBroker) Claim(ctx context.Context, claim TaskClaim) ([]L func (broker *MemoryTaskBroker) AuthorizeObservation( ctx context.Context, checkerID string, + leaseToken string, observation healthDomain.Observation, now time.Time, ) error { - if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || now.IsZero() { + if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || !validTaskIdentifier(leaseToken) || now.IsZero() { return ErrInvalidTaskBroker } if err := ctx.Err(); err != nil { @@ -259,16 +271,17 @@ func (broker *MemoryTaskBroker) AuthorizeObservation( } broker.mu.Lock() defer broker.mu.Unlock() - return broker.authorizeObservationLocked(checkerID, normalized, now.UTC()) + return broker.authorizeObservationLocked(checkerID, leaseToken, normalized, now.UTC()) } func (broker *MemoryTaskBroker) CompleteObservation( ctx context.Context, checkerID string, + leaseToken string, observation healthDomain.Observation, now time.Time, ) error { - if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || now.IsZero() { + if ctx == nil || broker == nil || !validTaskIdentifier(checkerID) || !validTaskIdentifier(leaseToken) || now.IsZero() { return ErrInvalidTaskBroker } if err := ctx.Err(); err != nil { @@ -281,7 +294,7 @@ func (broker *MemoryTaskBroker) CompleteObservation( now = now.UTC() broker.mu.Lock() defer broker.mu.Unlock() - if err := broker.authorizeObservationLocked(checkerID, normalized, now); err != nil { + if err := broker.authorizeObservationLocked(checkerID, leaseToken, normalized, now); err != nil { return err } if _, complete := broker.finished[normalized.TaskID]; complete { @@ -289,12 +302,13 @@ func (broker *MemoryTaskBroker) CompleteObservation( } task := broker.tasks[normalized.TaskID] delete(broker.leases, normalized.TaskID) - broker.finished[normalized.TaskID] = taskCompletion{checkerID: checkerID, expiresAt: task.plan.Deadline} + broker.finished[normalized.TaskID] = taskCompletion{checkerID: checkerID, token: leaseToken, expiresAt: task.plan.Deadline} return nil } func (broker *MemoryTaskBroker) authorizeObservationLocked( checkerID string, + leaseToken string, observation healthDomain.Observation, now time.Time, ) error { @@ -307,7 +321,7 @@ func (broker *MemoryTaskBroker) authorizeObservationLocked( return ErrTaskObservation } if finished, complete := broker.finished[observation.TaskID]; complete { - if finished.checkerID != checkerID { + if finished.checkerID != checkerID || finished.token != leaseToken { return ErrTaskLeaseNotOwned } return nil @@ -316,7 +330,7 @@ func (broker *MemoryTaskBroker) authorizeObservationLocked( if !leased || !lease.expiresAt.After(now) { return ErrTaskLeaseExpired } - if lease.checkerID != checkerID { + if lease.checkerID != checkerID || lease.token != leaseToken { return ErrTaskLeaseNotOwned } return nil @@ -445,15 +459,23 @@ func deterministicTaskID(task PlannedTask) string { return "check_" + hex.EncodeToString(digest[:]) } -func leasedTask(taskID string, plan PlannedTask, material TaskMaterial) LeasedTask { +func leasedTask(taskID, leaseToken string, plan PlannedTask, material TaskMaterial) LeasedTask { return LeasedTask{ - TaskID: taskID, ProxyID: plan.Candidate.ProxyID, Protocol: material.Protocol, Host: material.Host, Port: material.Port, + TaskID: taskID, LeaseToken: leaseToken, ProxyID: plan.Candidate.ProxyID, Protocol: material.Protocol, Host: material.Host, Port: material.Port, SecretRef: material.SecretRef, CredentialVersion: material.CredentialVersion, Username: material.Username, Password: material.Password, Level: plan.Candidate.Level, RoutingName: plan.Candidate.RoutingName, TargetURL: plan.Candidate.TargetURL, Deadline: plan.Deadline.UTC(), Attempts: plan.Attempts, } } +func newLeaseToken() (string, error) { + var entropy [24]byte + if _, err := rand.Read(entropy[:]); err != nil { + return "", err + } + return "lease_" + hex.EncodeToString(entropy[:]), nil +} + func taskMatchesObservation(task PlannedTask, observation healthDomain.Observation) bool { return task.Candidate.ProxyID == observation.ProxyID && task.Candidate.Level == observation.Level && task.Candidate.RoutingName == observation.RoutingName && task.Candidate.TargetURL == observation.TargetURL diff --git a/internal/controller/health/task_broker_test.go b/internal/controller/health/task_broker_test.go index 814721f..92cf40b 100644 --- a/internal/controller/health/task_broker_test.go +++ b/internal/controller/health/task_broker_test.go @@ -42,16 +42,16 @@ func TestMemoryTaskBrokerClaimsSupportedTasksAndFencesObservations(t *testing.T) TaskID: claimed[0].TaskID, ProxyID: "basic", Level: healthDomain.LevelBasic, Success: true, Latency: time.Millisecond, ObservedAt: now.Add(time.Second), } - if err := broker.AuthorizeObservation(context.Background(), "checker-b", observation, now.Add(time.Second)); !errors.Is(err, ErrTaskLeaseNotOwned) { + if err := broker.AuthorizeObservation(context.Background(), "checker-b", claimed[0].LeaseToken, observation, now.Add(time.Second)); !errors.Is(err, ErrTaskLeaseNotOwned) { t.Fatalf("AuthorizeObservation(other checker) error = %v, want ErrTaskLeaseNotOwned", err) } - if err := broker.AuthorizeObservation(context.Background(), "checker-a", observation, now.Add(time.Second)); err != nil { + if err := broker.AuthorizeObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(time.Second)); err != nil { t.Fatalf("AuthorizeObservation(owner): %v", err) } - if err := broker.CompleteObservation(context.Background(), "checker-a", observation, now.Add(time.Second)); err != nil { + if err := broker.CompleteObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(time.Second)); err != nil { t.Fatalf("CompleteObservation(): %v", err) } - if err := broker.AuthorizeObservation(context.Background(), "checker-a", observation, now.Add(2*time.Second)); err != nil { + if err := broker.AuthorizeObservation(context.Background(), "checker-a", claimed[0].LeaseToken, observation, now.Add(2*time.Second)); err != nil { t.Fatalf("AuthorizeObservation(replay): %v", err) } } @@ -82,7 +82,7 @@ func TestMemoryTaskBrokerReclaimsExpiredLeaseWithoutExceedingCheckerLimit(t *tes } now = now.Add(2 * time.Second) reclaimed, err := broker.Claim(context.Background(), TaskClaim{CheckerID: "checker-b", InstanceID: "instance-b", MaxInFlight: 1, SupportedLevels: []healthDomain.Level{healthDomain.LevelBasic}}) - if err != nil || len(reclaimed) != 1 || reclaimed[0].TaskID != first[0].TaskID { + if err != nil || len(reclaimed) != 1 || reclaimed[0].TaskID != first[0].TaskID || reclaimed[0].LeaseToken == first[0].LeaseToken { t.Fatalf("reclaimed Claim() = (%+v, %v)", reclaimed, err) } }