feat: expose checker task metrics
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run

This commit is contained in:
youfak 2026-08-02 09:24:46 +08:00
parent 2166214777
commit b35092ccdd
10 changed files with 258 additions and 3 deletions

View File

@ -39,6 +39,8 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
库存预留,以及 `AVAILABLE -> EXTRACTED` 的一次性独占提取。 库存预留,以及 `AVAILABLE -> EXTRACTED` 的一次性独占提取。
- **Controller 入口**`proxy-controller` 已装配 Distribution、Admin、Metrics、 - **Controller 入口**`proxy-controller` 已装配 Distribution、Admin、Metrics、
PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。 PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。
- **Checker 指标**Metrics 启用时暴露 Checker 任务下发与 Observation 接受/拒绝计数;
标签仅使用固定检查级别和结果,不记录 Proxy、IP、URL 或凭据。
- **PostgreSQL 管理面**持久化配置版本、Upstream/Routing 管理状态、Admin - **PostgreSQL 管理面**持久化配置版本、Upstream/Routing 管理状态、Admin
审计与 Outbox不保存 Proxy 明细或逐次提取记录。 审计与 Outbox不保存 Proxy 明细或逐次提取记录。
- **Gateway 组件**HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、 - **Gateway 组件**HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、

View File

@ -510,8 +510,11 @@ Runtime指纹已更新但本地源尚未同步时停止旧 Provider待源
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez` Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`
`/readyz``/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由 `/readyz``/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标Provider、提取和容量等 网络策略限制。当前 `/metrics` 包含 Go/进程基础指标,以及 Checker 的
业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为 `proxy_pool_checker_tasks_dispatched_total{level}`
`proxy_pool_checker_observations_total{level,result}`。`level` 固定为 BASIC、EGRESS、TARGET
`result` 固定为 accepted、rejected绝不包含 Proxy、IP、Checker、Client、URL 或凭据标签。
Provider、提取和容量等业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
服务流量门槛PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址 服务流量门槛PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
变更需要重启 Controller。 变更需要重启 Controller。

View File

@ -332,6 +332,12 @@ Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
- [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor, - [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
NAT/conntrack, and incident runbooks. NAT/conntrack, and incident runbooks.
当前进度2026-08-02已接入 Checker 任务流指标
`proxy_pool_checker_tasks_dispatched_total{level}`
`proxy_pool_checker_observations_total{level,result}`;标签值仅允许固定的
`BASIC`、`EGRESS`、`TARGET` 级别及 `accepted`、`rejected` 结果。Provider、提取、
容量指标与密钥安全的结构化日志仍待补齐。
## Task 14: Documentation, Examples, and Diagrams ## Task 14: Documentation, Examples, and Diagrams
**Files:** `docs/**`, `examples/**`, `diagrams/**` **Files:** `docs/**`, `examples/**`, `diagrams/**`

View File

@ -128,6 +128,14 @@ kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=10m
探针不得执行 Provider 请求或完整数据库扫描。 探针不得执行 Provider 请求或完整数据库扫描。
Checker 指标使用固定标签集:
- `proxy_pool_checker_tasks_dispatched_total{level}`:成功写入 Checker 任务流后的任务数。
- `proxy_pool_checker_observations_total{level,result}`:已解码的 Observation 被 Controller 接受或拒绝的数量。
`level` 仅为 BASIC、EGRESS、TARGET`result` 仅为 accepted、rejected。不得将
Proxy ID、IP、Checker ID、目标 URL、Client ID 或凭据加入指标标签。
### 3.4 发布顺序与兼容性 ### 3.4 发布顺序与兼容性
1. 先做向前兼容数据库迁移。 1. 先做向前兼容数据库迁移。

View File

@ -23,6 +23,7 @@ import (
"proxy-pool/internal/controller/worker" "proxy-pool/internal/controller/worker"
"proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction" extractionDomain "proxy-pool/internal/domain/extraction"
healthDomain "proxy-pool/internal/domain/health"
ownershipDomain "proxy-pool/internal/domain/ownership" ownershipDomain "proxy-pool/internal/domain/ownership"
"proxy-pool/internal/domain/workerruntime" "proxy-pool/internal/domain/workerruntime"
"proxy-pool/internal/platform/admission" "proxy-pool/internal/platform/admission"
@ -219,10 +220,16 @@ func runWithWorkerFactory(
} }
dependencies.AdminService = service dependencies.AdminService = service
} }
var checkerMetrics healthDomain.TaskMetricsObserver
if loaded.Value.Metrics.Enabled { if loaded.Value.Metrics.Enabled {
if nilInterface(opened.metricsReadiness) { if nilInterface(opened.metricsReadiness) {
return errors.Join(ErrStartup, ErrInvalidOptions) return errors.Join(ErrStartup, ErrInvalidOptions)
} }
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
if collectorErr != nil {
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
}
checkerMetrics = collector
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{ handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness, Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
}) })
@ -289,6 +296,7 @@ func runWithWorkerFactory(
return fmt.Errorf("%w: build Checker identity authorizer: %w", ErrStartup, identityErr) return fmt.Errorf("%w: build Checker identity authorizer: %w", ErrStartup, identityErr)
} }
checkerOptions := controllerHealth.DefaultGRPCHandlerOptions() checkerOptions := controllerHealth.DefaultGRPCHandlerOptions()
checkerOptions.Metrics = checkerMetrics
if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) { if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) {
checkerOptions.TaskBroker = tasks checkerOptions.TaskBroker = tasks
} }

View File

@ -31,6 +31,7 @@ type GRPCHandlerOptions struct {
MaxObservationsPerBatch int MaxObservationsPerBatch int
MaxTasksPerClaim int MaxTasksPerClaim int
TaskBroker TaskBroker TaskBroker TaskBroker
Metrics healthDomain.TaskMetricsObserver
Now func() time.Time Now func() time.Time
} }
@ -122,6 +123,9 @@ func (handler *GRPCHandler) StreamCheckTasks(
if sendErr := stream.Send(wire); sendErr != nil { if sendErr := stream.Send(wire); sendErr != nil {
return sendErr return sendErr
} }
if handler.options.Metrics != nil {
handler.options.Metrics.ObserveTaskDispatch(task.Level, 1)
}
} }
return nil return nil
} }
@ -160,10 +164,16 @@ func (handler *GRPCHandler) ReportObservations(
} }
if err == nil { if err == nil {
response.Accepted++ response.Accepted++
if handler.options.Metrics != nil {
handler.options.Metrics.ObserveObservation(observation.Level, healthDomain.ObservationMetricAccepted)
}
continue continue
} }
if rejectedObservationError(err) { if rejectedObservationError(err) {
response.Rejected++ response.Rejected++
if handler.options.Metrics != nil && observation.Level != "" {
handler.options.Metrics.ObserveObservation(observation.Level, healthDomain.ObservationMetricRejected)
}
continue continue
} }
return nil, healthGRPCError(err) return nil, healthGRPCError(err)

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"io" "io"
"net" "net"
"sync"
"testing" "testing"
"time" "time"
@ -127,8 +128,9 @@ func TestGRPCHandlerStreamsLeasedTasksAndFencesReportedFacts(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewReducer(): %v", err) t.Fatalf("NewReducer(): %v", err)
} }
metrics := &recordingTaskMetrics{}
handler, err := NewGRPCHandler(reducer, &recordingCheckerIdentity{}, GRPCHandlerOptions{ handler, err := NewGRPCHandler(reducer, &recordingCheckerIdentity{}, GRPCHandlerOptions{
MaxObservationsPerBatch: 2, MaxTasksPerClaim: 2, TaskBroker: broker, Now: func() time.Time { return now }, MaxObservationsPerBatch: 2, MaxTasksPerClaim: 2, TaskBroker: broker, Metrics: metrics, Now: func() time.Time { return now },
}) })
if err != nil { if err != nil {
t.Fatalf("NewGRPCHandler(): %v", err) t.Fatalf("NewGRPCHandler(): %v", err)
@ -171,6 +173,11 @@ func TestGRPCHandlerStreamsLeasedTasksAndFencesReportedFacts(t *testing.T) {
if err != nil || rejected.GetRejected() != 1 || len(global.commands) != 1 { if err != nil || rejected.GetRejected() != 1 || len(global.commands) != 1 {
t.Fatalf("ReportObservations(other checker) = (%+v, %v), global=%d", rejected, err, len(global.commands)) t.Fatalf("ReportObservations(other checker) = (%+v, %v), global=%d", rejected, err, len(global.commands))
} }
if metrics.dispatches(healthDomain.LevelBasic) != 1 ||
metrics.observations(healthDomain.LevelBasic, healthDomain.ObservationMetricAccepted) != 1 ||
metrics.observations(healthDomain.LevelBasic, healthDomain.ObservationMetricRejected) != 1 {
t.Fatalf("Checker metrics = %+v", metrics)
}
} }
func TestGRPCHandlerStreamsEgressProbeURLAndAcceptsGlobalFact(t *testing.T) { func TestGRPCHandlerStreamsEgressProbeURLAndAcceptsGlobalFact(t *testing.T) {
@ -265,6 +272,42 @@ type recordingCheckerIdentity struct {
err error err error
} }
type recordingTaskMetrics struct {
mu sync.Mutex
dispatch map[healthDomain.Level]int
observation map[string]int
}
func (metrics *recordingTaskMetrics) ObserveTaskDispatch(level healthDomain.Level, count int) {
metrics.mu.Lock()
defer metrics.mu.Unlock()
if metrics.dispatch == nil {
metrics.dispatch = make(map[healthDomain.Level]int)
}
metrics.dispatch[level] += count
}
func (metrics *recordingTaskMetrics) ObserveObservation(level healthDomain.Level, result healthDomain.ObservationMetricResult) {
metrics.mu.Lock()
defer metrics.mu.Unlock()
if metrics.observation == nil {
metrics.observation = make(map[string]int)
}
metrics.observation[string(level)+"\x00"+string(result)]++
}
func (metrics *recordingTaskMetrics) dispatches(level healthDomain.Level) int {
metrics.mu.Lock()
defer metrics.mu.Unlock()
return metrics.dispatch[level]
}
func (metrics *recordingTaskMetrics) observations(level healthDomain.Level, result healthDomain.ObservationMetricResult) int {
metrics.mu.Lock()
defer metrics.mu.Unlock()
return metrics.observation[string(level)+"\x00"+string(result)]
}
func (identity *recordingCheckerIdentity) AuthorizeChecker(_ context.Context, checkerID string) error { func (identity *recordingCheckerIdentity) AuthorizeChecker(_ context.Context, checkerID string) error {
identity.checkerID = checkerID identity.checkerID = checkerID
return identity.err return identity.err

View File

@ -0,0 +1,16 @@
package health
// TaskMetricsObserver accepts only the low-cardinality dimensions emitted by
// Checker task processing. Implementations must not add task, proxy, checker,
// endpoint, target URL, or credential values as metric labels.
type TaskMetricsObserver interface {
ObserveTaskDispatch(Level, int)
ObserveObservation(Level, ObservationMetricResult)
}
type ObservationMetricResult string
const (
ObservationMetricAccepted ObservationMetricResult = "accepted"
ObservationMetricRejected ObservationMetricResult = "rejected"
)

View File

@ -0,0 +1,80 @@
package metrics
import (
"errors"
"fmt"
"github.com/prometheus/client_golang/prometheus"
healthDomain "proxy-pool/internal/domain/health"
)
// CheckerCollector exposes bounded Checker task-flow counters. The only
// labels are the three fixed health levels and accepted/rejected outcome.
type CheckerCollector struct {
tasksDispatched *prometheus.CounterVec
observations *prometheus.CounterVec
}
var _ healthDomain.TaskMetricsObserver = (*CheckerCollector)(nil)
func NewCheckerCollector(registerer prometheus.Registerer) (*CheckerCollector, error) {
if registerer == nil {
return nil, ErrInvalidDependencies
}
tasks, err := registerCounterVec(registerer, prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "proxy_pool", Subsystem: "checker", Name: "tasks_dispatched_total",
Help: "Number of checker tasks dispatched after a successful task-stream send.",
}, []string{"level"}))
if err != nil {
return nil, err
}
observations, err := registerCounterVec(registerer, prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "proxy_pool", Subsystem: "checker", Name: "observations_total",
Help: "Number of checker observations accepted or rejected after decoding a known level.",
}, []string{"level", "result"}))
if err != nil {
return nil, err
}
return &CheckerCollector{tasksDispatched: tasks, observations: observations}, nil
}
func (collector *CheckerCollector) ObserveTaskDispatch(level healthDomain.Level, count int) {
if collector == nil || collector.tasksDispatched == nil || count <= 0 || !validCheckerLevel(level) {
return
}
collector.tasksDispatched.WithLabelValues(string(level)).Add(float64(count))
}
func (collector *CheckerCollector) ObserveObservation(level healthDomain.Level, result healthDomain.ObservationMetricResult) {
if collector == nil || collector.observations == nil || !validCheckerLevel(level) ||
(result != healthDomain.ObservationMetricAccepted && result != healthDomain.ObservationMetricRejected) {
return
}
collector.observations.WithLabelValues(string(level), string(result)).Inc()
}
func registerCounterVec(registerer prometheus.Registerer, candidate *prometheus.CounterVec) (*prometheus.CounterVec, error) {
if err := registerer.Register(candidate); err == nil {
return candidate, nil
} else {
var registered prometheus.AlreadyRegisteredError
if !errors.As(err, &registered) {
return nil, fmt.Errorf("register checker metrics: %w", err)
}
existing, ok := registered.ExistingCollector.(*prometheus.CounterVec)
if !ok {
return nil, fmt.Errorf("register checker metrics: existing collector has unexpected type")
}
return existing, nil
}
}
func validCheckerLevel(level healthDomain.Level) bool {
switch level {
case healthDomain.LevelBasic, healthDomain.LevelEgress, healthDomain.LevelTarget:
return true
default:
return false
}
}

View File

@ -0,0 +1,79 @@
package metrics
import (
"testing"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
healthDomain "proxy-pool/internal/domain/health"
)
func TestCheckerCollectorRecordsOnlyFixedDimensions(t *testing.T) {
registry := prometheus.NewRegistry()
collector, err := NewCheckerCollector(registry)
if err != nil {
t.Fatalf("NewCheckerCollector() = %v", err)
}
collector.ObserveTaskDispatch(healthDomain.LevelBasic, 2)
collector.ObserveTaskDispatch(healthDomain.LevelTarget, 1)
collector.ObserveTaskDispatch("invalid", 99)
collector.ObserveObservation(healthDomain.LevelBasic, healthDomain.ObservationMetricAccepted)
collector.ObserveObservation(healthDomain.LevelTarget, healthDomain.ObservationMetricRejected)
collector.ObserveObservation("invalid", healthDomain.ObservationMetricAccepted)
assertMetricValue(t, registry, "proxy_pool_checker_tasks_dispatched_total", map[string]string{"level": "BASIC"}, 2)
assertMetricValue(t, registry, "proxy_pool_checker_tasks_dispatched_total", map[string]string{"level": "TARGET"}, 1)
assertMetricValue(t, registry, "proxy_pool_checker_observations_total", map[string]string{
"level": "BASIC", "result": "accepted",
}, 1)
assertMetricValue(t, registry, "proxy_pool_checker_observations_total", map[string]string{
"level": "TARGET", "result": "rejected",
}, 1)
}
func TestNewCheckerCollectorReusesRegisteredCollectors(t *testing.T) {
registry := prometheus.NewRegistry()
first, err := NewCheckerCollector(registry)
if err != nil {
t.Fatalf("first NewCheckerCollector() = %v", err)
}
second, err := NewCheckerCollector(registry)
if err != nil {
t.Fatalf("second NewCheckerCollector() = %v", err)
}
first.ObserveTaskDispatch(healthDomain.LevelEgress, 1)
second.ObserveTaskDispatch(healthDomain.LevelEgress, 1)
assertMetricValue(t, registry, "proxy_pool_checker_tasks_dispatched_total", map[string]string{"level": "EGRESS"}, 2)
}
func assertMetricValue(t *testing.T, registry *prometheus.Registry, name string, labels map[string]string, want float64) {
t.Helper()
metrics, err := registry.Gather()
if err != nil {
t.Fatalf("Gather() = %v", err)
}
for _, family := range metrics {
if family.GetName() != name {
continue
}
for _, metric := range family.GetMetric() {
if metricLabelsMatch(metric.GetLabel(), labels) && metric.GetCounter().GetValue() == want {
return
}
}
}
t.Fatalf("metric %s labels=%v value=%v was not found", name, labels, want)
}
func metricLabelsMatch(labels []*dto.LabelPair, want map[string]string) bool {
if len(labels) != len(want) {
return false
}
for _, label := range labels {
if want[label.GetName()] != label.GetValue() {
return false
}
}
return true
}