From 7f51e333c5ca19a3dbdd2e2c9aa52549840a6bf3 Mon Sep 17 00:00:00 2001 From: youfak Date: Fri, 7 Aug 2026 17:04:46 +0800 Subject: [PATCH] fix: align observability assets with registered metrics --- deploy/README.md | 5 + deploy/compose_test.go | 93 +++++++++++++++++++ .../dashboards/proxy-pool-overview.json | 22 +++-- deploy/prometheus/rules/proxy-pool.yml | 81 ++++++++++------ progress.md | 3 + 5 files changed, 168 insertions(+), 36 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index ccb14be..d917b83 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -34,3 +34,8 @@ PostgreSQL Adapter 和对应执行脚本完成前，不把数据库契约记为 发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁， 并完成代表性集群容量验证。 + +Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway +请求/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain；不按 Proxy、IP、 +Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝 +不存在的指标与禁止标签，指标改名或新增面板时必须同步更新该契约。 diff --git a/deploy/compose_test.go b/deploy/compose_test.go index c35052a..bea2cd8 100644 --- a/deploy/compose_test.go +++ b/deploy/compose_test.go @@ -1,6 +1,7 @@ package deploy import ( + "encoding/json" "os" "path/filepath" "regexp" @@ -11,6 +12,47 @@ import ( "go.yaml.in/yaml/v4" ) +var observableMetricNames = map[string]struct{}{ + "proxy_pool_checker_observations_total": {}, + "proxy_pool_checker_tasks_dispatched_total": {}, + "proxy_pool_controller_capacity_active_upstreams": {}, + "proxy_pool_controller_capacity_available_slots": {}, + "proxy_pool_controller_capacity_effective_slots": {}, + "proxy_pool_controller_capacity_inventory_reads_total": {}, + "proxy_pool_controller_capacity_managed_proxies": {}, + "proxy_pool_controller_capacity_pending_expected_proxies": {}, + "proxy_pool_controller_drain_candidates_total": {}, + "proxy_pool_controller_drains_started_total": {}, + "proxy_pool_controller_extraction_requested_proxies_total": {}, + "proxy_pool_controller_extraction_requests_total": {}, + "proxy_pool_controller_extraction_returned_proxies_total": {}, + "proxy_pool_controller_provider_fetch_results_total": {}, + "proxy_pool_controller_provider_new_proxies_total": {}, + "proxy_pool_controller_provider_valid_candidates_total": {}, + "proxy_pool_gateway_active_tunnels": {}, + "proxy_pool_gateway_capacity_invariant_violations_total": {}, + "proxy_pool_gateway_outcome_queue_dropped_total": {}, + "proxy_pool_gateway_outcomes_total": {}, + "proxy_pool_gateway_requests_in_flight": {}, + "proxy_pool_gateway_requests_total": {}, +} + +type dashboardDocument struct { + Panels []struct { + Targets []struct { + Expression string `json:"expr"` + } `json:"targets"` + } `json:"panels"` +} + +type alertRulesDocument struct { + Groups []struct { + Rules []struct { + Expression string `yaml:"expr"` + } `yaml:"rules"` + } `yaml:"groups"` +} + type composeDocument struct { Services map[string]composeService `yaml:"services"` Volumes map[string]any `yaml:"volumes"` @@ -231,6 +273,57 @@ func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) { } } +func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) { + dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json") + if err != nil { + t.Fatalf("read overview dashboard: %v", err) + } + var dashboard dashboardDocument + if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil { + t.Fatalf("parse overview dashboard: %v", err) + } + expressions := make([]string, 0, len(dashboard.Panels)) + for _, panel := range dashboard.Panels { + for _, target := range panel.Targets { + expressions = append(expressions, target.Expression) + } + } + rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml") + if err != nil { + t.Fatalf("read Prometheus rules: %v", err) + } + var rules alertRulesDocument + if err := yaml.Unmarshal(rulesPayload, &rules); err != nil { + t.Fatalf("parse Prometheus rules: %v", err) + } + for _, group := range rules.Groups { + for _, rule := range group.Rules { + expressions = append(expressions, rule.Expression) + } + } + metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`) + forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`) + for _, expression := range expressions { + for _, name := range metricPattern.FindAllString(expression, -1) { + if _, exists := observableMetricNames[name]; !exists { + t.Errorf("observability expression references unregistered metric %q: %s", name, expression) + } + } + if forbiddenLabel.MatchString(expression) { + t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression) + } + } + for _, required := range []string{ + "proxy_pool_gateway_requests_total", "proxy_pool_controller_capacity_available_slots", + "proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total", + "proxy_pool_checker_observations_total", + } { + if !strings.Contains(strings.Join(expressions, "\n"), required) { + t.Errorf("observability assets do not cover %s", required) + } + } +} + func loadComposeDocument(t *testing.T) composeDocument { t.Helper() return loadComposeFile(t, "docker-compose.yml") diff --git a/deploy/grafana/dashboards/proxy-pool-overview.json b/deploy/grafana/dashboards/proxy-pool-overview.json index 95992d2..baca390 100644 --- a/deploy/grafana/dashboards/proxy-pool-overview.json +++ b/deploy/grafana/dashboards/proxy-pool-overview.json @@ -3,13 +3,19 @@ "editable": true, "graphTooltip": 1, "panels": [ - {"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]}, - {"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]}, - {"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]}, - {"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]}, - {"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]}, - {"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]}, - {"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]} + {"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]}, + {"type":"timeseries","title":"Gateway In Flight","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"{{protocol}}"}]}, + {"type":"timeseries","title":"Active CONNECT Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]}, + {"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]}, + {"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]}, + {"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]}, + {"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]}, + {"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]}, + {"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]}, + {"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]}, + {"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]}, + {"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]}, + {"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]} ], "schemaVersion": 41, "tags": ["proxy-pool"], @@ -17,5 +23,5 @@ "time": {"from":"now-6h","to":"now"}, "title": "Proxy Pool Overview", "uid": "proxy-pool-overview", - "version": 1 + "version": 2 } diff --git a/deploy/prometheus/rules/proxy-pool.yml b/deploy/prometheus/rules/proxy-pool.yml index 3727d6a..a1d947b 100644 --- a/deploy/prometheus/rules/proxy-pool.yml +++ b/deploy/prometheus/rules/proxy-pool.yml @@ -1,40 +1,65 @@ groups: - name: proxy-pool rules: - - alert: ProxyPoolGatewayHighErrorRate + - alert: ProxyPoolGatewayOutcomeFailureRate expr: | - sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m])) - / clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02 + sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m])) + / clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1 for: 10m labels: severity: warning annotations: - summary: Gateway 错误率持续高于 2% - - alert: ProxyPoolGatewaySnapshotStale - expr: proxy_pool_snapshot_age_seconds > 60 - for: 2m - labels: - severity: critical - annotations: - summary: Gateway Snapshot 已超过安全陈旧时间 - - alert: ProxyPoolNoAvailableSlots - expr: sum(proxy_pool_available_slots) == 0 - for: 1m - labels: - severity: critical - annotations: - summary: Gateway 可分配容量耗尽 - - alert: ProxyPoolProviderFetchErrors - expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2 - for: 10m - labels: - severity: warning - annotations: - summary: Provider Fetch 错误持续发生 - - alert: ProxyPoolExtractionConflict - expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0 + summary: Gateway 代理尝试失败比例持续高于 10% + - alert: ProxyPoolGatewayOutcomeQueueDrops + expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0 for: 5m labels: severity: warning annotations: - summary: 独占提取发生持续事务冲突 + summary: Gateway Outcome 本地队列发生丢弃 + - alert: ProxyPoolNoAvailableSlots + expr: | + sum(proxy_pool_controller_capacity_active_upstreams) > 0 + and sum(proxy_pool_controller_capacity_available_slots) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Controller 可分配容量耗尽 + - alert: ProxyPoolCapacityInventoryErrors + expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2 + for: 10m + labels: + severity: warning + annotations: + summary: Controller 容量库存读取持续失败 + - alert: ProxyPoolProviderFetchErrors + expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2 + for: 10m + labels: + severity: warning + annotations: + summary: Provider 获取持续失败 + - alert: ProxyPoolCheckerObservationRejections + expr: | + sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m])) + / clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: Checker Observation 拒绝比例持续高于 5% + - alert: ProxyPoolExtractionIdempotencyConflict + expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Distribution 幂等键冲突持续发生 + - alert: ProxyPoolControlPlaneTargetDown + expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: Proxy Pool 控制面目标不可抓取 diff --git a/progress.md b/progress.md index f1aadba..c5c1b97 100644 --- a/progress.md +++ b/progress.md @@ -17,6 +17,9 @@ - Health Scheduler 已与 PostgreSQL Admin 管理态对齐：每轮以同一配置 revision 合并启用状态， 被管理态停用的 Upstream 不再读取 Redis due-index，也不会创建 BASIC、EGRESS、TARGET 任务； 管理态停用 Routing 则仅阻止该 Routing 的 TARGET 任务；revision 不匹配、状态缺失或重复时失败关闭。 +- Grafana Overview 与 Prometheus 告警已从早期失效指标迁移到当前代码注册的低基数指标，覆盖 + Gateway、Controller 容量、Provider、Extraction、Checker 与 Drain。部署测试会解析仪表盘和 + 规则，拒绝未注册指标以及 `upstream`/`worker` 聚合或 selector 标签，防止观测资产再次漂移。 - 全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、`go build ./...`、 Protobuf descriptor、Kustomize Base 渲染及开发证书 SAN/SPIFFE 校验均通过。Compose 容器端到端启动在拉取 Dockerfile 前端与监控镜像时受 Docker Desktop HTTPS 代理缺失阻断，