fix: align observability assets with registered metrics

deploy/README.md

@@ -34,3 +34,8 @@ PostgreSQL Adapter 和对应执行脚本完成前，不把数据库契约记为
 
 发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁，
 并完成代表性集群容量验证。
+
+Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway
+请求/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain；不按 Proxy、IP、
+Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝
+不存在的指标与禁止标签，指标改名或新增面板时必须同步更新该契约。

deploy/compose_test.go

@@ -1,6 +1,7 @@
 package deploy
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -11,6 +12,47 @@ import (
 	"go.yaml.in/yaml/v4"
 )
 
+var observableMetricNames = map[string]struct{}{
+	"proxy_pool_checker_observations_total":                    {},
+	"proxy_pool_checker_tasks_dispatched_total":                {},
+	"proxy_pool_controller_capacity_active_upstreams":          {},
+	"proxy_pool_controller_capacity_available_slots":           {},
+	"proxy_pool_controller_capacity_effective_slots":           {},
+	"proxy_pool_controller_capacity_inventory_reads_total":     {},
+	"proxy_pool_controller_capacity_managed_proxies":           {},
+	"proxy_pool_controller_capacity_pending_expected_proxies":  {},
+	"proxy_pool_controller_drain_candidates_total":             {},
+	"proxy_pool_controller_drains_started_total":               {},
+	"proxy_pool_controller_extraction_requested_proxies_total": {},
+	"proxy_pool_controller_extraction_requests_total":          {},
+	"proxy_pool_controller_extraction_returned_proxies_total":  {},
+	"proxy_pool_controller_provider_fetch_results_total":       {},
+	"proxy_pool_controller_provider_new_proxies_total":         {},
+	"proxy_pool_controller_provider_valid_candidates_total":    {},
+	"proxy_pool_gateway_active_tunnels":                        {},
+	"proxy_pool_gateway_capacity_invariant_violations_total":   {},
+	"proxy_pool_gateway_outcome_queue_dropped_total":           {},
+	"proxy_pool_gateway_outcomes_total":                        {},
+	"proxy_pool_gateway_requests_in_flight":                    {},
+	"proxy_pool_gateway_requests_total":                        {},
+}
+
+type dashboardDocument struct {
+	Panels []struct {
+		Targets []struct {
+			Expression string `json:"expr"`
+		} `json:"targets"`
+	} `json:"panels"`
+}
+
+type alertRulesDocument struct {
+	Groups []struct {
+		Rules []struct {
+			Expression string `yaml:"expr"`
+		} `yaml:"rules"`
+	} `yaml:"groups"`
+}
+
 type composeDocument struct {
 	Services map[string]composeService `yaml:"services"`
 	Volumes  map[string]any            `yaml:"volumes"`
@@ -231,6 +273,57 @@ func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
 	}
 }
 
+func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
+	dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
+	if err != nil {
+		t.Fatalf("read overview dashboard: %v", err)
+	}
+	var dashboard dashboardDocument
+	if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
+		t.Fatalf("parse overview dashboard: %v", err)
+	}
+	expressions := make([]string, 0, len(dashboard.Panels))
+	for _, panel := range dashboard.Panels {
+		for _, target := range panel.Targets {
+			expressions = append(expressions, target.Expression)
+		}
+	}
+	rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
+	if err != nil {
+		t.Fatalf("read Prometheus rules: %v", err)
+	}
+	var rules alertRulesDocument
+	if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
+		t.Fatalf("parse Prometheus rules: %v", err)
+	}
+	for _, group := range rules.Groups {
+		for _, rule := range group.Rules {
+			expressions = append(expressions, rule.Expression)
+		}
+	}
+	metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
+	forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
+	for _, expression := range expressions {
+		for _, name := range metricPattern.FindAllString(expression, -1) {
+			if _, exists := observableMetricNames[name]; !exists {
+				t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
+			}
+		}
+		if forbiddenLabel.MatchString(expression) {
+			t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
+		}
+	}
+	for _, required := range []string{
+		"proxy_pool_gateway_requests_total", "proxy_pool_controller_capacity_available_slots",
+		"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
+		"proxy_pool_checker_observations_total",
+	} {
+		if !strings.Contains(strings.Join(expressions, "\n"), required) {
+			t.Errorf("observability assets do not cover %s", required)
+		}
+	}
+}
+
 func loadComposeDocument(t *testing.T) composeDocument {
 	t.Helper()
 	return loadComposeFile(t, "docker-compose.yml")

deploy/grafana/dashboards/proxy-pool-overview.json

@@ -3,13 +3,19 @@
   "editable": true,
   "graphTooltip": 1,
   "panels": [
-    {"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
+    {"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]},
-    {"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
+    {"type":"timeseries","title":"Gateway In Flight","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"{{protocol}}"}]},
-    {"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
+    {"type":"timeseries","title":"Active CONNECT Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]},
-    {"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
+    {"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]},
-    {"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
+    {"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]},
-    {"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
+    {"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]},
-    {"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
+    {"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]},
+    {"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]},
+    {"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]},
+    {"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]},
+    {"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]},
+    {"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]},
+    {"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]}
   ],
   "schemaVersion": 41,
   "tags": ["proxy-pool"],
@@ -17,5 +23,5 @@
   "time": {"from":"now-6h","to":"now"},
   "title": "Proxy Pool Overview",
   "uid": "proxy-pool-overview",
-  "version": 1
+  "version": 2
 }

deploy/prometheus/rules/proxy-pool.yml

@@ -1,40 +1,65 @@
 groups:
   - name: proxy-pool
     rules:
-      - alert: ProxyPoolGatewayHighErrorRate
+      - alert: ProxyPoolGatewayOutcomeFailureRate
         expr: |
-          sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
+          sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m]))
-          / clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
+          / clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1
         for: 10m
         labels:
           severity: warning
         annotations:
-          summary: Gateway 错误率持续高于 2%
+          summary: Gateway 代理尝试失败比例持续高于 10%
-      - alert: ProxyPoolGatewaySnapshotStale
+      - alert: ProxyPoolGatewayOutcomeQueueDrops
-        expr: proxy_pool_snapshot_age_seconds > 60
+        expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: Gateway Snapshot 已超过安全陈旧时间
-      - alert: ProxyPoolNoAvailableSlots
-        expr: sum(proxy_pool_available_slots) == 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: Gateway 可分配容量耗尽
-      - alert: ProxyPoolProviderFetchErrors
-        expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
-        for: 10m
-        labels:
-          severity: warning
-        annotations:
-          summary: Provider Fetch 错误持续发生
-      - alert: ProxyPoolExtractionConflict
-        expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: 独占提取发生持续事务冲突
+          summary: Gateway Outcome 本地队列发生丢弃
+      - alert: ProxyPoolNoAvailableSlots
+        expr: |
+          sum(proxy_pool_controller_capacity_active_upstreams) > 0
+          and sum(proxy_pool_controller_capacity_available_slots) == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: Controller 可分配容量耗尽
+      - alert: ProxyPoolCapacityInventoryErrors
+        expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Controller 容量库存读取持续失败
+      - alert: ProxyPoolProviderFetchErrors
+        expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Provider 获取持续失败
+      - alert: ProxyPoolCheckerObservationRejections
+        expr: |
+          sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m]))
+          / clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Checker Observation 拒绝比例持续高于 5%
+      - alert: ProxyPoolExtractionIdempotencyConflict
+        expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Distribution 幂等键冲突持续发生
+      - alert: ProxyPoolControlPlaneTargetDown
+        expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: Proxy Pool 控制面目标不可抓取

progress.md

@@ -17,6 +17,9 @@
 - Health Scheduler 已与 PostgreSQL Admin 管理态对齐：每轮以同一配置 revision 合并启用状态，
   被管理态停用的 Upstream 不再读取 Redis due-index，也不会创建 BASIC、EGRESS、TARGET 任务；
   管理态停用 Routing 则仅阻止该 Routing 的 TARGET 任务；revision 不匹配、状态缺失或重复时失败关闭。
+- Grafana Overview 与 Prometheus 告警已从早期失效指标迁移到当前代码注册的低基数指标，覆盖
+  Gateway、Controller 容量、Provider、Extraction、Checker 与 Drain。部署测试会解析仪表盘和
+  规则，拒绝未注册指标以及 `upstream`/`worker` 聚合或 selector 标签，防止观测资产再次漂移。
 - 全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、`go build ./...`、
   Protobuf descriptor、Kustomize Base 渲染及开发证书 SAN/SPIFFE 校验均通过。Compose
   容器端到端启动在拉取 Dockerfile 前端与监控镜像时受 Docker Desktop HTTPS 代理缺失阻断，