fix: align observability assets with registered metrics
This commit is contained in:
youfak
parent
a191ca5837
commit
7f51e333c5
@ -34,3 +34,8 @@ PostgreSQL Adapter 和对应执行脚本完成前，不把数据库契约记为
|
||||
|
||||
发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁，
|
||||
并完成代表性集群容量验证。
|
||||
|
||||
Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway
|
||||
请求/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain；不按 Proxy、IP、
|
||||
Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝
|
||||
不存在的指标与禁止标签，指标改名或新增面板时必须同步更新该契约。
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@ -11,6 +12,47 @@ import (
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
var observableMetricNames = map[string]struct{}{
|
||||
"proxy_pool_checker_observations_total": {},
|
||||
"proxy_pool_checker_tasks_dispatched_total": {},
|
||||
"proxy_pool_controller_capacity_active_upstreams": {},
|
||||
"proxy_pool_controller_capacity_available_slots": {},
|
||||
"proxy_pool_controller_capacity_effective_slots": {},
|
||||
"proxy_pool_controller_capacity_inventory_reads_total": {},
|
||||
"proxy_pool_controller_capacity_managed_proxies": {},
|
||||
"proxy_pool_controller_capacity_pending_expected_proxies": {},
|
||||
"proxy_pool_controller_drain_candidates_total": {},
|
||||
"proxy_pool_controller_drains_started_total": {},
|
||||
"proxy_pool_controller_extraction_requested_proxies_total": {},
|
||||
"proxy_pool_controller_extraction_requests_total": {},
|
||||
"proxy_pool_controller_extraction_returned_proxies_total": {},
|
||||
"proxy_pool_controller_provider_fetch_results_total": {},
|
||||
"proxy_pool_controller_provider_new_proxies_total": {},
|
||||
"proxy_pool_controller_provider_valid_candidates_total": {},
|
||||
"proxy_pool_gateway_active_tunnels": {},
|
||||
"proxy_pool_gateway_capacity_invariant_violations_total": {},
|
||||
"proxy_pool_gateway_outcome_queue_dropped_total": {},
|
||||
"proxy_pool_gateway_outcomes_total": {},
|
||||
"proxy_pool_gateway_requests_in_flight": {},
|
||||
"proxy_pool_gateway_requests_total": {},
|
||||
}
|
||||
|
||||
type dashboardDocument struct {
|
||||
Panels []struct {
|
||||
Targets []struct {
|
||||
Expression string `json:"expr"`
|
||||
} `json:"targets"`
|
||||
} `json:"panels"`
|
||||
}
|
||||
|
||||
type alertRulesDocument struct {
|
||||
Groups []struct {
|
||||
Rules []struct {
|
||||
Expression string `yaml:"expr"`
|
||||
} `yaml:"rules"`
|
||||
} `yaml:"groups"`
|
||||
}
|
||||
|
||||
type composeDocument struct {
|
||||
Services map[string]composeService `yaml:"services"`
|
||||
Volumes map[string]any `yaml:"volumes"`
|
||||
@ -231,6 +273,57 @@ func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
|
||||
dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read overview dashboard: %v", err)
|
||||
}
|
||||
var dashboard dashboardDocument
|
||||
if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
|
||||
t.Fatalf("parse overview dashboard: %v", err)
|
||||
}
|
||||
expressions := make([]string, 0, len(dashboard.Panels))
|
||||
for _, panel := range dashboard.Panels {
|
||||
for _, target := range panel.Targets {
|
||||
expressions = append(expressions, target.Expression)
|
||||
}
|
||||
}
|
||||
rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("read Prometheus rules: %v", err)
|
||||
}
|
||||
var rules alertRulesDocument
|
||||
if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
|
||||
t.Fatalf("parse Prometheus rules: %v", err)
|
||||
}
|
||||
for _, group := range rules.Groups {
|
||||
for _, rule := range group.Rules {
|
||||
expressions = append(expressions, rule.Expression)
|
||||
}
|
||||
}
|
||||
metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
|
||||
forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
|
||||
for _, expression := range expressions {
|
||||
for _, name := range metricPattern.FindAllString(expression, -1) {
|
||||
if _, exists := observableMetricNames[name]; !exists {
|
||||
t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
|
||||
}
|
||||
}
|
||||
if forbiddenLabel.MatchString(expression) {
|
||||
t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
|
||||
}
|
||||
}
|
||||
for _, required := range []string{
|
||||
"proxy_pool_gateway_requests_total", "proxy_pool_controller_capacity_available_slots",
|
||||
"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
|
||||
"proxy_pool_checker_observations_total",
|
||||
} {
|
||||
if !strings.Contains(strings.Join(expressions, "\n"), required) {
|
||||
t.Errorf("observability assets do not cover %s", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func loadComposeDocument(t *testing.T) composeDocument {
|
||||
t.Helper()
|
||||
return loadComposeFile(t, "docker-compose.yml")
|
||||
|
||||
@ -3,13 +3,19 @@
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"panels": [
|
||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
|
||||
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
|
||||
{"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
|
||||
{"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
|
||||
{"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
|
||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]},
|
||||
{"type":"timeseries","title":"Gateway In Flight","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"{{protocol}}"}]},
|
||||
{"type":"timeseries","title":"Active CONNECT Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]},
|
||||
{"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]},
|
||||
{"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]},
|
||||
{"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]},
|
||||
{"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]},
|
||||
{"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]},
|
||||
{"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]},
|
||||
{"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]},
|
||||
{"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]},
|
||||
{"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]},
|
||||
{"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]}
|
||||
],
|
||||
"schemaVersion": 41,
|
||||
"tags": ["proxy-pool"],
|
||||
@ -17,5 +23,5 @@
|
||||
"time": {"from":"now-6h","to":"now"},
|
||||
"title": "Proxy Pool Overview",
|
||||
"uid": "proxy-pool-overview",
|
||||
"version": 1
|
||||
"version": 2
|
||||
}
|
||||
|
||||
@ -1,40 +1,65 @@
|
||||
groups:
|
||||
- name: proxy-pool
|
||||
rules:
|
||||
- alert: ProxyPoolGatewayHighErrorRate
|
||||
- alert: ProxyPoolGatewayOutcomeFailureRate
|
||||
expr: |
|
||||
sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
|
||||
sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gateway 错误率持续高于 2%
|
||||
- alert: ProxyPoolGatewaySnapshotStale
|
||||
expr: proxy_pool_snapshot_age_seconds > 60
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Gateway Snapshot 已超过安全陈旧时间
|
||||
- alert: ProxyPoolNoAvailableSlots
|
||||
expr: sum(proxy_pool_available_slots) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Gateway 可分配容量耗尽
|
||||
- alert: ProxyPoolProviderFetchErrors
|
||||
expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Provider Fetch 错误持续发生
|
||||
- alert: ProxyPoolExtractionConflict
|
||||
expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
|
||||
summary: Gateway 代理尝试失败比例持续高于 10%
|
||||
- alert: ProxyPoolGatewayOutcomeQueueDrops
|
||||
expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 独占提取发生持续事务冲突
|
||||
summary: Gateway Outcome 本地队列发生丢弃
|
||||
- alert: ProxyPoolNoAvailableSlots
|
||||
expr: |
|
||||
sum(proxy_pool_controller_capacity_active_upstreams) > 0
|
||||
and sum(proxy_pool_controller_capacity_available_slots) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Controller 可分配容量耗尽
|
||||
- alert: ProxyPoolCapacityInventoryErrors
|
||||
expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Controller 容量库存读取持续失败
|
||||
- alert: ProxyPoolProviderFetchErrors
|
||||
expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Provider 获取持续失败
|
||||
- alert: ProxyPoolCheckerObservationRejections
|
||||
expr: |
|
||||
sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Checker Observation 拒绝比例持续高于 5%
|
||||
- alert: ProxyPoolExtractionIdempotencyConflict
|
||||
expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Distribution 幂等键冲突持续发生
|
||||
- alert: ProxyPoolControlPlaneTargetDown
|
||||
expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Proxy Pool 控制面目标不可抓取
|
||||
|
||||
@ -17,6 +17,9 @@
|
||||
- Health Scheduler 已与 PostgreSQL Admin 管理态对齐：每轮以同一配置 revision 合并启用状态，
|
||||
被管理态停用的 Upstream 不再读取 Redis due-index，也不会创建 BASIC、EGRESS、TARGET 任务；
|
||||
管理态停用 Routing 则仅阻止该 Routing 的 TARGET 任务；revision 不匹配、状态缺失或重复时失败关闭。
|
||||
- Grafana Overview 与 Prometheus 告警已从早期失效指标迁移到当前代码注册的低基数指标，覆盖
|
||||
Gateway、Controller 容量、Provider、Extraction、Checker 与 Drain。部署测试会解析仪表盘和
|
||||
规则，拒绝未注册指标以及 `upstream`/`worker` 聚合或 selector 标签，防止观测资产再次漂移。
|
||||
- 全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、`go build ./...`、
|
||||
Protobuf descriptor、Kustomize Base 渲染及开发证书 SAN/SPIFFE 校验均通过。Compose
|
||||
容器端到端启动在拉取 Dockerfile 前端与监控镜像时受 Docker Desktop HTTPS 代理缺失阻断，
|
||||
|
||||
Loading…
Reference in New Issue
Block a user