diff --git a/deploy/compose_test.go b/deploy/compose_test.go index f30072d..425121b 100644 --- a/deploy/compose_test.go +++ b/deploy/compose_test.go @@ -50,6 +50,7 @@ type alertRulesDocument struct { Groups []struct { Rules []struct { Expression string `yaml:"expr"` + Alert string `yaml:"alert"` } `yaml:"rules"` } `yaml:"groups"` } @@ -297,9 +298,11 @@ func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) { if err := yaml.Unmarshal(rulesPayload, &rules); err != nil { t.Fatalf("parse Prometheus rules: %v", err) } + alerts := make([]string, 0) for _, group := range rules.Groups { for _, rule := range group.Rules { expressions = append(expressions, rule.Expression) + alerts = append(alerts, rule.Alert) } } metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`) @@ -324,6 +327,9 @@ func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) { t.Errorf("observability assets do not cover %s", required) } } + if !slices.Contains(alerts, "ProxyPoolGatewayHighLatency") { + t.Error("Prometheus rules do not alert on Gateway p99 latency") + } } func loadComposeDocument(t *testing.T) composeDocument { diff --git a/deploy/prometheus/rules/proxy-pool.yml b/deploy/prometheus/rules/proxy-pool.yml index a1d947b..962ee4c 100644 --- a/deploy/prometheus/rules/proxy-pool.yml +++ b/deploy/prometheus/rules/proxy-pool.yml @@ -10,6 +10,15 @@ groups: severity: warning annotations: summary: Gateway 代理尝试失败比例持续高于 10% + - alert: ProxyPoolGatewayHighLatency + expr: | + histogram_quantile(0.99, + sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m]))) > 5 + for: 10m + labels: + severity: warning + annotations: + summary: Gateway p99 请求时延持续高于 5 秒 - alert: ProxyPoolGatewayOutcomeQueueDrops expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0 for: 5m diff --git a/progress.md b/progress.md index b787534..1b4101c 100644 --- a/progress.md +++ b/progress.md @@ -21,7 +21,8 @@ Gateway、Controller 容量、Provider、Extraction、Checker 与 Drain。部署测试会解析仪表盘和 规则,拒绝未注册指标以及 `upstream`/`worker` 聚合或 selector 标签,防止观测资产再次漂移。 - Gateway 新增按固定 `HTTP`/`CONNECT` protocol 标签聚合的请求总耗时 Histogram,覆盖从准入到 - HTTP 响应或 CONNECT 隧道结束的完整生命周期;Grafana Overview 已恢复该真实指标的 p99 面板。 + HTTP 响应或 CONNECT 隧道结束的完整生命周期;Grafana Overview 已恢复该真实指标的 p99 面板, + Prometheus 以相同固定标签集监控持续高于 5 秒的 p99。 - 全仓 `go test -count=1 -timeout 60s ./...`、`go vet ./...`、`go build ./...`、 Protobuf descriptor、Kustomize Base 渲染及开发证书 SAN/SPIFFE 校验均通过。Compose 容器端到端启动在拉取 Dockerfile 前端与监控镜像时受 Docker Desktop HTTPS 代理缺失阻断,