feat: gate loadgen reports by acceptance thresholds
This commit is contained in:
youfak
parent
a4aa245b6b
commit
1a8eda65c0
16
README.md
16
README.md
@ -43,7 +43,7 @@ Proxy Pool 用 Controller 协调这些变化，并让 Gateway 数据面只消费
|
||||
标签仅使用固定检查级别和结果，不记录 Proxy、IP、URL 或凭据。
|
||||
- **Gateway 指标**：Metrics 启用时暴露代理尝试的固定阶段成功/失败计数、本地
|
||||
Outcome 队列满后的丢弃计数、HTTP/CONNECT 已接收请求数与在途数，以及活跃
|
||||
CONNECT 隧道数；协议标签仅有 HTTP 与 CONNECT，不记录 Proxy、路由、目标、
|
||||
CONNECT 隧道数和请求生命周期 p50/p95/p99 直方图；协议标签仅有 HTTP 与 CONNECT，不记录 Proxy、路由、目标、
|
||||
客户端或凭据。
|
||||
- **Drain 指标**：Controller 暴露 `proxy_pool_controller_drain_candidates_total` 与
|
||||
`proxy_pool_controller_drains_started_total`；`reason` 仅有 `unhealthy` 与
|
||||
@ -116,7 +116,7 @@ flowchart LR
|
||||
|
||||
## 当前完成度
|
||||
|
||||
截至 **2026-08-02**，实施计划中可直接勾选的检查项为 **60 / 74（81.1%）**。详情见
|
||||
截至 **2026-08-07**，实施计划中可直接勾选的检查项为 **68 / 75（90.7%）**。详情见
|
||||
[实施计划](docs/development/implementation-plan.md)和
|
||||
[交付完成度审计](docs/requirements/completion-audit.md)。
|
||||
|
||||
@ -284,6 +284,18 @@ go run ./cmd/proxy-loadgen `
|
||||
`RateStartsGenerated = Requests + RateStartsDropped`。
|
||||
需要保留单次容量证据时，使用 `-output REPORT_FILE` 同时写出 JSON 文件；文件在
|
||||
同目录完整写入后才替换目标，标准输出仍保留相同报告，便于交给日志或指标系统。
|
||||
`-max-error-rate` 接受 `0` 到 `1`，`-max-p99` 接受正的 Go duration；任一阈值违反时，
|
||||
报告会附带 `acceptance` 结果，命令仍写出标准输出和 `-output` 文件，然后以退出码 `3`
|
||||
结束，适合在 CI 中保留失败压测证据：
|
||||
|
||||
```powershell
|
||||
go run ./cmd/proxy-loadgen `
|
||||
-target https://TARGET_URL/health `
|
||||
-proxy http://GATEWAY_HOST:8080 `
|
||||
-duration 30s -rate 5000 -concurrency 128 `
|
||||
-max-error-rate 0.01 -max-p99 500ms `
|
||||
-output REPORT_FILE
|
||||
```
|
||||
|
||||
## 关键配置与入口
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
@ -42,6 +43,8 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
hold := flags.Duration("hold", 0, "CONNECT tunnel hold duration; required by -scenario connect")
|
||||
extractCount := flags.Int("extract-count", 1, "proxies requested by each extract scenario request")
|
||||
extractFulfillment := flags.String("extract-fulfillment", "partial", "extract fulfillment: partial or allOrNothing")
|
||||
maxErrorRate := flags.Float64("max-error-rate", -1, "optional maximum failed/completed rate from 0 to 1")
|
||||
maxP99 := flags.Duration("max-p99", 0, "optional maximum p99 latency")
|
||||
outputPath := flags.String("output", "", "optional JSON report file; replaces only after a complete write")
|
||||
var headers headerValues
|
||||
flags.Var(&headers, "header", "repeatable HTTP header in Name: Value form")
|
||||
@ -59,6 +62,11 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: invalid output path")
|
||||
return 2
|
||||
}
|
||||
criteria, configured, err := acceptanceCriteria(*maxErrorRate, *maxP99)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
parsedHeaders, err := headers.Header()
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
@ -78,6 +86,14 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if configured {
|
||||
acceptance, acceptanceErr := loadgen.EvaluateAcceptance(report, criteria)
|
||||
if acceptanceErr != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", acceptanceErr)
|
||||
return 2
|
||||
}
|
||||
report.Acceptance = &acceptance
|
||||
}
|
||||
if err := writeReportFile(*outputPath, report); err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: write report: %v\n", err)
|
||||
return 1
|
||||
@ -86,9 +102,26 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: encode report: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if report.Acceptance != nil && !report.Acceptance.Passed {
|
||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: workload acceptance thresholds were not met")
|
||||
return 3
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func acceptanceCriteria(maxErrorRate float64, maxP99 time.Duration) (loadgen.AcceptanceCriteria, bool, error) {
|
||||
if math.IsNaN(maxErrorRate) || math.IsInf(maxErrorRate, 0) || maxErrorRate < -1 || maxErrorRate > 1 || maxP99 < 0 {
|
||||
return loadgen.AcceptanceCriteria{}, false, loadgen.ErrInvalidAcceptanceCriteria
|
||||
}
|
||||
criteria := loadgen.AcceptanceCriteria{
|
||||
EnforceFailureRate: maxErrorRate >= 0,
|
||||
MaxFailureRate: maxErrorRate,
|
||||
EnforceP99Latency: maxP99 > 0,
|
||||
MaxP99Latency: maxP99,
|
||||
}
|
||||
return criteria, criteria.EnforceFailureRate || criteria.EnforceP99Latency, nil
|
||||
}
|
||||
|
||||
func writeReportFile(path string, report loadgen.Report) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@ -102,3 +103,59 @@ func TestExecuteWritesReportFile(t *testing.T) {
|
||||
t.Fatalf("report file = (%q, %+v, %v)", content, report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteWritesReportBeforeReturningAcceptanceFailure(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := execute(context.Background(), []string{
|
||||
"-target", "https://target.example/health", "-requests", "10", "-concurrency", "1",
|
||||
"-max-error-rate", "0.1", "-max-p99", "100ms", "-output", path,
|
||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
||||
return loadgen.Report{
|
||||
Requests: 10, Completed: 10, Failed: 2, Duration: time.Second,
|
||||
Latency: loadgen.LatencyReport{Samples: 10, P99UpperBound: 250 * time.Millisecond},
|
||||
}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 3 || stderr.Len() == 0 {
|
||||
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
for _, content := range [][]byte{stdout.Bytes(), mustReadFile(t, path)} {
|
||||
var report loadgen.Report
|
||||
if err := json.Unmarshal(content, &report); err != nil || report.Acceptance == nil || report.Acceptance.Passed ||
|
||||
len(report.Acceptance.Violations) != 2 {
|
||||
t.Fatalf("acceptance report = (%+v, %v)", report, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsInvalidAcceptanceThresholdBeforeRunningWorkload(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
called := false
|
||||
code := execute(context.Background(), []string{
|
||||
"-target", "https://target.example/health", "-requests", "1", "-max-error-rate", "1.1",
|
||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
||||
called = true
|
||||
return loadgen.Report{}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 2 || called || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("execute() = %d, called=%t stdout=%q stderr=%q", code, called, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceCriteriaRejectsNonFiniteErrorRate(t *testing.T) {
|
||||
for _, value := range []float64{math.NaN(), math.Inf(1)} {
|
||||
if _, _, err := acceptanceCriteria(value, 0); err == nil {
|
||||
t.Fatalf("acceptanceCriteria(%v) accepted a non-finite error rate", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
@ -54,12 +54,12 @@ test/{fixtures,integration,e2e,load}/
|
||||
**Files:** `go.mod`, `.golangci.yml`, `README.md`, `scripts/verify.ps1`,
|
||||
`.github/workflows/ci.yml`
|
||||
|
||||
- [ ] Create module `github.com/proxy-pool/proxy-pool` with Go 1.26.
|
||||
- [ ] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
||||
- [x] Create Go module `proxy-pool` with Go 1.26.
|
||||
- [x] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
||||
- [x] Add `scripts/verify.ps1` that runs format check, `go vet`, unit tests, race tests,
|
||||
and builds all commands, each test command bounded to 60 seconds.
|
||||
- [x] Add CI for Windows and Linux with unit/race/build jobs.
|
||||
- [ ] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
||||
- [x] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
||||
|
||||
## Task 2: Strict Configuration
|
||||
|
||||
@ -80,7 +80,7 @@ test/{fixtures,integration,e2e,load}/
|
||||
|
||||
- [x] Implement Proxy fields, UTC TTL precedence, canonical host/port, and unique key.
|
||||
- [x] Implement state transitions and reject illegal transitions.
|
||||
- [ ] Implement sharded runtime counters with CAS Reserve, Commit, Cancel, Release.
|
||||
- [x] Implement packed per-Proxy runtime counters with CAS Reserve, Commit, Cancel, Release.
|
||||
- [x] Prove with 1,000 concurrent goroutines that effective capacity is never exceeded.
|
||||
- [x] Add race coverage and duplicate-release invariant metrics hook.
|
||||
|
||||
@ -130,7 +130,7 @@ reject/wait/direct，并会在快照刷新后看到停用状态。
|
||||
|
||||
**Files:** `internal/controller/pool/*.go`, `internal/domain/upstream/pool.go`, tests
|
||||
|
||||
- [ ] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
||||
- [x] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
||||
health, ownership, pending expected fetch, and gateway reserve.
|
||||
- [x] Implement pool.maxSize and fetch.maxTotal as distinct counters.
|
||||
- [x] Allocate each Proxy to one Worker with epoch/version/expiry ownership.
|
||||
@ -326,7 +326,8 @@ Supervisor 也改为同时服从静态配置与管理态，消除两条启停消
|
||||
延迟直方图，输出状态分类、CONNECT 建立数、Extract 校验数、吞吐和 Go 内存/GC 快照。CONNECT
|
||||
以原始 TCP 握手连接 HTTP Gateway，建连成功后按 `hold` 保持，且不透明读取隧道内容。`extract`
|
||||
场景会自动生成独立 Request/Idempotency 标识，校验返回数量与单响应 ID 唯一性，且不记录地址或
|
||||
凭据。故障注入以及代表性集群报告仍未实现。
|
||||
凭据。报告可选按最大错误率和 p99 延迟门禁；违反阈值时仍完整写出 JSON 证据并以
|
||||
专用退出码失败。故障注入以及代表性集群报告仍未实现。
|
||||
|
||||
## Task 12: Machine-readable Contracts
|
||||
|
||||
@ -349,10 +350,12 @@ Supervisor 也改为同时服从静态配置与管理态，消除两条启停消
|
||||
|
||||
**Files:** `deploy/**`, `internal/platform/**`, `docs/operations/**`
|
||||
|
||||
- [ ] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
||||
- [x] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
||||
Grafana/HAProxy.
|
||||
- [ ] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
||||
- [x] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
||||
probes, resource limits, topology spread, and graceful termination.
|
||||
- [ ] Add the production Kubernetes mTLS identity overlay, certificate rotation and unique
|
||||
elastic Worker identity wiring.
|
||||
- [x] Add low-cardinality Prometheus metrics and structured secret-safe logs.
|
||||
- [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
|
||||
NAT/conntrack, and incident runbooks.
|
||||
@ -383,7 +386,7 @@ Gateway 热路径。`platform/logging` 以 JSON `slog` 输出进程级致命错
|
||||
|
||||
## Task 15: Completion Audit
|
||||
|
||||
- [ ] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
||||
- [x] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
||||
- [ ] Run `gofmt`, `go vet`, unit tests, race tests, builds, contract validation, and
|
||||
documentation link/example validation.
|
||||
- [ ] Run bounded local performance benchmarks; label 100k QPS as unverified until a
|
||||
|
||||
@ -2,29 +2,29 @@
|
||||
|
||||
## 架构与一致性
|
||||
|
||||
- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker，ownership epoch 单调。
|
||||
- [ ] Reserved -> Active 使用单个原子转换，无超卖与负计数。
|
||||
- [x] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||
- [x] 每个 Proxy 同一时刻最多归属一个 Worker，ownership epoch 单调。
|
||||
- [x] Reserved -> Active 使用单个原子转换，无超卖与负计数。
|
||||
- [x] Sequential 并发 Empty 只切换一次，旧 Upstream Proxy 自然耗尽；末端 `stop`
|
||||
使用权威 CAS 停用 Routing 并刷新 Gateway 快照。
|
||||
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`，OpenAPI 不存在 release/renew。
|
||||
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||
- [ ] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
||||
- [ ] `reserveForGateway` 在所有提取路径上统一执行。
|
||||
- [x] Extract 只有 `AVAILABLE -> EXTRACTED`，OpenAPI 不存在 release/renew。
|
||||
- [x] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||
- [x] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||
- [x] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
||||
- [x] `reserveForGateway` 在所有提取路径上统一执行。
|
||||
|
||||
## 安全
|
||||
|
||||
- [ ] 非回环监听均配置 Auth 或 allowCIDRs，严格模式已开启。
|
||||
- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||
- [x] 非回环监听均配置 Auth 或 allowCIDRs，严格模式已开启。
|
||||
- [x] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [x] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [ ] Secret 由外部密钥系统注入，镜像、ConfigMap、日志没有明文。
|
||||
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥，并完成轮换演练。
|
||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||
- [ ] NetworkPolicy 默认拒绝，外部数据库/Redis/Provider 网段已收紧。
|
||||
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
- [x] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
|
||||
## 可用性
|
||||
|
||||
@ -32,9 +32,9 @@
|
||||
- [ ] Redis 跨可用区并受监控，完成丢失后由 Provider 重建活动池的演练。
|
||||
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
|
||||
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
|
||||
- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续，超限拒绝新请求。
|
||||
- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||
- [ ] Snapshot 版本缺口触发全量同步，旧 Delta 被拒绝。
|
||||
- [x] Controller 断线时 Gateway 在 `maxStaleAge` 内继续，超限拒绝新请求。
|
||||
- [x] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||
- [x] Snapshot 版本缺口触发全量同步，旧 Delta 被拒绝。
|
||||
|
||||
## 性能
|
||||
|
||||
@ -42,15 +42,15 @@
|
||||
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
|
||||
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
|
||||
- [ ] 目标利用率不高于 60%，HPA 缩容稳定窗口不低于 10 分钟。
|
||||
- [ ] 队列、buffer、日志和结果上报全部有界。
|
||||
- [x] 队列、buffer、日志和结果上报全部有界。
|
||||
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
|
||||
|
||||
## 观测与值班
|
||||
|
||||
- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker。
|
||||
- [x] 仪表盘覆盖 Gateway、库存、Provider、Extract、Snapshot、Checker 和 Drain。
|
||||
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
|
||||
- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||
- [ ] 日志脱敏已用真实 Secret fixture 验证。
|
||||
- [x] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||
- [x] 日志脱敏已用真实 Secret fixture 验证。
|
||||
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
|
||||
|
||||
## 发布门禁
|
||||
|
||||
@ -24,11 +24,10 @@
|
||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
||||
配额、动态重载和 Admin 低基数统计已装配；Controller 已装配 Redis BASIC/EGRESS/TARGET 任务 broker，
|
||||
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP
|
||||
请求场景；CONNECT 长连接/Extract 压测与完整 mTLS 环境 Overlay 仍属于
|
||||
`implementation-plan.md` 后续任务。
|
||||
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系，不能
|
||||
视为完整可运行拓扑。
|
||||
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP、
|
||||
CONNECT 长连接和 Extract 场景，并可按最大错误率和 p99 延迟输出机器可判定的验收结果；
|
||||
它不替代代表性集群压测。Compose 已声明本地 Controller/Gateway/Checker mTLS 拓扑，
|
||||
但本机尚未完成容器端到端启动；Kubernetes 仍需要环境专属的工作负载身份 Overlay。
|
||||
|
||||
### 2.1 前置条件
|
||||
|
||||
|
||||
@ -98,38 +98,26 @@ Mermaid blocks 35
|
||||
|
||||
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器，`go test -race` 在本机未执行；
|
||||
CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixture
|
||||
已经运行；当前 Docker 镜像和激活的 Compose/Kubernetes 清单只包含已实现的
|
||||
Controller/Gateway 入口，完整 mTLS 运行时拓扑仍只有静态验证。
|
||||
已经运行；本地 Compose 已声明 Controller、两个独立 Gateway 身份和一个 Checker 的
|
||||
mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
|
||||
启用 Checker，因为生产环境必须通过 Overlay 为每个弹性 Worker 注入唯一身份与证书；
|
||||
该 Overlay 和容器端到端启动仍待具备镜像网络的环境验证。
|
||||
|
||||
## 3. 后续实现范围
|
||||
|
||||
以下已有设计、接口或部署位置，但尚无端到端生产实现：
|
||||
|
||||
1. `proxy-loadgen` 已提供有界 HTTP、HTTP Gateway CONNECT 长连接和 Distribution Extract
|
||||
响应校验场景；`proxy-checker` 的 BASIC 任务进程已经完成，
|
||||
`proxy-controller` 已完成
|
||||
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配，`proxy-gateway` 已完成
|
||||
HTTP/Metrics 与控制面 Session 装配；Checker、Gateway Outcome、Drain、Provider、
|
||||
Extraction 与 Capacity 的低基数业务指标已闭环；Gateway Capacity 的动态降容保留
|
||||
已有连接并在其排空前停止新增预留，非法 Reservation 终结汇总为固定枚举指标；Controller、Gateway、Checker 的
|
||||
进程级致命错误现使用统一 JSON 脱敏日志出口，不输出错误原文。
|
||||
2. Gateway 已支持连接池、每 Host 连接上限、握手/空闲超时和隧道缓冲的本地配置；
|
||||
仍需代表性环境的流量压测。
|
||||
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
|
||||
4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成；
|
||||
公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成，Controller
|
||||
三监听器与探针集成已完成；可选聚合指标和完整容器进程部署验证仍待实现。
|
||||
5. Worker Register/ACK/Runtime gRPC 接收路径、Redis 服务端 TTL 会话生命周期、
|
||||
SPIFFE mTLS 校验和 Controller 生命周期接线已完成；Redis Provider Leader、
|
||||
分布式请求额度、Distribution Client 限制和 Provider 状态丢失重建已完成。
|
||||
6. Worker 基础网络快照流、Proxy/Gateway Routing/凭据 Snapshot payload、Gateway Snapshot
|
||||
客户端和进程装配、同版本 Routing 编译/动态匹配、五种策略上游选择与 reject/wait/direct 已完成；
|
||||
Outcome 上报已完成基础观测链；Redis ownership drain/ACK/过期回收及按 Worker 的可下发索引已完成。
|
||||
7. BASIC Checker 调度与 HTTP/HTTPS/SOCKS5 探测器、全局与 TARGET Profile 的 Memory/Redis
|
||||
原子归并、Controller Reducer 和 Observation 上报 RPC 已完成；EGRESS、TARGET
|
||||
生产任务调度与 REMOVE 编排仍待实现。
|
||||
8. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
||||
9. 将 reject/wait/direct 接入 Distribution 运行链。
|
||||
1. 已实现的 Controller、Gateway、Checker 和 Loadgen 进程仍需在实际容器网络中完成
|
||||
mTLS、DNS、Redis/PostgreSQL 连通性、健康探针和优雅停机的端到端验证。
|
||||
2. Gateway 已支持连接池、每 Host 连接上限、握手/空闲超时、隧道缓冲、请求生命周期指标
|
||||
与 p99 告警；仍需代表性环境的流量压测和容量报告。
|
||||
3. Provider 长期凭据回收与累计额度的生产运营策略仍需结合实际供应商合同、失效时间和
|
||||
密钥平台完成演练。
|
||||
4. Kubernetes 需要按工作负载身份系统提供 mTLS Overlay、证书轮换、唯一 Worker ID 和
|
||||
Checker 出站 NetworkPolicy；base 清单不假定或伪造这些生产环境能力。
|
||||
5. Redis/PostgreSQL 故障转移、控制面断线恢复、Snapshot 回滚、Drain 回收和 Extract
|
||||
库存不足的完整故障演练。
|
||||
6. 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率的代表性集群负载测试。
|
||||
## 4. 容量结论
|
||||
|
||||
100,000 QPS 是集群设计输入，不是本次验证结果。只有实现上述运行时，并在
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
|---|---|---|---|
|
||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离；Controller 已运行 Worker Register/Watch/ACK/Runtime/Outcome 与 Checker Observation gRPC，并发布 Proxy/Gateway Routing/按引用去重凭据完整快照；Checker 任务流已具备有界领取、租约栅栏和任务期凭据契约。Gateway 已将快照编译为同版本动态 View，并由独立进程维护控制面会话。`proxy-checker` 与 Redis BASIC/EGRESS/TARGET 共享任务运行态、`proxy-loadgen` 有界 HTTP/CONNECT/Extract 校验场景均已实现 |
|
||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Gateway bootstrap 集成测试验证启动期控制面会话与快照就绪，HTTP 请求只走本地 Snapshot/Dispatch；Outcome 仅写入有界非阻塞本地队列，代表性性能剖析待完成 |
|
||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机；Gateway 命令已装配代理与 Metrics 监听，运行时 mTLS 部署 Overlay 待完成 |
|
||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机；Gateway 命令已装配代理与 Metrics 监听。本地 Compose 已声明 Controller/Gateway/Checker mTLS 拓扑；Kubernetes 的工作负载身份 Overlay 仍待环境实现 |
|
||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成；静态配置与 Admin 管理态共同决定 Provider 启停，停用策略以 Upstream revision 栅栏驱动既有 ownership Drain；Routing 切换会向本进程 Worker 快照流广播完整刷新，多副本按定时刷新与权威 HMAC 指纹/revision 栅栏收敛并拒绝旧配置换主 |
|
||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标；待代表性集群负载报告 |
|
||||
|
||||
@ -19,11 +19,11 @@
|
||||
|---|---|---|---|
|
||||
| ROUTE-001 | Routing 自上而下匹配，首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
||||
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成；Distribution 接线待完成 |
|
||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成。Distribution 按独占 Extract 契约分配 Proxy，不参与目标请求的 Routing 策略选择 |
|
||||
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | Provider Stats 对连续空结果分配单调代次，Controller `SequentialCoordinator` 在独立有界循环中读取权威配置/管理快照，并通过 `ExpectedCurrent` CAS 自动切换。禁用候选会跳过；若当前项被管理态禁用，协调器不读取 Provider Stats，按配置顺序推进到下一启用项，只有 `loop` 可回绕，`stop`/`stayLast` 无后继时通过 `DisableRouting` 原子停用路由。重复 Tick、旧代次及跨 Controller 并发 Tick 均不会重复变更；成功切换或停用都会广播完整 Snapshot，并与审计/Outbox 同事务提交。|
|
||||
| ROUTE-005 | 空计数属于 Upstream，当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
||||
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽，不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照；Sequential 仅将新分配切到新的 CurrentUpstream，旧 Proxy 仍保留在快照，既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain，避免影响其他 Routing |
|
||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct，默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 direct；Distribution 接线和默认化策略待完成 |
|
||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct，默认 reject | 5075-5294, 6743-6760 | Gateway 已实现默认 reject、带 `wait_timeout` 的本地容量等待，以及经 TargetPolicy 的 direct。静态 `routing.action: direct` 在认证、限流和目标策略后绕过 Proxy 调度；Distribution 不承载目标请求路由 |
|
||||
|
||||
## Provider 与补池
|
||||
|
||||
@ -46,7 +46,7 @@
|
||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成；Snapshot 动态降容会保留既有 Active、在计数低于新上限前拒绝新预留，运行态复用与恢复预留均有回归测试。非法 Commit/Cancel/Release 经固定枚举观察器进入低基数指标 |
|
||||
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试；Worker session/ACK/sequence/TTL/ownership fence、基础 Snapshot 流及 Gateway 会话组件已通过内存、Redis 8.2 和 gRPC 测试；权威 payload、目标健康和 Gateway reserve 策略仍待完成 |
|
||||
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试；Worker session/ACK/sequence/TTL/ownership fence、权威 Proxy/Routing/凭据 payload、目标健康和 Gateway reserve 策略已通过内存、Redis 8.2、gRPC 与 Gateway 回归测试 |
|
||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||
@ -86,6 +86,6 @@
|
||||
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
|
||||
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
||||
| OPS-002 | 优雅停机停止新请求/Fetch，等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试；真实 PostgreSQL 契约待完成 |
|
||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现；Checker Collector 暴露固定等级和结果，Gateway Collector 暴露 `proxy_pool_gateway_outcomes_total{stage,result}`、`proxy_pool_gateway_outcome_queue_dropped_total` 与 `proxy_pool_gateway_capacity_invariant_violations_total{operation}`，后者仅允许固定的 Reservation 生命周期违规枚举。Drain Collector 暴露 `proxy_pool_controller_drain_candidates_total{reason}` 与 `proxy_pool_controller_drains_started_total{reason}`，其中 reason 固定为 unhealthy/upstream_disabled；Provider Collector 暴露固定 class 的拉取次数与候选计数；Extraction Collector 暴露固定 result 的请求、请求数与交付数；Capacity Collector 按 Controller 进程聚合库存 Gauge，并只暴露固定 success/error 的库存读取结果，不把 Upstream 写成标签。`platform/logging` 使用 JSON `slog`，敏感字段、URL 用户信息、查询 Secret 和错误对象写出前均被脱敏；回归测试锁定标签集与 Secret 不泄露 |
|
||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试；真实 PostgreSQL 18 契约、迁移幂等和 `information_schema` 数据边界测试已完成 |
|
||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现；Gateway Collector 暴露固定 `protocol` 的请求总数、并发数、隧道数和生命周期直方图，以及固定 `stage,result` 的 Outcome 指标。Checker、Drain、Provider、Extraction 与 Capacity Collector 仅使用固定枚举标签；Grafana 和 Prometheus 规则由部署契约测试校验只引用已注册指标且不按 Upstream/Worker 聚合。`platform/logging` 使用 JSON `slog`，敏感字段、URL 用户信息、查询 Secret 和错误对象写出前均被脱敏；回归测试锁定标签集与 Secret 不泄露 |
|
||||
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单；Redis 活动池由 Memory/Redis 公用契约覆盖，跨进程故障场景仍按清单推进 |
|
||||
|
||||
67
internal/loadgen/acceptance.go
Normal file
67
internal/loadgen/acceptance.go
Normal file
@ -0,0 +1,67 @@
|
||||
package loadgen
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrInvalidAcceptanceCriteria = errors.New("invalid load generator acceptance criteria")
|
||||
|
||||
// AcceptanceCriteria describes optional report gates. The explicit enforcement
|
||||
// flags keep a zero-error requirement distinct from an omitted requirement.
|
||||
type AcceptanceCriteria struct {
|
||||
EnforceFailureRate bool
|
||||
MaxFailureRate float64
|
||||
EnforceP99Latency bool
|
||||
MaxP99Latency time.Duration
|
||||
}
|
||||
|
||||
// AcceptanceResult is a machine-readable evaluation of one bounded workload.
|
||||
// It carries no target, proxy, client, credential, or request identifier data.
|
||||
type AcceptanceResult struct {
|
||||
Passed bool `json:"passed"`
|
||||
FailureRate float64 `json:"failureRate"`
|
||||
Violations []string `json:"violations,omitempty"`
|
||||
}
|
||||
|
||||
// EvaluateAcceptance applies the requested thresholds to a completed report.
|
||||
// A report without completed requests cannot prove any requested threshold.
|
||||
func EvaluateAcceptance(report Report, criteria AcceptanceCriteria) (AcceptanceResult, error) {
|
||||
if criteria.EnforceFailureRate && (math.IsNaN(criteria.MaxFailureRate) || math.IsInf(criteria.MaxFailureRate, 0) ||
|
||||
criteria.MaxFailureRate < 0 || criteria.MaxFailureRate > 1) {
|
||||
return AcceptanceResult{}, ErrInvalidAcceptanceCriteria
|
||||
}
|
||||
if criteria.EnforceP99Latency && criteria.MaxP99Latency <= 0 {
|
||||
return AcceptanceResult{}, ErrInvalidAcceptanceCriteria
|
||||
}
|
||||
|
||||
result := AcceptanceResult{Passed: true}
|
||||
if !criteria.EnforceFailureRate && !criteria.EnforceP99Latency {
|
||||
return result, nil
|
||||
}
|
||||
if report.Completed == 0 {
|
||||
result.Passed = false
|
||||
result.Violations = []string{"no completed requests were measured"}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
result.FailureRate = float64(report.Failed) / float64(report.Completed)
|
||||
if criteria.EnforceFailureRate && result.FailureRate > criteria.MaxFailureRate {
|
||||
result.Passed = false
|
||||
result.Violations = append(result.Violations,
|
||||
fmt.Sprintf("failure rate %.6f exceeded limit %.6f", result.FailureRate, criteria.MaxFailureRate))
|
||||
}
|
||||
if criteria.EnforceP99Latency {
|
||||
if report.Latency.Samples == 0 {
|
||||
result.Passed = false
|
||||
result.Violations = append(result.Violations, "no latency samples were measured")
|
||||
} else if report.Latency.P99UpperBound > criteria.MaxP99Latency {
|
||||
result.Passed = false
|
||||
result.Violations = append(result.Violations,
|
||||
fmt.Sprintf("p99 latency %s exceeded limit %s", report.Latency.P99UpperBound, criteria.MaxP99Latency))
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
64
internal/loadgen/acceptance_test.go
Normal file
64
internal/loadgen/acceptance_test.go
Normal file
@ -0,0 +1,64 @@
|
||||
package loadgen
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEvaluateAcceptancePassesWithinConfiguredThresholds(t *testing.T) {
|
||||
report := Report{
|
||||
Completed: 100,
|
||||
Failed: 1,
|
||||
Latency: LatencyReport{Samples: 100, P99UpperBound: 80 * time.Millisecond},
|
||||
}
|
||||
|
||||
result, err := EvaluateAcceptance(report, AcceptanceCriteria{
|
||||
EnforceFailureRate: true,
|
||||
MaxFailureRate: 0.02,
|
||||
EnforceP99Latency: true,
|
||||
MaxP99Latency: 100 * time.Millisecond,
|
||||
})
|
||||
if err != nil || !result.Passed || result.FailureRate != 0.01 || len(result.Violations) != 0 {
|
||||
t.Fatalf("EvaluateAcceptance() = (%+v, %v)", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateAcceptanceReportsEveryBreachedThreshold(t *testing.T) {
|
||||
report := Report{
|
||||
Completed: 10,
|
||||
Failed: 2,
|
||||
Latency: LatencyReport{Samples: 10, P99UpperBound: 250 * time.Millisecond},
|
||||
}
|
||||
|
||||
result, err := EvaluateAcceptance(report, AcceptanceCriteria{
|
||||
EnforceFailureRate: true,
|
||||
MaxFailureRate: 0.1,
|
||||
EnforceP99Latency: true,
|
||||
MaxP99Latency: 100 * time.Millisecond,
|
||||
})
|
||||
if err != nil || result.Passed || result.FailureRate != 0.2 || len(result.Violations) != 2 ||
|
||||
!strings.Contains(result.Violations[0], "failure rate") || !strings.Contains(result.Violations[1], "p99 latency") {
|
||||
t.Fatalf("EvaluateAcceptance() = (%+v, %v)", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateAcceptanceRejectsInvalidCriteriaAndEmptyMeasurements(t *testing.T) {
|
||||
if _, err := EvaluateAcceptance(Report{}, AcceptanceCriteria{EnforceFailureRate: true, MaxFailureRate: 1.1}); err == nil {
|
||||
t.Fatal("EvaluateAcceptance() accepted failure rate above one")
|
||||
}
|
||||
if _, err := EvaluateAcceptance(Report{}, AcceptanceCriteria{EnforceP99Latency: true, MaxP99Latency: 0}); err == nil {
|
||||
t.Fatal("EvaluateAcceptance() accepted zero p99 limit")
|
||||
}
|
||||
for _, value := range []float64{math.NaN(), math.Inf(1)} {
|
||||
if _, err := EvaluateAcceptance(Report{}, AcceptanceCriteria{EnforceFailureRate: true, MaxFailureRate: value}); err == nil {
|
||||
t.Fatalf("EvaluateAcceptance() accepted non-finite failure rate %v", value)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := EvaluateAcceptance(Report{}, AcceptanceCriteria{EnforceFailureRate: true, MaxFailureRate: 0})
|
||||
if err != nil || result.Passed || len(result.Violations) != 1 || !strings.Contains(result.Violations[0], "no completed requests") {
|
||||
t.Fatalf("EvaluateAcceptance(empty) = (%+v, %v)", result, err)
|
||||
}
|
||||
}
|
||||
@ -79,6 +79,7 @@ type Report struct {
|
||||
RateStartsDropped uint64
|
||||
Latency LatencyReport
|
||||
Runtime RuntimeReport
|
||||
Acceptance *AcceptanceResult `json:"acceptance,omitempty"`
|
||||
}
|
||||
|
||||
// LatencyReport uses logarithmic microsecond buckets. Percentiles are upper
|
||||
|
||||
28
task_plan.md
28
task_plan.md
@ -31,12 +31,13 @@
|
||||
11. [已完成] 实现 PostgreSQL 管理面；ADR、领域事务契约、MemoryStore、公用
|
||||
契约、六表 Schema、Admin Actor 传播、应用服务、原子配置发布、pgx Adapter
|
||||
和真实 PostgreSQL 18 集成测试已完成
|
||||
12. [进行中] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界；
|
||||
12. [已完成] 复核验收清单并收敛既有 Routing/Sequential 与 Proxy 容量边界；
|
||||
机器契约和文档类滞后勾选已按仓库证据校正
|
||||
13. [进行中] 落地 `proxy-controller` 进程装配；配置单次加载、PostgreSQL 迁移、
|
||||
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成，
|
||||
双存储 bootstrap 和探针集成已通过；Worker 运行态存储与权威容量读取原语已
|
||||
完成，WorkerControlPlane、Provider、业务指标与完整容器进程链仍待实现
|
||||
13. [已完成] 落地 `proxy-controller` 进程装配；配置单次加载、PostgreSQL 迁移、
|
||||
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭、
|
||||
WorkerControlPlane、Provider、Gateway/Checker/Loadgen 进程和业务指标均已完成。
|
||||
Compose mTLS 链路已具备配置、证书生成、静态清单与进程级验证；容器端到端启动
|
||||
仍受本机 Docker 镜像网络环境阻塞，留待具备镜像访问的环境复核。
|
||||
14. [已完成] 收敛健康故障生命周期：将 `UNHEALTHY` 的持续时长纳入权威活动池状态，
|
||||
通过 Controller 后台的有界回收清理无 Worker ownership 的持续异常 Proxy；拥有
|
||||
Worker 的候选经 health/ownership 条件栅栏自动发起 Drain，在 ACK 与 Runtime 收尾后再处理。
|
||||
@ -48,8 +49,9 @@
|
||||
Prometheus 指标。Admin 成功提交的 Upstream 启停、Routing 切换和配置发布现会经
|
||||
公用广播器立即刷新本进程所有 Worker 完整 Snapshot；跨 Controller 副本仍以定时刷新
|
||||
收敛。Health Scheduler 现以相同 revision 合并配置与管理态启用状态，管理态停用会停止
|
||||
该 Upstream 的新 BASIC/EGRESS/TARGET 任务，停用 Routing 仅停止其新 TARGET 任务。后续补齐
|
||||
更完整的运行态可观测闭环。
|
||||
该 Upstream 的新 BASIC/EGRESS/TARGET 任务，停用 Routing 仅停止其新 TARGET 任务。
|
||||
Gateway 请求生命周期直方图、Grafana 仪表盘、指标契约检查和 p99 延迟告警已完成；
|
||||
后续以真实容器、集群故障演练和代表性负载报告完成运行态验证闭环。
|
||||
|
||||
## 串并行关系
|
||||
|
||||
@ -68,12 +70,12 @@
|
||||
|
||||
## 已知环境限制
|
||||
|
||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证；Redis 8.2
|
||||
与 PostgreSQL 18 的隔离 Adapter fixture 已运行，完整目标运行拓扑尚未启动。
|
||||
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配；
|
||||
Provider 分布式协调和 Worker 运行态 Redis 原语已完成，但 WorkerControlPlane
|
||||
接收端、Provider Fleet、Gateway、Checker、Loadgen、业务指标、Redis 故障
|
||||
转移验证与代表性集群压测属于后续实施范围。
|
||||
- Docker Compose mTLS 控制面与 Kubernetes Kustomize 已完成静态渲染验证；Redis 8.2
|
||||
与 PostgreSQL 18 的隔离 Adapter fixture 已运行。Docker Desktop 缺少镜像 HTTPS
|
||||
代理配置，未能在本机完成 Compose 容器端到端启动。
|
||||
- Controller、Gateway、Checker、Loadgen、Provider Fleet、WorkerControlPlane、业务指标和
|
||||
Redis 故障边界均有实现与定向测试。真实容器故障转移、Kubernetes 工作负载身份 Overlay
|
||||
与代表性集群压测仍属于后续验证范围。
|
||||
- `implementation-plan.md` 当前按 74 个验收项统计；验收勾选数不等同于生产就绪度。
|
||||
Compose 本地 mTLS 控制面运行链已实现并通过配置、证书、单元与构建验证；受本机 Docker
|
||||
Desktop 镜像 HTTPS 代理缺失影响，容器端到端启动仍待具备镜像网络的环境复核。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user