feat: reload control plane trust bundles

deploy/kubernetes/overlays/development-mtls/README.md

@@ -54,3 +54,5 @@ session、ownership 和证书角色边界。
 
 生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书，并将该副本的
 身份注入 `worker_id`、`instance_id`、Gateway Client TLS 或 Checker Client TLS。
+应用会在每次新控制面 TLS 握手读取更新后的叶证书和信任根；已有 gRPC 流仍按原会话保留，
+根轮换时先投放新旧根的重叠信任包，再按 PDB 滚动排空旧连接。

docs/configuration/reference.md

@@ -302,9 +302,9 @@ controlPlane:
 - `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验，但必须使用独立的 Checker
   证书。Controller 分别验证 `.../worker/<worker-id>` 与
   `.../checker/<checker-id>` SPIFFE URI，不能跨角色复用证书。
-- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书和私钥；
+- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和
-  文件暂时不可读时继续使用最后一次有效叶证书。既有 gRPC 流不强制重握手，根 CA 轮换仍需
+  信任根；文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手，根 CA
-  通过包含新旧根的重叠信任包和滚动发布完成。
+  轮换仍需先发布包含新旧根的重叠信任包，并在宽限期后滚动排空旧连接。
 
 Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、

docs/development/implementation-plan.md

@@ -377,8 +377,8 @@ CI 另有独立 Deployment job，在占位凭据下渲染 Compose，并使用 `k
 渲染 Kubernetes base 和单副本 development mTLS Overlay；它不启动容器、不访问真实存储或密钥。
 development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
 网络策略，但固定 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本，不能代替生产证书轮换。
-控制面 TLS 叶证书已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效证书；
+控制面 TLS 叶证书和信任根已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效材料；
-生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI、根信任轮换和弹性身份注入。
+生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。
 
 ## Task 14: Documentation, Examples, and Diagrams
 

docs/operations/runbook.md

@@ -350,11 +350,10 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
 
 Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。
 
-控制面 mTLS 的叶证书和私钥会在每次新的 TLS 握手时重新读取；Kubernetes 投影 Secret 或
+控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取；Kubernetes 投影
-CSI 写入短暂不完整时保留最后一次有效证书。既有 gRPC 长连接不会被强制替换，因此应在新
+Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换，
-叶证书生效后确认新连接成功，再按 PDB 约束滚动重启或排空旧 Worker。`serverCAFile` 与
+因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功，再按 PDB 约束滚动
-Controller `clientCAFile` 的信任根在进程启动时加载；根 CA 轮换必须先发布包含新旧根的
+重启或排空旧 Worker，最后才撤销旧根并使旧流重新认证。
-重叠信任包，再滚动所有角色，最后才撤销旧根。
 
 ## 10. 主机与网络容量基线
 

docs/requirements/completion-audit.md

@@ -105,9 +105,9 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt
 mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
 启用 Checker；`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
 提供固定开发身份、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
-Controller、Gateway 与 Checker 的叶证书已在每个新 TLS 握手时重载，文件短暂不一致时
+Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载，文件短暂
-保留最后一次有效证书。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一
+不一致时保留最后一次有效材料。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入
-SPIFFE URI、根信任轮换与证书，且容器端到端启动仍待具备镜像网络的环境验证。
+唯一 SPIFFE URI，且容器端到端启动仍待具备镜像网络的环境验证。
 
 运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算，以及
 主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求；这些文档不替代真实环境

internal/controller/worker/server.go

@@ -3,11 +3,9 @@ package worker
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"errors"
 	"fmt"
 	"net"
-	"os"
 	"reflect"
 	"strings"
 	"time"
@@ -164,21 +162,31 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize
 		if err != nil {
 			return nil, nil, fmt.Errorf("%w: load server certificate: %v", ErrInvalidServer, err)
 		}
-		caPEM, err := os.ReadFile(controlPlane.TLS.ClientCAFile)
+		trust, err := tlsreload.NewTrustProvider(controlPlane.TLS.ClientCAFile)
 		if err != nil {
-			return nil, nil, fmt.Errorf("%w: read client ca: %v", ErrInvalidServer, err)
+			return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err)
 		}
-		clientCAs := x509.NewCertPool()
+		clientCAs, err := trust.Pool()
-		if !clientCAs.AppendCertsFromPEM(caPEM) {
+		if err != nil {
-			return nil, nil, fmt.Errorf("%w: parse client ca", ErrInvalidServer)
+			return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err)
 		}
-		transport := credentials.NewTLS(&tls.Config{
+		configuration := &tls.Config{
 			MinVersion:     tls.VersionTLS13,
 			GetCertificate: certificate.ServerCertificate,
 			ClientAuth:     tls.RequireAndVerifyClientCert,
 			ClientCAs:      clientCAs,
-		})
+		}
-		return identity, []grpc.ServerOption{grpc.Creds(transport)}, nil
+		configuration.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) {
+			updatedCAs, loadErr := trust.Pool()
+			if loadErr != nil {
+				return nil, loadErr
+			}
+			current := configuration.Clone()
+			current.GetConfigForClient = nil
+			current.ClientCAs = updatedCAs
+			return current, nil
+		}
+		return identity, []grpc.ServerOption{grpc.Creds(credentials.NewTLS(configuration))}, nil
 	default:
 		return nil, nil, fmt.Errorf("%w: unsupported tls mode", ErrInvalidServer)
 	}

internal/controlplane/clienttransport/transport.go

@@ -4,11 +4,9 @@ package clienttransport
 
 import (
 	"crypto/tls"
-	"crypto/x509"
 	"errors"
 	"fmt"
 	"net"
-	"os"
 	"strconv"
 	"strings"
 
@@ -51,18 +49,18 @@ func New(
 		if err != nil {
 			return nil, fmt.Errorf("load %s control-plane certificate: %w", role, err)
 		}
-		caPEM, err := os.ReadFile(clientTLS.ServerCAFile)
+		trust, err := tlsreload.NewTrustProvider(clientTLS.ServerCAFile)
 		if err != nil {
-			return nil, fmt.Errorf("read %s control-plane CA: %w", role, err)
+			return nil, fmt.Errorf("load %s control-plane CA: %w", role, err)
-		}
-		roots := x509.NewCertPool()
-		if !roots.AppendCertsFromPEM(caPEM) {
-			return nil, fmt.Errorf("parse %s control-plane CA", role)
 		}
 		host, _, _ := net.SplitHostPort(address)
+		serverName := strings.Trim(host, "[]")
 		return credentials.NewTLS(&tls.Config{
 			MinVersion: tls.VersionTLS13, GetClientCertificate: certificate.ClientCertificate,
-			RootCAs: roots, ServerName: strings.Trim(host, "[]"),
+			ServerName: serverName,
+			// VerifyConnection retains standard verification semantics while allowing
+			// the CA bundle to reload for the next control-plane handshake.
+			InsecureSkipVerify: true, VerifyConnection: trust.VerifyServer(serverName),
 		}), nil
 	default:
 		return nil, fmt.Errorf("%w: unsupported control-plane tls mode", ErrInvalidTransport)

internal/controlplane/tlsreload/trust.go

@@ -0,0 +1,90 @@
+package tlsreload
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+)
+
+var ErrInvalidTrustPath = errors.New("invalid TLS trust bundle path")
+
+// TrustProvider reloads a PEM trust bundle for each new control-plane
+// handshake. It keeps the previous valid bundle while a projected volume is
+// momentarily incomplete during an update.
+type TrustProvider struct {
+	path string
+
+	mu   sync.RWMutex
+	last *x509.CertPool
+}
+
+func NewTrustProvider(path string) (*TrustProvider, error) {
+	if path == "" {
+		return nil, ErrInvalidTrustPath
+	}
+	pool, err := loadTrustPool(path)
+	if err != nil {
+		return nil, err
+	}
+	return &TrustProvider{path: path, last: pool}, nil
+}
+
+// Pool returns the latest valid roots. Returned pools are immutable after
+// construction, so callers can safely use them for an in-flight handshake.
+func (provider *TrustProvider) Pool() (*x509.CertPool, error) {
+	if provider == nil {
+		return nil, ErrInvalidTrustPath
+	}
+	pool, err := loadTrustPool(provider.path)
+	if err == nil {
+		provider.mu.Lock()
+		provider.last = pool
+		provider.mu.Unlock()
+		return pool, nil
+	}
+	provider.mu.RLock()
+	fallback := provider.last
+	provider.mu.RUnlock()
+	if fallback == nil {
+		return nil, err
+	}
+	return fallback, nil
+}
+
+// VerifyServer returns a TLS verifier that uses the latest trust bundle while
+// retaining normal DNS-name and server-authentication verification.
+func (provider *TrustProvider) VerifyServer(serverName string) func(tls.ConnectionState) error {
+	return func(state tls.ConnectionState) error {
+		if len(state.PeerCertificates) == 0 {
+			return errors.New("control-plane server did not provide a certificate")
+		}
+		roots, err := provider.Pool()
+		if err != nil {
+			return fmt.Errorf("load control-plane trust bundle: %w", err)
+		}
+		intermediates := x509.NewCertPool()
+		for _, certificate := range state.PeerCertificates[1:] {
+			intermediates.AddCert(certificate)
+		}
+		_, err = state.PeerCertificates[0].Verify(x509.VerifyOptions{
+			DNSName: serverName, Roots: roots, Intermediates: intermediates,
+			KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		})
+		return err
+	}
+}
+
+func loadTrustPool(path string) (*x509.CertPool, error) {
+	payload, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(payload) {
+		return nil, errors.New("parse TLS trust bundle")
+	}
+	return pool, nil
+}

internal/controlplane/tlsreload/trust_test.go

@@ -0,0 +1,120 @@
+package tlsreload
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestTrustProviderReloadsRootsAndKeepsLastKnownGoodBundle(t *testing.T) {
+	directory := t.TempDir()
+	trustPath := filepath.Join(directory, "ca.crt")
+	firstAuthority := newTestAuthority(t, "first")
+	firstServer := firstAuthority.server(t, "controller")
+	writeTrustBundle(t, trustPath, firstAuthority.certificate)
+
+	provider, err := NewTrustProvider(trustPath)
+	if err != nil {
+		t.Fatalf("NewTrustProvider() error = %v", err)
+	}
+	verify := provider.VerifyServer("controller")
+	if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err != nil {
+		t.Fatalf("verify first server with first root: %v", err)
+	}
+
+	secondAuthority := newTestAuthority(t, "second")
+	secondServer := secondAuthority.server(t, "controller")
+	writeTrustBundle(t, trustPath, secondAuthority.certificate)
+	if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err == nil {
+		t.Fatal("verify old server after trust rotation error = nil")
+	}
+	if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil {
+		t.Fatalf("verify second server with second root: %v", err)
+	}
+
+	if err := os.WriteFile(trustPath, []byte("incomplete"), 0o600); err != nil {
+		t.Fatalf("break trust bundle: %v", err)
+	}
+	if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil {
+		t.Fatalf("verify with last known good trust bundle: %v", err)
+	}
+}
+
+func TestTrustProviderRejectsEmptyPath(t *testing.T) {
+	if _, err := NewTrustProvider(""); err != ErrInvalidTrustPath {
+		t.Fatalf("NewTrustProvider(empty path) error = %v, want %v", err, ErrInvalidTrustPath)
+	}
+}
+
+type testAuthority struct {
+	certificate *x509.Certificate
+	privateKey  *ecdsa.PrivateKey
+}
+
+func newTestAuthority(t *testing.T, commonName string) testAuthority {
+	t.Helper()
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate authority key: %v", err)
+	}
+	now := time.Now()
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName},
+		NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true,
+		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		t.Fatalf("create authority certificate: %v", err)
+	}
+	certificate, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse authority certificate: %v", err)
+	}
+	return testAuthority{certificate: certificate, privateKey: privateKey}
+}
+
+func (authority testAuthority) server(t *testing.T, serverName string) *x509.Certificate {
+	t.Helper()
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate server key: %v", err)
+	}
+	now := time.Now()
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: serverName},
+		DNSNames: []string{serverName}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
+		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, authority.certificate, &privateKey.PublicKey, authority.privateKey)
+	if err != nil {
+		t.Fatalf("create server certificate: %v", err)
+	}
+	certificate, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse server certificate: %v", err)
+	}
+	return certificate
+}
+
+func writeTrustBundle(t *testing.T, path string, certificate *x509.Certificate) {
+	t.Helper()
+	if certificate == nil {
+		t.Fatal("write trust bundle: certificate is nil")
+	}
+	payload := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
+	if err := os.WriteFile(path, payload, 0o600); err != nil {
+		t.Fatalf("write trust bundle: %v", err)
+	}
+}