feat: reload control plane leaf certificates

docs/configuration/reference.md

@@ -302,6 +302,9 @@ controlPlane:
 - `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验，但必须使用独立的 Checker
   证书。Controller 分别验证 `.../worker/<worker-id>` 与
   `.../checker/<checker-id>` SPIFFE URI，不能跨角色复用证书。
+- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书和私钥；
+  文件暂时不可读时继续使用最后一次有效叶证书。既有 gRPC 流不强制重握手，根 CA 轮换仍需
+  通过包含新旧根的重叠信任包和滚动发布完成。
 
 Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、

docs/development/implementation-plan.md

@@ -377,6 +377,8 @@ CI 另有独立 Deployment job，在占位凭据下渲染 Compose，并使用 `k
 渲染 Kubernetes base 和单副本 development mTLS Overlay；它不启动容器、不访问真实存储或密钥。
 development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
 网络策略，但固定 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本，不能代替生产证书轮换。
+控制面 TLS 叶证书已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效证书；
+生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI、根信任轮换和弹性身份注入。
 
 ## Task 14: Documentation, Examples, and Diagrams
 

docs/operations/runbook.md

@@ -350,6 +350,12 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
 
 Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。
 
+控制面 mTLS 的叶证书和私钥会在每次新的 TLS 握手时重新读取；Kubernetes 投影 Secret 或
+CSI 写入短暂不完整时保留最后一次有效证书。既有 gRPC 长连接不会被强制替换，因此应在新
+叶证书生效后确认新连接成功，再按 PDB 约束滚动重启或排空旧 Worker。`serverCAFile` 与
+Controller `clientCAFile` 的信任根在进程启动时加载；根 CA 轮换必须先发布包含新旧根的
+重叠信任包，再滚动所有角色，最后才撤销旧根。
+
 ## 10. 主机与网络容量基线
 
 在预发布压测和每次生产扩容前，记录 Gateway 节点、容器运行时和 Pod 内的下列只读基线；

docs/requirements/completion-audit.md

@@ -105,8 +105,9 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt
 mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
 启用 Checker；`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
 提供固定开发身份、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
-生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一、可轮换的身份与证书，且
+Controller、Gateway 与 Checker 的叶证书已在每个新 TLS 握手时重载，文件短暂不一致时
-容器端到端启动仍待具备镜像网络的环境验证。
+保留最后一次有效证书。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一
+SPIFFE URI、根信任轮换与证书，且容器端到端启动仍待具备镜像网络的环境验证。
 
 运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算，以及
 主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求；这些文档不替代真实环境

internal/controller/worker/server.go

@@ -14,6 +14,7 @@ import (
 
 	controlplanev1 "proxy-pool/gen/controlplane/v1"
 	"proxy-pool/internal/config"
+	"proxy-pool/internal/controlplane/tlsreload"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -159,7 +160,7 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize
 		if err != nil {
 			return nil, nil, fmt.Errorf("%w: %v", ErrInvalidServer, err)
 		}
-		certificate, err := tls.LoadX509KeyPair(controlPlane.TLS.CertFile, controlPlane.TLS.KeyFile)
+		certificate, err := tlsreload.NewCertificateProvider(controlPlane.TLS.CertFile, controlPlane.TLS.KeyFile)
 		if err != nil {
 			return nil, nil, fmt.Errorf("%w: load server certificate: %v", ErrInvalidServer, err)
 		}
@@ -173,7 +174,7 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize
 		}
 		transport := credentials.NewTLS(&tls.Config{
 			MinVersion:     tls.VersionTLS13,
-			Certificates: []tls.Certificate{certificate},
+			GetCertificate: certificate.ServerCertificate,
 			ClientAuth:     tls.RequireAndVerifyClientCert,
 			ClientCAs:      clientCAs,
 		})

internal/controlplane/clienttransport/transport.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"proxy-pool/internal/config"
+	"proxy-pool/internal/controlplane/tlsreload"
 
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
@@ -46,7 +47,7 @@ func New(
 		if clientTLS.CertFile == "" || clientTLS.KeyFile == "" || clientTLS.ServerCAFile == "" {
 			return nil, fmt.Errorf("%w: controlPlane.%s is required for mtls", ErrInvalidTransport, role)
 		}
-		certificate, err := tls.LoadX509KeyPair(clientTLS.CertFile, clientTLS.KeyFile)
+		certificate, err := tlsreload.NewCertificateProvider(clientTLS.CertFile, clientTLS.KeyFile)
 		if err != nil {
 			return nil, fmt.Errorf("load %s control-plane certificate: %w", role, err)
 		}
@@ -60,8 +61,8 @@ func New(
 		}
 		host, _, _ := net.SplitHostPort(address)
 		return credentials.NewTLS(&tls.Config{
-			MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{certificate}, RootCAs: roots,
+			MinVersion: tls.VersionTLS13, GetClientCertificate: certificate.ClientCertificate,
-			ServerName: strings.Trim(host, "[]"),
+			RootCAs: roots, ServerName: strings.Trim(host, "[]"),
 		}), nil
 	default:
 		return nil, fmt.Errorf("%w: unsupported control-plane tls mode", ErrInvalidTransport)

internal/controlplane/tlsreload/certificate.go

@@ -0,0 +1,61 @@
+// Package tlsreload provides last-known-good leaf certificate reloads for the
+// low-frequency control-plane TLS handshakes.
+package tlsreload
+
+import (
+	"crypto/tls"
+	"errors"
+	"sync"
+)
+
+var ErrInvalidCertificatePaths = errors.New("invalid TLS certificate paths")
+
+// CertificateProvider reloads a certificate/key pair for each new handshake.
+// Kubernetes projected volumes update file pairs atomically, but retaining the
+// last valid pair also bridges short writer or filesystem visibility gaps.
+type CertificateProvider struct {
+	certificateFile string
+	keyFile         string
+
+	mu   sync.RWMutex
+	last tls.Certificate
+}
+
+func NewCertificateProvider(certificateFile, keyFile string) (*CertificateProvider, error) {
+	if certificateFile == "" || keyFile == "" {
+		return nil, ErrInvalidCertificatePaths
+	}
+	certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+	return &CertificateProvider{certificateFile: certificateFile, keyFile: keyFile, last: certificate}, nil
+}
+
+func (provider *CertificateProvider) ServerCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return provider.certificate()
+}
+
+func (provider *CertificateProvider) ClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	return provider.certificate()
+}
+
+func (provider *CertificateProvider) certificate() (*tls.Certificate, error) {
+	if provider == nil {
+		return nil, ErrInvalidCertificatePaths
+	}
+	certificate, err := tls.LoadX509KeyPair(provider.certificateFile, provider.keyFile)
+	if err == nil {
+		provider.mu.Lock()
+		provider.last = certificate
+		provider.mu.Unlock()
+		return &certificate, nil
+	}
+	provider.mu.RLock()
+	fallback := provider.last
+	provider.mu.RUnlock()
+	if len(fallback.Certificate) == 0 {
+		return nil, err
+	}
+	return &fallback, nil
+}

internal/controlplane/tlsreload/certificate_test.go

@@ -0,0 +1,97 @@
+package tlsreload
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestCertificateProviderReloadsPairsAndKeepsLastKnownGoodCertificate(t *testing.T) {
+	directory := t.TempDir()
+	certificatePath := filepath.Join(directory, "tls.crt")
+	keyPath := filepath.Join(directory, "tls.key")
+	writeCertificatePair(t, certificatePath, keyPath, "first")
+
+	provider, err := NewCertificateProvider(certificatePath, keyPath)
+	if err != nil {
+		t.Fatalf("NewCertificateProvider() error = %v", err)
+	}
+	certificate, err := provider.ServerCertificate(nil)
+	if subject := certificateSubject(t, certificate, err); subject != "first" {
+		t.Fatalf("initial certificate subject = %q, want first", subject)
+	}
+
+	writeCertificatePair(t, certificatePath, keyPath, "second")
+	certificate, err = provider.ClientCertificate(nil)
+	if subject := certificateSubject(t, certificate, err); subject != "second" {
+		t.Fatalf("reloaded certificate subject = %q, want second", subject)
+	}
+
+	if err := os.WriteFile(keyPath, []byte("incomplete"), 0o600); err != nil {
+		t.Fatalf("break private key: %v", err)
+	}
+	certificate, err = provider.ServerCertificate(nil)
+	if subject := certificateSubject(t, certificate, err); subject != "second" {
+		t.Fatalf("fallback certificate subject = %q, want second", subject)
+	}
+}
+
+func TestCertificateProviderRejectsMissingPaths(t *testing.T) {
+	if _, err := NewCertificateProvider("", "key"); err != ErrInvalidCertificatePaths {
+		t.Fatalf("NewCertificateProvider(empty certificate) error = %v, want %v", err, ErrInvalidCertificatePaths)
+	}
+	if _, err := NewCertificateProvider("certificate", ""); err != ErrInvalidCertificatePaths {
+		t.Fatalf("NewCertificateProvider(empty key) error = %v, want %v", err, ErrInvalidCertificatePaths)
+	}
+}
+
+func certificateSubject(t *testing.T, certificate *tls.Certificate, err error) string {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("load certificate: %v", err)
+	}
+	parsed, err := x509.ParseCertificate(certificate.Certificate[0])
+	if err != nil {
+		t.Fatalf("parse certificate: %v", err)
+	}
+	return parsed.Subject.CommonName
+}
+
+func writeCertificatePair(t *testing.T, certificatePath, keyPath, commonName string) {
+	t.Helper()
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate private key: %v", err)
+	}
+	now := time.Now()
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName},
+		NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		t.Fatalf("create certificate: %v", err)
+	}
+	certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("marshal private key: %v", err)
+	}
+	key := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER})
+	if err := os.WriteFile(certificatePath, certificate, 0o600); err != nil {
+		t.Fatalf("write certificate: %v", err)
+	}
+	if err := os.WriteFile(keyPath, key, 0o600); err != nil {
+		t.Fatalf("write private key: %v", err)
+	}
+}