diff --git a/docs/configuration/reference.md b/docs/configuration/reference.md index 78e6566..45ade46 100644 --- a/docs/configuration/reference.md +++ b/docs/configuration/reference.md @@ -302,6 +302,9 @@ controlPlane: - `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验，但必须使用独立的 Checker 证书。Controller 分别验证 `.../worker/` 与 `.../checker/` SPIFFE URI，不能跨角色复用证书。 +- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书和私钥； + 文件暂时不可读时继续使用最后一次有效叶证书。既有 gRPC 流不强制重握手，根 CA 轮换仍需 + 通过包含新旧根的重叠信任包和滚动发布完成。 Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、 diff --git a/docs/development/implementation-plan.md b/docs/development/implementation-plan.md index c810af4..d1b7e5a 100644 --- a/docs/development/implementation-plan.md +++ b/docs/development/implementation-plan.md @@ -377,6 +377,8 @@ CI 另有独立 Deployment job，在占位凭据下渲染 Compose，并使用 `k 渲染 Kubernetes base 和单副本 development mTLS Overlay；它不启动容器、不访问真实存储或密钥。 development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小 网络策略，但固定 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本，不能代替生产证书轮换。 +控制面 TLS 叶证书已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效证书； +生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI、根信任轮换和弹性身份注入。 ## Task 14: Documentation, Examples, and Diagrams diff --git a/docs/operations/runbook.md b/docs/operations/runbook.md index 5fefcc9..2cdcb26 100644 --- a/docs/operations/runbook.md +++ b/docs/operations/runbook.md @@ -350,6 +350,12 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。 +控制面 mTLS 的叶证书和私钥会在每次新的 TLS 握手时重新读取；Kubernetes 投影 Secret 或 +CSI 写入短暂不完整时保留最后一次有效证书。既有 gRPC 长连接不会被强制替换，因此应在新 +叶证书生效后确认新连接成功，再按 PDB 约束滚动重启或排空旧 Worker。`serverCAFile` 与 +Controller `clientCAFile` 的信任根在进程启动时加载；根 CA 轮换必须先发布包含新旧根的 +重叠信任包，再滚动所有角色，最后才撤销旧根。 + ## 10. 主机与网络容量基线 在预发布压测和每次生产扩容前，记录 Gateway 节点、容器运行时和 Pod 内的下列只读基线； diff --git a/docs/requirements/completion-audit.md b/docs/requirements/completion-audit.md index 6e5af4f..65b6655 100644 --- a/docs/requirements/completion-audit.md +++ b/docs/requirements/completion-audit.md @@ -105,8 +105,9 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接 启用 Checker；`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker 提供固定开发身份、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。 -生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一、可轮换的身份与证书，且 -容器端到端启动仍待具备镜像网络的环境验证。 +Controller、Gateway 与 Checker 的叶证书已在每个新 TLS 握手时重载，文件短暂不一致时 +保留最后一次有效证书。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一 +SPIFFE URI、根信任轮换与证书，且容器端到端启动仍待具备镜像网络的环境验证。 运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算，以及 主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求；这些文档不替代真实环境 diff --git a/internal/controller/worker/server.go b/internal/controller/worker/server.go index 0eb6375..7da2fab 100644 --- a/internal/controller/worker/server.go +++ b/internal/controller/worker/server.go @@ -14,6 +14,7 @@ import ( controlplanev1 "proxy-pool/gen/controlplane/v1" "proxy-pool/internal/config" + "proxy-pool/internal/controlplane/tlsreload" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -159,7 +160,7 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize if err != nil { return nil, nil, fmt.Errorf("%w: %v", ErrInvalidServer, err) } - certificate, err := tls.LoadX509KeyPair(controlPlane.TLS.CertFile, controlPlane.TLS.KeyFile) + certificate, err := tlsreload.NewCertificateProvider(controlPlane.TLS.CertFile, controlPlane.TLS.KeyFile) if err != nil { return nil, nil, fmt.Errorf("%w: load server certificate: %v", ErrInvalidServer, err) } @@ -172,10 +173,10 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize return nil, nil, fmt.Errorf("%w: parse client ca", ErrInvalidServer) } transport := credentials.NewTLS(&tls.Config{ - MinVersion: tls.VersionTLS13, - Certificates: []tls.Certificate{certificate}, - ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: clientCAs, + MinVersion: tls.VersionTLS13, + GetCertificate: certificate.ServerCertificate, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, }) return identity, []grpc.ServerOption{grpc.Creds(transport)}, nil default: diff --git a/internal/controlplane/clienttransport/transport.go b/internal/controlplane/clienttransport/transport.go index f603828..1d446ff 100644 --- a/internal/controlplane/clienttransport/transport.go +++ b/internal/controlplane/clienttransport/transport.go @@ -13,6 +13,7 @@ import ( "strings" "proxy-pool/internal/config" + "proxy-pool/internal/controlplane/tlsreload" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" @@ -46,7 +47,7 @@ func New( if clientTLS.CertFile == "" || clientTLS.KeyFile == "" || clientTLS.ServerCAFile == "" { return nil, fmt.Errorf("%w: controlPlane.%s is required for mtls", ErrInvalidTransport, role) } - certificate, err := tls.LoadX509KeyPair(clientTLS.CertFile, clientTLS.KeyFile) + certificate, err := tlsreload.NewCertificateProvider(clientTLS.CertFile, clientTLS.KeyFile) if err != nil { return nil, fmt.Errorf("load %s control-plane certificate: %w", role, err) } @@ -60,8 +61,8 @@ func New( } host, _, _ := net.SplitHostPort(address) return credentials.NewTLS(&tls.Config{ - MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{certificate}, RootCAs: roots, - ServerName: strings.Trim(host, "[]"), + MinVersion: tls.VersionTLS13, GetClientCertificate: certificate.ClientCertificate, + RootCAs: roots, ServerName: strings.Trim(host, "[]"), }), nil default: return nil, fmt.Errorf("%w: unsupported control-plane tls mode", ErrInvalidTransport) diff --git a/internal/controlplane/tlsreload/certificate.go b/internal/controlplane/tlsreload/certificate.go new file mode 100644 index 0000000..6afce3d --- /dev/null +++ b/internal/controlplane/tlsreload/certificate.go @@ -0,0 +1,61 @@ +// Package tlsreload provides last-known-good leaf certificate reloads for the +// low-frequency control-plane TLS handshakes. +package tlsreload + +import ( + "crypto/tls" + "errors" + "sync" +) + +var ErrInvalidCertificatePaths = errors.New("invalid TLS certificate paths") + +// CertificateProvider reloads a certificate/key pair for each new handshake. +// Kubernetes projected volumes update file pairs atomically, but retaining the +// last valid pair also bridges short writer or filesystem visibility gaps. +type CertificateProvider struct { + certificateFile string + keyFile string + + mu sync.RWMutex + last tls.Certificate +} + +func NewCertificateProvider(certificateFile, keyFile string) (*CertificateProvider, error) { + if certificateFile == "" || keyFile == "" { + return nil, ErrInvalidCertificatePaths + } + certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile) + if err != nil { + return nil, err + } + return &CertificateProvider{certificateFile: certificateFile, keyFile: keyFile, last: certificate}, nil +} + +func (provider *CertificateProvider) ServerCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return provider.certificate() +} + +func (provider *CertificateProvider) ClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return provider.certificate() +} + +func (provider *CertificateProvider) certificate() (*tls.Certificate, error) { + if provider == nil { + return nil, ErrInvalidCertificatePaths + } + certificate, err := tls.LoadX509KeyPair(provider.certificateFile, provider.keyFile) + if err == nil { + provider.mu.Lock() + provider.last = certificate + provider.mu.Unlock() + return &certificate, nil + } + provider.mu.RLock() + fallback := provider.last + provider.mu.RUnlock() + if len(fallback.Certificate) == 0 { + return nil, err + } + return &fallback, nil +} diff --git a/internal/controlplane/tlsreload/certificate_test.go b/internal/controlplane/tlsreload/certificate_test.go new file mode 100644 index 0000000..caf8574 --- /dev/null +++ b/internal/controlplane/tlsreload/certificate_test.go @@ -0,0 +1,97 @@ +package tlsreload + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +func TestCertificateProviderReloadsPairsAndKeepsLastKnownGoodCertificate(t *testing.T) { + directory := t.TempDir() + certificatePath := filepath.Join(directory, "tls.crt") + keyPath := filepath.Join(directory, "tls.key") + writeCertificatePair(t, certificatePath, keyPath, "first") + + provider, err := NewCertificateProvider(certificatePath, keyPath) + if err != nil { + t.Fatalf("NewCertificateProvider() error = %v", err) + } + certificate, err := provider.ServerCertificate(nil) + if subject := certificateSubject(t, certificate, err); subject != "first" { + t.Fatalf("initial certificate subject = %q, want first", subject) + } + + writeCertificatePair(t, certificatePath, keyPath, "second") + certificate, err = provider.ClientCertificate(nil) + if subject := certificateSubject(t, certificate, err); subject != "second" { + t.Fatalf("reloaded certificate subject = %q, want second", subject) + } + + if err := os.WriteFile(keyPath, []byte("incomplete"), 0o600); err != nil { + t.Fatalf("break private key: %v", err) + } + certificate, err = provider.ServerCertificate(nil) + if subject := certificateSubject(t, certificate, err); subject != "second" { + t.Fatalf("fallback certificate subject = %q, want second", subject) + } +} + +func TestCertificateProviderRejectsMissingPaths(t *testing.T) { + if _, err := NewCertificateProvider("", "key"); err != ErrInvalidCertificatePaths { + t.Fatalf("NewCertificateProvider(empty certificate) error = %v, want %v", err, ErrInvalidCertificatePaths) + } + if _, err := NewCertificateProvider("certificate", ""); err != ErrInvalidCertificatePaths { + t.Fatalf("NewCertificateProvider(empty key) error = %v, want %v", err, ErrInvalidCertificatePaths) + } +} + +func certificateSubject(t *testing.T, certificate *tls.Certificate, err error) string { + t.Helper() + if err != nil { + t.Fatalf("load certificate: %v", err) + } + parsed, err := x509.ParseCertificate(certificate.Certificate[0]) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + return parsed.Subject.CommonName +} + +func writeCertificatePair(t *testing.T, certificatePath, keyPath, commonName string) { + t.Helper() + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate private key: %v", err) + } + now := time.Now() + template := x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatalf("marshal private key: %v", err) + } + key := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER}) + if err := os.WriteFile(certificatePath, certificate, 0o600); err != nil { + t.Fatalf("write certificate: %v", err) + } + if err := os.WriteFile(keyPath, key, 0o600); err != nil { + t.Fatalf("write private key: %v", err) + } +}