diff --git a/deploy/kubernetes/overlays/development-mtls/README.md b/deploy/kubernetes/overlays/development-mtls/README.md index e05b06c..ec1ee02 100644 --- a/deploy/kubernetes/overlays/development-mtls/README.md +++ b/deploy/kubernetes/overlays/development-mtls/README.md @@ -54,3 +54,5 @@ session、ownership 和证书角色边界。 生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书，并将该副本的 身份注入 `worker_id`、`instance_id`、Gateway Client TLS 或 Checker Client TLS。 +应用会在每次新控制面 TLS 握手读取更新后的叶证书和信任根；已有 gRPC 流仍按原会话保留， +根轮换时先投放新旧根的重叠信任包，再按 PDB 滚动排空旧连接。 diff --git a/docs/configuration/reference.md b/docs/configuration/reference.md index 45ade46..2a17275 100644 --- a/docs/configuration/reference.md +++ b/docs/configuration/reference.md @@ -302,9 +302,9 @@ controlPlane: - `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验，但必须使用独立的 Checker 证书。Controller 分别验证 `.../worker/` 与 `.../checker/` SPIFFE URI，不能跨角色复用证书。 -- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书和私钥； - 文件暂时不可读时继续使用最后一次有效叶证书。既有 gRPC 流不强制重握手，根 CA 轮换仍需 - 通过包含新旧根的重叠信任包和滚动发布完成。 +- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和 + 信任根；文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手，根 CA + 轮换仍需先发布包含新旧根的重叠信任包，并在宽限期后滚动排空旧连接。 Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、 diff --git a/docs/development/implementation-plan.md b/docs/development/implementation-plan.md index d1b7e5a..0d10be6 100644 --- a/docs/development/implementation-plan.md +++ b/docs/development/implementation-plan.md @@ -377,8 +377,8 @@ CI 另有独立 Deployment job，在占位凭据下渲染 Compose，并使用 `k 渲染 Kubernetes base 和单副本 development mTLS Overlay；它不启动容器、不访问真实存储或密钥。 development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小 网络策略，但固定 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本，不能代替生产证书轮换。 -控制面 TLS 叶证书已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效证书； -生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI、根信任轮换和弹性身份注入。 +控制面 TLS 叶证书和信任根已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效材料； +生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。 ## Task 14: Documentation, Examples, and Diagrams diff --git a/docs/operations/runbook.md b/docs/operations/runbook.md index 2cdcb26..68f4752 100644 --- a/docs/operations/runbook.md +++ b/docs/operations/runbook.md @@ -350,11 +350,10 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。 -控制面 mTLS 的叶证书和私钥会在每次新的 TLS 握手时重新读取；Kubernetes 投影 Secret 或 -CSI 写入短暂不完整时保留最后一次有效证书。既有 gRPC 长连接不会被强制替换，因此应在新 -叶证书生效后确认新连接成功，再按 PDB 约束滚动重启或排空旧 Worker。`serverCAFile` 与 -Controller `clientCAFile` 的信任根在进程启动时加载；根 CA 轮换必须先发布包含新旧根的 -重叠信任包，再滚动所有角色，最后才撤销旧根。 +控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取；Kubernetes 投影 +Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换， +因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功，再按 PDB 约束滚动 +重启或排空旧 Worker，最后才撤销旧根并使旧流重新认证。 ## 10. 主机与网络容量基线 diff --git a/docs/requirements/completion-audit.md b/docs/requirements/completion-audit.md index 65b6655..4a882e5 100644 --- a/docs/requirements/completion-audit.md +++ b/docs/requirements/completion-audit.md @@ -105,9 +105,9 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接 启用 Checker；`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker 提供固定开发身份、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。 -Controller、Gateway 与 Checker 的叶证书已在每个新 TLS 握手时重载，文件短暂不一致时 -保留最后一次有效证书。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一 -SPIFFE URI、根信任轮换与证书，且容器端到端启动仍待具备镜像网络的环境验证。 +Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载，文件短暂 +不一致时保留最后一次有效材料。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入 +唯一 SPIFFE URI，且容器端到端启动仍待具备镜像网络的环境验证。 运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算，以及 主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求；这些文档不替代真实环境 diff --git a/internal/controller/worker/server.go b/internal/controller/worker/server.go index 7da2fab..cc8b162 100644 --- a/internal/controller/worker/server.go +++ b/internal/controller/worker/server.go @@ -3,11 +3,9 @@ package worker import ( "context" "crypto/tls" - "crypto/x509" "errors" "fmt" "net" - "os" "reflect" "strings" "time" @@ -164,21 +162,31 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize if err != nil { return nil, nil, fmt.Errorf("%w: load server certificate: %v", ErrInvalidServer, err) } - caPEM, err := os.ReadFile(controlPlane.TLS.ClientCAFile) + trust, err := tlsreload.NewTrustProvider(controlPlane.TLS.ClientCAFile) if err != nil { - return nil, nil, fmt.Errorf("%w: read client ca: %v", ErrInvalidServer, err) + return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err) } - clientCAs := x509.NewCertPool() - if !clientCAs.AppendCertsFromPEM(caPEM) { - return nil, nil, fmt.Errorf("%w: parse client ca", ErrInvalidServer) + clientCAs, err := trust.Pool() + if err != nil { + return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err) } - transport := credentials.NewTLS(&tls.Config{ + configuration := &tls.Config{ MinVersion: tls.VersionTLS13, GetCertificate: certificate.ServerCertificate, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs, - }) - return identity, []grpc.ServerOption{grpc.Creds(transport)}, nil + } + configuration.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + updatedCAs, loadErr := trust.Pool() + if loadErr != nil { + return nil, loadErr + } + current := configuration.Clone() + current.GetConfigForClient = nil + current.ClientCAs = updatedCAs + return current, nil + } + return identity, []grpc.ServerOption{grpc.Creds(credentials.NewTLS(configuration))}, nil default: return nil, nil, fmt.Errorf("%w: unsupported tls mode", ErrInvalidServer) } diff --git a/internal/controlplane/clienttransport/transport.go b/internal/controlplane/clienttransport/transport.go index 1d446ff..ac8bcc0 100644 --- a/internal/controlplane/clienttransport/transport.go +++ b/internal/controlplane/clienttransport/transport.go @@ -4,11 +4,9 @@ package clienttransport import ( "crypto/tls" - "crypto/x509" "errors" "fmt" "net" - "os" "strconv" "strings" @@ -51,18 +49,18 @@ func New( if err != nil { return nil, fmt.Errorf("load %s control-plane certificate: %w", role, err) } - caPEM, err := os.ReadFile(clientTLS.ServerCAFile) + trust, err := tlsreload.NewTrustProvider(clientTLS.ServerCAFile) if err != nil { - return nil, fmt.Errorf("read %s control-plane CA: %w", role, err) - } - roots := x509.NewCertPool() - if !roots.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("parse %s control-plane CA", role) + return nil, fmt.Errorf("load %s control-plane CA: %w", role, err) } host, _, _ := net.SplitHostPort(address) + serverName := strings.Trim(host, "[]") return credentials.NewTLS(&tls.Config{ MinVersion: tls.VersionTLS13, GetClientCertificate: certificate.ClientCertificate, - RootCAs: roots, ServerName: strings.Trim(host, "[]"), + ServerName: serverName, + // VerifyConnection retains standard verification semantics while allowing + // the CA bundle to reload for the next control-plane handshake. + InsecureSkipVerify: true, VerifyConnection: trust.VerifyServer(serverName), }), nil default: return nil, fmt.Errorf("%w: unsupported control-plane tls mode", ErrInvalidTransport) diff --git a/internal/controlplane/tlsreload/trust.go b/internal/controlplane/tlsreload/trust.go new file mode 100644 index 0000000..0825f27 --- /dev/null +++ b/internal/controlplane/tlsreload/trust.go @@ -0,0 +1,90 @@ +package tlsreload + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "sync" +) + +var ErrInvalidTrustPath = errors.New("invalid TLS trust bundle path") + +// TrustProvider reloads a PEM trust bundle for each new control-plane +// handshake. It keeps the previous valid bundle while a projected volume is +// momentarily incomplete during an update. +type TrustProvider struct { + path string + + mu sync.RWMutex + last *x509.CertPool +} + +func NewTrustProvider(path string) (*TrustProvider, error) { + if path == "" { + return nil, ErrInvalidTrustPath + } + pool, err := loadTrustPool(path) + if err != nil { + return nil, err + } + return &TrustProvider{path: path, last: pool}, nil +} + +// Pool returns the latest valid roots. Returned pools are immutable after +// construction, so callers can safely use them for an in-flight handshake. +func (provider *TrustProvider) Pool() (*x509.CertPool, error) { + if provider == nil { + return nil, ErrInvalidTrustPath + } + pool, err := loadTrustPool(provider.path) + if err == nil { + provider.mu.Lock() + provider.last = pool + provider.mu.Unlock() + return pool, nil + } + provider.mu.RLock() + fallback := provider.last + provider.mu.RUnlock() + if fallback == nil { + return nil, err + } + return fallback, nil +} + +// VerifyServer returns a TLS verifier that uses the latest trust bundle while +// retaining normal DNS-name and server-authentication verification. +func (provider *TrustProvider) VerifyServer(serverName string) func(tls.ConnectionState) error { + return func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return errors.New("control-plane server did not provide a certificate") + } + roots, err := provider.Pool() + if err != nil { + return fmt.Errorf("load control-plane trust bundle: %w", err) + } + intermediates := x509.NewCertPool() + for _, certificate := range state.PeerCertificates[1:] { + intermediates.AddCert(certificate) + } + _, err = state.PeerCertificates[0].Verify(x509.VerifyOptions{ + DNSName: serverName, Roots: roots, Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + return err + } +} + +func loadTrustPool(path string) (*x509.CertPool, error) { + payload, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(payload) { + return nil, errors.New("parse TLS trust bundle") + } + return pool, nil +} diff --git a/internal/controlplane/tlsreload/trust_test.go b/internal/controlplane/tlsreload/trust_test.go new file mode 100644 index 0000000..b64ef43 --- /dev/null +++ b/internal/controlplane/tlsreload/trust_test.go @@ -0,0 +1,120 @@ +package tlsreload + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +func TestTrustProviderReloadsRootsAndKeepsLastKnownGoodBundle(t *testing.T) { + directory := t.TempDir() + trustPath := filepath.Join(directory, "ca.crt") + firstAuthority := newTestAuthority(t, "first") + firstServer := firstAuthority.server(t, "controller") + writeTrustBundle(t, trustPath, firstAuthority.certificate) + + provider, err := NewTrustProvider(trustPath) + if err != nil { + t.Fatalf("NewTrustProvider() error = %v", err) + } + verify := provider.VerifyServer("controller") + if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err != nil { + t.Fatalf("verify first server with first root: %v", err) + } + + secondAuthority := newTestAuthority(t, "second") + secondServer := secondAuthority.server(t, "controller") + writeTrustBundle(t, trustPath, secondAuthority.certificate) + if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err == nil { + t.Fatal("verify old server after trust rotation error = nil") + } + if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil { + t.Fatalf("verify second server with second root: %v", err) + } + + if err := os.WriteFile(trustPath, []byte("incomplete"), 0o600); err != nil { + t.Fatalf("break trust bundle: %v", err) + } + if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil { + t.Fatalf("verify with last known good trust bundle: %v", err) + } +} + +func TestTrustProviderRejectsEmptyPath(t *testing.T) { + if _, err := NewTrustProvider(""); err != ErrInvalidTrustPath { + t.Fatalf("NewTrustProvider(empty path) error = %v, want %v", err, ErrInvalidTrustPath) + } +} + +type testAuthority struct { + certificate *x509.Certificate + privateKey *ecdsa.PrivateKey +} + +func newTestAuthority(t *testing.T, commonName string) testAuthority { + t.Helper() + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate authority key: %v", err) + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey) + if err != nil { + t.Fatalf("create authority certificate: %v", err) + } + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse authority certificate: %v", err) + } + return testAuthority{certificate: certificate, privateKey: privateKey} +} + +func (authority testAuthority) server(t *testing.T, serverName string) *x509.Certificate { + t.Helper() + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate server key: %v", err) + } + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: serverName}, + DNSNames: []string{serverName}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, authority.certificate, &privateKey.PublicKey, authority.privateKey) + if err != nil { + t.Fatalf("create server certificate: %v", err) + } + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse server certificate: %v", err) + } + return certificate +} + +func writeTrustBundle(t *testing.T, path string, certificate *x509.Certificate) { + t.Helper() + if certificate == nil { + t.Fatal("write trust bundle: certificate is nil") + } + payload := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}) + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatalf("write trust bundle: %v", err) + } +}