feat: reload control plane trust bundles
This commit is contained in:
youfak
parent
72d0517eae
commit
cc30399535
@ -54,3 +54,5 @@ session、ownership 和证书角色边界。
|
||||
|
||||
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书，并将该副本的
|
||||
身份注入 `worker_id`、`instance_id`、Gateway Client TLS 或 Checker Client TLS。
|
||||
应用会在每次新控制面 TLS 握手读取更新后的叶证书和信任根；已有 gRPC 流仍按原会话保留，
|
||||
根轮换时先投放新旧根的重叠信任包，再按 PDB 滚动排空旧连接。
|
||||
|
||||
@ -302,9 +302,9 @@ controlPlane:
|
||||
- `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验，但必须使用独立的 Checker
|
||||
证书。Controller 分别验证 `.../worker/<worker-id>` 与
|
||||
`.../checker/<checker-id>` SPIFFE URI，不能跨角色复用证书。
|
||||
- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书和私钥；
|
||||
文件暂时不可读时继续使用最后一次有效叶证书。既有 gRPC 流不强制重握手，根 CA 轮换仍需
|
||||
通过包含新旧根的重叠信任包和滚动发布完成。
|
||||
- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和
|
||||
信任根；文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手，根 CA
|
||||
轮换仍需先发布包含新旧根的重叠信任包，并在宽限期后滚动排空旧连接。
|
||||
|
||||
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
||||
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||
|
||||
@ -377,8 +377,8 @@ CI 另有独立 Deployment job，在占位凭据下渲染 Compose，并使用 `k
|
||||
渲染 Kubernetes base 和单副本 development mTLS Overlay；它不启动容器、不访问真实存储或密钥。
|
||||
development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
|
||||
网络策略，但固定 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本，不能代替生产证书轮换。
|
||||
控制面 TLS 叶证书已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效证书；
|
||||
生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI、根信任轮换和弹性身份注入。
|
||||
控制面 TLS 叶证书和信任根已在每次新握手时从文件加载，短暂文件不一致时保留最后一次有效材料；
|
||||
生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。
|
||||
|
||||
## Task 14: Documentation, Examples, and Diagrams
|
||||
|
||||
|
||||
@ -350,11 +350,10 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
|
||||
|
||||
Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。
|
||||
|
||||
控制面 mTLS 的叶证书和私钥会在每次新的 TLS 握手时重新读取；Kubernetes 投影 Secret 或
|
||||
CSI 写入短暂不完整时保留最后一次有效证书。既有 gRPC 长连接不会被强制替换，因此应在新
|
||||
叶证书生效后确认新连接成功，再按 PDB 约束滚动重启或排空旧 Worker。`serverCAFile` 与
|
||||
Controller `clientCAFile` 的信任根在进程启动时加载；根 CA 轮换必须先发布包含新旧根的
|
||||
重叠信任包，再滚动所有角色，最后才撤销旧根。
|
||||
控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取；Kubernetes 投影
|
||||
Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换，
|
||||
因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功，再按 PDB 约束滚动
|
||||
重启或排空旧 Worker，最后才撤销旧根并使旧流重新认证。
|
||||
|
||||
## 10. 主机与网络容量基线
|
||||
|
||||
|
||||
@ -105,9 +105,9 @@ CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixt
|
||||
mTLS 运行拓扑，并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
|
||||
启用 Checker；`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
|
||||
提供固定开发身份、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
|
||||
Controller、Gateway 与 Checker 的叶证书已在每个新 TLS 握手时重载，文件短暂不一致时
|
||||
保留最后一次有效证书。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一
|
||||
SPIFFE URI、根信任轮换与证书，且容器端到端启动仍待具备镜像网络的环境验证。
|
||||
Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载，文件短暂
|
||||
不一致时保留最后一次有效材料。生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入
|
||||
唯一 SPIFFE URI，且容器端到端启动仍待具备镜像网络的环境验证。
|
||||
|
||||
运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算，以及
|
||||
主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求；这些文档不替代真实环境
|
||||
|
||||
@ -3,11 +3,9 @@ package worker
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
@ -164,21 +162,31 @@ func serverTransportOptions(controlPlane config.ControlPlane) (IdentityAuthorize
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: load server certificate: %v", ErrInvalidServer, err)
|
||||
}
|
||||
caPEM, err := os.ReadFile(controlPlane.TLS.ClientCAFile)
|
||||
trust, err := tlsreload.NewTrustProvider(controlPlane.TLS.ClientCAFile)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: read client ca: %v", ErrInvalidServer, err)
|
||||
return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err)
|
||||
}
|
||||
clientCAs := x509.NewCertPool()
|
||||
if !clientCAs.AppendCertsFromPEM(caPEM) {
|
||||
return nil, nil, fmt.Errorf("%w: parse client ca", ErrInvalidServer)
|
||||
clientCAs, err := trust.Pool()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: load client ca: %v", ErrInvalidServer, err)
|
||||
}
|
||||
transport := credentials.NewTLS(&tls.Config{
|
||||
configuration := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
GetCertificate: certificate.ServerCertificate,
|
||||
ClientAuth: tls.RequireAndVerifyClientCert,
|
||||
ClientCAs: clientCAs,
|
||||
})
|
||||
return identity, []grpc.ServerOption{grpc.Creds(transport)}, nil
|
||||
}
|
||||
configuration.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
updatedCAs, loadErr := trust.Pool()
|
||||
if loadErr != nil {
|
||||
return nil, loadErr
|
||||
}
|
||||
current := configuration.Clone()
|
||||
current.GetConfigForClient = nil
|
||||
current.ClientCAs = updatedCAs
|
||||
return current, nil
|
||||
}
|
||||
return identity, []grpc.ServerOption{grpc.Creds(credentials.NewTLS(configuration))}, nil
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("%w: unsupported tls mode", ErrInvalidServer)
|
||||
}
|
||||
|
||||
@ -4,11 +4,9 @@ package clienttransport
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@ -51,18 +49,18 @@ func New(
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load %s control-plane certificate: %w", role, err)
|
||||
}
|
||||
caPEM, err := os.ReadFile(clientTLS.ServerCAFile)
|
||||
trust, err := tlsreload.NewTrustProvider(clientTLS.ServerCAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s control-plane CA: %w", role, err)
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("parse %s control-plane CA", role)
|
||||
return nil, fmt.Errorf("load %s control-plane CA: %w", role, err)
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(address)
|
||||
serverName := strings.Trim(host, "[]")
|
||||
return credentials.NewTLS(&tls.Config{
|
||||
MinVersion: tls.VersionTLS13, GetClientCertificate: certificate.ClientCertificate,
|
||||
RootCAs: roots, ServerName: strings.Trim(host, "[]"),
|
||||
ServerName: serverName,
|
||||
// VerifyConnection retains standard verification semantics while allowing
|
||||
// the CA bundle to reload for the next control-plane handshake.
|
||||
InsecureSkipVerify: true, VerifyConnection: trust.VerifyServer(serverName),
|
||||
}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: unsupported control-plane tls mode", ErrInvalidTransport)
|
||||
|
||||
90
internal/controlplane/tlsreload/trust.go
Normal file
90
internal/controlplane/tlsreload/trust.go
Normal file
@ -0,0 +1,90 @@
|
||||
package tlsreload
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var ErrInvalidTrustPath = errors.New("invalid TLS trust bundle path")
|
||||
|
||||
// TrustProvider reloads a PEM trust bundle for each new control-plane
|
||||
// handshake. It keeps the previous valid bundle while a projected volume is
|
||||
// momentarily incomplete during an update.
|
||||
type TrustProvider struct {
|
||||
path string
|
||||
|
||||
mu sync.RWMutex
|
||||
last *x509.CertPool
|
||||
}
|
||||
|
||||
func NewTrustProvider(path string) (*TrustProvider, error) {
|
||||
if path == "" {
|
||||
return nil, ErrInvalidTrustPath
|
||||
}
|
||||
pool, err := loadTrustPool(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &TrustProvider{path: path, last: pool}, nil
|
||||
}
|
||||
|
||||
// Pool returns the latest valid roots. Returned pools are immutable after
|
||||
// construction, so callers can safely use them for an in-flight handshake.
|
||||
func (provider *TrustProvider) Pool() (*x509.CertPool, error) {
|
||||
if provider == nil {
|
||||
return nil, ErrInvalidTrustPath
|
||||
}
|
||||
pool, err := loadTrustPool(provider.path)
|
||||
if err == nil {
|
||||
provider.mu.Lock()
|
||||
provider.last = pool
|
||||
provider.mu.Unlock()
|
||||
return pool, nil
|
||||
}
|
||||
provider.mu.RLock()
|
||||
fallback := provider.last
|
||||
provider.mu.RUnlock()
|
||||
if fallback == nil {
|
||||
return nil, err
|
||||
}
|
||||
return fallback, nil
|
||||
}
|
||||
|
||||
// VerifyServer returns a TLS verifier that uses the latest trust bundle while
|
||||
// retaining normal DNS-name and server-authentication verification.
|
||||
func (provider *TrustProvider) VerifyServer(serverName string) func(tls.ConnectionState) error {
|
||||
return func(state tls.ConnectionState) error {
|
||||
if len(state.PeerCertificates) == 0 {
|
||||
return errors.New("control-plane server did not provide a certificate")
|
||||
}
|
||||
roots, err := provider.Pool()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load control-plane trust bundle: %w", err)
|
||||
}
|
||||
intermediates := x509.NewCertPool()
|
||||
for _, certificate := range state.PeerCertificates[1:] {
|
||||
intermediates.AddCert(certificate)
|
||||
}
|
||||
_, err = state.PeerCertificates[0].Verify(x509.VerifyOptions{
|
||||
DNSName: serverName, Roots: roots, Intermediates: intermediates,
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
})
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func loadTrustPool(path string) (*x509.CertPool, error) {
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(payload) {
|
||||
return nil, errors.New("parse TLS trust bundle")
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
120
internal/controlplane/tlsreload/trust_test.go
Normal file
120
internal/controlplane/tlsreload/trust_test.go
Normal file
@ -0,0 +1,120 @@
|
||||
package tlsreload
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTrustProviderReloadsRootsAndKeepsLastKnownGoodBundle(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
trustPath := filepath.Join(directory, "ca.crt")
|
||||
firstAuthority := newTestAuthority(t, "first")
|
||||
firstServer := firstAuthority.server(t, "controller")
|
||||
writeTrustBundle(t, trustPath, firstAuthority.certificate)
|
||||
|
||||
provider, err := NewTrustProvider(trustPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewTrustProvider() error = %v", err)
|
||||
}
|
||||
verify := provider.VerifyServer("controller")
|
||||
if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err != nil {
|
||||
t.Fatalf("verify first server with first root: %v", err)
|
||||
}
|
||||
|
||||
secondAuthority := newTestAuthority(t, "second")
|
||||
secondServer := secondAuthority.server(t, "controller")
|
||||
writeTrustBundle(t, trustPath, secondAuthority.certificate)
|
||||
if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{firstServer}}); err == nil {
|
||||
t.Fatal("verify old server after trust rotation error = nil")
|
||||
}
|
||||
if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil {
|
||||
t.Fatalf("verify second server with second root: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(trustPath, []byte("incomplete"), 0o600); err != nil {
|
||||
t.Fatalf("break trust bundle: %v", err)
|
||||
}
|
||||
if err := verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{secondServer}}); err != nil {
|
||||
t.Fatalf("verify with last known good trust bundle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrustProviderRejectsEmptyPath(t *testing.T) {
|
||||
if _, err := NewTrustProvider(""); err != ErrInvalidTrustPath {
|
||||
t.Fatalf("NewTrustProvider(empty path) error = %v, want %v", err, ErrInvalidTrustPath)
|
||||
}
|
||||
}
|
||||
|
||||
type testAuthority struct {
|
||||
certificate *x509.Certificate
|
||||
privateKey *ecdsa.PrivateKey
|
||||
}
|
||||
|
||||
func newTestAuthority(t *testing.T, commonName string) testAuthority {
|
||||
t.Helper()
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate authority key: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName},
|
||||
NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
|
||||
IsCA: true, BasicConstraintsValid: true,
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create authority certificate: %v", err)
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
t.Fatalf("parse authority certificate: %v", err)
|
||||
}
|
||||
return testAuthority{certificate: certificate, privateKey: privateKey}
|
||||
}
|
||||
|
||||
func (authority testAuthority) server(t *testing.T, serverName string) *x509.Certificate {
|
||||
t.Helper()
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate server key: %v", err)
|
||||
}
|
||||
now := time.Now()
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: serverName},
|
||||
DNSNames: []string{serverName}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, authority.certificate, &privateKey.PublicKey, authority.privateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("create server certificate: %v", err)
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
t.Fatalf("parse server certificate: %v", err)
|
||||
}
|
||||
return certificate
|
||||
}
|
||||
|
||||
func writeTrustBundle(t *testing.T, path string, certificate *x509.Certificate) {
|
||||
t.Helper()
|
||||
if certificate == nil {
|
||||
t.Fatal("write trust bundle: certificate is nil")
|
||||
}
|
||||
payload := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw})
|
||||
if err := os.WriteFile(path, payload, 0o600); err != nil {
|
||||
t.Fatalf("write trust bundle: %v", err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user