feat: derive worker identity from spiffe certificates
This commit is contained in:
parent
cc30399535
commit
25218e8d27
@ -216,6 +216,10 @@ go run ./cmd/proxy-gateway -config CONFIG_FILE `
|
|||||||
|
|
||||||
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
|
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
|
||||||
|
生产工作负载可设置 `PROXY_POOL_AUTO_IDENTITY=true`,省略 Worker 和 Instance ID;
|
||||||
|
Gateway 会从挂载的 `gatewayTLS` 证书解析
|
||||||
|
`spiffe://<trust-domain>/<environment>/worker/<worker-id>`,并将 `<worker-id>`
|
||||||
|
作为默认实例身份。Controller 仍会将请求 ID 与证书 URI 严格比对。
|
||||||
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
|
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
|
||||||
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
|
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
|
||||||
|
|
||||||
@ -228,7 +232,9 @@ go run ./cmd/proxy-checker -config CONFIG_FILE `
|
|||||||
|
|
||||||
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、
|
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、
|
||||||
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
|
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
|
||||||
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。它不会访问 Redis/PostgreSQL;生产
|
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。`PROXY_POOL_AUTO_IDENTITY=true` 会从
|
||||||
|
`checkerTLS` 的 `.../checker/<checker-id>` URI 自动派生 Checker 与缺省实例身份。
|
||||||
|
它不会访问 Redis/PostgreSQL;生产
|
||||||
Controller 在启用控制面时装配 Redis 共享任务 broker,并按启用的 Upstream 调度
|
Controller 在启用控制面时装配 Redis 共享任务 broker,并按启用的 Upstream 调度
|
||||||
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
|
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
|
||||||
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置;启用 Admin 时只调度配置与
|
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置;启用 Admin 时只调度配置与
|
||||||
|
|||||||
@ -23,6 +23,7 @@ const (
|
|||||||
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
|
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
|
||||||
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
|
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
|
||||||
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
|
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
|
||||||
|
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
||||||
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
|
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -42,6 +43,7 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
|||||||
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
|
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
|
||||||
checkerID := flags.String("checker-id", "", "unique Checker identifier")
|
checkerID := flags.String("checker-id", "", "unique Checker identifier")
|
||||||
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
|
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
|
||||||
|
autoIdentity := flags.Bool("auto-identity", false, "derive Checker and missing instance IDs from the mTLS SPIFFE certificate")
|
||||||
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
|
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
|
||||||
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
|
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
@ -64,20 +66,27 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
|||||||
*maxInFlight = value
|
*maxInFlight = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !flagWasSet(flags, "auto-identity") {
|
||||||
|
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
||||||
|
*autoIdentity = value
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
supportedLevels, err := parseLevels(*levels)
|
supportedLevels, err := parseLevels(*levels)
|
||||||
|
missingIdentity := !*autoIdentity && (!validValue(*checkerID) || !validValue(*instanceID))
|
||||||
|
invalidProvidedIdentity := (*checkerID != "" && !validValue(*checkerID)) || (*instanceID != "" && !validValue(*instanceID))
|
||||||
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||||
!validValue(*checkerID) || !validValue(*instanceID) || *maxInFlight <= 0 {
|
missingIdentity || invalidProvidedIdentity || *maxInFlight <= 0 {
|
||||||
_, _ = fmt.Fprintf(stderr,
|
_, _ = fmt.Fprintf(stderr,
|
||||||
"proxy-checker: -config, -control-plane, -checker-id, -instance-id and positive -max-in-flight are required; "+
|
"proxy-checker: -config, -control-plane and positive -max-in-flight are required; -checker-id/-instance-id are required unless -auto-identity is enabled; "+
|
||||||
"environment fallbacks: %s, %s, %s, %s, %s\n",
|
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
||||||
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment,
|
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment, autoIdentityEnvironment,
|
||||||
)
|
)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
err = run(ctx, bootstrap.Options{
|
err = run(ctx, bootstrap.Options{
|
||||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||||
CheckerID: *checkerID, InstanceID: *instanceID, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
CheckerID: *checkerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
||||||
})
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
@ -86,6 +95,16 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
||||||
|
set := false
|
||||||
|
flags.Visit(func(item *flag.Flag) {
|
||||||
|
if item.Name == name {
|
||||||
|
set = true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return set
|
||||||
|
}
|
||||||
|
|
||||||
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
|
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
|
||||||
levels := make([]controlplanev1.CheckLevel, 0, 3)
|
levels := make([]controlplanev1.CheckLevel, 0, 3)
|
||||||
seen := make(map[controlplanev1.CheckLevel]struct{})
|
seen := make(map[controlplanev1.CheckLevel]struct{})
|
||||||
|
|||||||
@ -24,6 +24,21 @@ func TestExecuteUsesFlagsAndPassesCheckerIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutePassesAutoCheckerIdentityFromEnvironment(t *testing.T) {
|
||||||
|
received := bootstrap.Options{}
|
||||||
|
values := map[string]string{
|
||||||
|
configEnvironment: "checker.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
||||||
|
maxInFlightEnvironment: "2", autoIdentityEnvironment: "true",
|
||||||
|
}
|
||||||
|
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
||||||
|
received = options
|
||||||
|
return nil
|
||||||
|
}, io.Discard)
|
||||||
|
if code != 0 || !received.AutoIdentity || received.CheckerID != "" || received.InstanceID != "" {
|
||||||
|
t.Fatalf("execute(auto identity) = (%d, %+v)", code, received)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
|
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
|
||||||
code := execute(context.Background(), []string{
|
code := execute(context.Background(), []string{
|
||||||
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",
|
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
@ -22,6 +23,7 @@ const (
|
|||||||
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
|
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
|
||||||
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
|
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
|
||||||
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
|
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
|
||||||
|
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
||||||
zoneEnvironment = "PROXY_POOL_ZONE"
|
zoneEnvironment = "PROXY_POOL_ZONE"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -48,6 +50,7 @@ func execute(
|
|||||||
clusterID := flags.String("cluster-id", "", "cluster identifier")
|
clusterID := flags.String("cluster-id", "", "cluster identifier")
|
||||||
workerID := flags.String("worker-id", "", "unique Worker identifier")
|
workerID := flags.String("worker-id", "", "unique Worker identifier")
|
||||||
instanceID := flags.String("instance-id", "", "unique process instance identifier")
|
instanceID := flags.String("instance-id", "", "unique process instance identifier")
|
||||||
|
autoIdentity := flags.Bool("auto-identity", false, "derive Worker and missing instance IDs from the mTLS SPIFFE certificate")
|
||||||
zone := flags.String("zone", "", "availability zone identifier")
|
zone := flags.String("zone", "", "availability zone identifier")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
@ -66,19 +69,26 @@ func execute(
|
|||||||
setIfEmpty(workerID, getenv(workerIDEnvironment))
|
setIfEmpty(workerID, getenv(workerIDEnvironment))
|
||||||
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
|
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
|
||||||
setIfEmpty(zone, getenv(zoneEnvironment))
|
setIfEmpty(zone, getenv(zoneEnvironment))
|
||||||
|
if !flagWasSet(flags, "auto-identity") {
|
||||||
|
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
||||||
|
*autoIdentity = value
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
missingIdentity := !*autoIdentity && (!validValue(*workerID) || !validValue(*instanceID))
|
||||||
|
invalidProvidedIdentity := (*workerID != "" && !validValue(*workerID)) || (*instanceID != "" && !validValue(*instanceID))
|
||||||
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||||
!validValue(*clusterID) || !validValue(*workerID) || !validValue(*instanceID) || !validValue(*zone) {
|
!validValue(*clusterID) || missingIdentity || invalidProvidedIdentity || !validValue(*zone) {
|
||||||
_, _ = fmt.Fprintf(stderr,
|
_, _ = fmt.Fprintf(stderr,
|
||||||
"proxy-gateway: -config, -control-plane, -cluster-id, -worker-id, -instance-id and -zone are required; "+
|
"proxy-gateway: -config, -control-plane, -cluster-id and -zone are required; -worker-id/-instance-id are required unless -auto-identity is enabled; "+
|
||||||
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
"environment fallbacks: %s, %s, %s, %s, %s, %s, %s\n",
|
||||||
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment,
|
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment, autoIdentityEnvironment,
|
||||||
)
|
)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
err := run(ctx, bootstrap.Options{
|
err := run(ctx, bootstrap.Options{
|
||||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||||
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, Zone: *zone,
|
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, Zone: *zone,
|
||||||
})
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
@ -87,6 +97,16 @@ func execute(
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
||||||
|
set := false
|
||||||
|
flags.Visit(func(item *flag.Flag) {
|
||||||
|
if item.Name == name {
|
||||||
|
set = true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return set
|
||||||
|
}
|
||||||
|
|
||||||
func setIfEmpty(target *string, value string) {
|
func setIfEmpty(target *string, value string) {
|
||||||
if target != nil && *target == "" {
|
if target != nil && *target == "" {
|
||||||
*target = value
|
*target = value
|
||||||
|
|||||||
@ -55,6 +55,21 @@ func TestExecuteReadsGatewayOptionsFromEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutePassesAutoGatewayIdentityFromEnvironment(t *testing.T) {
|
||||||
|
captured := bootstrap.Options{}
|
||||||
|
values := map[string]string{
|
||||||
|
configEnvironment: "gateway.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
||||||
|
clusterIDEnvironment: "cluster-a", zoneEnvironment: "zone-a", autoIdentityEnvironment: "true",
|
||||||
|
}
|
||||||
|
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
||||||
|
captured = options
|
||||||
|
return nil
|
||||||
|
}, io.Discard)
|
||||||
|
if code != 0 || !captured.AutoIdentity || captured.WorkerID != "" || captured.InstanceID != "" {
|
||||||
|
t.Fatalf("execute(auto identity) = (%d, %+v)", code, captured)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
|
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@ -15,10 +15,14 @@ docker compose -f deploy/docker-compose.yml up -d --build
|
|||||||
```
|
```
|
||||||
|
|
||||||
Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/development-mtls`
|
Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/development-mtls`
|
||||||
启用单副本 Controller/Gateway/Checker mTLS 拓扑,挂载三组外部 TLS Secret,并固定
|
启用单副本 Controller/Gateway/Checker mTLS 拓扑,挂载三组外部 TLS Secret,并从
|
||||||
`gateway-a`、`checker-a` 身份。具体创建 Secret 和部署步骤见
|
`gateway-a`、`checker-a` 的证书 URI 自动派生身份。具体创建 Secret 和部署步骤见
|
||||||
[development mTLS overlay](kubernetes/overlays/development-mtls/README.md)。生产 Overlay
|
[development mTLS overlay](kubernetes/overlays/development-mtls/README.md)。生产 Overlay
|
||||||
必须为每个弹性 Gateway/Checker 工作负载配置唯一身份与证书轮换,不能复用固定开发证书。
|
必须为每个弹性 Gateway/Checker 工作负载配置唯一身份与证书轮换,不能复用固定开发证书。
|
||||||
|
身份系统只需把每 Pod 的叶证书、私钥和信任包写入配置指定路径,并签发唯一
|
||||||
|
`.../worker/<worker-id>` 或 `.../checker/<checker-id>` URI;启动命令设置
|
||||||
|
`-auto-identity` 后由进程派生逻辑/缺省实例 ID,不依赖手工环境变量。开发 Compose 和
|
||||||
|
此 Overlay 已启用该模式;其中证书只适用于本地开发,不代表生产身份签发与挂载实现。
|
||||||
|
|
||||||
当前可执行验证:
|
当前可执行验证:
|
||||||
|
|
||||||
|
|||||||
@ -187,13 +187,11 @@ func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.
|
|||||||
}
|
}
|
||||||
for _, expected := range []struct {
|
for _, expected := range []struct {
|
||||||
name string
|
name string
|
||||||
identity string
|
|
||||||
instance string
|
|
||||||
certificate string
|
certificate string
|
||||||
}{
|
}{
|
||||||
{name: "gateway-a", identity: "gateway-a", instance: "gateway-a-1", certificate: "gateway-a"},
|
{name: "gateway-a", certificate: "gateway-a"},
|
||||||
{name: "gateway-b", identity: "gateway-b", instance: "gateway-b-1", certificate: "gateway-b"},
|
{name: "gateway-b", certificate: "gateway-b"},
|
||||||
{name: "checker", identity: "checker-a", instance: "checker-a-1", certificate: "checker-a"},
|
{name: "checker", certificate: "checker-a"},
|
||||||
} {
|
} {
|
||||||
service, ok := document.Services[expected.name]
|
service, ok := document.Services[expected.name]
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -205,15 +203,17 @@ func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.
|
|||||||
t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
|
t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
|
||||||
}
|
}
|
||||||
if expected.name == "checker" {
|
if expected.name == "checker" {
|
||||||
if service.Environment["PROXY_POOL_CHECKER_ID"] != expected.identity ||
|
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
||||||
service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != expected.instance ||
|
service.Environment["PROXY_POOL_CHECKER_ID"] != "" ||
|
||||||
|
service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != "" ||
|
||||||
service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
|
service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
|
||||||
t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
|
t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if service.Environment["PROXY_POOL_WORKER_ID"] != expected.identity ||
|
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
||||||
service.Environment["PROXY_POOL_INSTANCE_ID"] != expected.instance ||
|
service.Environment["PROXY_POOL_WORKER_ID"] != "" ||
|
||||||
|
service.Environment["PROXY_POOL_INSTANCE_ID"] != "" ||
|
||||||
service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
|
service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
|
||||||
t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
|
t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -26,8 +26,7 @@ services:
|
|||||||
<<: *app-environment
|
<<: *app-environment
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||||
PROXY_POOL_CLUSTER_ID: compose-local
|
PROXY_POOL_CLUSTER_ID: compose-local
|
||||||
PROXY_POOL_WORKER_ID: gateway-a
|
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||||
PROXY_POOL_INSTANCE_ID: gateway-a-1
|
|
||||||
PROXY_POOL_ZONE: compose-local
|
PROXY_POOL_ZONE: compose-local
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
@ -50,8 +49,7 @@ services:
|
|||||||
<<: *app-environment
|
<<: *app-environment
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||||
PROXY_POOL_CLUSTER_ID: compose-local
|
PROXY_POOL_CLUSTER_ID: compose-local
|
||||||
PROXY_POOL_WORKER_ID: gateway-b
|
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||||
PROXY_POOL_INSTANCE_ID: gateway-b-1
|
|
||||||
PROXY_POOL_ZONE: compose-local
|
PROXY_POOL_ZONE: compose-local
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
@ -98,8 +96,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
<<: *app-environment
|
<<: *app-environment
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||||
PROXY_POOL_CHECKER_ID: checker-a
|
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||||
PROXY_POOL_CHECKER_INSTANCE_ID: checker-a-1
|
|
||||||
PROXY_POOL_CHECKER_MAX_IN_FLIGHT: "200"
|
PROXY_POOL_CHECKER_MAX_IN_FLIGHT: "200"
|
||||||
volumes:
|
volumes:
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
|
|||||||
@ -1,7 +1,8 @@
|
|||||||
# Kubernetes development mTLS overlay
|
# Kubernetes development mTLS overlay
|
||||||
|
|
||||||
该 Overlay 在 base 资源之上启用 Controller 控制面 mTLS,并启动一个固定身份的 Gateway
|
该 Overlay 在 base 资源之上启用 Controller 控制面 mTLS,并启动一个 Gateway 和
|
||||||
和 Checker。它只用于开发或预发布的单副本控制面验证,不是生产弹性身份方案。
|
Checker。进程通过 `PROXY_POOL_AUTO_IDENTITY=true` 从各自证书的 URI SAN 派生身份;
|
||||||
|
它只用于开发或预发布的单副本控制面验证,不是生产弹性身份方案。
|
||||||
|
|
||||||
## Prepare certificates
|
## Prepare certificates
|
||||||
|
|
||||||
@ -47,12 +48,12 @@ kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=5m
|
|||||||
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
|
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
|
||||||
```
|
```
|
||||||
|
|
||||||
Overlay 的 Gateway 固定为 `worker_id=gateway-a`,Checker 固定为
|
Overlay 的 Gateway 从 `gateway-a` 证书派生 `worker_id=gateway-a`,Checker 从
|
||||||
`checker_id=checker-a`,并把 Gateway HPA 限制为 `minReplicas=maxReplicas=1`。
|
`checker-a` 证书派生 `checker_id=checker-a`,并把 Gateway HPA 限制为
|
||||||
因此不得在此 Overlay 上增加副本数或放宽 HPA;重复使用同一 Worker 身份会破坏
|
`minReplicas=maxReplicas=1`。因此不得在此 Overlay 上增加副本数或放宽 HPA;重复使用
|
||||||
session、ownership 和证书角色边界。
|
同一证书和 Worker 身份会破坏 session、ownership 和证书角色边界。
|
||||||
|
|
||||||
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书,并将该副本的
|
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书,并将该副本的
|
||||||
身份注入 `worker_id`、`instance_id`、Gateway Client TLS 或 Checker Client TLS。
|
身份注入 Gateway Client TLS 或 Checker Client TLS,并启用自动身份派生。应用会在每次新
|
||||||
应用会在每次新控制面 TLS 握手读取更新后的叶证书和信任根;已有 gRPC 流仍按原会话保留,
|
控制面 TLS 握手读取更新后的叶证书和信任根;已有 gRPC 流仍按原会话保留,根轮换时先投放
|
||||||
根轮换时先投放新旧根的重叠信任包,再按 PDB 滚动排空旧连接。
|
新旧根的重叠信任包,再按 PDB 滚动排空旧连接。
|
||||||
|
|||||||
@ -11,8 +11,7 @@ spec:
|
|||||||
- name: checker
|
- name: checker
|
||||||
env:
|
env:
|
||||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
||||||
- {name: PROXY_POOL_CHECKER_ID, value: checker-a}
|
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
||||||
- {name: PROXY_POOL_CHECKER_INSTANCE_ID, value: checker-a-1}
|
|
||||||
- {name: PROXY_POOL_CHECKER_MAX_IN_FLIGHT, value: "200"}
|
- {name: PROXY_POOL_CHECKER_MAX_IN_FLIGHT, value: "200"}
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: controlplane-checker-tls
|
- name: controlplane-checker-tls
|
||||||
|
|||||||
@ -12,8 +12,7 @@ spec:
|
|||||||
env:
|
env:
|
||||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
||||||
- {name: PROXY_POOL_CLUSTER_ID, value: kubernetes-development}
|
- {name: PROXY_POOL_CLUSTER_ID, value: kubernetes-development}
|
||||||
- {name: PROXY_POOL_WORKER_ID, value: gateway-a}
|
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
||||||
- {name: PROXY_POOL_INSTANCE_ID, value: gateway-a-1}
|
|
||||||
- {name: PROXY_POOL_ZONE, value: kubernetes-development}
|
- {name: PROXY_POOL_ZONE, value: kubernetes-development}
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: controlplane-gateway-tls
|
- name: controlplane-gateway-tls
|
||||||
|
|||||||
@ -6,7 +6,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestKubernetesDevelopmentMTLSOverlayRendersStaticWorkerIdentities(t *testing.T) {
|
func TestKubernetesDevelopmentMTLSOverlayRendersCertificateDerivedIdentities(t *testing.T) {
|
||||||
kubectl, err := exec.LookPath("kubectl")
|
kubectl, err := exec.LookPath("kubectl")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
||||||
@ -22,8 +22,8 @@ func TestKubernetesDevelopmentMTLSOverlayRendersStaticWorkerIdentities(t *testin
|
|||||||
"mode: mtls",
|
"mode: mtls",
|
||||||
"trustDomain: proxy-pool.local",
|
"trustDomain: proxy-pool.local",
|
||||||
"value: controller:8443",
|
"value: controller:8443",
|
||||||
"value: gateway-a",
|
"name: PROXY_POOL_AUTO_IDENTITY",
|
||||||
"value: checker-a",
|
"value: \"true\"",
|
||||||
"secretName: proxy-pool-controlplane-server-tls",
|
"secretName: proxy-pool-controlplane-server-tls",
|
||||||
"secretName: proxy-pool-controlplane-gateway-tls",
|
"secretName: proxy-pool-controlplane-gateway-tls",
|
||||||
"secretName: proxy-pool-controlplane-checker-tls",
|
"secretName: proxy-pool-controlplane-checker-tls",
|
||||||
@ -36,4 +36,9 @@ func TestKubernetesDevelopmentMTLSOverlayRendersStaticWorkerIdentities(t *testin
|
|||||||
t.Errorf("development mTLS overlay does not render %q", required)
|
t.Errorf("development mTLS overlay does not render %q", required)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, forbidden := range []string{"name: PROXY_POOL_WORKER_ID", "name: PROXY_POOL_INSTANCE_ID", "name: PROXY_POOL_CHECKER_ID", "name: PROXY_POOL_CHECKER_INSTANCE_ID"} {
|
||||||
|
if strings.Contains(rendered, forbidden) {
|
||||||
|
t.Errorf("development mTLS overlay still renders manual identity %q", forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -236,6 +236,9 @@ Gateway 不复用 `controlPlane.listen` 作为客户端地址。`listen` 是 Con
|
|||||||
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
|
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
|
||||||
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
|
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
|
||||||
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
|
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
|
||||||
|
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
||||||
|
证书中唯一的 `.../worker/<worker-id>` SPIFFE URI 派生 Worker ID,并在未显式配置时
|
||||||
|
使用同一值作为 Instance ID。
|
||||||
|
|
||||||
当 `controlPlane.tls.mode=mtls` 时,Gateway 使用 `controlPlane.gatewayTLS` 中独立的
|
当 `controlPlane.tls.mode=mtls` 时,Gateway 使用 `controlPlane.gatewayTLS` 中独立的
|
||||||
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
|
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
|
||||||
@ -249,6 +252,8 @@ SPIFFE Worker 身份校验。
|
|||||||
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
|
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
|
||||||
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
|
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
|
||||||
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
|
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
|
||||||
|
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
||||||
|
证书中唯一的 `.../checker/<checker-id>` SPIFFE URI 派生 Checker ID 和缺省 Instance ID。
|
||||||
|
|
||||||
当 `controlPlane.tls.mode=mtls` 时,Checker 使用 `controlPlane.checkerTLS` 的独立
|
当 `controlPlane.tls.mode=mtls` 时,Checker 使用 `controlPlane.checkerTLS` 的独立
|
||||||
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的
|
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的
|
||||||
|
|||||||
@ -309,9 +309,12 @@ controlPlane:
|
|||||||
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
||||||
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
|
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
|
||||||
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。Compose 本地模板使用固定
|
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。Compose 本地模板从被忽略的
|
||||||
Worker/Checker 身份和被忽略的开发证书目录;Kubernetes Base 保持
|
开发证书目录派生 Worker/Checker 身份;Kubernetes Base 保持
|
||||||
`controlPlane.enabled: false`,必须由具备每工作负载唯一身份的 mTLS Overlay 启用。
|
`controlPlane.enabled: false`,必须由具备每工作负载唯一身份的 mTLS Overlay 启用。
|
||||||
|
在 mTLS 生产工作负载中,`PROXY_POOL_AUTO_IDENTITY=true` 可省略 `WORKER_ID` 和
|
||||||
|
`INSTANCE_ID`:进程仅接受其 `gatewayTLS` 叶证书中唯一的
|
||||||
|
`spiffe://<trust-domain>/<environment>/worker/<worker-id>` URI,并以该 ID 作为缺省实例。
|
||||||
|
|
||||||
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`、
|
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`、
|
||||||
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
|
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
|
||||||
@ -325,6 +328,10 @@ Routing 也不会再产生其 TARGET 任务;revision 不一致或状态不完
|
|||||||
上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
||||||
新启用的上游无需重启 Controller。
|
新启用的上游无需重启 Controller。
|
||||||
|
|
||||||
|
`PROXY_POOL_AUTO_IDENTITY=true` 对 Checker 使用同样规则:仅接受 `checkerTLS` 中唯一的
|
||||||
|
`spiffe://<trust-domain>/<environment>/checker/<checker-id>` URI,并在未显式设置时复用其
|
||||||
|
`checker-id` 作为实例 ID。
|
||||||
|
|
||||||
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
|
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
|
||||||
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
|
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
|
||||||
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
|
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
|
||||||
|
|||||||
@ -23,6 +23,7 @@ import (
|
|||||||
"proxy-pool/internal/checker/probe"
|
"proxy-pool/internal/checker/probe"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controlplane/clienttransport"
|
"proxy-pool/internal/controlplane/clienttransport"
|
||||||
|
"proxy-pool/internal/controlplane/tlsreload"
|
||||||
"proxy-pool/internal/domain/workerruntime"
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
"proxy-pool/internal/platform/httpserver"
|
"proxy-pool/internal/platform/httpserver"
|
||||||
"proxy-pool/internal/platform/lifecycle"
|
"proxy-pool/internal/platform/lifecycle"
|
||||||
@ -41,6 +42,7 @@ type Options struct {
|
|||||||
ControlPlaneAddress string
|
ControlPlaneAddress string
|
||||||
CheckerID string
|
CheckerID string
|
||||||
InstanceID string
|
InstanceID string
|
||||||
|
AutoIdentity bool
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
SupportedLevels []controlplanev1.CheckLevel
|
SupportedLevels []controlplanev1.CheckLevel
|
||||||
GRPCTransport credentials.TransportCredentials
|
GRPCTransport credentials.TransportCredentials
|
||||||
@ -59,6 +61,10 @@ func Run(ctx context.Context, options Options) error {
|
|||||||
if !configuration.ControlPlane.Enabled {
|
if !configuration.ControlPlane.Enabled {
|
||||||
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
|
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
|
||||||
}
|
}
|
||||||
|
options, err = resolveIdentity(configuration, options)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
runtime, err := newRuntime(ctx, configuration, options)
|
runtime, err := newRuntime(ctx, configuration, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: %w", ErrStartup, err)
|
return fmt.Errorf("%w: %w", ErrStartup, err)
|
||||||
@ -124,13 +130,44 @@ func newRuntime(ctx context.Context, configuration *config.Config, options Optio
|
|||||||
func validateOptions(ctx context.Context, options Options) error {
|
func validateOptions(ctx context.Context, options Options) error {
|
||||||
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
||||||
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
|
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
|
||||||
!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) ||
|
(options.CheckerID != "" && !workerruntime.ValidIdentifier(options.CheckerID)) ||
|
||||||
|
(options.InstanceID != "" && !workerruntime.ValidIdentifier(options.InstanceID)) ||
|
||||||
|
(!options.AutoIdentity && (!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID))) ||
|
||||||
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
|
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
|
||||||
return ErrInvalidOptions
|
return ErrInvalidOptions
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveIdentity(configuration *config.Config, options Options) (Options, error) {
|
||||||
|
if configuration == nil {
|
||||||
|
return Options{}, ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if !options.AutoIdentity {
|
||||||
|
return options, nil
|
||||||
|
}
|
||||||
|
if configuration.ControlPlane.TLS.Mode != "mtls" {
|
||||||
|
return Options{}, errors.Join(ErrInvalidOptions, errors.New("auto identity requires controlPlane mTLS"))
|
||||||
|
}
|
||||||
|
identity, err := tlsreload.ResolveSPIFFEIdentity(
|
||||||
|
configuration.ControlPlane.CheckerTLS.CertFile, configuration.ControlPlane.CheckerTLS.KeyFile,
|
||||||
|
configuration.ControlPlane.TLS.TrustDomain, configuration.ControlPlane.TLS.Environment, "checker",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return Options{}, fmt.Errorf("resolve checker SPIFFE identity: %w", err)
|
||||||
|
}
|
||||||
|
if options.CheckerID == "" {
|
||||||
|
options.CheckerID = identity
|
||||||
|
}
|
||||||
|
if options.InstanceID == "" {
|
||||||
|
options.InstanceID = identity
|
||||||
|
}
|
||||||
|
if !workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) || options.CheckerID != identity {
|
||||||
|
return Options{}, errors.Join(ErrInvalidOptions, errors.New("checker identity does not match SPIFFE certificate"))
|
||||||
|
}
|
||||||
|
return options, nil
|
||||||
|
}
|
||||||
|
|
||||||
type runtime struct {
|
type runtime struct {
|
||||||
connection *grpc.ClientConn
|
connection *grpc.ClientConn
|
||||||
group *lifecycle.Group
|
group *lifecycle.Group
|
||||||
|
|||||||
57
internal/controlplane/tlsreload/identity.go
Normal file
57
internal/controlplane/tlsreload/identity.go
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
package tlsreload
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidSPIFFEIdentity = errors.New("invalid SPIFFE workload identity")
|
||||||
|
|
||||||
|
// ResolveSPIFFEIdentity returns the logical Worker or Checker ID encoded in a
|
||||||
|
// role-specific SPIFFE URI SAN. It only accepts the exact identity shape that
|
||||||
|
// the Controller authorizer accepts for a control-plane request.
|
||||||
|
func ResolveSPIFFEIdentity(certificateFile, keyFile, trustDomain, environment, role string) (string, error) {
|
||||||
|
if certificateFile == "" || keyFile == "" || trustDomain == "" || environment == "" || (role != "worker" && role != "checker") {
|
||||||
|
return "", ErrInvalidSPIFFEIdentity
|
||||||
|
}
|
||||||
|
certificate, err := tls.LoadX509KeyPair(certificateFile, keyFile)
|
||||||
|
if err != nil || len(certificate.Certificate) == 0 {
|
||||||
|
return "", fmt.Errorf("%w: load client certificate", ErrInvalidSPIFFEIdentity)
|
||||||
|
}
|
||||||
|
leaf, err := x509.ParseCertificate(certificate.Certificate[0])
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%w: parse client certificate", ErrInvalidSPIFFEIdentity)
|
||||||
|
}
|
||||||
|
var identity string
|
||||||
|
for _, uri := range leaf.URIs {
|
||||||
|
candidate, ok := spiffeIdentity(uri, trustDomain, environment, role)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if identity != "" {
|
||||||
|
return "", ErrInvalidSPIFFEIdentity
|
||||||
|
}
|
||||||
|
identity = candidate
|
||||||
|
}
|
||||||
|
if !workerruntime.ValidIdentifier(identity) {
|
||||||
|
return "", ErrInvalidSPIFFEIdentity
|
||||||
|
}
|
||||||
|
return identity, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func spiffeIdentity(uri *url.URL, trustDomain, environment, role string) (string, bool) {
|
||||||
|
if uri == nil || uri.Scheme != "spiffe" || uri.Host != trustDomain || uri.RawQuery != "" || uri.Fragment != "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
segments := strings.Split(strings.Trim(uri.Path, "/"), "/")
|
||||||
|
if len(segments) != 3 || segments[0] != environment || segments[1] != role || !workerruntime.ValidIdentifier(segments[2]) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return segments[2], true
|
||||||
|
}
|
||||||
80
internal/controlplane/tlsreload/identity_test.go
Normal file
80
internal/controlplane/tlsreload/identity_test.go
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
package tlsreload
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ecdsa"
|
||||||
|
"crypto/elliptic"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/x509"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"encoding/pem"
|
||||||
|
"math/big"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveSPIFFEIdentityUsesExactRoleSpecificURI(t *testing.T) {
|
||||||
|
certificatePath, keyPath := writeWorkloadCertificate(t, "gateway-a", "spiffe://proxy-pool.local/production/worker/gateway-a")
|
||||||
|
identity, err := ResolveSPIFFEIdentity(certificatePath, keyPath, "proxy-pool.local", "production", "worker")
|
||||||
|
if err != nil || identity != "gateway-a" {
|
||||||
|
t.Fatalf("ResolveSPIFFEIdentity() = (%q, %v), want (gateway-a, nil)", identity, err)
|
||||||
|
}
|
||||||
|
if _, err := ResolveSPIFFEIdentity(certificatePath, keyPath, "proxy-pool.local", "production", "checker"); err == nil {
|
||||||
|
t.Fatal("ResolveSPIFFEIdentity(checker) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSPIFFEIdentityRejectsMultipleMatchingURIs(t *testing.T) {
|
||||||
|
certificatePath, keyPath := writeWorkloadCertificate(t, "gateway-a",
|
||||||
|
"spiffe://proxy-pool.local/production/worker/gateway-a",
|
||||||
|
"spiffe://proxy-pool.local/production/worker/gateway-b",
|
||||||
|
)
|
||||||
|
if _, err := ResolveSPIFFEIdentity(certificatePath, keyPath, "proxy-pool.local", "production", "worker"); err == nil {
|
||||||
|
t.Fatal("ResolveSPIFFEIdentity(multiple identities) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeWorkloadCertificate(t *testing.T, commonName string, identityURIs ...string) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
directory := t.TempDir()
|
||||||
|
certificatePath := filepath.Join(directory, "tls.crt")
|
||||||
|
keyPath := filepath.Join(directory, "tls.key")
|
||||||
|
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generate workload key: %v", err)
|
||||||
|
}
|
||||||
|
identities := make([]*url.URL, 0, len(identityURIs))
|
||||||
|
for _, raw := range identityURIs {
|
||||||
|
identity, parseErr := url.Parse(raw)
|
||||||
|
if parseErr != nil {
|
||||||
|
t.Fatalf("parse workload identity: %v", parseErr)
|
||||||
|
}
|
||||||
|
identities = append(identities, identity)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
template := &x509.Certificate{
|
||||||
|
SerialNumber: big.NewInt(now.UnixNano()), Subject: pkix.Name{CommonName: commonName},
|
||||||
|
URIs: identities, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour),
|
||||||
|
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||||
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||||
|
}
|
||||||
|
der, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create workload certificate: %v", err)
|
||||||
|
}
|
||||||
|
certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||||
|
privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal workload key: %v", err)
|
||||||
|
}
|
||||||
|
key := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER})
|
||||||
|
if err := os.WriteFile(certificatePath, certificate, 0o600); err != nil {
|
||||||
|
t.Fatalf("write workload certificate: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(keyPath, key, 0o600); err != nil {
|
||||||
|
t.Fatalf("write workload key: %v", err)
|
||||||
|
}
|
||||||
|
return certificatePath, keyPath
|
||||||
|
}
|
||||||
@ -19,6 +19,7 @@ import (
|
|||||||
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controlplane/clienttransport"
|
"proxy-pool/internal/controlplane/clienttransport"
|
||||||
|
"proxy-pool/internal/controlplane/tlsreload"
|
||||||
outcomeDomain "proxy-pool/internal/domain/outcome"
|
outcomeDomain "proxy-pool/internal/domain/outcome"
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
"proxy-pool/internal/domain/workerruntime"
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
@ -57,6 +58,7 @@ type Options struct {
|
|||||||
ClusterID string
|
ClusterID string
|
||||||
WorkerID string
|
WorkerID string
|
||||||
InstanceID string
|
InstanceID string
|
||||||
|
AutoIdentity bool
|
||||||
Zone string
|
Zone string
|
||||||
Labels map[string]string
|
Labels map[string]string
|
||||||
HTTP httpserver.Options
|
HTTP httpserver.Options
|
||||||
@ -83,6 +85,10 @@ func Run(ctx context.Context, options Options) error {
|
|||||||
if !configuration.Gateway.Enabled || !configuration.ControlPlane.Enabled {
|
if !configuration.Gateway.Enabled || !configuration.ControlPlane.Enabled {
|
||||||
return errors.Join(ErrInvalidOptions, errors.New("gateway and controlPlane must be enabled"))
|
return errors.Join(ErrInvalidOptions, errors.New("gateway and controlPlane must be enabled"))
|
||||||
}
|
}
|
||||||
|
options, err = resolveIdentity(configuration, options)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: %w", ErrStartup, err)
|
||||||
|
}
|
||||||
|
|
||||||
runtime, err := newRuntime(ctx, configuration, options)
|
runtime, err := newRuntime(ctx, configuration, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -94,8 +100,11 @@ func Run(ctx context.Context, options Options) error {
|
|||||||
|
|
||||||
func validateOptions(ctx context.Context, options Options) error {
|
func validateOptions(ctx context.Context, options Options) error {
|
||||||
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
||||||
nilInterface(options.Resolver) || !workerruntime.ValidIdentifier(options.ClusterID) || !workerruntime.ValidIdentifier(options.WorkerID) ||
|
nilInterface(options.Resolver) || !workerruntime.ValidIdentifier(options.ClusterID) ||
|
||||||
!workerruntime.ValidIdentifier(options.InstanceID) || !workerruntime.ValidIdentifier(options.Zone) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) {
|
(options.WorkerID != "" && !workerruntime.ValidIdentifier(options.WorkerID)) ||
|
||||||
|
(options.InstanceID != "" && !workerruntime.ValidIdentifier(options.InstanceID)) ||
|
||||||
|
(!options.AutoIdentity && (!workerruntime.ValidIdentifier(options.WorkerID) || !workerruntime.ValidIdentifier(options.InstanceID))) ||
|
||||||
|
!workerruntime.ValidIdentifier(options.Zone) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) {
|
||||||
return ErrInvalidOptions
|
return ErrInvalidOptions
|
||||||
}
|
}
|
||||||
if options.GatewayListener == nil && options.MetricsListener != nil {
|
if options.GatewayListener == nil && options.MetricsListener != nil {
|
||||||
@ -107,6 +116,35 @@ func validateOptions(ctx context.Context, options Options) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveIdentity(configuration *config.Config, options Options) (Options, error) {
|
||||||
|
if configuration == nil {
|
||||||
|
return Options{}, ErrInvalidOptions
|
||||||
|
}
|
||||||
|
if !options.AutoIdentity {
|
||||||
|
return options, nil
|
||||||
|
}
|
||||||
|
if configuration.ControlPlane.TLS.Mode != "mtls" {
|
||||||
|
return Options{}, errors.Join(ErrInvalidOptions, errors.New("auto identity requires controlPlane mTLS"))
|
||||||
|
}
|
||||||
|
identity, err := tlsreload.ResolveSPIFFEIdentity(
|
||||||
|
configuration.ControlPlane.GatewayTLS.CertFile, configuration.ControlPlane.GatewayTLS.KeyFile,
|
||||||
|
configuration.ControlPlane.TLS.TrustDomain, configuration.ControlPlane.TLS.Environment, "worker",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return Options{}, fmt.Errorf("resolve gateway SPIFFE identity: %w", err)
|
||||||
|
}
|
||||||
|
if options.WorkerID == "" {
|
||||||
|
options.WorkerID = identity
|
||||||
|
}
|
||||||
|
if options.InstanceID == "" {
|
||||||
|
options.InstanceID = identity
|
||||||
|
}
|
||||||
|
if !workerruntime.ValidIdentifier(options.WorkerID) || !workerruntime.ValidIdentifier(options.InstanceID) || options.WorkerID != identity {
|
||||||
|
return Options{}, errors.Join(ErrInvalidOptions, errors.New("gateway identity does not match SPIFFE certificate"))
|
||||||
|
}
|
||||||
|
return options, nil
|
||||||
|
}
|
||||||
|
|
||||||
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
|
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
|
||||||
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
|
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
|
||||||
return nil, ErrInvalidOptions
|
return nil, ErrInvalidOptions
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user