338 lines
11 KiB
Go
338 lines
11 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
|
"proxy-pool/internal/config"
|
|
"proxy-pool/internal/controlplane/snapshotwire"
|
|
"proxy-pool/internal/gateway/snapshot"
|
|
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
"google.golang.org/protobuf/types/known/durationpb"
|
|
"google.golang.org/protobuf/types/known/emptypb"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
func TestRunServesGatewayOnlyAfterApplyingControlPlaneSnapshot(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
target := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
if request.URL.Path != "/through-gateway" {
|
|
t.Fatalf("target path = %q, want /through-gateway", request.URL.Path)
|
|
}
|
|
_, _ = io.WriteString(writer, "direct-route")
|
|
}))
|
|
t.Cleanup(target.Close)
|
|
|
|
controlListener := mustListen(t)
|
|
controlServer := grpc.NewServer()
|
|
controlplanev1.RegisterWorkerControlPlaneServer(controlServer, &snapshotServer{
|
|
snapshot: testSnapshot(t, target.URL),
|
|
})
|
|
go func() { _ = controlServer.Serve(controlListener) }()
|
|
t.Cleanup(func() {
|
|
controlServer.Stop()
|
|
_ = controlListener.Close()
|
|
})
|
|
|
|
proxyListener := mustListen(t)
|
|
metricsListener := mustListen(t)
|
|
configPath := writeConfig(t, target.URL)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
result := make(chan error, 1)
|
|
go func() {
|
|
result <- Run(ctx, Options{
|
|
ConfigPath: configPath,
|
|
Resolver: config.OSResolver{},
|
|
ControlPlaneAddress: controlListener.Addr().String(),
|
|
ClusterID: "cluster-a",
|
|
WorkerID: "worker-a",
|
|
InstanceID: "instance-a",
|
|
Zone: "zone-a",
|
|
GatewayListener: proxyListener,
|
|
MetricsListener: metricsListener,
|
|
GRPCTransport: insecure.NewCredentials(),
|
|
ReconnectInitialDelay: 5 * time.Millisecond,
|
|
ReconnectMaxDelay: 20 * time.Millisecond,
|
|
})
|
|
}()
|
|
|
|
if err := waitForStatus("http://"+metricsListener.Addr().String()+"/readyz", http.StatusOK); err != nil {
|
|
select {
|
|
case runErr := <-result:
|
|
t.Fatalf("gateway did not become ready: %v; Run() error = %v", err, runErr)
|
|
default:
|
|
t.Fatalf("gateway did not become ready: %v", err)
|
|
}
|
|
}
|
|
metricsResponse, err := http.Get("http://" + metricsListener.Addr().String() + "/metrics")
|
|
if err != nil {
|
|
t.Fatalf("GET gateway metrics: %v", err)
|
|
}
|
|
metricsBody, err := io.ReadAll(metricsResponse.Body)
|
|
_ = metricsResponse.Body.Close()
|
|
if err != nil {
|
|
t.Fatalf("ReadAll(gateway metrics) = %v", err)
|
|
}
|
|
if metricsResponse.StatusCode != http.StatusOK || !strings.Contains(string(metricsBody), "proxy_pool_gateway_outcome_queue_dropped_total") {
|
|
t.Fatalf("gateway metrics = (%d, %q), want outcome queue metric", metricsResponse.StatusCode, metricsBody)
|
|
}
|
|
proxyURL, err := url.Parse("http://" + proxyListener.Addr().String())
|
|
if err != nil {
|
|
t.Fatalf("Parse(proxy URL): %v", err)
|
|
}
|
|
response, err := (&http.Client{Timeout: time.Second, Transport: &http.Transport{
|
|
Proxy: http.ProxyURL(proxyURL),
|
|
}}).Get(target.URL + "/through-gateway")
|
|
if err != nil {
|
|
t.Fatalf("GET through gateway: %v", err)
|
|
}
|
|
defer response.Body.Close()
|
|
body, err := io.ReadAll(response.Body)
|
|
if err != nil {
|
|
t.Fatalf("ReadAll(gateway response): %v", err)
|
|
}
|
|
if response.StatusCode != http.StatusOK || string(body) != "direct-route" {
|
|
t.Fatalf("gateway response = (%d, %q)", response.StatusCode, body)
|
|
}
|
|
|
|
cancel()
|
|
select {
|
|
case err := <-result:
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("Run() error = %v", err)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("Run() did not stop after cancellation")
|
|
}
|
|
}
|
|
|
|
func TestRunRejectsMissingControlPlaneEndpoint(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := Run(context.Background(), Options{
|
|
ConfigPath: "config.yaml", Resolver: config.OSResolver{},
|
|
ClusterID: "cluster-a", WorkerID: "worker-a", InstanceID: "instance-a", Zone: "zone-a",
|
|
})
|
|
if !errors.Is(err, ErrInvalidOptions) {
|
|
t.Fatalf("Run() error = %v, want ErrInvalidOptions", err)
|
|
}
|
|
}
|
|
|
|
func TestControlPlaneTransportRequiresDedicatedGatewayTLS(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
_, err := controlPlaneTransport(config.ControlPlane{
|
|
TLS: config.ControlPlaneTLS{Mode: "mtls"},
|
|
}, "controller.example:8443", nil)
|
|
if !errors.Is(err, ErrInvalidOptions) {
|
|
t.Fatalf("controlPlaneTransport() error = %v, want ErrInvalidOptions", err)
|
|
}
|
|
}
|
|
|
|
func TestGatewayTransportConfigMapsListenerSettings(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := gatewayTransportConfig(config.Listener{Transport: config.GatewayTransport{
|
|
DialTimeout: config.Duration(time.Second),
|
|
HandshakeTimeout: config.Duration(2 * time.Second),
|
|
ResponseHeaderTimeout: config.Duration(3 * time.Second),
|
|
IdleConnTimeout: config.Duration(time.Minute),
|
|
MaxIdleConns: 500,
|
|
MaxIdleConnsPerHost: 20,
|
|
MaxConnsPerHost: 10,
|
|
TunnelBufferBytes: 32 << 10,
|
|
TunnelIdleTimeout: config.Duration(5 * time.Minute),
|
|
}})
|
|
|
|
if got.DialTimeout != time.Second ||
|
|
got.HandshakeTimeout != 2*time.Second ||
|
|
got.ResponseHeaderTimeout != 3*time.Second ||
|
|
got.IdleConnTimeout != time.Minute ||
|
|
got.MaxIdleConns != 500 ||
|
|
got.MaxIdleConnsPerHost != 20 ||
|
|
got.MaxConnsPerHost != 10 ||
|
|
got.TunnelBufferBytes != 32<<10 ||
|
|
got.TunnelIdleTimeout != 5*time.Minute {
|
|
t.Fatalf("gatewayTransportConfig() = %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotReadinessRequiresCurrentSnapshot(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
store := snapshot.NewStore("cluster-a", "worker-a")
|
|
now := time.Now().UTC()
|
|
readiness := snapshotReadiness{store: store, now: func() time.Time { return now }}
|
|
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
|
t.Fatalf("Ready() before snapshot error = %v, want ErrNotReady", err)
|
|
}
|
|
|
|
if err := store.Apply(snapshot.Envelope{
|
|
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
|
Checksum: snapshot.ChecksumWithRouting(nil, nil), ValidUntil: now.Add(time.Second),
|
|
}); err != nil {
|
|
t.Fatalf("Apply() error = %v", err)
|
|
}
|
|
if err := readiness.Ready(context.Background()); err != nil {
|
|
t.Fatalf("Ready() after snapshot error = %v", err)
|
|
}
|
|
now = now.Add(2 * time.Second)
|
|
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
|
t.Fatalf("Ready() after expiry error = %v, want ErrNotReady", err)
|
|
}
|
|
}
|
|
|
|
type snapshotServer struct {
|
|
controlplanev1.UnimplementedWorkerControlPlaneServer
|
|
snapshot *controlplanev1.WorkerSnapshot
|
|
}
|
|
|
|
func (server *snapshotServer) RegisterWorker(_ context.Context, request *controlplanev1.RegisterWorkerRequest) (*controlplanev1.RegisterWorkerResponse, error) {
|
|
if request.GetWorkerId() != "worker-a" || request.GetInstanceId() != "instance-a" || request.GetZone() != "zone-a" {
|
|
return nil, errors.New("unexpected worker registration")
|
|
}
|
|
return &controlplanev1.RegisterWorkerResponse{
|
|
WorkerId: request.GetWorkerId(), SessionId: "session-a", OwnershipEpoch: 1,
|
|
HeartbeatInterval: durationpb.New(time.Hour), MaxStaleAge: durationpb.New(2 * time.Hour),
|
|
}, nil
|
|
}
|
|
|
|
func (server *snapshotServer) WatchSnapshots(_ *controlplanev1.WatchSnapshotsRequest, stream grpc.ServerStreamingServer[controlplanev1.SnapshotEnvelope]) error {
|
|
if err := stream.Send(&controlplanev1.SnapshotEnvelope{Payload: &controlplanev1.SnapshotEnvelope_Full{Full: server.snapshot}}); err != nil {
|
|
return err
|
|
}
|
|
<-stream.Context().Done()
|
|
return stream.Context().Err()
|
|
}
|
|
|
|
func (server *snapshotServer) AcknowledgeSnapshot(context.Context, *controlplanev1.AcknowledgeSnapshotRequest) (*emptypb.Empty, error) {
|
|
return &emptypb.Empty{}, nil
|
|
}
|
|
|
|
func (server *snapshotServer) ReportRuntime(context.Context, *controlplanev1.ReportRuntimeRequest) (*controlplanev1.ReportRuntimeResponse, error) {
|
|
return &controlplanev1.ReportRuntimeResponse{AcceptedOwnershipEpoch: 1}, nil
|
|
}
|
|
|
|
func testSnapshot(t *testing.T, _ string) *controlplanev1.WorkerSnapshot {
|
|
t.Helper()
|
|
generated := time.Now().UTC()
|
|
result := &controlplanev1.WorkerSnapshot{
|
|
Version: 1, OwnershipEpoch: 1, GeneratedAt: timestamppb.New(generated), ValidUntil: timestamppb.New(generated.Add(time.Minute)),
|
|
Routing: []*controlplanev1.RoutingRule{{
|
|
Name: "gateway-default", Enabled: true, HostRegex: ".+", Upstreams: []string{"provider-a"},
|
|
Strategy: &controlplanev1.RoutingStrategy{Type: controlplanev1.StrategyType_STRATEGY_TYPE_RANDOM},
|
|
OnUnavailable: controlplanev1.UnavailableAction_UNAVAILABLE_ACTION_DIRECT,
|
|
}},
|
|
}
|
|
checksum, err := snapshotwire.Checksum(result)
|
|
if err != nil {
|
|
t.Fatalf("Checksum() error = %v", err)
|
|
}
|
|
result.Checksum = checksum[:]
|
|
return result
|
|
}
|
|
|
|
func writeConfig(t *testing.T, target string) string {
|
|
t.Helper()
|
|
parsed, err := url.Parse(target)
|
|
if err != nil {
|
|
t.Fatalf("Parse(target URL): %v", err)
|
|
}
|
|
port := parsed.Port()
|
|
if port == "" {
|
|
t.Fatalf("target URL has no explicit port: %q", target)
|
|
}
|
|
path := filepath.Join(t.TempDir(), "gateway.yaml")
|
|
content := fmt.Sprintf(`version: 1
|
|
security:
|
|
requireProtectionOnPublicListen: false
|
|
gateway:
|
|
enabled: true
|
|
listen: 127.0.0.1:0
|
|
auth: {mode: none}
|
|
destinationPolicy:
|
|
denyPrivateNetworks: false
|
|
denyLoopback: false
|
|
denyLinkLocal: false
|
|
allowedPorts: [%s]
|
|
distribution:
|
|
enabled: false
|
|
admin:
|
|
enabled: false
|
|
controlPlane:
|
|
enabled: true
|
|
listen: 127.0.0.1:0
|
|
protocolVersion: 1
|
|
heartbeatInterval: 1s
|
|
sessionTTL: 3s
|
|
maxStaleAge: 2s
|
|
maxMessageBytes: 1048576
|
|
maxRuntimeCounters: 10
|
|
maxConcurrentStreams: 10
|
|
tls: {mode: disabled}
|
|
metrics:
|
|
enabled: true
|
|
listen: 127.0.0.1:0
|
|
upstreams:
|
|
provider-a:
|
|
enabled: true
|
|
exposure: [gateway]
|
|
provider: {billingMode: fetch, protocols: [http]}
|
|
api: {url: https://provider.invalid/api, method: GET, auth: {type: none}}
|
|
proxyAuth: {type: response}
|
|
pool: {maxSize: 1}
|
|
capacity: {maxConcurrencyPerProxy: 2}
|
|
refill: {reconcileInterval: 1s, minimumAvailableSlots: 1, targetAvailableSlots: 2}
|
|
lifecycle: {ttl: 1m, allocationSafetyMargin: 1s}
|
|
fetch: {estimatedIPsPerCall: 1, timeout: 1s, maxAttempts: 1, maxInFlight: 1}
|
|
`, port)
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatalf("WriteFile(): %v", err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func mustListen(t *testing.T) net.Listener {
|
|
t.Helper()
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("Listen(): %v", err)
|
|
}
|
|
return listener
|
|
}
|
|
|
|
func waitForStatus(target string, want int) error {
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
last := "no response"
|
|
for time.Now().Before(deadline) {
|
|
response, err := (&http.Client{Timeout: 100 * time.Millisecond}).Get(target)
|
|
if err == nil {
|
|
_ = response.Body.Close()
|
|
if response.StatusCode == want {
|
|
return nil
|
|
}
|
|
last = response.Status
|
|
} else {
|
|
last = err.Error()
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
return fmt.Errorf("GET %s did not return %d: last result %s", target, want, last)
|
|
}
|