proxy-pool/deploy/compose_test.go

package deploy

import (
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"testing"

	"go.yaml.in/yaml/v4"
)

var observableMetricNames = map[string]struct{}{
	"proxy_pool_checker_observations_total":                    {},
	"proxy_pool_checker_tasks_dispatched_total":                {},
	"proxy_pool_controller_capacity_active_upstreams":          {},
	"proxy_pool_controller_capacity_available_slots":           {},
	"proxy_pool_controller_capacity_effective_slots":           {},
	"proxy_pool_controller_capacity_inventory_reads_total":     {},
	"proxy_pool_controller_capacity_managed_proxies":           {},
	"proxy_pool_controller_capacity_pending_expected_proxies":  {},
	"proxy_pool_controller_drain_candidates_total":             {},
	"proxy_pool_controller_drains_started_total":               {},
	"proxy_pool_controller_extraction_requested_proxies_total": {},
	"proxy_pool_controller_extraction_requests_total":          {},
	"proxy_pool_controller_extraction_returned_proxies_total":  {},
	"proxy_pool_controller_provider_fetch_results_total":       {},
	"proxy_pool_controller_provider_new_proxies_total":         {},
	"proxy_pool_controller_provider_valid_candidates_total":    {},
	"proxy_pool_gateway_active_tunnels":                        {},
	"proxy_pool_gateway_capacity_invariant_violations_total":   {},
	"proxy_pool_gateway_outcome_queue_dropped_total":           {},
	"proxy_pool_gateway_outcomes_total":                        {},
	"proxy_pool_gateway_request_duration_seconds":              {},
	"proxy_pool_gateway_requests_in_flight":                    {},
	"proxy_pool_gateway_requests_total":                        {},
}

type dashboardDocument struct {
	Panels []struct {
		Targets []struct {
			Expression string `json:"expr"`
		} `json:"targets"`
	} `json:"panels"`
}

type alertRulesDocument struct {
	Groups []struct {
		Rules []struct {
			Expression string `yaml:"expr"`
			Alert      string `yaml:"alert"`
		} `yaml:"rules"`
	} `yaml:"groups"`
}

type composeDocument struct {
	Services map[string]composeService `yaml:"services"`
	Volumes  map[string]any            `yaml:"volumes"`
}

type composeService struct {
	Image       string            `yaml:"image"`
	Command     []string          `yaml:"command"`
	Ports       []string          `yaml:"ports"`
	Expose      []string          `yaml:"expose"`
	Volumes     []string          `yaml:"volumes"`
	Tmpfs       []string          `yaml:"tmpfs"`
	Environment map[string]string `yaml:"environment"`
	DependsOn   any               `yaml:"depends_on"`
}

func TestPostgresIntegrationFixtureIsIsolatedAndEphemeral(t *testing.T) {
	document := loadComposeFile(t, "docker-compose.test.yml")
	postgres, ok := document.Services["postgres"]
	if !ok {
		t.Fatal("docker-compose.test.yml has no postgres service")
	}
	if postgres.Image != "postgres:18-alpine" {
		t.Fatalf("postgres image = %q, want postgres:18-alpine", postgres.Image)
	}
	if !slices.Equal(postgres.Ports, []string{"127.0.0.1:15432:5432"}) {
		t.Fatalf("postgres ports = %v, want loopback test port", postgres.Ports)
	}
	if !slices.Contains(postgres.Tmpfs, "/var/lib/postgresql") || len(postgres.Volumes) != 0 {
		t.Fatalf("postgres tmpfs = %v, volumes = %v; want ephemeral PG18 data root", postgres.Tmpfs, postgres.Volumes)
	}
}

func TestRedisFixtureScriptUsesDedicatedComposeProject(t *testing.T) {
	payload, err := os.ReadFile("../scripts/test-redis.ps1")
	if err != nil {
		t.Fatalf("read test-redis.ps1: %v", err)
	}
	script := string(payload)
	for _, required := range []string{
		`-p $composeProject`,
		`up -d --wait --wait-timeout 60 redis`,
		`down --volumes --remove-orphans`,
	} {
		if !strings.Contains(script, required) {
			t.Errorf("test-redis.ps1 missing %q", required)
		}
	}
}

func TestPostgresFixtureScriptUsesDedicatedComposeProject(t *testing.T) {
	payload, err := os.ReadFile("../scripts/test-postgres.ps1")
	if err != nil {
		t.Fatalf("read test-postgres.ps1: %v", err)
	}
	script := string(payload)
	for _, required := range []string{
		`-p $composeProject`,
		`up -d --wait --wait-timeout 60 postgres`,
		`PROXY_POOL_TEST_POSTGRES_URL`,
		`go test -count=1 -tags=integration -timeout 60s ./internal/adapters/postgresadmin/...`,
		`down --volumes --remove-orphans`,
	} {
		if !strings.Contains(script, required) {
			t.Errorf("test-postgres.ps1 missing %q", required)
		}
	}
}

func TestControllerFixtureScriptStartsBothStoresInDedicatedProject(t *testing.T) {
	payload, err := os.ReadFile("../scripts/test-controller.ps1")
	if err != nil {
		t.Fatalf("read test-controller.ps1: %v", err)
	}
	script := string(payload)
	for _, required := range []string{
		`-p $composeProject`,
		`up -d --wait --wait-timeout 60 postgres redis`,
		`PROXY_POOL_TEST_POSTGRES_URL`,
		`PROXY_POOL_TEST_REDIS_URL`,
		`go test -count=1 -tags=integration -timeout 60s ./internal/controller/bootstrap`,
		`down --volumes --remove-orphans`,
	} {
		if !strings.Contains(script, required) {
			t.Errorf("test-controller.ps1 missing %q", required)
		}
	}
}

func TestLocalRedisIsExplicitlyEphemeral(t *testing.T) {
	document := loadComposeDocument(t)
	redis, ok := document.Services["redis"]
	if !ok {
		t.Fatal("docker-compose.yml has no redis service")
	}
	if value, ok := commandFlag(redis.Command, "--appendonly"); !ok || value != "no" {
		t.Fatalf("redis --appendonly = %q, %t; want no", value, ok)
	}
	if value, ok := commandFlag(redis.Command, "--save"); !ok || value != "" {
		t.Fatalf("redis --save = %q, %t; want empty schedule", value, ok)
	}
	if len(redis.Volumes) != 0 {
		t.Fatalf("redis volumes = %v; want no persistent mount", redis.Volumes)
	}
	if _, exists := document.Volumes["redis-data"]; exists {
		t.Fatal("docker-compose.yml still declares redis-data")
	}
}

func TestLocalGatewaysDoNotDependOnControlPlaneStorage(t *testing.T) {
	document := loadComposeDocument(t)
	for _, name := range []string{"gateway-a", "gateway-b"} {
		gateway, ok := document.Services[name]
		if !ok {
			t.Fatalf("docker-compose.yml has no %s service", name)
		}
		for _, storage := range []string{"postgres", "redis"} {
			if composeDependsOn(gateway.DependsOn, storage) {
				t.Errorf("%s depends on %s; gateway startup must be storage-independent", name, storage)
			}
		}
	}
}

func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.T) {
	document := loadComposeDocument(t)
	controller := document.Services["controller"]
	if !slices.Contains(controller.Expose, "8443") || !slices.Contains(controller.Volumes, "./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro") {
		t.Fatalf("controller does not expose and mount control-plane TLS: expose=%v volumes=%v", controller.Expose, controller.Volumes)
	}
	for _, expected := range []struct {
		name        string
		certificate string
	}{
		{name: "gateway-a", certificate: "gateway-a"},
		{name: "gateway-b", certificate: "gateway-b"},
		{name: "checker", certificate: "checker-a"},
	} {
		service, ok := document.Services[expected.name]
		if !ok {
			t.Fatalf("docker-compose.yml has no %s", expected.name)
		}
		if service.Environment["PROXY_POOL_CONTROL_PLANE_ADDRESS"] != "controller:8443" ||
			!slices.Contains(service.Volumes, "./.control-plane-tls/"+expected.certificate+":/run/proxy-pool-tls/client:ro") ||
			!composeDependsOn(service.DependsOn, "controller") {
			t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
		}
		if expected.name == "checker" {
			if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
				service.Environment["PROXY_POOL_CHECKER_ID"] != "" ||
				service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != "" ||
				service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
				t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
			}
			continue
		}
		if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
			service.Environment["PROXY_POOL_WORKER_ID"] != "" ||
			service.Environment["PROXY_POOL_INSTANCE_ID"] != "" ||
			service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
			t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
		}
	}
}

func TestLocalControlPlaneCertificateGeneratorUsesDistinctSPIFFERoles(t *testing.T) {
	payload, err := os.ReadFile("../scripts/generate-local-controlplane-certs.ps1")
	if err != nil {
		t.Fatalf("read generate-local-controlplane-certs.ps1: %v", err)
	}
	script := string(payload)
	for _, required := range []string{
		"OutputDirectory must be empty", "DNS:controller",
		"spiffe://proxy-pool.local/development/worker/gateway-a",
		"spiffe://proxy-pool.local/development/worker/gateway-b",
		"spiffe://proxy-pool.local/development/checker/checker-a",
	} {
		if !strings.Contains(script, required) {
			t.Errorf("certificate generator missing %q", required)
		}
	}
}

func TestDeploymentEntrypointsExistInSource(t *testing.T) {
	dockerfile, err := os.ReadFile("docker/Dockerfile")
	if err != nil {
		t.Fatalf("read docker/Dockerfile: %v", err)
	}
	entrypoints := regexp.MustCompile(`\./cmd/(proxy-[a-z0-9-]+)`).FindAllStringSubmatch(string(dockerfile), -1)
	if len(entrypoints) == 0 {
		t.Fatal("docker/Dockerfile has no proxy command build target")
	}
	if !slices.ContainsFunc(entrypoints, func(entrypoint []string) bool { return entrypoint[1] == "proxy-checker" }) {
		t.Fatal("docker/Dockerfile does not build proxy-checker")
	}
	for _, entrypoint := range entrypoints {
		assertCommandDirectory(t, entrypoint[1])
	}

	document := loadComposeDocument(t)
	for name, service := range document.Services {
		if len(service.Command) == 0 || !strings.HasPrefix(service.Command[0], "proxy-") {
			continue
		}
		if _, err := os.Stat(filepath.Join("..", "cmd", service.Command[0])); err != nil {
			t.Errorf("compose service %s command %q has no source entrypoint: %v", name, service.Command[0], err)
		}
	}
}

func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
	payload, err := os.ReadFile("kubernetes/base/kustomization.yaml")
	if err != nil {
		t.Fatalf("read kustomization.yaml: %v", err)
	}
	if strings.Contains(string(payload), "checker.yaml") {
		t.Fatal("kubernetes base includes checker.yaml without per-workload mTLS identity overlay")
	}
}

func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
	dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
	if err != nil {
		t.Fatalf("read overview dashboard: %v", err)
	}
	var dashboard dashboardDocument
	if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
		t.Fatalf("parse overview dashboard: %v", err)
	}
	expressions := make([]string, 0, len(dashboard.Panels))
	for _, panel := range dashboard.Panels {
		for _, target := range panel.Targets {
			expressions = append(expressions, target.Expression)
		}
	}
	rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
	if err != nil {
		t.Fatalf("read Prometheus rules: %v", err)
	}
	var rules alertRulesDocument
	if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
		t.Fatalf("parse Prometheus rules: %v", err)
	}
	alerts := make([]string, 0)
	for _, group := range rules.Groups {
		for _, rule := range group.Rules {
			expressions = append(expressions, rule.Expression)
			alerts = append(alerts, rule.Alert)
		}
	}
	metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
	forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
	for _, expression := range expressions {
		for _, name := range metricPattern.FindAllString(expression, -1) {
			baseName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_bucket"), "_count"), "_sum")
			if _, exists := observableMetricNames[baseName]; !exists {
				t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
			}
		}
		if forbiddenLabel.MatchString(expression) {
			t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
		}
	}
	for _, required := range []string{
		"proxy_pool_gateway_requests_total", "proxy_pool_gateway_request_duration_seconds", "proxy_pool_controller_capacity_available_slots",
		"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
		"proxy_pool_checker_observations_total",
	} {
		if !strings.Contains(strings.Join(expressions, "\n"), required) {
			t.Errorf("observability assets do not cover %s", required)
		}
	}
	if !slices.Contains(alerts, "ProxyPoolGatewayHighLatency") {
		t.Error("Prometheus rules do not alert on Gateway p99 latency")
	}
}

func loadComposeDocument(t *testing.T) composeDocument {
	t.Helper()
	return loadComposeFile(t, "docker-compose.yml")
}

func loadComposeFile(t *testing.T, name string) composeDocument {
	t.Helper()
	payload, err := os.ReadFile(name)
	if err != nil {
		t.Fatalf("read %s: %v", name, err)
	}
	var document composeDocument
	if err := yaml.Unmarshal(payload, &document); err != nil {
		t.Fatalf("parse %s: %v", name, err)
	}
	return document
}

func composeDependsOn(value any, service string) bool {
	switch dependencies := value.(type) {
	case map[string]any:
		_, exists := dependencies[service]
		return exists
	case []any:
		for _, dependency := range dependencies {
			if dependency == service {
				return true
			}
		}
	}
	return false
}

func commandFlag(command []string, name string) (string, bool) {
	for index := 0; index+1 < len(command); index++ {
		if command[index] == name {
			return command[index+1], true
		}
	}
	return "", false
}

func assertCommandDirectory(t *testing.T, command string) {
	t.Helper()
	info, err := os.Stat(filepath.Join("..", "cmd", command))
	if err != nil || !info.IsDir() {
		t.Errorf("build command %q has no source directory", command)
	}
}