289 lines
9.1 KiB
Go
289 lines
9.1 KiB
Go
package metrics
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
outcomeDomain "proxy-pool/internal/domain/outcome"
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
)
|
|
|
|
// GatewayCollector exposes fixed-cardinality request-path outcome metrics.
|
|
// Proxy, route, destination, client and credential values are never labels.
|
|
type GatewayCollector struct {
|
|
outcomes *prometheus.CounterVec
|
|
dropped prometheus.Counter
|
|
invariants *prometheus.CounterVec
|
|
httpRequests prometheus.Counter
|
|
connectRequests prometheus.Counter
|
|
httpInFlight prometheus.Gauge
|
|
connectInFlight prometheus.Gauge
|
|
httpDuration prometheus.Observer
|
|
connectDuration prometheus.Observer
|
|
activeTunnels prometheus.Gauge
|
|
}
|
|
|
|
var (
|
|
_ outcomeDomain.MetricsObserver = (*GatewayCollector)(nil)
|
|
_ proxyDomain.CapacityInvariantObserver = (*GatewayCollector)(nil)
|
|
)
|
|
|
|
func NewGatewayCollector(registerer prometheus.Registerer) (*GatewayCollector, error) {
|
|
if registerer == nil {
|
|
return nil, ErrInvalidDependencies
|
|
}
|
|
outcomes, err := registerCounterVec(registerer, prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "outcomes_total",
|
|
Help: "Number of Gateway proxy attempts by furthest completed stage and result.",
|
|
}, []string{"stage", "result"}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dropped, err := registerCounter(registerer, prometheus.NewCounter(prometheus.CounterOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "outcome_queue_dropped_total",
|
|
Help: "Number of Gateway outcome observations dropped after the local queue was full.",
|
|
}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
invariants, err := registerCounterVec(registerer, prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "capacity_invariant_violations_total",
|
|
Help: "Number of invalid Gateway local proxy capacity lifecycle transitions.",
|
|
}, []string{"operation"}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
requests, err := registerCounterVec(registerer, prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "requests_total",
|
|
Help: "Number of accepted Gateway requests by fixed proxy protocol.",
|
|
}, []string{"protocol"}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inFlight, err := registerGaugeVec(registerer, prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "requests_in_flight",
|
|
Help: "Number of accepted Gateway requests currently executing by fixed proxy protocol.",
|
|
}, []string{"protocol"}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
durations, err := registerHistogramVec(registerer, prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "request_duration_seconds",
|
|
Help: "Gateway request duration from admission to completion by fixed proxy protocol.",
|
|
Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300},
|
|
}, []string{"protocol"}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
activeTunnels, err := registerGauge(registerer, prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: "proxy_pool", Subsystem: "gateway", Name: "active_tunnels",
|
|
Help: "Number of established CONNECT tunnels currently relaying through this Gateway Worker.",
|
|
}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &GatewayCollector{
|
|
outcomes: outcomes, dropped: dropped, invariants: invariants,
|
|
httpRequests: requests.WithLabelValues("HTTP"), connectRequests: requests.WithLabelValues("CONNECT"),
|
|
httpInFlight: inFlight.WithLabelValues("HTTP"), connectInFlight: inFlight.WithLabelValues("CONNECT"),
|
|
httpDuration: durations.WithLabelValues("HTTP"), connectDuration: durations.WithLabelValues("CONNECT"),
|
|
activeTunnels: activeTunnels,
|
|
}, nil
|
|
}
|
|
|
|
func (collector *GatewayCollector) Observe(event outcomeDomain.Event) {
|
|
if collector == nil || collector.outcomes == nil || !validGatewayStage(event.Stage) {
|
|
return
|
|
}
|
|
result := "failure"
|
|
if event.Success {
|
|
result = "success"
|
|
}
|
|
collector.outcomes.WithLabelValues(gatewayStageLabel(event.Stage), result).Inc()
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveDropped() {
|
|
if collector == nil || collector.dropped == nil {
|
|
return
|
|
}
|
|
collector.dropped.Inc()
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveCapacityInvariant(event proxyDomain.CapacityInvariant) {
|
|
if collector == nil || collector.invariants == nil || !validCapacityInvariantViolation(event.Violation) {
|
|
return
|
|
}
|
|
collector.invariants.WithLabelValues(string(event.Violation)).Inc()
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveRequestStarted(protocol string) {
|
|
if collector == nil {
|
|
return
|
|
}
|
|
switch protocol {
|
|
case "HTTP":
|
|
if collector.httpRequests != nil {
|
|
collector.httpRequests.Inc()
|
|
}
|
|
if collector.httpInFlight != nil {
|
|
collector.httpInFlight.Inc()
|
|
}
|
|
case "CONNECT":
|
|
if collector.connectRequests != nil {
|
|
collector.connectRequests.Inc()
|
|
}
|
|
if collector.connectInFlight != nil {
|
|
collector.connectInFlight.Inc()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveRequestFinished(protocol string) {
|
|
if collector == nil {
|
|
return
|
|
}
|
|
switch protocol {
|
|
case "HTTP":
|
|
if collector.httpInFlight != nil {
|
|
collector.httpInFlight.Dec()
|
|
}
|
|
case "CONNECT":
|
|
if collector.connectInFlight != nil {
|
|
collector.connectInFlight.Dec()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveRequestDuration(protocol string, duration time.Duration) {
|
|
if collector == nil || duration < 0 {
|
|
return
|
|
}
|
|
switch protocol {
|
|
case "HTTP":
|
|
if collector.httpDuration != nil {
|
|
collector.httpDuration.Observe(duration.Seconds())
|
|
}
|
|
case "CONNECT":
|
|
if collector.connectDuration != nil {
|
|
collector.connectDuration.Observe(duration.Seconds())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveTunnelOpened() {
|
|
if collector != nil && collector.activeTunnels != nil {
|
|
collector.activeTunnels.Inc()
|
|
}
|
|
}
|
|
|
|
func (collector *GatewayCollector) ObserveTunnelClosed() {
|
|
if collector != nil && collector.activeTunnels != nil {
|
|
collector.activeTunnels.Dec()
|
|
}
|
|
}
|
|
|
|
func registerCounter(registerer prometheus.Registerer, candidate prometheus.Counter) (prometheus.Counter, error) {
|
|
if err := registerer.Register(candidate); err == nil {
|
|
return candidate, nil
|
|
} else {
|
|
var registered prometheus.AlreadyRegisteredError
|
|
if !errors.As(err, ®istered) {
|
|
return nil, fmt.Errorf("register counter: %w", err)
|
|
}
|
|
existing, ok := registered.ExistingCollector.(prometheus.Counter)
|
|
if !ok {
|
|
return nil, fmt.Errorf("register counter: existing collector has unexpected type")
|
|
}
|
|
return existing, nil
|
|
}
|
|
}
|
|
|
|
func registerGauge(registerer prometheus.Registerer, candidate prometheus.Gauge) (prometheus.Gauge, error) {
|
|
if err := registerer.Register(candidate); err == nil {
|
|
return candidate, nil
|
|
} else {
|
|
var registered prometheus.AlreadyRegisteredError
|
|
if !errors.As(err, ®istered) {
|
|
return nil, fmt.Errorf("register gauge: %w", err)
|
|
}
|
|
existing, ok := registered.ExistingCollector.(prometheus.Gauge)
|
|
if !ok {
|
|
return nil, fmt.Errorf("register gauge: existing collector has unexpected type")
|
|
}
|
|
return existing, nil
|
|
}
|
|
}
|
|
|
|
func registerGaugeVec(registerer prometheus.Registerer, candidate *prometheus.GaugeVec) (*prometheus.GaugeVec, error) {
|
|
if err := registerer.Register(candidate); err == nil {
|
|
return candidate, nil
|
|
} else {
|
|
var registered prometheus.AlreadyRegisteredError
|
|
if !errors.As(err, ®istered) {
|
|
return nil, fmt.Errorf("register gauge vector: %w", err)
|
|
}
|
|
existing, ok := registered.ExistingCollector.(*prometheus.GaugeVec)
|
|
if !ok {
|
|
return nil, fmt.Errorf("register gauge vector: existing collector has unexpected type")
|
|
}
|
|
return existing, nil
|
|
}
|
|
}
|
|
|
|
func registerHistogramVec(registerer prometheus.Registerer, candidate *prometheus.HistogramVec) (*prometheus.HistogramVec, error) {
|
|
if err := registerer.Register(candidate); err == nil {
|
|
return candidate, nil
|
|
} else {
|
|
var registered prometheus.AlreadyRegisteredError
|
|
if !errors.As(err, ®istered) {
|
|
return nil, fmt.Errorf("register histogram vector: %w", err)
|
|
}
|
|
existing, ok := registered.ExistingCollector.(*prometheus.HistogramVec)
|
|
if !ok {
|
|
return nil, fmt.Errorf("register histogram vector: existing collector has unexpected type")
|
|
}
|
|
return existing, nil
|
|
}
|
|
}
|
|
|
|
func validGatewayStage(stage outcomeDomain.Stage) bool {
|
|
switch stage {
|
|
case outcomeDomain.StageDial, outcomeDomain.StageProxyHandshake,
|
|
outcomeDomain.StageResponseHeaders, outcomeDomain.StageTunnel:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func gatewayStageLabel(stage outcomeDomain.Stage) string {
|
|
switch stage {
|
|
case outcomeDomain.StageDial:
|
|
return "DIAL"
|
|
case outcomeDomain.StageProxyHandshake:
|
|
return "PROXY_HANDSHAKE"
|
|
case outcomeDomain.StageResponseHeaders:
|
|
return "RESPONSE_HEADERS"
|
|
case outcomeDomain.StageTunnel:
|
|
return "TUNNEL"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func validCapacityInvariantViolation(violation proxyDomain.CapacityInvariantViolation) bool {
|
|
switch violation {
|
|
case proxyDomain.CapacityInvariantCommitAlreadyCommitted,
|
|
proxyDomain.CapacityInvariantCommitFinished,
|
|
proxyDomain.CapacityInvariantCancelFinished,
|
|
proxyDomain.CapacityInvariantReleaseBeforeCommit,
|
|
proxyDomain.CapacityInvariantReleaseFinished:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|