Files
wild-central/internal/haproxy/config.go
Paul Payne 6aa1e7d438 Resilient HAProxy config: isolate broken services on validation failure
When haproxy -c fails, parse ALERT line numbers, map to service domains
via # service: comments in generated config, exclude broken services,
and regenerate. One retry — prevents a single bad registration from
blocking all config updates.
2026-07-10 07:00:27 +00:00

637 lines
20 KiB
Go

package haproxy
import (
"bufio"
"fmt"
"log/slog"
"net"
"os"
"os/exec"
"sort"
"strconv"
"strings"
"time"
"github.com/wild-cloud/wild-central/internal/services"
)
// InstanceRoute represents a Wild Cloud instance's ingress routing configuration
// InstanceRoute represents an L4 TCP passthrough route (k8s instance).
type InstanceRoute struct {
Name string `json:"name"` // identifier for backend naming
Domain string `json:"domain"` // e.g. "cloud.payne.io"
BackendIP string `json:"backendIP"` // k8s load balancer IP
Subdomains bool `json:"subdomains"` // if true, also match *.domain
}
// CustomRoute represents a custom TCP proxy rule for non-Wild Cloud services
type CustomRoute struct {
Name string // e.g. "civil_ssh"
Port int // external listen port
Backend string // e.g. "192.168.8.10:22"
}
// HTTPRouteBackend maps path prefixes to a specific backend with its own L7 options.
type HTTPRouteBackend struct {
Paths []string // path prefixes (empty = catch-all)
Backend string // host:port
HealthPath string // optional health check path
Headers *services.HeaderConfig // per-route headers
IPAllow []string // per-route CIDR whitelist
}
// HTTPRoute represents an L7 HTTP reverse proxy route.
// Central terminates TLS using a wildcard certificate and proxies by Host header.
// Routes are evaluated in order — path-specific before catch-all.
type HTTPRoute struct {
Name string // e.g. "my-api"
Domain string // e.g. "my-api.payne.io"
Subdomains bool // if true, also match *.domain
Routes []HTTPRouteBackend // ordered path→backend mappings
}
const defaultConfigPath = "/etc/haproxy/haproxy.cfg"
const defaultSocketPath = "/var/run/haproxy/admin.sock"
// Manager handles HAProxy configuration generation and service management
type Manager struct {
configPath string
socketPath string
}
// NewManager creates a new HAProxy manager
func NewManager(configPath string) *Manager {
if configPath == "" {
configPath = defaultConfigPath
}
return &Manager{configPath: configPath, socketPath: defaultSocketPath}
}
// SetSocketPath overrides the HAProxy stats socket path (useful for testing)
func (m *Manager) SetSocketPath(path string) {
m.socketPath = path
}
// GetConfigPath returns the HAProxy config file path
func (m *Manager) GetConfigPath() string {
return m.configPath
}
// GenerateOpts holds optional parameters for HAProxy config generation.
type GenerateOpts struct {
HTTPRoutes []HTTPRoute // L7 HTTP reverse proxy routes (TLS terminated by Central)
CertsDir string // directory containing per-domain PEM files for L7 TLS
}
// Generate creates a complete HAProxy configuration from instance routes, custom routes,
// and optional config. Uses L4 TCP mode with SNI inspection for HTTPS —
// TLS terminates at each k8s cluster's traefik. HTTP routes add an L7 frontend
// where Central terminates TLS with a wildcard cert and reverse-proxies by Host header.
func (m *Manager) Generate(instances []InstanceRoute, custom []CustomRoute, centralDomain ...string) string {
return m.GenerateWithOpts(instances, custom, GenerateOpts{})
}
// GenerateWithOpts creates a complete HAProxy configuration with full options.
func (m *Manager) GenerateWithOpts(instances []InstanceRoute, custom []CustomRoute, opts GenerateOpts) string {
var sb strings.Builder
sb.WriteString(`# Wild Cloud HAProxy Configuration
# Managed by Wild Cloud Central API — do not edit manually
global
log /dev/log local0 info
stats socket /var/run/haproxy/admin.sock mode 666 level admin expose-fd listeners
stats timeout 30s
maxconn 50000
daemon
defaults
log global
mode tcp
option tcplog
option dontlognull
timeout connect 5s
timeout client 50s
timeout server 50s
# Stats page (LAN access only — not proxied externally)
frontend stats
bind *:8404
mode http
stats enable
stats uri /stats
stats refresh 10s
# HTTP: redirect all to HTTPS
frontend http_in
bind *:80
mode http
redirect scheme https code 301
`)
hasL4Routes := len(instances) > 0 || len(opts.HTTPRoutes) > 0
if hasL4Routes {
sb.WriteString("# HTTPS: SNI-based L4 routing (TLS passes through to k8s traefik)\n")
sb.WriteString("frontend https_in\n")
sb.WriteString(" bind *:443\n")
sb.WriteString(" mode tcp\n")
sb.WriteString(" tcp-request inspect-delay 5s\n")
sb.WriteString(" tcp-request content accept if { req_ssl_hello_type 1 }\n")
sb.WriteString("\n")
// ACL ordering is critical. Process in this order:
// 1. L7 HTTP exact matches (central, wild-cloud app) → L7 termination
// 2. L4 exact matches (payne.io, civilsociety.dev) → instance backends
// 3. L4 wildcard matches (*.cloud.payne.io) → instance backends
// 4. Default → L7 termination (if any HTTP routes exist)
// 1. L7 HTTP services — route to L7 TLS termination frontend
if len(opts.HTTPRoutes) > 0 {
sb.WriteString(" # L7 HTTP services (→ TLS termination)\n")
// Exact matches first
for _, route := range opts.HTTPRoutes {
if route.Subdomains {
continue // handled below
}
aclName := "is_l7_" + sanitizeName(route.Name)
fmt.Fprintf(&sb, " # service: %s\n", route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName)
}
// Wildcard matches after exact
for _, route := range opts.HTTPRoutes {
if !route.Subdomains {
continue
}
aclName := "is_l7_" + sanitizeName(route.Name)
fmt.Fprintf(&sb, " # service: %s\n", route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName)
}
sb.WriteString("\n")
}
// 2. L4 exact matches — instances with subdomains:false
hasExact := false
for _, inst := range instances {
if inst.Subdomains {
continue // handled in step 3
}
hasExact = true
aclName := "is_" + sanitizeName(inst.Name)
fmt.Fprintf(&sb, " # service: %s\n", inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName)
}
if hasExact {
sb.WriteString("\n")
}
// 3. L4 wildcard matches — instances with subdomains:true
sb.WriteString(" # L4 instance routes (wildcard subdomain matching)\n")
for _, inst := range instances {
if !inst.Subdomains {
continue // already handled in step 2
}
aclName := "is_" + sanitizeName(inst.Name)
fmt.Fprintf(&sb, " # service: %s\n", inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName)
}
sb.WriteString("\n")
// 4. Default → L7 termination fallback
if len(opts.HTTPRoutes) > 0 {
sb.WriteString(" default_backend be_l7_termination\n")
}
sb.WriteString("\n")
// Instance L4 backends
for _, inst := range instances {
fmt.Fprintf(&sb, "# service: %s\n", inst.Domain)
fmt.Fprintf(&sb, "backend be_https_%s\n", sanitizeName(inst.Name))
sb.WriteString(" mode tcp\n")
sb.WriteString(" option tcp-check\n")
fmt.Fprintf(&sb, " server s0 %s:443 check\n", inst.BackendIP)
sb.WriteString("\n")
}
// L7 TLS termination backend + frontend (for HTTP routes)
if len(opts.HTTPRoutes) > 0 {
// Load all PEM files from the certs directory — HAProxy serves
// the right cert per-SNI. Each service gets its own <domain>.pem.
certPath := opts.CertsDir
if certPath == "" {
certPath = "/etc/haproxy/certs/"
}
sb.WriteString("backend be_l7_termination\n")
sb.WriteString(" mode tcp\n")
sb.WriteString(" server s0 127.0.0.1:8443\n")
sb.WriteString("\n")
sb.WriteString("# L7 HTTP frontend: TLS termination + Host-based routing\n")
sb.WriteString("frontend l7_https\n")
fmt.Fprintf(&sb, " bind 127.0.0.1:8443 ssl crt %s\n", certPath)
sb.WriteString(" mode http\n")
sb.WriteString("\n")
// Host ACLs for all services
for _, httpRoute := range opts.HTTPRoutes {
hostACL := "host_" + sanitizeName(httpRoute.Name)
fmt.Fprintf(&sb, " # service: %s\n", httpRoute.Domain)
if httpRoute.Subdomains {
fmt.Fprintf(&sb, " acl %s hdr(host) -i -m end .%s\n", hostACL, httpRoute.Domain)
fmt.Fprintf(&sb, " acl %s hdr(host) -i -m str %s\n", hostACL, httpRoute.Domain)
} else {
fmt.Fprintf(&sb, " acl %s hdr(host) -i %s\n", hostACL, httpRoute.Domain)
}
}
sb.WriteString("\n")
// Per-route ACLs (IP whitelisting, headers) and routing rules
for _, httpRoute := range opts.HTTPRoutes {
hostACL := "host_" + sanitizeName(httpRoute.Name)
routeName := sanitizeName(httpRoute.Name)
for i, rb := range httpRoute.Routes {
aclSuffix := fmt.Sprintf("%s_%d", routeName, i)
// IP whitelisting for this route
if len(rb.IPAllow) > 0 {
ipACL := "ip_" + aclSuffix
fmt.Fprintf(&sb, " acl %s src %s\n", ipACL, strings.Join(rb.IPAllow, " "))
if len(rb.Paths) > 0 {
pathACL := "path_" + aclSuffix
fmt.Fprintf(&sb, " acl %s path_beg %s\n", pathACL, strings.Join(rb.Paths, " "))
fmt.Fprintf(&sb, " http-request deny if %s %s !%s\n", hostACL, pathACL, ipACL)
} else {
fmt.Fprintf(&sb, " http-request deny if %s !%s\n", hostACL, ipACL)
}
}
// Request headers
if rb.Headers != nil {
for _, k := range sortedKeys(rb.Headers.Request) {
fmt.Fprintf(&sb, " http-request set-header %s %q if %s\n", k, rb.Headers.Request[k], hostACL)
}
}
// Response headers
if rb.Headers != nil {
for _, k := range sortedKeys(rb.Headers.Response) {
fmt.Fprintf(&sb, " http-response set-header %s %q if %s\n", k, rb.Headers.Response[k], hostACL)
}
}
}
}
sb.WriteString("\n")
// Routing rules: path-specific first, then catch-all
for _, httpRoute := range opts.HTTPRoutes {
hostACL := "host_" + sanitizeName(httpRoute.Name)
routeName := sanitizeName(httpRoute.Name)
// Path-specific routes first (most specific wins)
for i, rb := range httpRoute.Routes {
if len(rb.Paths) == 0 {
continue
}
backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i)
pathACL := fmt.Sprintf("path_%s_%d", routeName, i)
fmt.Fprintf(&sb, " acl %s path_beg %s\n", pathACL, strings.Join(rb.Paths, " "))
fmt.Fprintf(&sb, " use_backend %s if %s %s\n", backendName, hostACL, pathACL)
}
// Catch-all route last
for i, rb := range httpRoute.Routes {
if len(rb.Paths) > 0 {
continue
}
backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i)
fmt.Fprintf(&sb, " use_backend %s if %s\n", backendName, hostACL)
}
}
sb.WriteString("\n")
// L7 backends — one per route
for _, httpRoute := range opts.HTTPRoutes {
routeName := sanitizeName(httpRoute.Name)
for i, rb := range httpRoute.Routes {
backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i)
fmt.Fprintf(&sb, "# service: %s\n", httpRoute.Domain)
fmt.Fprintf(&sb, "backend %s\n", backendName)
sb.WriteString(" mode http\n")
if rb.HealthPath != "" {
fmt.Fprintf(&sb, " option httpchk GET %s\n", rb.HealthPath)
fmt.Fprintf(&sb, " server s0 %s check\n", rb.Backend)
} else {
fmt.Fprintf(&sb, " server s0 %s\n", rb.Backend)
}
sb.WriteString("\n")
}
}
}
}
for _, route := range custom {
fmt.Fprintf(&sb, "# Custom route: %s\n", route.Name)
fmt.Fprintf(&sb, "frontend fe_%s\n", sanitizeName(route.Name))
fmt.Fprintf(&sb, " bind *:%d\n", route.Port)
sb.WriteString(" mode tcp\n")
fmt.Fprintf(&sb, " default_backend be_%s\n", sanitizeName(route.Name))
sb.WriteString("\n")
fmt.Fprintf(&sb, "backend be_%s\n", sanitizeName(route.Name))
sb.WriteString(" mode tcp\n")
fmt.Fprintf(&sb, " server s0 %s\n", route.Backend)
sb.WriteString("\n")
}
return sb.String()
}
// sortedKeys returns the keys of a map in sorted order for deterministic output.
func sortedKeys(m map[string]string) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// ParseValidationErrors extracts line numbers from haproxy -c ALERT output.
// HAProxy format: [ALERT] (pid) : config : parsing [/path/to/file:LINE]: message
func ParseValidationErrors(output string) []int {
var lines []int
for _, line := range strings.Split(output, "\n") {
if !strings.Contains(line, "[ALERT]") || !strings.Contains(line, "parsing") {
continue
}
// Find the bracketed path:line — e.g., [/tmp/test.cfg:111]
bracketStart := strings.LastIndex(line, "[")
bracketEnd := strings.Index(line[bracketStart:], "]")
if bracketStart < 0 || bracketEnd < 0 {
continue
}
bracket := line[bracketStart+1 : bracketStart+bracketEnd]
// Extract line number after the last colon in the bracket
colonIdx := strings.LastIndex(bracket, ":")
if colonIdx < 0 {
continue
}
if n, err := strconv.Atoi(bracket[colonIdx+1:]); err == nil {
lines = append(lines, n)
}
}
return lines
}
// FindBrokenServices maps failing line numbers to service domains using
// "# service: domain" comment markers in the generated config.
// For each failing line, scans backwards to find the nearest marker.
func FindBrokenServices(config string, failingLines []int) []string {
configLines := strings.Split(config, "\n")
seen := map[string]bool{}
var broken []string
for _, lineNum := range failingLines {
if lineNum < 1 || lineNum > len(configLines) {
continue
}
// Scan backwards from the failing line to find the nearest service marker
for i := lineNum - 1; i >= 0; i-- {
trimmed := strings.TrimSpace(configLines[i])
if strings.HasPrefix(trimmed, "# service: ") {
domain := strings.TrimPrefix(trimmed, "# service: ")
if !seen[domain] {
seen[domain] = true
broken = append(broken, domain)
}
break
}
}
}
return broken
}
// sanitizeName converts a name to a valid HAProxy identifier (alphanumeric + underscore)
func sanitizeName(name string) string {
return strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
return r
}
return '_'
}, name)
}
// GetListenPorts returns all TCP/HTTP ports HAProxy is configured to listen on.
// Used to keep nftables rules in sync.
func (m *Manager) GetListenPorts(instances []InstanceRoute, custom []CustomRoute) []int {
ports := []int{80, 443, 8404}
for _, route := range custom {
ports = append(ports, route.Port)
}
return ports
}
// ValidateConfig tests an HAProxy configuration file for syntax errors
func (m *Manager) ValidateConfig(configPath string) error {
cmd := exec.Command("haproxy", "-c", "-f", configPath)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("haproxy config validation failed: %w (output: %s)", err, string(output))
}
return nil
}
// WriteConfig validates and atomically writes the HAProxy configuration
func (m *Manager) WriteConfig(content string) error {
tempFile := m.configPath + ".tmp"
if err := os.WriteFile(tempFile, []byte(content), 0644); err != nil {
return fmt.Errorf("writing temp config: %w", err)
}
if err := m.ValidateConfig(tempFile); err != nil {
os.Remove(tempFile)
return err
}
if err := os.Rename(tempFile, m.configPath); err != nil {
os.Remove(tempFile)
return fmt.Errorf("installing config: %w", err)
}
slog.Info("haproxy config written", "component", "haproxy", "path", m.configPath)
return nil
}
// ReloadService gracefully reloads HAProxy (starts it if not running)
func (m *Manager) ReloadService() error {
cmd := exec.Command("systemctl", "reload-or-restart", "haproxy.service")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to reload haproxy: %w (output: %s)", err, string(output))
}
slog.Info("haproxy service reloaded", "component", "haproxy")
return nil
}
// RestartService performs a full restart of HAProxy
func (m *Manager) RestartService() error {
cmd := exec.Command("systemctl", "restart", "haproxy.service")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to restart haproxy: %w (output: %s)", err, string(output))
}
slog.Info("haproxy service restarted", "component", "haproxy")
return nil
}
// ServiceStatus represents the status of the HAProxy service
type ServiceStatus struct {
Status string `json:"status"`
PID int `json:"pid"`
ConfigFile string `json:"configFile"`
LastRestart time.Time `json:"lastRestart"`
}
// GetStatus returns the current HAProxy service status
func (m *Manager) GetStatus() (*ServiceStatus, error) {
status := &ServiceStatus{
ConfigFile: m.configPath,
}
cmd := exec.Command("systemctl", "is-active", "haproxy.service")
output, err := cmd.Output()
if err != nil {
status.Status = "inactive"
return status, nil
}
status.Status = strings.TrimSpace(string(output))
if status.Status == "active" {
cmd = exec.Command("systemctl", "show", "haproxy.service", "--property=MainPID")
if output, err := cmd.Output(); err == nil {
parts := strings.Split(strings.TrimSpace(string(output)), "=")
if len(parts) == 2 {
if pid, err := strconv.Atoi(parts[1]); err == nil {
status.PID = pid
}
}
}
cmd = exec.Command("systemctl", "show", "haproxy.service", "--property=ActiveEnterTimestamp")
if output, err := cmd.Output(); err == nil {
parts := strings.Split(strings.TrimSpace(string(output)), "=")
if len(parts) == 2 {
if t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", parts[1]); err == nil {
status.LastRestart = t
}
}
}
}
return status, nil
}
// ReadConfig reads the current HAProxy configuration file
func (m *Manager) ReadConfig() (string, error) {
data, err := os.ReadFile(m.configPath)
if err != nil {
return "", fmt.Errorf("reading haproxy config: %w", err)
}
return string(data), nil
}
// BackendStat holds per-backend connection counts from the HAProxy stats socket
type BackendStat struct {
Name string `json:"name"`
Status string `json:"status"`
ActiveConns int `json:"activeConns"`
Rate int `json:"rate"`
PeakRate int `json:"peakRate"`
TotalConns int64 `json:"totalConns"`
Errors int64 `json:"errors"`
}
// GetStats reads live connection stats from the HAProxy stats socket.
// Returns an empty slice (not an error) when HAProxy is not running.
func (m *Manager) GetStats() ([]BackendStat, error) {
conn, err := net.Dial("unix", m.socketPath)
if err != nil {
// HAProxy not running or socket not yet created — not a hard error
return []BackendStat{}, nil
}
defer conn.Close()
if _, err := fmt.Fprintf(conn, "show stat\n"); err != nil {
return nil, fmt.Errorf("sending stats command: %w", err)
}
// CSV format: # pxname,svname,...,status,...,scur,...,stot,...,eresp,...
var stats []BackendStat
scanner := bufio.NewScanner(conn)
var headers []string
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "#") {
// Header line: "# pxname,svname,..."
headers = strings.Split(strings.TrimPrefix(line, "# "), ",")
continue
}
if line == "" || len(headers) == 0 {
continue
}
fields := strings.Split(line, ",")
if len(fields) < len(headers) {
continue
}
idx := func(name string) int {
for i, h := range headers {
if h == name {
return i
}
}
return -1
}
// Only report backends (svname == "BACKEND")
svIdx := idx("svname")
if svIdx < 0 || fields[svIdx] != "BACKEND" {
continue
}
stat := BackendStat{
Name: fields[idx("pxname")],
Status: fields[idx("status")],
}
if i := idx("scur"); i >= 0 {
fmt.Sscanf(fields[i], "%d", &stat.ActiveConns)
}
if i := idx("rate"); i >= 0 {
fmt.Sscanf(fields[i], "%d", &stat.Rate)
}
if i := idx("rate_max"); i >= 0 {
fmt.Sscanf(fields[i], "%d", &stat.PeakRate)
}
if i := idx("stot"); i >= 0 {
fmt.Sscanf(fields[i], "%d", &stat.TotalConns)
}
if i := idx("eresp"); i >= 0 {
fmt.Sscanf(fields[i], "%d", &stat.Errors)
}
stats = append(stats, stat)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("reading stats: %w", err)
}
return stats, nil
}