package haproxy import ( "bufio" "fmt" "log/slog" "net" "os" "os/exec" "sort" "strconv" "strings" "time" "github.com/wild-cloud/wild-central/internal/domains" ) // InstanceRoute represents a Wild Cloud instance's ingress routing configuration // InstanceRoute represents an L4 TCP passthrough route (k8s instance). type InstanceRoute struct { Name string `json:"name"` // identifier for backend naming Domain string `json:"domain"` // e.g. "cloud.payne.io" BackendIP string `json:"backendIP"` // k8s load balancer IP Subdomains bool `json:"subdomains"` // if true, also match *.domain } // CustomRoute represents a custom TCP proxy rule for non-Wild Cloud services type CustomRoute struct { Name string // e.g. "civil_ssh" Port int // external listen port Backend string // e.g. "192.168.8.10:22" } // HTTPRouteBackend maps path prefixes to a specific backend with its own L7 options. type HTTPRouteBackend struct { Paths []string // path prefixes (empty = catch-all) Backend string // host:port HealthPath string // optional health check path Headers *domains.HeaderConfig // per-route headers IPAllow []string // per-route CIDR whitelist } // HTTPRoute represents an L7 HTTP reverse proxy route. // Central terminates TLS using a wildcard certificate and proxies by Host header. // Routes are evaluated in order — path-specific before catch-all. type HTTPRoute struct { Name string // e.g. "my-api" Domain string // e.g. "my-api.payne.io" Subdomains bool // if true, also match *.domain Routes []HTTPRouteBackend // ordered path→backend mappings } const defaultConfigPath = "/etc/haproxy/haproxy.cfg" const defaultSocketPath = "/var/run/haproxy/admin.sock" // Manager handles HAProxy configuration generation and service management type Manager struct { configPath string socketPath string } // NewManager creates a new HAProxy manager func NewManager(configPath string) *Manager { if configPath == "" { configPath = defaultConfigPath } return &Manager{configPath: configPath, socketPath: defaultSocketPath} } // SetSocketPath overrides the HAProxy stats socket path (useful for testing) func (m *Manager) SetSocketPath(path string) { m.socketPath = path } // GetConfigPath returns the HAProxy config file path func (m *Manager) GetConfigPath() string { return m.configPath } // GenerateOpts holds optional parameters for HAProxy config generation. type GenerateOpts struct { HTTPRoutes []HTTPRoute // L7 HTTP reverse proxy routes (TLS terminated by Central) CertsDir string // directory containing per-domain PEM files for L7 TLS } // Generate creates a complete HAProxy configuration from instance routes, custom routes, // and optional config. Uses L4 TCP mode with SNI inspection for HTTPS — // TLS terminates at each k8s cluster's traefik. HTTP routes add an L7 frontend // where Central terminates TLS with a wildcard cert and reverse-proxies by Host header. func (m *Manager) Generate(instances []InstanceRoute, custom []CustomRoute, centralDomain ...string) string { return m.GenerateWithOpts(instances, custom, GenerateOpts{}) } // GenerateWithOpts creates a complete HAProxy configuration with full options. func (m *Manager) GenerateWithOpts(instances []InstanceRoute, custom []CustomRoute, opts GenerateOpts) string { var sb strings.Builder sb.WriteString(`# Wild Cloud HAProxy Configuration # Managed by Wild Cloud Central API — do not edit manually global log /dev/log local0 info stats socket /var/run/haproxy/admin.sock mode 666 level admin expose-fd listeners stats timeout 30s maxconn 50000 daemon defaults log global mode tcp option tcplog option dontlognull timeout connect 5s timeout client 50s timeout server 50s # Stats page (LAN access only — not proxied externally) frontend stats bind *:8404 mode http stats enable stats uri /stats stats refresh 10s # HTTP: redirect all to HTTPS frontend http_in bind *:80 mode http redirect scheme https code 301 `) hasL4Routes := len(instances) > 0 || len(opts.HTTPRoutes) > 0 if hasL4Routes { sb.WriteString("# HTTPS: SNI-based L4 routing (TLS passes through to k8s traefik)\n") sb.WriteString("frontend https_in\n") sb.WriteString(" bind *:443\n") sb.WriteString(" mode tcp\n") sb.WriteString(" tcp-request inspect-delay 5s\n") sb.WriteString(" tcp-request content accept if { req_ssl_hello_type 1 }\n") sb.WriteString("\n") // ACL ordering is critical. Process in this order: // 1. L7 HTTP exact matches (central, wild-cloud app) → L7 termination // 2. L4 exact matches (payne.io, civilsociety.dev) → instance backends // 3. L4 wildcard matches (*.cloud.payne.io) → instance backends // 4. Default → L7 termination (if any HTTP routes exist) // 1. L7 HTTP services — route to L7 TLS termination frontend if len(opts.HTTPRoutes) > 0 { sb.WriteString(" # L7 HTTP services (→ TLS termination)\n") // Exact matches first for _, route := range opts.HTTPRoutes { if route.Subdomains { continue // handled below } aclName := "is_l7_" + sanitizeName(route.Name) fmt.Fprintf(&sb, " # service: %s\n", route.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain) fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName) } // Wildcard matches after exact for _, route := range opts.HTTPRoutes { if !route.Subdomains { continue } aclName := "is_l7_" + sanitizeName(route.Name) fmt.Fprintf(&sb, " # service: %s\n", route.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, route.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain) fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName) } sb.WriteString("\n") } // 2. L4 exact matches — instances with subdomains:false hasExact := false for _, inst := range instances { if inst.Subdomains { continue // handled in step 3 } hasExact = true aclName := "is_" + sanitizeName(inst.Name) fmt.Fprintf(&sb, " # service: %s\n", inst.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain) fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName) } if hasExact { sb.WriteString("\n") } // 3. L4 wildcard matches — instances with subdomains:true sb.WriteString(" # L4 instance routes (wildcard subdomain matching)\n") for _, inst := range instances { if !inst.Subdomains { continue // already handled in step 2 } aclName := "is_" + sanitizeName(inst.Name) fmt.Fprintf(&sb, " # service: %s\n", inst.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, inst.Domain) fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain) fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName) } sb.WriteString("\n") // 4. Default → L7 termination fallback if len(opts.HTTPRoutes) > 0 { sb.WriteString(" default_backend be_l7_termination\n") } sb.WriteString("\n") // Instance L4 backends for _, inst := range instances { fmt.Fprintf(&sb, "# service: %s\n", inst.Domain) fmt.Fprintf(&sb, "backend be_https_%s\n", sanitizeName(inst.Name)) sb.WriteString(" mode tcp\n") sb.WriteString(" option tcp-check\n") fmt.Fprintf(&sb, " server s0 %s:443 check\n", inst.BackendIP) sb.WriteString("\n") } // L7 TLS termination backend + frontend (for HTTP routes) if len(opts.HTTPRoutes) > 0 { // Load all PEM files from the certs directory — HAProxy serves // the right cert per-SNI. Each service gets its own .pem. certPath := opts.CertsDir if certPath == "" { certPath = "/etc/haproxy/certs/" } sb.WriteString("backend be_l7_termination\n") sb.WriteString(" mode tcp\n") sb.WriteString(" server s0 127.0.0.1:8443\n") sb.WriteString("\n") sb.WriteString("# L7 HTTP frontend: TLS termination + Host-based routing\n") sb.WriteString("frontend l7_https\n") fmt.Fprintf(&sb, " bind 127.0.0.1:8443 ssl crt %s\n", certPath) sb.WriteString(" mode http\n") sb.WriteString("\n") // Host ACLs for all services for _, httpRoute := range opts.HTTPRoutes { hostACL := "host_" + sanitizeName(httpRoute.Name) fmt.Fprintf(&sb, " # service: %s\n", httpRoute.Domain) if httpRoute.Subdomains { fmt.Fprintf(&sb, " acl %s hdr(host) -i -m end .%s\n", hostACL, httpRoute.Domain) fmt.Fprintf(&sb, " acl %s hdr(host) -i -m str %s\n", hostACL, httpRoute.Domain) } else { fmt.Fprintf(&sb, " acl %s hdr(host) -i %s\n", hostACL, httpRoute.Domain) } } sb.WriteString("\n") // Per-route ACLs (IP whitelisting, headers) and routing rules for _, httpRoute := range opts.HTTPRoutes { hostACL := "host_" + sanitizeName(httpRoute.Name) routeName := sanitizeName(httpRoute.Name) for i, rb := range httpRoute.Routes { aclSuffix := fmt.Sprintf("%s_%d", routeName, i) // IP whitelisting for this route if len(rb.IPAllow) > 0 { ipACL := "ip_" + aclSuffix fmt.Fprintf(&sb, " acl %s src %s\n", ipACL, strings.Join(rb.IPAllow, " ")) if len(rb.Paths) > 0 { pathACL := "path_" + aclSuffix fmt.Fprintf(&sb, " acl %s path_beg %s\n", pathACL, strings.Join(rb.Paths, " ")) fmt.Fprintf(&sb, " http-request deny if %s %s !%s\n", hostACL, pathACL, ipACL) } else { fmt.Fprintf(&sb, " http-request deny if %s !%s\n", hostACL, ipACL) } } // Request headers if rb.Headers != nil { for _, k := range sortedKeys(rb.Headers.Request) { fmt.Fprintf(&sb, " http-request set-header %s %q if %s\n", k, rb.Headers.Request[k], hostACL) } } // Response headers if rb.Headers != nil { for _, k := range sortedKeys(rb.Headers.Response) { fmt.Fprintf(&sb, " http-response set-header %s %q if %s\n", k, rb.Headers.Response[k], hostACL) } } } } sb.WriteString("\n") // Routing rules: path-specific first, then catch-all for _, httpRoute := range opts.HTTPRoutes { hostACL := "host_" + sanitizeName(httpRoute.Name) routeName := sanitizeName(httpRoute.Name) // Path-specific routes first (most specific wins) for i, rb := range httpRoute.Routes { if len(rb.Paths) == 0 { continue } backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i) pathACL := fmt.Sprintf("path_%s_%d", routeName, i) fmt.Fprintf(&sb, " acl %s path_beg %s\n", pathACL, strings.Join(rb.Paths, " ")) fmt.Fprintf(&sb, " use_backend %s if %s %s\n", backendName, hostACL, pathACL) } // Catch-all route last for i, rb := range httpRoute.Routes { if len(rb.Paths) > 0 { continue } backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i) fmt.Fprintf(&sb, " use_backend %s if %s\n", backendName, hostACL) } } sb.WriteString("\n") // L7 backends — one per route for _, httpRoute := range opts.HTTPRoutes { routeName := sanitizeName(httpRoute.Name) for i, rb := range httpRoute.Routes { backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i) fmt.Fprintf(&sb, "# service: %s\n", httpRoute.Domain) fmt.Fprintf(&sb, "backend %s\n", backendName) sb.WriteString(" mode http\n") if rb.HealthPath != "" { fmt.Fprintf(&sb, " option httpchk GET %s\n", rb.HealthPath) fmt.Fprintf(&sb, " server s0 %s check\n", rb.Backend) } else { fmt.Fprintf(&sb, " server s0 %s\n", rb.Backend) } sb.WriteString("\n") } } } } for _, route := range custom { fmt.Fprintf(&sb, "# Custom route: %s\n", route.Name) fmt.Fprintf(&sb, "frontend fe_%s\n", sanitizeName(route.Name)) fmt.Fprintf(&sb, " bind *:%d\n", route.Port) sb.WriteString(" mode tcp\n") fmt.Fprintf(&sb, " default_backend be_%s\n", sanitizeName(route.Name)) sb.WriteString("\n") fmt.Fprintf(&sb, "backend be_%s\n", sanitizeName(route.Name)) sb.WriteString(" mode tcp\n") fmt.Fprintf(&sb, " server s0 %s\n", route.Backend) sb.WriteString("\n") } return sb.String() } // sortedKeys returns the keys of a map in sorted order for deterministic output. func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } sort.Strings(keys) return keys } // ParseValidationErrors extracts line numbers from haproxy -c ALERT output. // HAProxy format: [ALERT] (pid) : config : parsing [/path/to/file:LINE]: message func ParseValidationErrors(output string) []int { var lines []int for _, line := range strings.Split(output, "\n") { if !strings.Contains(line, "[ALERT]") || !strings.Contains(line, "parsing") { continue } // Find the bracketed path:line — e.g., [/tmp/test.cfg:111] bracketStart := strings.LastIndex(line, "[") bracketEnd := strings.Index(line[bracketStart:], "]") if bracketStart < 0 || bracketEnd < 0 { continue } bracket := line[bracketStart+1 : bracketStart+bracketEnd] // Extract line number after the last colon in the bracket colonIdx := strings.LastIndex(bracket, ":") if colonIdx < 0 { continue } if n, err := strconv.Atoi(bracket[colonIdx+1:]); err == nil { lines = append(lines, n) } } return lines } // FindBrokenServices maps failing line numbers to service domains using // "# service: domain" comment markers in the generated config. // For each failing line, scans backwards to find the nearest marker. func FindBrokenServices(config string, failingLines []int) []string { configLines := strings.Split(config, "\n") seen := map[string]bool{} var broken []string for _, lineNum := range failingLines { if lineNum < 1 || lineNum > len(configLines) { continue } // Scan backwards from the failing line to find the nearest service marker for i := lineNum - 1; i >= 0; i-- { trimmed := strings.TrimSpace(configLines[i]) if strings.HasPrefix(trimmed, "# service: ") { domain := strings.TrimPrefix(trimmed, "# service: ") if !seen[domain] { seen[domain] = true broken = append(broken, domain) } break } } } return broken } // sanitizeName converts a name to a valid HAProxy identifier (alphanumeric + underscore) func sanitizeName(name string) string { return strings.Map(func(r rune) rune { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { return r } return '_' }, name) } // GetListenPorts returns all TCP/HTTP ports HAProxy is configured to listen on. // Used to keep nftables rules in sync. func (m *Manager) GetListenPorts(instances []InstanceRoute, custom []CustomRoute) []int { ports := []int{80, 443, 8404} for _, route := range custom { ports = append(ports, route.Port) } return ports } // ValidateConfig tests an HAProxy configuration file for syntax errors func (m *Manager) ValidateConfig(configPath string) error { cmd := exec.Command("haproxy", "-c", "-f", configPath) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("haproxy config validation failed: %w (output: %s)", err, string(output)) } return nil } // WriteConfig validates and atomically writes the HAProxy configuration func (m *Manager) WriteConfig(content string) error { tempFile := m.configPath + ".tmp" if err := os.WriteFile(tempFile, []byte(content), 0644); err != nil { return fmt.Errorf("writing temp config: %w", err) } if err := m.ValidateConfig(tempFile); err != nil { os.Remove(tempFile) return err } if err := os.Rename(tempFile, m.configPath); err != nil { os.Remove(tempFile) return fmt.Errorf("installing config: %w", err) } slog.Info("haproxy config written", "component", "haproxy", "path", m.configPath) return nil } // ReloadService gracefully reloads HAProxy (starts it if not running) func (m *Manager) ReloadService() error { cmd := exec.Command("systemctl", "reload-or-restart", "haproxy.service") output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to reload haproxy: %w (output: %s)", err, string(output)) } slog.Info("haproxy service reloaded", "component", "haproxy") return nil } // RestartService performs a full restart of HAProxy func (m *Manager) RestartService() error { cmd := exec.Command("systemctl", "restart", "haproxy.service") output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to restart haproxy: %w (output: %s)", err, string(output)) } slog.Info("haproxy service restarted", "component", "haproxy") return nil } // ServiceStatus represents the status of the HAProxy service type ServiceStatus struct { Status string `json:"status"` PID int `json:"pid"` ConfigFile string `json:"configFile"` LastRestart time.Time `json:"lastRestart"` } // GetStatus returns the current HAProxy service status func (m *Manager) GetStatus() (*ServiceStatus, error) { status := &ServiceStatus{ ConfigFile: m.configPath, } cmd := exec.Command("systemctl", "is-active", "haproxy.service") output, err := cmd.Output() if err != nil { status.Status = "inactive" return status, nil } status.Status = strings.TrimSpace(string(output)) if status.Status == "active" { cmd = exec.Command("systemctl", "show", "haproxy.service", "--property=MainPID") if output, err := cmd.Output(); err == nil { parts := strings.Split(strings.TrimSpace(string(output)), "=") if len(parts) == 2 { if pid, err := strconv.Atoi(parts[1]); err == nil { status.PID = pid } } } cmd = exec.Command("systemctl", "show", "haproxy.service", "--property=ActiveEnterTimestamp") if output, err := cmd.Output(); err == nil { parts := strings.Split(strings.TrimSpace(string(output)), "=") if len(parts) == 2 { if t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", parts[1]); err == nil { status.LastRestart = t } } } } return status, nil } // ReadConfig reads the current HAProxy configuration file func (m *Manager) ReadConfig() (string, error) { data, err := os.ReadFile(m.configPath) if err != nil { return "", fmt.Errorf("reading haproxy config: %w", err) } return string(data), nil } // BackendStat holds per-backend connection counts from the HAProxy stats socket type BackendStat struct { Name string `json:"name"` Status string `json:"status"` ActiveConns int `json:"activeConns"` Rate int `json:"rate"` PeakRate int `json:"peakRate"` TotalConns int64 `json:"totalConns"` Errors int64 `json:"errors"` } // GetStats reads live connection stats from the HAProxy stats socket. // Returns an empty slice (not an error) when HAProxy is not running. func (m *Manager) GetStats() ([]BackendStat, error) { conn, err := net.Dial("unix", m.socketPath) if err != nil { // HAProxy not running or socket not yet created — not a hard error return []BackendStat{}, nil } defer conn.Close() if _, err := fmt.Fprintf(conn, "show stat\n"); err != nil { return nil, fmt.Errorf("sending stats command: %w", err) } // CSV format: # pxname,svname,...,status,...,scur,...,stot,...,eresp,... var stats []BackendStat scanner := bufio.NewScanner(conn) var headers []string for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "#") { // Header line: "# pxname,svname,..." headers = strings.Split(strings.TrimPrefix(line, "# "), ",") continue } if line == "" || len(headers) == 0 { continue } fields := strings.Split(line, ",") if len(fields) < len(headers) { continue } idx := func(name string) int { for i, h := range headers { if h == name { return i } } return -1 } // Only report backends (svname == "BACKEND") svIdx := idx("svname") if svIdx < 0 || fields[svIdx] != "BACKEND" { continue } stat := BackendStat{ Name: fields[idx("pxname")], Status: fields[idx("status")], } if i := idx("scur"); i >= 0 { fmt.Sscanf(fields[i], "%d", &stat.ActiveConns) } if i := idx("rate"); i >= 0 { fmt.Sscanf(fields[i], "%d", &stat.Rate) } if i := idx("rate_max"); i >= 0 { fmt.Sscanf(fields[i], "%d", &stat.PeakRate) } if i := idx("stot"); i >= 0 { fmt.Sscanf(fields[i], "%d", &stat.TotalConns) } if i := idx("eresp"); i >= 0 { fmt.Sscanf(fields[i], "%d", &stat.Errors) } stats = append(stats, stat) } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("reading stats: %w", err) } return stats, nil }