wild-cloud/wild-central
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Resilient HAProxy config: isolate broken services on validation failure

Browse Source 
When haproxy -c fails, parse ALERT line numbers, map to service domains
via # service: comments in generated config, exclude broken services,
and regenerate. One retry — prevents a single bad registration from
blocking all config updates.
This commit is contained in:
Paul Payne
2026-07-10 07:00:27 +00:00
parent 68d6fde80d
commit 6aa1e7d438
4 changed files with 221 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/haproxy/config.go
View File

@@ -156,6 +156,7 @@ frontend http_in
continue // handled below
}
aclName := "is_l7_" + sanitizeName(route.Name)
fmt.Fprintf(&sb, " # service: %s\n", route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName)
}
@@ -165,6 +166,7 @@ frontend http_in
continue
}
aclName := "is_l7_" + sanitizeName(route.Name)
fmt.Fprintf(&sb, " # service: %s\n", route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, route.Domain)
fmt.Fprintf(&sb, " use_backend be_l7_termination if %s\n", aclName)
@@ -180,6 +182,7 @@ frontend http_in
}
hasExact = true
aclName := "is_" + sanitizeName(inst.Name)
fmt.Fprintf(&sb, " # service: %s\n", inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName)
}
@@ -194,6 +197,7 @@ frontend http_in
continue // already handled in step 2
}
aclName := "is_" + sanitizeName(inst.Name)
fmt.Fprintf(&sb, " # service: %s\n", inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m end .%s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " acl %s req_ssl_sni -m str %s\n", aclName, inst.Domain)
fmt.Fprintf(&sb, " use_backend be_https_%s if %s\n", sanitizeName(inst.Name), aclName)
@@ -209,6 +213,7 @@ frontend http_in
// Instance L4 backends
for _, inst := range instances {
fmt.Fprintf(&sb, "# service: %s\n", inst.Domain)
fmt.Fprintf(&sb, "backend be_https_%s\n", sanitizeName(inst.Name))
sb.WriteString(" mode tcp\n")
sb.WriteString(" option tcp-check\n")
@@ -239,6 +244,7 @@ frontend http_in
// Host ACLs for all services
for _, httpRoute := range opts.HTTPRoutes {
hostACL := "host_" + sanitizeName(httpRoute.Name)
fmt.Fprintf(&sb, " # service: %s\n", httpRoute.Domain)
if httpRoute.Subdomains {
fmt.Fprintf(&sb, " acl %s hdr(host) -i -m end .%s\n", hostACL, httpRoute.Domain)
fmt.Fprintf(&sb, " acl %s hdr(host) -i -m str %s\n", hostACL, httpRoute.Domain)
@@ -319,6 +325,7 @@ frontend http_in
routeName := sanitizeName(httpRoute.Name)
for i, rb := range httpRoute.Routes {
backendName := fmt.Sprintf("be_l7_%s_%d", routeName, i)
fmt.Fprintf(&sb, "# service: %s\n", httpRoute.Domain)
fmt.Fprintf(&sb, "backend %s\n", backendName)
sb.WriteString(" mode http\n")
if rb.HealthPath != "" {
@@ -359,6 +366,61 @@ func sortedKeys(m map[string]string) []string {
return keys
}
// ParseValidationErrors extracts line numbers from haproxy -c ALERT output.
// HAProxy format: [ALERT] (pid) : config : parsing [/path/to/file:LINE]: message
func ParseValidationErrors(output string) []int {
var lines []int
for _, line := range strings.Split(output, "\n") {
if !strings.Contains(line, "[ALERT]") || !strings.Contains(line, "parsing") {
continue
}
// Find the bracketed path:line — e.g., [/tmp/test.cfg:111]
bracketStart := strings.LastIndex(line, "[")
bracketEnd := strings.Index(line[bracketStart:], "]")
if bracketStart < 0 || bracketEnd < 0 {
continue
}
bracket := line[bracketStart+1 : bracketStart+bracketEnd]
// Extract line number after the last colon in the bracket
colonIdx := strings.LastIndex(bracket, ":")
if colonIdx < 0 {
continue
}
if n, err := strconv.Atoi(bracket[colonIdx+1:]); err == nil {
lines = append(lines, n)
}
}
return lines
}
// FindBrokenServices maps failing line numbers to service domains using
// "# service: domain" comment markers in the generated config.
// For each failing line, scans backwards to find the nearest marker.
func FindBrokenServices(config string, failingLines []int) []string {
configLines := strings.Split(config, "\n")
seen := map[string]bool{}
var broken []string
for _, lineNum := range failingLines {
if lineNum < 1 || lineNum > len(configLines) {
continue
}
// Scan backwards from the failing line to find the nearest service marker
for i := lineNum - 1; i >= 0; i-- {
trimmed := strings.TrimSpace(configLines[i])
if strings.HasPrefix(trimmed, "# service: ") {
domain := strings.TrimPrefix(trimmed, "# service: ")
if !seen[domain] {
seen[domain] = true
broken = append(broken, domain)
}
break
}
}
}
return broken
}
// sanitizeName converts a name to a valid HAProxy identifier (alphanumeric + underscore)
func sanitizeName(name string) string {
return strings.Map(func(r rune) rune {