wild-cloud/wild-central
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Resilient HAProxy config: isolate broken services on validation failure

Browse Source 
When haproxy -c fails, parse ALERT line numbers, map to service domains
via # service: comments in generated config, exclude broken services,
and regenerate. One retry — prevents a single bad registration from
blocking all config updates.
This commit is contained in:
Paul Payne
2026-07-10 07:00:27 +00:00
parent 68d6fde80d
commit 6aa1e7d438
4 changed files with 221 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/api/v1/helpers.go
View File

@@ -122,13 +122,51 @@ func (api *API) reconcileNetworking() {
}
// Generate and write HAProxy config
haproxyCfg := api.haproxy.GenerateWithOpts(instanceRoutes, nil, haproxy.GenerateOpts{
genOpts := haproxy.GenerateOpts{
HTTPRoutes: activeHTTPRoutes,
CertsDir: certsDir,
})
}
haproxyCfg := api.haproxy.GenerateWithOpts(instanceRoutes, nil, genOpts)
if err := api.haproxy.WriteConfig(haproxyCfg); err != nil {
slog.Error("reconcile: failed to write HAProxy config", "error", err)
// Validation failed — identify and exclude broken services, then retry
brokenDomains := haproxy.FindBrokenServices(haproxyCfg, haproxy.ParseValidationErrors(err.Error()))
if len(brokenDomains) > 0 {
slog.Error("reconcile: excluding broken services and retrying",
"broken", brokenDomains, "error", err)
exclude := map[string]bool{}
for _, d := range brokenDomains {
exclude[d] = true
}
var filteredInstances []haproxy.InstanceRoute
for _, r := range instanceRoutes {
if !exclude[r.Domain] {
filteredInstances = append(filteredInstances, r)
}
}
var filteredHTTP []haproxy.HTTPRoute
for _, r := range activeHTTPRoutes {
if !exclude[r.Domain] {
filteredHTTP = append(filteredHTTP, r)
}
}
genOpts.HTTPRoutes = filteredHTTP
haproxyCfg = api.haproxy.GenerateWithOpts(filteredInstances, nil, genOpts)
if err := api.haproxy.WriteConfig(haproxyCfg); err != nil {
slog.Error("reconcile: retry also failed", "error", err)
} else {
if err := api.haproxy.ReloadService(); err != nil {
slog.Warn("reconcile: failed to reload HAProxy", "error", err)
} else {
api.broadcastHaproxyEvent("haproxy:config", "HAProxy config regenerated (excluded broken services)")
}
}
} else {
slog.Error("reconcile: failed to write HAProxy config (no broken services identified)", "error", err)
}
} else {
if err := api.haproxy.ReloadService(); err != nil {
slog.Warn("reconcile: failed to reload HAProxy", "error", err)