wild-cloud/wild-central
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add SafeApply lifecycle, health tracking, convergence loop, and startup checks

Browse Source 
SafeApply pattern (validate → backup → write → reload → verify → rollback):
- HAProxy: SafeApply wraps existing validate+write+reload with backup and
  post-reload health check; rolls back to .bak on failure
- dnsmasq: SafeApply validates via dnsmasq --test, backs up, atomic writes,
  restarts, verifies daemon is active; rolls back on failure
- nftables: SafeApply validates via nft -c, backs up, atomic writes, applies
  to kernel, verifies table loaded; rolls back on failure

Health tracking:
- Add SubsystemHealth and Health structs to reconciler
- Track per-subsystem status (ok/degraded/error) after each reconcile
- Detect recovery: previous error → current ok broadcasts recovery event
- GET /api/v1/health/reconcile endpoint exposes health state
- HAProxy tracks excluded domains as "degraded" state

SSE error events:
- Broadcast haproxy:error, dnsmasq:error on SafeApply failure
- Broadcast haproxy:recovered, dnsmasq:recovered on recovery from error

Convergence loop:
- 5-minute periodic reconcile drives system toward desired state
- Catches config drift, daemon crashes, transient failures
- Serialized by reconcile mutex — no race with event-driven reconciles

Startup:
- CheckPrerequisites verifies required (dnsmasq, haproxy) and optional
  (wg, authelia, cscli, cloudflared, nft) binaries before first reconcile
This commit is contained in:
Paul Payne
2026-07-14 11:44:44 +00:00
parent 88acd437a1
commit a0ab8faa1c
7 changed files with 276 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
internal/reconcile/reconciler.go
View File

@@ -26,13 +26,15 @@ import (
// HAProxyManager is the subset of haproxy.Manager used by reconciliation.
type HAProxyManager interface {
GenerateWithOpts(instances []haproxy.L4Route, custom []haproxy.CustomRoute, opts haproxy.GenerateOpts) string
SafeApply(content string) error
WriteConfig(content string) error
ReloadService() error
}
// DNSManager is the subset of dnsmasq.Manager used by reconciliation.
type DNSManager interface {
UpdateConfig(cfg *config.State, entries []dnsmasq.DNSEntry, restart bool) error
Generate(cfg *config.State, entries []dnsmasq.DNSEntry) string
SafeApply(content string) error
SetFilterConfPath(path string)
GetStatus() (*dnsmasq.Status, error)
}
@@ -68,9 +70,24 @@ type EventBroadcaster interface {
// that the reconciler doesn't own.
type GenerateAutheliaConfigFn func(state *config.State) error
// SubsystemHealth records the outcome of the last reconciliation for one subsystem.
type SubsystemHealth struct {
Status string `json:"status"` // "ok", "degraded", "error"
Message string `json:"message,omitempty"`
}
// Health records the outcome of the last reconciliation across all subsystems.
type Health struct {
HAProxy SubsystemHealth `json:"haproxy"`
DNS SubsystemHealth `json:"dns"`
UpdatedAt time.Time `json:"updatedAt"`
ExcludedDomains []string `json:"excludedDomains,omitempty"`
}
// Reconciler orchestrates config regeneration when domains change.
type Reconciler struct {
mu sync.Mutex // serializes concurrent Reconcile() calls
health Health
Domains DomainManager
HAProxy HAProxyManager
DNS DNSManager
@@ -82,6 +99,13 @@ type Reconciler struct {
GenerateAutheliaConfig GenerateAutheliaConfigFn
}
// GetHealth returns a snapshot of the last reconciliation health state.
func (r *Reconciler) GetHealth() Health {
r.mu.Lock()
defer r.mu.Unlock()
return r.health
}
// Reconcile reads all registered domains and regenerates dnsmasq DNS entries
// and HAProxy routes to match. Serialized by mutex — concurrent calls wait
// rather than racing on config files.
@@ -119,6 +143,8 @@ func (r *Reconciler) Reconcile() {
}
}
previousHealth := r.health
r.writeHAProxyConfig(l4Routes, activeHTTPRoutes, globalCfg)
// Build and apply DNS config
@@ -141,9 +167,13 @@ func (r *Reconciler) Reconcile() {
r.DNS.SetFilterConfPath("")
}
if err := r.DNS.UpdateConfig(globalCfg, dnsEntries, true); err != nil {
dnsConfig := r.DNS.Generate(globalCfg, dnsEntries)
if err := r.DNS.SafeApply(dnsConfig); err != nil {
slog.Error("failed to update dnsmasq", "component", "reconcile", "error", err)
r.health.DNS = SubsystemHealth{Status: "error", Message: err.Error()}
r.broadcastEvent("dnsmasq:error", err.Error())
} else {
r.health.DNS = SubsystemHealth{Status: "ok"}
r.broadcastDNSEvent("dnsmasq:config", "DNS config regenerated from registered domains")
}
@@ -153,10 +183,22 @@ func (r *Reconciler) Reconcile() {
r.DDNS.Trigger()
r.health.UpdatedAt = time.Now()
// Detect recoveries
if previousHealth.HAProxy.Status == "error" && r.health.HAProxy.Status == "ok" {
r.broadcastEvent("haproxy:recovered", "HAProxy recovered")
}
if previousHealth.DNS.Status == "error" && r.health.DNS.Status == "ok" {
r.broadcastEvent("dnsmasq:recovered", "DNS recovered")
}
slog.Info("networking updated", "component", "reconcile",
"domains", len(doms),
"l4Routes", len(l4Routes),
"l7Routes", len(httpRoutes),
"haproxy", r.health.HAProxy.Status,
"dns", r.health.DNS.Status,
)
}
@@ -229,8 +271,8 @@ func buildDNSEntries(doms []domains.Domain, centralIP string) []dnsmasq.DNSEntry
return entries
}
// writeHAProxyConfig generates, validates, and writes the HAProxy config.
// On validation failure, it identifies broken domains and retries without them.
// writeHAProxyConfig generates and applies the HAProxy config via SafeApply.
// On validation failure, identifies broken domains and retries without them.
func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes []haproxy.HTTPRoute, globalCfg *config.State) {
genOpts := haproxy.GenerateOpts{
HTTPRoutes: httpRoutes,
@@ -254,7 +296,8 @@ func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes [
cfg := r.HAProxy.GenerateWithOpts(l4Routes, nil, genOpts)
if err := r.HAProxy.WriteConfig(cfg); err != nil {
if err := r.HAProxy.SafeApply(cfg); err != nil {
// SafeApply handles rollback internally. Try to identify broken domains and retry.
brokenDomains := haproxy.FindBrokenServices(cfg, haproxy.ParseValidationErrors(err.Error()))
if len(brokenDomains) > 0 {
slog.Error("excluding broken domains and retrying",
@@ -280,19 +323,23 @@ func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes [
genOpts.HTTPRoutes = filteredHTTP
cfg = r.HAProxy.GenerateWithOpts(filteredL4, nil, genOpts)
if err := r.HAProxy.WriteConfig(cfg); err != nil {
if err := r.HAProxy.SafeApply(cfg); err != nil {
slog.Error("retry also failed", "component", "reconcile", "error", err)
} else if err := r.HAProxy.ReloadService(); err != nil {
slog.Warn("failed to reload HAProxy", "component", "reconcile", "error", err)
r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
r.broadcastEvent("haproxy:error", err.Error())
} else {
r.health.HAProxy = SubsystemHealth{Status: "degraded", Message: fmt.Sprintf("excluded domains: %v", brokenDomains)}
r.health.ExcludedDomains = brokenDomains
r.broadcastEvent("haproxy:config", "HAProxy config regenerated (excluded broken domains)")
}
} else {
slog.Error("failed to write HAProxy config (no broken domains identified)", "component", "reconcile", "error", err)
slog.Error("failed to apply HAProxy config (no broken domains identified)", "component", "reconcile", "error", err)
r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
r.broadcastEvent("haproxy:error", err.Error())
}
} else if err := r.HAProxy.ReloadService(); err != nil {
slog.Warn("failed to reload HAProxy", "component", "reconcile", "error", err)
} else {
r.health.HAProxy = SubsystemHealth{Status: "ok"}
r.health.ExcludedDomains = nil
r.broadcastEvent("haproxy:config", "HAProxy config regenerated from registered domains")
}
}

22
internal/reconcile/reconciler_test.go
View File

@@ -36,22 +36,26 @@ func (s *stubHAProxy) GenerateWithOpts(l4 []haproxy.L4Route, _ []haproxy.CustomR
s.lastL4Routes = l4
return "generated-config"
}
func (s *stubHAProxy) WriteConfig(c string) error { s.writtenConfig = c; return s.writeErr }
func (s *stubHAProxy) SafeApply(c string) error { s.writtenConfig = c; return s.writeErr }
func (s *stubHAProxy) WriteConfig(c string) error { s.writtenConfig = c; return s.writeErr }
func (s *stubHAProxy) ReloadService() error { s.reloadCalled = true; return nil }
type stubDNS struct {
entries []dnsmasq.DNSEntry
filterPath string
updateCalled bool
entries []dnsmasq.DNSEntry
filterPath string
applyCalled bool
}
func (s *stubDNS) UpdateConfig(_ *config.State, entries []dnsmasq.DNSEntry, _ bool) error {
func (s *stubDNS) Generate(_ *config.State, entries []dnsmasq.DNSEntry) string {
s.entries = entries
s.updateCalled = true
return "generated-dns-config"
}
func (s *stubDNS) SafeApply(_ string) error {
s.applyCalled = true
return nil
}
func (s *stubDNS) SetFilterConfPath(p string) { s.filterPath = p }
func (s *stubDNS) GetStatus() (*dnsmasq.Status, error) { return nil, nil }
func (s *stubDNS) SetFilterConfPath(p string) { s.filterPath = p }
func (s *stubDNS) GetStatus() (*dnsmasq.Status, error) { return nil, nil }
type stubAuth struct {
userCount int
@@ -188,7 +192,7 @@ func TestReconcile_EmptyDomains(t *testing.T) {
r, hp, dns, ddns := newTestReconciler(t, nil)
r.Reconcile()
if !dns.updateCalled {
if !dns.applyCalled {
t.Error("expected dnsmasq update")
}
if len(dns.entries) != 0 {