Add SafeApply lifecycle, health tracking, convergence loop, and startup checks

SafeApply pattern (validate → backup → write → reload → verify → rollback):
- HAProxy: SafeApply wraps existing validate+write+reload with backup and
  post-reload health check; rolls back to .bak on failure
- dnsmasq: SafeApply validates via dnsmasq --test, backs up, atomic writes,
  restarts, verifies daemon is active; rolls back on failure
- nftables: SafeApply validates via nft -c, backs up, atomic writes, applies
  to kernel, verifies table loaded; rolls back on failure

Health tracking:
- Add SubsystemHealth and Health structs to reconciler
- Track per-subsystem status (ok/degraded/error) after each reconcile
- Detect recovery: previous error → current ok broadcasts recovery event
- GET /api/v1/health/reconcile endpoint exposes health state
- HAProxy tracks excluded domains as "degraded" state

SSE error events:
- Broadcast haproxy:error, dnsmasq:error on SafeApply failure
- Broadcast haproxy:recovered, dnsmasq:recovered on recovery from error

Convergence loop:
- 5-minute periodic reconcile drives system toward desired state
- Catches config drift, daemon crashes, transient failures
- Serialized by reconcile mutex — no race with event-driven reconciles

Startup:
- CheckPrerequisites verifies required (dnsmasq, haproxy) and optional
  (wg, authelia, cscli, cloudflared, nft) binaries before first reconcile
This commit is contained in:
2026-07-14 11:44:44 +00:00
parent 88acd437a1
commit a0ab8faa1c
7 changed files with 276 additions and 20 deletions

View File

@@ -300,6 +300,62 @@ log-dhcp
return sb.String()
}
// SafeApply validates, backs up, writes, restarts, and verifies the dnsmasq config.
// On restart or verification failure, rolls back to the previous config.
func (g *Manager) SafeApply(content string) error {
// Validate via temp file
tmpFile := g.configPath + ".validate"
if err := os.WriteFile(tmpFile, []byte(content), 0644); err != nil {
return fmt.Errorf("writing validation file: %w", err)
}
defer os.Remove(tmpFile)
if err := g.ValidateConfig(tmpFile); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
// Backup current config
if storage.FileExists(g.configPath) {
_ = storage.CopyFile(g.configPath, g.configPath+".bak")
}
// Write atomically
if err := storage.WriteFileAtomic(g.configPath, []byte(content), 0644); err != nil {
return fmt.Errorf("write failed: %w", err)
}
// Restart daemon
if err := g.RestartService(); err != nil {
g.rollback()
return fmt.Errorf("restart failed (rolled back): %w", err)
}
// Verify daemon is running
if err := g.verify(); err != nil {
g.rollback()
return fmt.Errorf("health check failed (rolled back): %w", err)
}
return nil
}
func (g *Manager) rollback() {
bakPath := g.configPath + ".bak"
if storage.FileExists(bakPath) {
_ = os.Rename(bakPath, g.configPath)
_ = g.RestartService()
slog.Warn("rolled back to previous config", "component", "dnsmasq", "path", g.configPath)
}
}
func (g *Manager) verify() error {
cmd := exec.Command("systemctl", "is-active", "--quiet", "dnsmasq.service")
if err := cmd.Run(); err != nil {
return fmt.Errorf("dnsmasq not active after restart")
}
return nil
}
// ValidateConfig tests a dnsmasq configuration file for syntax errors
func (g *Manager) ValidateConfig(configPath string) error {
cmd := exec.Command("dnsmasq", "--test", "-C", configPath)