Add SafeApply lifecycle, health tracking, convergence loop, and startup checks
SafeApply pattern (validate → backup → write → reload → verify → rollback): - HAProxy: SafeApply wraps existing validate+write+reload with backup and post-reload health check; rolls back to .bak on failure - dnsmasq: SafeApply validates via dnsmasq --test, backs up, atomic writes, restarts, verifies daemon is active; rolls back on failure - nftables: SafeApply validates via nft -c, backs up, atomic writes, applies to kernel, verifies table loaded; rolls back on failure Health tracking: - Add SubsystemHealth and Health structs to reconciler - Track per-subsystem status (ok/degraded/error) after each reconcile - Detect recovery: previous error → current ok broadcasts recovery event - GET /api/v1/health/reconcile endpoint exposes health state - HAProxy tracks excluded domains as "degraded" state SSE error events: - Broadcast haproxy:error, dnsmasq:error on SafeApply failure - Broadcast haproxy:recovered, dnsmasq:recovered on recovery from error Convergence loop: - 5-minute periodic reconcile drives system toward desired state - Catches config drift, daemon crashes, transient failures - Serialized by reconcile mutex — no race with event-driven reconciles Startup: - CheckPrerequisites verifies required (dnsmasq, haproxy) and optional (wg, authelia, cscli, cloudflared, nft) binaries before first reconcile
This commit is contained in:
@@ -131,6 +131,62 @@ func (m *Manager) ValidateRules(rulesPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SafeApply validates, backs up, writes, and applies nftables rules.
|
||||
// On apply failure, rolls back to the previous rules.
|
||||
func (m *Manager) SafeApply(content string) error {
|
||||
// Validate syntax
|
||||
tmpFile := "/tmp/wild-cloud-nft-safeapply.tmp"
|
||||
if err := os.WriteFile(tmpFile, []byte(content), 0644); err != nil {
|
||||
return fmt.Errorf("writing validation file: %w", err)
|
||||
}
|
||||
defer os.Remove(tmpFile)
|
||||
|
||||
if err := m.ValidateRules(tmpFile); err != nil {
|
||||
return fmt.Errorf("validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Backup current rules
|
||||
if storage.FileExists(m.rulesPath) {
|
||||
_ = storage.CopyFile(m.rulesPath, m.rulesPath+".bak")
|
||||
}
|
||||
|
||||
// Write atomically
|
||||
if err := storage.WriteFileAtomic(m.rulesPath, []byte(content), 0644); err != nil {
|
||||
return fmt.Errorf("write failed: %w", err)
|
||||
}
|
||||
|
||||
// Apply to kernel
|
||||
if err := m.ApplyRules(); err != nil {
|
||||
m.rollback()
|
||||
return fmt.Errorf("apply failed (rolled back): %w", err)
|
||||
}
|
||||
|
||||
// Verify rules loaded
|
||||
if err := m.verify(); err != nil {
|
||||
m.rollback()
|
||||
return fmt.Errorf("verification failed (rolled back): %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) rollback() {
|
||||
bakPath := m.rulesPath + ".bak"
|
||||
if storage.FileExists(bakPath) {
|
||||
_ = os.Rename(bakPath, m.rulesPath)
|
||||
_ = m.ApplyRules()
|
||||
slog.Warn("rolled back to previous rules", "component", "nftables", "path", m.rulesPath)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) verify() error {
|
||||
status, err := m.GetStatus()
|
||||
if err != nil || status == "" {
|
||||
return fmt.Errorf("nftables table not loaded after apply")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteRules validates the content then writes the nftables rules file.
|
||||
// Validation uses a temp file in /tmp (world-writable) to avoid needing
|
||||
// write permission on the /etc/nftables.d/ directory itself.
|
||||
|
||||
Reference in New Issue
Block a user