Add SafeApply lifecycle, health tracking, convergence loop, and startup checks
SafeApply pattern (validate → backup → write → reload → verify → rollback): - HAProxy: SafeApply wraps existing validate+write+reload with backup and post-reload health check; rolls back to .bak on failure - dnsmasq: SafeApply validates via dnsmasq --test, backs up, atomic writes, restarts, verifies daemon is active; rolls back on failure - nftables: SafeApply validates via nft -c, backs up, atomic writes, applies to kernel, verifies table loaded; rolls back on failure Health tracking: - Add SubsystemHealth and Health structs to reconciler - Track per-subsystem status (ok/degraded/error) after each reconcile - Detect recovery: previous error → current ok broadcasts recovery event - GET /api/v1/health/reconcile endpoint exposes health state - HAProxy tracks excluded domains as "degraded" state SSE error events: - Broadcast haproxy:error, dnsmasq:error on SafeApply failure - Broadcast haproxy:recovered, dnsmasq:recovered on recovery from error Convergence loop: - 5-minute periodic reconcile drives system toward desired state - Catches config drift, daemon crashes, transient failures - Serialized by reconcile mutex — no race with event-driven reconciles Startup: - CheckPrerequisites verifies required (dnsmasq, haproxy) and optional (wg, authelia, cscli, cloudflared, nft) binaries before first reconcile
This commit is contained in:
@@ -215,9 +215,51 @@ func (api *API) StartCentralStatusBroadcaster(startTime time.Time) {
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all Central API routes
|
||||
// ReconcileHealth returns the health state from the last reconciliation.
|
||||
func (api *API) ReconcileHealth(w http.ResponseWriter, r *http.Request) {
|
||||
respondJSON(w, http.StatusOK, api.reconciler.GetHealth())
|
||||
}
|
||||
|
||||
// CheckPrerequisites verifies that required daemons are available on the system.
|
||||
// Non-fatal — logs errors for required and info for optional missing binaries.
|
||||
func (api *API) CheckPrerequisites() {
|
||||
for _, bin := range []string{"dnsmasq", "haproxy"} {
|
||||
if _, err := exec.LookPath(bin); err != nil {
|
||||
slog.Error("required daemon not found", "component", "startup", "binary", bin)
|
||||
}
|
||||
}
|
||||
for _, bin := range []string{"wg", "authelia", "cscli", "cloudflared", "nft"} {
|
||||
if _, err := exec.LookPath(bin); err != nil {
|
||||
slog.Info("optional daemon not found (some features unavailable)", "component", "startup", "binary", bin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StartConvergenceLoop starts a periodic reconciliation loop that continuously
|
||||
// drives the system toward desired state. Catches config drift, daemon crashes,
|
||||
// and transient failures.
|
||||
func (api *API) StartConvergenceLoop(ctx gocontext.Context, interval time.Duration) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
api.reconciler.Reconcile()
|
||||
}
|
||||
}
|
||||
}()
|
||||
slog.Info("convergence loop started", "component", "reconcile", "interval", interval)
|
||||
}
|
||||
|
||||
func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.Use(RequestLoggingMiddleware)
|
||||
|
||||
// Health
|
||||
r.HandleFunc("/api/v1/health/reconcile", api.ReconcileHealth).Methods("GET")
|
||||
|
||||
// Resource-oriented settings (persisted in state.yaml)
|
||||
r.HandleFunc("/api/v1/operator", api.GetOperator).Methods("GET")
|
||||
r.HandleFunc("/api/v1/operator", api.UpdateOperator).Methods("PUT")
|
||||
|
||||
Reference in New Issue
Block a user