// Package reconcile orchestrates the domain→config→daemon pipeline. // When domains change, Reconcile regenerates HAProxy routes, dnsmasq DNS // entries, and related subsystem configs, then reloads the affected daemons. package reconcile import ( "fmt" "log/slog" "os" "path/filepath" "strings" "sync" "time" "github.com/wild-cloud/wild-central/internal/authelia" "github.com/wild-cloud/wild-central/internal/certbot" "github.com/wild-cloud/wild-central/internal/config" "github.com/wild-cloud/wild-central/internal/dnsmasq" "github.com/wild-cloud/wild-central/internal/domains" "github.com/wild-cloud/wild-central/internal/haproxy" "github.com/wild-cloud/wild-central/internal/network" "github.com/wild-cloud/wild-central/internal/sse" "github.com/wild-cloud/wild-central/internal/storage" ) // HAProxyManager is the subset of haproxy.Manager used by reconciliation. type HAProxyManager interface { GenerateWithOpts(instances []haproxy.L4Route, custom []haproxy.CustomRoute, opts haproxy.GenerateOpts) string SafeApply(content string) error WriteConfig(content string) error ReloadService() error } // DNSManager is the subset of dnsmasq.Manager used by reconciliation. type DNSManager interface { Generate(cfg *config.State, entries []dnsmasq.DNSEntry) string SafeApply(content string) error SetFilterConfPath(path string) GetStatus() (*dnsmasq.Status, error) } // DomainManager is the subset of domains.Manager used by reconciliation. type DomainManager interface { List() ([]domains.Domain, error) } // AuthManager is the subset of authelia.Manager used by reconciliation. type AuthManager interface { UserCount() int RestartService() error } // CertManager is the subset of certbot.Manager used by reconciliation. type CertManager interface { EnsureCredentials(apiToken string) error Provision(domain, email string) error BuildHAProxyCert(domain string) error } // DDNSRunner is the subset of ddns.Runner used by reconciliation. type DDNSRunner interface { Trigger() } // DNSFilterManager is the subset of dnsfilter.Manager used by reconciliation. type DNSFilterManager interface { HostsFilePath() string } // EventBroadcaster is the subset of sse.Manager used by reconciliation. type EventBroadcaster interface { Broadcast(event *sse.Event) } // GenerateAutheliaConfigFn generates the Authelia config from current state. // This is a callback because the logic depends on secrets and OIDC clients // that the reconciler doesn't own. type GenerateAutheliaConfigFn func(state *config.State) error // SubsystemHealth records the outcome of the last reconciliation for one subsystem. type SubsystemHealth struct { Status string `json:"status"` // "ok", "degraded", "error" Message string `json:"message,omitempty"` } // Health records the outcome of the last reconciliation across all subsystems. type Health struct { HAProxy SubsystemHealth `json:"haproxy"` DNS SubsystemHealth `json:"dns"` TLS SubsystemHealth `json:"tls"` UpdatedAt time.Time `json:"updatedAt"` ExcludedDomains []string `json:"excludedDomains,omitempty"` MissingCerts []string `json:"missingCerts,omitempty"` } // Reconciler orchestrates config regeneration when domains change. type Reconciler struct { mu sync.Mutex // serializes concurrent Reconcile() calls health Health Domains DomainManager HAProxy HAProxyManager DNS DNSManager Auth AuthManager Certs CertManager DDNS DDNSRunner DNSFilter DNSFilterManager SSE EventBroadcaster StatePath string GenerateAutheliaConfig GenerateAutheliaConfigFn GetCloudflareToken func() string // callback to read CF token from secrets } // GetHealth returns a snapshot of the last reconciliation health state. func (r *Reconciler) GetHealth() Health { r.mu.Lock() defer r.mu.Unlock() return r.health } // Reconcile reads all registered domains and regenerates dnsmasq DNS entries // and HAProxy routes to match. Serialized by mutex — concurrent calls wait // rather than racing on config files. func (r *Reconciler) Reconcile() { r.mu.Lock() defer r.mu.Unlock() doms, err := r.Domains.List() if err != nil { slog.Error("failed to list domains", "component", "reconcile", "error", err) return } globalCfg, err := config.LoadState(r.StatePath) if err != nil { if os.IsNotExist(err) { globalCfg = &config.State{} // first run, no state yet } else { slog.Error("failed to load state (refusing to reconcile with empty config)", "component", "reconcile", "error", err) return } } l4Routes, httpRoutes := buildRoutes(doms) cleanZeroByteCerts("/etc/haproxy/certs/") // Only include L7 HTTP routes for domains that have a valid cert. var activeHTTPRoutes []haproxy.HTTPRoute for _, route := range httpRoutes { if hasCertForDomain(route.Domain) { activeHTTPRoutes = append(activeHTTPRoutes, route) } else { slog.Warn("skipping L7 route (no cert)", "component", "reconcile", "domain", route.Domain) } } previousHealth := r.health r.writeHAProxyConfig(l4Routes, activeHTTPRoutes, globalCfg) // Build and apply DNS config centralIP := globalCfg.Cloud.Dnsmasq.IP if centralIP == "" { centralIP, _ = network.GetWildCentralIP() } if centralIP == "" { centralIP = "127.0.0.1" } dnsEntries := buildDNSEntries(doms, centralIP) if globalCfg.Cloud.DNSFilter.Enabled { hostsPath := r.DNSFilter.HostsFilePath() if storage.FileExists(hostsPath) { r.DNS.SetFilterConfPath(hostsPath) } } else { r.DNS.SetFilterConfPath("") } dnsConfig := r.DNS.Generate(globalCfg, dnsEntries) if err := r.DNS.SafeApply(dnsConfig); err != nil { slog.Error("failed to update dnsmasq", "component", "reconcile", "error", err) r.health.DNS = SubsystemHealth{Status: "error", Message: err.Error()} r.broadcastEvent("dnsmasq:error", err.Error()) } else { r.health.DNS = SubsystemHealth{Status: "ok"} r.broadcastDNSEvent("dnsmasq:config", "DNS config regenerated from registered domains") } if len(httpRoutes) > 0 { missingCerts := r.ensureTLSCerts(globalCfg, doms) r.health.MissingCerts = missingCerts if len(missingCerts) > 0 { r.health.TLS = SubsystemHealth{ Status: "degraded", Message: fmt.Sprintf("%d certs missing: %v", len(missingCerts), missingCerts), } } else { r.health.TLS = SubsystemHealth{Status: "ok"} } } else { r.health.TLS = SubsystemHealth{Status: "ok"} r.health.MissingCerts = nil } r.DDNS.Trigger() r.health.UpdatedAt = time.Now() // Detect recoveries if previousHealth.HAProxy.Status == "error" && r.health.HAProxy.Status == "ok" { r.broadcastEvent("haproxy:recovered", "HAProxy recovered") } if previousHealth.DNS.Status == "error" && r.health.DNS.Status == "ok" { r.broadcastEvent("dnsmasq:recovered", "DNS recovered") } if previousHealth.TLS.Status != "ok" && r.health.TLS.Status == "ok" { r.broadcastEvent("tls:recovered", "All TLS certificates provisioned") } slog.Info("networking updated", "component", "reconcile", "domains", len(doms), "l4Routes", len(l4Routes), "l7Routes", len(httpRoutes), "haproxy", r.health.HAProxy.Status, "dns", r.health.DNS.Status, "tls", r.health.TLS.Status, ) } // buildRoutes converts registered domains into HAProxy L4 and L7 route lists. func buildRoutes(doms []domains.Domain) ([]haproxy.L4Route, []haproxy.HTTPRoute) { var l4Routes []haproxy.L4Route var httpRoutes []haproxy.HTTPRoute for _, dom := range doms { switch dom.EffectiveBackendType() { case domains.BackendTCPPassthrough: l4Routes = append(l4Routes, haproxy.L4Route{ Name: dom.DomainName, Domain: dom.DomainName, BackendIP: extractHost(dom.EffectiveBackendAddress()), Subdomains: dom.Subdomains, }) case domains.BackendDNSOnly: // dns-only: no gateway routing, just DNS resolution case domains.BackendHTTP: var routeBackends []haproxy.HTTPRouteBackend for _, route := range dom.EffectiveRoutes() { rb := haproxy.HTTPRouteBackend{ Paths: route.Paths, Backend: route.Backend.Address, HealthPath: route.Backend.Health, Headers: route.Headers, IPAllow: route.IPAllow, } if dom.Auth != nil && dom.Auth.Enabled { rb.AuthEnabled = true rb.AuthPolicy = dom.Auth.Policy } routeBackends = append(routeBackends, rb) } httpRoutes = append(httpRoutes, haproxy.HTTPRoute{ Name: dom.DomainName, Domain: dom.DomainName, Subdomains: dom.Subdomains, Routes: routeBackends, }) } } return l4Routes, httpRoutes } // buildDNSEntries converts domains to dnsmasq entries with appropriate IPs. // HTTP domains point to centralIP (HAProxy terminates TLS). // TCP passthrough and DNS-only domains point to their backend IP directly. func buildDNSEntries(doms []domains.Domain, centralIP string) []dnsmasq.DNSEntry { var entries []dnsmasq.DNSEntry for _, dom := range doms { if dom.DomainName == "" { continue } dnsIP := centralIP bt := dom.EffectiveBackendType() if bt == domains.BackendTCPPassthrough || bt == domains.BackendDNSOnly { dnsIP = extractHost(dom.EffectiveBackendAddress()) } entries = append(entries, dnsmasq.DNSEntry{ Domain: dom.DomainName, IP: dnsIP, Wildcard: dom.Subdomains, }) } return entries } // writeHAProxyConfig generates and applies the HAProxy config via SafeApply. // On validation failure, identifies broken domains and retries without them. func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes []haproxy.HTTPRoute, globalCfg *config.State) { genOpts := haproxy.GenerateOpts{ HTTPRoutes: httpRoutes, CertsDir: "/etc/haproxy/certs/", } if globalCfg.Cloud.Authelia.Enabled && globalCfg.Cloud.Authelia.Domain != "" { genOpts.AuthEnabled = true genOpts.AuthBackend = "127.0.0.1:9091" genOpts.AuthDomain = globalCfg.Cloud.Authelia.Domain genOpts.AuthSessionDomain = authelia.SessionDomainFromAuthDomain(globalCfg.Cloud.Authelia.Domain) if r.GenerateAutheliaConfig != nil { if err := r.GenerateAutheliaConfig(globalCfg); err != nil { slog.Warn("failed to regenerate authelia config", "component", "reconcile", "error", err) } else if r.Auth.UserCount() > 0 { _ = r.Auth.RestartService() } } } cfg := r.HAProxy.GenerateWithOpts(l4Routes, nil, genOpts) if err := r.HAProxy.SafeApply(cfg); err != nil { // SafeApply handles rollback internally. Try to identify broken domains and retry. brokenDomains := haproxy.FindBrokenServices(cfg, haproxy.ParseValidationErrors(err.Error())) if len(brokenDomains) > 0 { slog.Error("excluding broken domains and retrying", "component", "reconcile", "broken", brokenDomains, "error", err) exclude := map[string]bool{} for _, d := range brokenDomains { exclude[d] = true } var filteredL4 []haproxy.L4Route for _, route := range l4Routes { if !exclude[route.Domain] { filteredL4 = append(filteredL4, route) } } var filteredHTTP []haproxy.HTTPRoute for _, route := range httpRoutes { if !exclude[route.Domain] { filteredHTTP = append(filteredHTTP, route) } } genOpts.HTTPRoutes = filteredHTTP cfg = r.HAProxy.GenerateWithOpts(filteredL4, nil, genOpts) if err := r.HAProxy.SafeApply(cfg); err != nil { slog.Error("retry also failed", "component", "reconcile", "error", err) r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()} r.broadcastEvent("haproxy:error", err.Error()) } else { r.health.HAProxy = SubsystemHealth{Status: "degraded", Message: fmt.Sprintf("excluded domains: %v", brokenDomains)} r.health.ExcludedDomains = brokenDomains r.broadcastEvent("haproxy:config", "HAProxy config regenerated (excluded broken domains)") } } else { slog.Error("failed to apply HAProxy config (no broken domains identified)", "component", "reconcile", "error", err) r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()} r.broadcastEvent("haproxy:error", err.Error()) } } else { r.health.HAProxy = SubsystemHealth{Status: "ok"} r.health.ExcludedDomains = nil r.broadcastEvent("haproxy:config", "HAProxy config regenerated from registered domains") } } // cleanZeroByteCerts removes 0-byte cert files that would poison HAProxy validation. func cleanZeroByteCerts(certsDir string) { entries, err := os.ReadDir(certsDir) if err != nil { return } for _, e := range entries { if strings.HasSuffix(e.Name(), ".pem") { if info, err := e.Info(); err == nil && info.Size() == 0 { slog.Warn("removing 0-byte cert file", "component", "reconcile", "file", e.Name()) _ = os.Remove(filepath.Join(certsDir, e.Name())) } } } } // broadcastEvent sends a simple SSE event. func (r *Reconciler) broadcastEvent(eventType, message string) { if r.SSE == nil { return } r.SSE.Broadcast(&sse.Event{ ID: fmt.Sprintf("%s-%d", strings.SplitN(eventType, ":", 2)[0], time.Now().UnixNano()), Type: eventType, InstanceName: "global", Timestamp: time.Now(), Data: map[string]any{"message": message}, }) } // broadcastDNSEvent sends a dnsmasq SSE event including current status. func (r *Reconciler) broadcastDNSEvent(eventType, message string) { if r.SSE == nil { return } status, err := r.DNS.GetStatus() if err != nil { status = nil } r.SSE.Broadcast(&sse.Event{ ID: fmt.Sprintf("dnsmasq-%d", time.Now().UnixNano()), Type: eventType, InstanceName: "global", Timestamp: time.Now(), Data: map[string]any{ "message": message, "status": status, }, }) } // ensureTLSCerts checks for missing certificates and auto-provisions them // when Cloudflare credentials and operator email are available. // Returns the list of domains still missing certs after provisioning attempts. func (r *Reconciler) ensureTLSCerts(globalCfg *config.State, doms []domains.Domain) []string { gatewayDomain := "" if parts := strings.SplitN(globalCfg.Cloud.Central.Domain, ".", 2); len(parts) == 2 { gatewayDomain = parts[1] } // Collect unique missing certs. Use hasCertForDomain which checks // both individual certs AND wildcard coverage (e.g., *.civilsociety.dev // covers git.civilsociety.dev — no individual cert needed). missing := map[string]bool{} for _, dom := range doms { if dom.TLS != domains.TLSTerminate || dom.DomainName == "" { continue } if hasCertForDomain(dom.DomainName) { continue } // Determine what cert to provision: wildcard for gateway subdomains, individual otherwise if gatewayDomain != "" && strings.HasSuffix(dom.DomainName, "."+gatewayDomain) { missing["*."+gatewayDomain] = true } else { missing[dom.DomainName] = true } } if len(missing) == 0 { return nil } // Can we auto-provision? email := globalCfg.Operator.Email cfToken := "" if r.GetCloudflareToken != nil { cfToken = r.GetCloudflareToken() } if cfToken == "" || email == "" || r.Certs == nil { // Can't auto-provision — report what's missing var names []string for domain := range missing { names = append(names, domain) slog.Warn("missing cert (auto-provision unavailable — configure Cloudflare token and operator email)", "component", "reconcile", "domain", domain) } return names } // Auto-provision missing certs if err := r.Certs.EnsureCredentials(cfToken); err != nil { slog.Error("failed to write certbot credentials", "component", "reconcile", "error", err) var names []string for domain := range missing { names = append(names, domain) } return names } var stillMissing []string for domain := range missing { provisionDomain := domain if strings.HasPrefix(domain, "*.") { provisionDomain = domain // certbot handles wildcard with -d *.example.com } slog.Info("auto-provisioning certificate", "component", "reconcile", "domain", provisionDomain) if err := r.Certs.Provision(provisionDomain, email); err != nil { slog.Error("cert auto-provision failed", "component", "reconcile", "domain", provisionDomain, "error", err) stillMissing = append(stillMissing, domain) } else { slog.Info("certificate provisioned", "component", "reconcile", "domain", provisionDomain) // Build HAProxy PEM for non-wildcard certs baseDomain := strings.TrimPrefix(domain, "*.") _ = r.Certs.BuildHAProxyCert(baseDomain) } } return stillMissing } // hasCertForDomain checks if a valid (non-empty) cert exists for a domain — // either an individual cert (.pem) or a wildcard cert that covers it. func hasCertForDomain(domain string) bool { if isValidCertFile(certbot.HAProxyCertPath(domain)) { return true } parts := strings.SplitN(domain, ".", 2) if len(parts) == 2 { if isValidCertFile(certbot.HAProxyCertPath(parts[1])) { return true } } return false } func isValidCertFile(path string) bool { info, err := os.Stat(path) return err == nil && info.Size() > 0 } func extractHost(addr string) string { for i := len(addr) - 1; i >= 0; i-- { if addr[i] == ':' { return addr[:i] } } return addr }