Paul Payne
8dd9e117bc
The auto-provisioner was checking for individual cert files (e.g., git.civilsociety.dev.pem) without checking if a wildcard cert already covers the domain (*.civilsociety.dev.pem). This caused redundant individual certs to be provisioned for domains already covered by a wildcard. Now uses hasCertForDomain() which checks both individual AND wildcard cert coverage before deciding a cert is missing.
534 lines
17 KiB
Go
534 lines
17 KiB
Go
// Package reconcile orchestrates the domain→config→daemon pipeline.
|
|
// When domains change, Reconcile regenerates HAProxy routes, dnsmasq DNS
|
|
// entries, and related subsystem configs, then reloads the affected daemons.
|
|
package reconcile
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/wild-cloud/wild-central/internal/authelia"
|
|
"github.com/wild-cloud/wild-central/internal/certbot"
|
|
"github.com/wild-cloud/wild-central/internal/config"
|
|
"github.com/wild-cloud/wild-central/internal/dnsmasq"
|
|
"github.com/wild-cloud/wild-central/internal/domains"
|
|
"github.com/wild-cloud/wild-central/internal/haproxy"
|
|
"github.com/wild-cloud/wild-central/internal/network"
|
|
"github.com/wild-cloud/wild-central/internal/sse"
|
|
"github.com/wild-cloud/wild-central/internal/storage"
|
|
)
|
|
|
|
// HAProxyManager is the subset of haproxy.Manager used by reconciliation.
|
|
type HAProxyManager interface {
|
|
GenerateWithOpts(instances []haproxy.L4Route, custom []haproxy.CustomRoute, opts haproxy.GenerateOpts) string
|
|
SafeApply(content string) error
|
|
WriteConfig(content string) error
|
|
ReloadService() error
|
|
}
|
|
|
|
// DNSManager is the subset of dnsmasq.Manager used by reconciliation.
|
|
type DNSManager interface {
|
|
Generate(cfg *config.State, entries []dnsmasq.DNSEntry) string
|
|
SafeApply(content string) error
|
|
SetFilterConfPath(path string)
|
|
GetStatus() (*dnsmasq.Status, error)
|
|
}
|
|
|
|
// DomainManager is the subset of domains.Manager used by reconciliation.
|
|
type DomainManager interface {
|
|
List() ([]domains.Domain, error)
|
|
}
|
|
|
|
// AuthManager is the subset of authelia.Manager used by reconciliation.
|
|
type AuthManager interface {
|
|
UserCount() int
|
|
RestartService() error
|
|
}
|
|
|
|
// CertManager is the subset of certbot.Manager used by reconciliation.
|
|
type CertManager interface {
|
|
EnsureCredentials(apiToken string) error
|
|
Provision(domain, email string) error
|
|
BuildHAProxyCert(domain string) error
|
|
}
|
|
|
|
// DDNSRunner is the subset of ddns.Runner used by reconciliation.
|
|
type DDNSRunner interface {
|
|
Trigger()
|
|
}
|
|
|
|
// DNSFilterManager is the subset of dnsfilter.Manager used by reconciliation.
|
|
type DNSFilterManager interface {
|
|
HostsFilePath() string
|
|
}
|
|
|
|
// EventBroadcaster is the subset of sse.Manager used by reconciliation.
|
|
type EventBroadcaster interface {
|
|
Broadcast(event *sse.Event)
|
|
}
|
|
|
|
// GenerateAutheliaConfigFn generates the Authelia config from current state.
|
|
// This is a callback because the logic depends on secrets and OIDC clients
|
|
// that the reconciler doesn't own.
|
|
type GenerateAutheliaConfigFn func(state *config.State) error
|
|
|
|
// SubsystemHealth records the outcome of the last reconciliation for one subsystem.
|
|
type SubsystemHealth struct {
|
|
Status string `json:"status"` // "ok", "degraded", "error"
|
|
Message string `json:"message,omitempty"`
|
|
}
|
|
|
|
// Health records the outcome of the last reconciliation across all subsystems.
|
|
type Health struct {
|
|
HAProxy SubsystemHealth `json:"haproxy"`
|
|
DNS SubsystemHealth `json:"dns"`
|
|
TLS SubsystemHealth `json:"tls"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
ExcludedDomains []string `json:"excludedDomains,omitempty"`
|
|
MissingCerts []string `json:"missingCerts,omitempty"`
|
|
}
|
|
|
|
// Reconciler orchestrates config regeneration when domains change.
|
|
type Reconciler struct {
|
|
mu sync.Mutex // serializes concurrent Reconcile() calls
|
|
health Health
|
|
Domains DomainManager
|
|
HAProxy HAProxyManager
|
|
DNS DNSManager
|
|
Auth AuthManager
|
|
Certs CertManager
|
|
DDNS DDNSRunner
|
|
DNSFilter DNSFilterManager
|
|
SSE EventBroadcaster
|
|
StatePath string
|
|
GenerateAutheliaConfig GenerateAutheliaConfigFn
|
|
GetCloudflareToken func() string // callback to read CF token from secrets
|
|
}
|
|
|
|
// GetHealth returns a snapshot of the last reconciliation health state.
|
|
func (r *Reconciler) GetHealth() Health {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.health
|
|
}
|
|
|
|
// Reconcile reads all registered domains and regenerates dnsmasq DNS entries
|
|
// and HAProxy routes to match. Serialized by mutex — concurrent calls wait
|
|
// rather than racing on config files.
|
|
func (r *Reconciler) Reconcile() {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
doms, err := r.Domains.List()
|
|
if err != nil {
|
|
slog.Error("failed to list domains", "component", "reconcile", "error", err)
|
|
return
|
|
}
|
|
|
|
globalCfg, err := config.LoadState(r.StatePath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
globalCfg = &config.State{} // first run, no state yet
|
|
} else {
|
|
slog.Error("failed to load state (refusing to reconcile with empty config)", "component", "reconcile", "error", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
l4Routes, httpRoutes := buildRoutes(doms)
|
|
|
|
cleanZeroByteCerts("/etc/haproxy/certs/")
|
|
|
|
// Only include L7 HTTP routes for domains that have a valid cert.
|
|
var activeHTTPRoutes []haproxy.HTTPRoute
|
|
for _, route := range httpRoutes {
|
|
if hasCertForDomain(route.Domain) {
|
|
activeHTTPRoutes = append(activeHTTPRoutes, route)
|
|
} else {
|
|
slog.Warn("skipping L7 route (no cert)", "component", "reconcile", "domain", route.Domain)
|
|
}
|
|
}
|
|
|
|
previousHealth := r.health
|
|
|
|
r.writeHAProxyConfig(l4Routes, activeHTTPRoutes, globalCfg)
|
|
|
|
// Build and apply DNS config
|
|
centralIP := globalCfg.Cloud.Dnsmasq.IP
|
|
if centralIP == "" {
|
|
centralIP, _ = network.GetWildCentralIP()
|
|
}
|
|
if centralIP == "" {
|
|
centralIP = "127.0.0.1"
|
|
}
|
|
|
|
dnsEntries := buildDNSEntries(doms, centralIP)
|
|
|
|
if globalCfg.Cloud.DNSFilter.Enabled {
|
|
hostsPath := r.DNSFilter.HostsFilePath()
|
|
if storage.FileExists(hostsPath) {
|
|
r.DNS.SetFilterConfPath(hostsPath)
|
|
}
|
|
} else {
|
|
r.DNS.SetFilterConfPath("")
|
|
}
|
|
|
|
dnsConfig := r.DNS.Generate(globalCfg, dnsEntries)
|
|
if err := r.DNS.SafeApply(dnsConfig); err != nil {
|
|
slog.Error("failed to update dnsmasq", "component", "reconcile", "error", err)
|
|
r.health.DNS = SubsystemHealth{Status: "error", Message: err.Error()}
|
|
r.broadcastEvent("dnsmasq:error", err.Error())
|
|
} else {
|
|
r.health.DNS = SubsystemHealth{Status: "ok"}
|
|
r.broadcastDNSEvent("dnsmasq:config", "DNS config regenerated from registered domains")
|
|
}
|
|
|
|
if len(httpRoutes) > 0 {
|
|
missingCerts := r.ensureTLSCerts(globalCfg, doms)
|
|
r.health.MissingCerts = missingCerts
|
|
if len(missingCerts) > 0 {
|
|
r.health.TLS = SubsystemHealth{
|
|
Status: "degraded",
|
|
Message: fmt.Sprintf("%d certs missing: %v", len(missingCerts), missingCerts),
|
|
}
|
|
} else {
|
|
r.health.TLS = SubsystemHealth{Status: "ok"}
|
|
}
|
|
} else {
|
|
r.health.TLS = SubsystemHealth{Status: "ok"}
|
|
r.health.MissingCerts = nil
|
|
}
|
|
|
|
r.DDNS.Trigger()
|
|
|
|
r.health.UpdatedAt = time.Now()
|
|
|
|
// Detect recoveries
|
|
if previousHealth.HAProxy.Status == "error" && r.health.HAProxy.Status == "ok" {
|
|
r.broadcastEvent("haproxy:recovered", "HAProxy recovered")
|
|
}
|
|
if previousHealth.DNS.Status == "error" && r.health.DNS.Status == "ok" {
|
|
r.broadcastEvent("dnsmasq:recovered", "DNS recovered")
|
|
}
|
|
if previousHealth.TLS.Status != "ok" && r.health.TLS.Status == "ok" {
|
|
r.broadcastEvent("tls:recovered", "All TLS certificates provisioned")
|
|
}
|
|
|
|
slog.Info("networking updated", "component", "reconcile",
|
|
"domains", len(doms),
|
|
"l4Routes", len(l4Routes),
|
|
"l7Routes", len(httpRoutes),
|
|
"haproxy", r.health.HAProxy.Status,
|
|
"dns", r.health.DNS.Status,
|
|
"tls", r.health.TLS.Status,
|
|
)
|
|
}
|
|
|
|
// buildRoutes converts registered domains into HAProxy L4 and L7 route lists.
|
|
func buildRoutes(doms []domains.Domain) ([]haproxy.L4Route, []haproxy.HTTPRoute) {
|
|
var l4Routes []haproxy.L4Route
|
|
var httpRoutes []haproxy.HTTPRoute
|
|
|
|
for _, dom := range doms {
|
|
switch dom.EffectiveBackendType() {
|
|
case domains.BackendTCPPassthrough:
|
|
l4Routes = append(l4Routes, haproxy.L4Route{
|
|
Name: dom.DomainName,
|
|
Domain: dom.DomainName,
|
|
BackendIP: extractHost(dom.EffectiveBackendAddress()),
|
|
Subdomains: dom.Subdomains,
|
|
})
|
|
case domains.BackendDNSOnly:
|
|
// dns-only: no gateway routing, just DNS resolution
|
|
case domains.BackendHTTP:
|
|
var routeBackends []haproxy.HTTPRouteBackend
|
|
for _, route := range dom.EffectiveRoutes() {
|
|
rb := haproxy.HTTPRouteBackend{
|
|
Paths: route.Paths,
|
|
Backend: route.Backend.Address,
|
|
HealthPath: route.Backend.Health,
|
|
Headers: route.Headers,
|
|
IPAllow: route.IPAllow,
|
|
}
|
|
if dom.Auth != nil && dom.Auth.Enabled {
|
|
rb.AuthEnabled = true
|
|
rb.AuthPolicy = dom.Auth.Policy
|
|
}
|
|
routeBackends = append(routeBackends, rb)
|
|
}
|
|
httpRoutes = append(httpRoutes, haproxy.HTTPRoute{
|
|
Name: dom.DomainName,
|
|
Domain: dom.DomainName,
|
|
Subdomains: dom.Subdomains,
|
|
Routes: routeBackends,
|
|
})
|
|
}
|
|
}
|
|
|
|
return l4Routes, httpRoutes
|
|
}
|
|
|
|
// buildDNSEntries converts domains to dnsmasq entries with appropriate IPs.
|
|
// HTTP domains point to centralIP (HAProxy terminates TLS).
|
|
// TCP passthrough and DNS-only domains point to their backend IP directly.
|
|
func buildDNSEntries(doms []domains.Domain, centralIP string) []dnsmasq.DNSEntry {
|
|
var entries []dnsmasq.DNSEntry
|
|
for _, dom := range doms {
|
|
if dom.DomainName == "" {
|
|
continue
|
|
}
|
|
|
|
dnsIP := centralIP
|
|
bt := dom.EffectiveBackendType()
|
|
if bt == domains.BackendTCPPassthrough || bt == domains.BackendDNSOnly {
|
|
dnsIP = extractHost(dom.EffectiveBackendAddress())
|
|
}
|
|
|
|
entries = append(entries, dnsmasq.DNSEntry{
|
|
Domain: dom.DomainName,
|
|
IP: dnsIP,
|
|
Wildcard: dom.Subdomains,
|
|
})
|
|
}
|
|
return entries
|
|
}
|
|
|
|
// writeHAProxyConfig generates and applies the HAProxy config via SafeApply.
|
|
// On validation failure, identifies broken domains and retries without them.
|
|
func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes []haproxy.HTTPRoute, globalCfg *config.State) {
|
|
genOpts := haproxy.GenerateOpts{
|
|
HTTPRoutes: httpRoutes,
|
|
CertsDir: "/etc/haproxy/certs/",
|
|
}
|
|
|
|
if globalCfg.Cloud.Authelia.Enabled && globalCfg.Cloud.Authelia.Domain != "" {
|
|
genOpts.AuthEnabled = true
|
|
genOpts.AuthBackend = "127.0.0.1:9091"
|
|
genOpts.AuthDomain = globalCfg.Cloud.Authelia.Domain
|
|
genOpts.AuthSessionDomain = authelia.SessionDomainFromAuthDomain(globalCfg.Cloud.Authelia.Domain)
|
|
|
|
if r.GenerateAutheliaConfig != nil {
|
|
if err := r.GenerateAutheliaConfig(globalCfg); err != nil {
|
|
slog.Warn("failed to regenerate authelia config", "component", "reconcile", "error", err)
|
|
} else if r.Auth.UserCount() > 0 {
|
|
_ = r.Auth.RestartService()
|
|
}
|
|
}
|
|
}
|
|
|
|
cfg := r.HAProxy.GenerateWithOpts(l4Routes, nil, genOpts)
|
|
|
|
if err := r.HAProxy.SafeApply(cfg); err != nil {
|
|
// SafeApply handles rollback internally. Try to identify broken domains and retry.
|
|
brokenDomains := haproxy.FindBrokenServices(cfg, haproxy.ParseValidationErrors(err.Error()))
|
|
if len(brokenDomains) > 0 {
|
|
slog.Error("excluding broken domains and retrying",
|
|
"component", "reconcile", "broken", brokenDomains, "error", err)
|
|
|
|
exclude := map[string]bool{}
|
|
for _, d := range brokenDomains {
|
|
exclude[d] = true
|
|
}
|
|
|
|
var filteredL4 []haproxy.L4Route
|
|
for _, route := range l4Routes {
|
|
if !exclude[route.Domain] {
|
|
filteredL4 = append(filteredL4, route)
|
|
}
|
|
}
|
|
var filteredHTTP []haproxy.HTTPRoute
|
|
for _, route := range httpRoutes {
|
|
if !exclude[route.Domain] {
|
|
filteredHTTP = append(filteredHTTP, route)
|
|
}
|
|
}
|
|
|
|
genOpts.HTTPRoutes = filteredHTTP
|
|
cfg = r.HAProxy.GenerateWithOpts(filteredL4, nil, genOpts)
|
|
if err := r.HAProxy.SafeApply(cfg); err != nil {
|
|
slog.Error("retry also failed", "component", "reconcile", "error", err)
|
|
r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
|
|
r.broadcastEvent("haproxy:error", err.Error())
|
|
} else {
|
|
r.health.HAProxy = SubsystemHealth{Status: "degraded", Message: fmt.Sprintf("excluded domains: %v", brokenDomains)}
|
|
r.health.ExcludedDomains = brokenDomains
|
|
r.broadcastEvent("haproxy:config", "HAProxy config regenerated (excluded broken domains)")
|
|
}
|
|
} else {
|
|
slog.Error("failed to apply HAProxy config (no broken domains identified)", "component", "reconcile", "error", err)
|
|
r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
|
|
r.broadcastEvent("haproxy:error", err.Error())
|
|
}
|
|
} else {
|
|
r.health.HAProxy = SubsystemHealth{Status: "ok"}
|
|
r.health.ExcludedDomains = nil
|
|
r.broadcastEvent("haproxy:config", "HAProxy config regenerated from registered domains")
|
|
}
|
|
}
|
|
|
|
// cleanZeroByteCerts removes 0-byte cert files that would poison HAProxy validation.
|
|
func cleanZeroByteCerts(certsDir string) {
|
|
entries, err := os.ReadDir(certsDir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
if strings.HasSuffix(e.Name(), ".pem") {
|
|
if info, err := e.Info(); err == nil && info.Size() == 0 {
|
|
slog.Warn("removing 0-byte cert file", "component", "reconcile", "file", e.Name())
|
|
_ = os.Remove(filepath.Join(certsDir, e.Name()))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// broadcastEvent sends a simple SSE event.
|
|
func (r *Reconciler) broadcastEvent(eventType, message string) {
|
|
if r.SSE == nil {
|
|
return
|
|
}
|
|
r.SSE.Broadcast(&sse.Event{
|
|
ID: fmt.Sprintf("%s-%d", strings.SplitN(eventType, ":", 2)[0], time.Now().UnixNano()),
|
|
Type: eventType,
|
|
InstanceName: "global",
|
|
Timestamp: time.Now(),
|
|
Data: map[string]any{"message": message},
|
|
})
|
|
}
|
|
|
|
// broadcastDNSEvent sends a dnsmasq SSE event including current status.
|
|
func (r *Reconciler) broadcastDNSEvent(eventType, message string) {
|
|
if r.SSE == nil {
|
|
return
|
|
}
|
|
status, err := r.DNS.GetStatus()
|
|
if err != nil {
|
|
status = nil
|
|
}
|
|
r.SSE.Broadcast(&sse.Event{
|
|
ID: fmt.Sprintf("dnsmasq-%d", time.Now().UnixNano()),
|
|
Type: eventType,
|
|
InstanceName: "global",
|
|
Timestamp: time.Now(),
|
|
Data: map[string]any{
|
|
"message": message,
|
|
"status": status,
|
|
},
|
|
})
|
|
}
|
|
|
|
// ensureTLSCerts checks for missing certificates and auto-provisions them
|
|
// when Cloudflare credentials and operator email are available.
|
|
// Returns the list of domains still missing certs after provisioning attempts.
|
|
func (r *Reconciler) ensureTLSCerts(globalCfg *config.State, doms []domains.Domain) []string {
|
|
gatewayDomain := ""
|
|
if parts := strings.SplitN(globalCfg.Cloud.Central.Domain, ".", 2); len(parts) == 2 {
|
|
gatewayDomain = parts[1]
|
|
}
|
|
|
|
// Collect unique missing certs. Use hasCertForDomain which checks
|
|
// both individual certs AND wildcard coverage (e.g., *.civilsociety.dev
|
|
// covers git.civilsociety.dev — no individual cert needed).
|
|
missing := map[string]bool{}
|
|
for _, dom := range doms {
|
|
if dom.TLS != domains.TLSTerminate || dom.DomainName == "" {
|
|
continue
|
|
}
|
|
if hasCertForDomain(dom.DomainName) {
|
|
continue
|
|
}
|
|
// Determine what cert to provision: wildcard for gateway subdomains, individual otherwise
|
|
if gatewayDomain != "" && strings.HasSuffix(dom.DomainName, "."+gatewayDomain) {
|
|
missing["*."+gatewayDomain] = true
|
|
} else {
|
|
missing[dom.DomainName] = true
|
|
}
|
|
}
|
|
|
|
if len(missing) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Can we auto-provision?
|
|
email := globalCfg.Operator.Email
|
|
cfToken := ""
|
|
if r.GetCloudflareToken != nil {
|
|
cfToken = r.GetCloudflareToken()
|
|
}
|
|
|
|
if cfToken == "" || email == "" || r.Certs == nil {
|
|
// Can't auto-provision — report what's missing
|
|
var names []string
|
|
for domain := range missing {
|
|
names = append(names, domain)
|
|
slog.Warn("missing cert (auto-provision unavailable — configure Cloudflare token and operator email)",
|
|
"component", "reconcile", "domain", domain)
|
|
}
|
|
return names
|
|
}
|
|
|
|
// Auto-provision missing certs
|
|
if err := r.Certs.EnsureCredentials(cfToken); err != nil {
|
|
slog.Error("failed to write certbot credentials", "component", "reconcile", "error", err)
|
|
var names []string
|
|
for domain := range missing {
|
|
names = append(names, domain)
|
|
}
|
|
return names
|
|
}
|
|
|
|
var stillMissing []string
|
|
for domain := range missing {
|
|
provisionDomain := domain
|
|
if strings.HasPrefix(domain, "*.") {
|
|
provisionDomain = domain // certbot handles wildcard with -d *.example.com
|
|
}
|
|
|
|
slog.Info("auto-provisioning certificate", "component", "reconcile", "domain", provisionDomain)
|
|
if err := r.Certs.Provision(provisionDomain, email); err != nil {
|
|
slog.Error("cert auto-provision failed", "component", "reconcile", "domain", provisionDomain, "error", err)
|
|
stillMissing = append(stillMissing, domain)
|
|
} else {
|
|
slog.Info("certificate provisioned", "component", "reconcile", "domain", provisionDomain)
|
|
// Build HAProxy PEM for non-wildcard certs
|
|
baseDomain := strings.TrimPrefix(domain, "*.")
|
|
_ = r.Certs.BuildHAProxyCert(baseDomain)
|
|
}
|
|
}
|
|
return stillMissing
|
|
}
|
|
|
|
// hasCertForDomain checks if a valid (non-empty) cert exists for a domain —
|
|
// either an individual cert (<domain>.pem) or a wildcard cert that covers it.
|
|
func hasCertForDomain(domain string) bool {
|
|
if isValidCertFile(certbot.HAProxyCertPath(domain)) {
|
|
return true
|
|
}
|
|
parts := strings.SplitN(domain, ".", 2)
|
|
if len(parts) == 2 {
|
|
if isValidCertFile(certbot.HAProxyCertPath(parts[1])) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isValidCertFile(path string) bool {
|
|
info, err := os.Stat(path)
|
|
return err == nil && info.Size() > 0
|
|
}
|
|
|
|
func extractHost(addr string) string {
|
|
for i := len(addr) - 1; i >= 0; i-- {
|
|
if addr[i] == ':' {
|
|
return addr[:i]
|
|
}
|
|
}
|
|
return addr
|
|
}
|