wild-central/internal/reconcile/reconciler.go

// Package reconcile orchestrates the domain→config→daemon pipeline.
// When domains change, Reconcile regenerates HAProxy routes, dnsmasq DNS
// entries, and related subsystem configs, then reloads the affected daemons.
package reconcile

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/wild-cloud/wild-central/internal/authelia"
	"github.com/wild-cloud/wild-central/internal/certbot"
	"github.com/wild-cloud/wild-central/internal/config"
	"github.com/wild-cloud/wild-central/internal/ddns"
	"github.com/wild-cloud/wild-central/internal/dnsmasq"
	"github.com/wild-cloud/wild-central/internal/domains"
	"github.com/wild-cloud/wild-central/internal/haproxy"
	"github.com/wild-cloud/wild-central/internal/network"
	"github.com/wild-cloud/wild-central/internal/sse"
	"github.com/wild-cloud/wild-central/internal/storage"
)

// HAProxyManager is the subset of haproxy.Manager used by reconciliation.
type HAProxyManager interface {
	GenerateWithOpts(instances []haproxy.L4Route, custom []haproxy.CustomRoute, opts haproxy.GenerateOpts) string
	SafeApply(content string) error
	WriteConfig(content string) error
	ReloadService() error
}

// DNSManager is the subset of dnsmasq.Manager used by reconciliation.
type DNSManager interface {
	Generate(cfg *config.State, entries []dnsmasq.DNSEntry) string
	SafeApply(content string) error
	SetFilterConfPath(path string)
	GetStatus() (*dnsmasq.Status, error)
}

// DomainManager is the subset of domains.Manager used by reconciliation.
type DomainManager interface {
	List() ([]domains.Domain, error)
}

// AuthManager is the subset of authelia.Manager used by reconciliation.
type AuthManager interface {
	UserCount() int
	RestartService() error
}

// CertManager is the subset of certbot.Manager used by reconciliation.
type CertManager interface {
	EnsureCredentials(apiToken string) error
	Provision(domain, email string) error
	BuildHAProxyCert(domain string) error
}

// DDNSRunner is the subset of ddns.Runner used by reconciliation.
type DDNSRunner interface {
	Trigger()
}

// DNSFilterManager is the subset of dnsfilter.Manager used by reconciliation.
type DNSFilterManager interface {
	HostsFilePath() string
}

// EventBroadcaster is the subset of sse.Manager used by reconciliation.
type EventBroadcaster interface {
	Broadcast(event *sse.Event)
}

// GenerateAutheliaConfigFn generates the Authelia config from current state.
// This is a callback because the logic depends on secrets and OIDC clients
// that the reconciler doesn't own.
type GenerateAutheliaConfigFn func(state *config.State) error

// SubsystemHealth records the outcome of the last reconciliation for one subsystem.
type SubsystemHealth struct {
	Status  string `json:"status"`            // "ok", "degraded", "error"
	Message string `json:"message,omitempty"`
}

// Health records the outcome of the last reconciliation across all subsystems.
type Health struct {
	HAProxy         SubsystemHealth `json:"haproxy"`
	DNS             SubsystemHealth `json:"dns"`
	TLS             SubsystemHealth `json:"tls"`
	UpdatedAt       time.Time       `json:"updatedAt"`
	ExcludedDomains []string        `json:"excludedDomains,omitempty"`
	MissingCerts    []string        `json:"missingCerts,omitempty"`
}

// Reconciler orchestrates config regeneration when domains change.
type Reconciler struct {
	mu                     sync.Mutex // serializes concurrent Reconcile() calls
	health                 Health
	Domains                DomainManager
	HAProxy                HAProxyManager
	DNS                    DNSManager
	Auth                   AuthManager
	Certs                  CertManager
	DDNS                   DDNSRunner
	DNSFilter              DNSFilterManager
	SSE                    EventBroadcaster
	StatePath              string
	GenerateAutheliaConfig GenerateAutheliaConfigFn
	GetCloudflareToken     func() string // callback to read CF token from secrets
}

// GetHealth returns a snapshot of the last reconciliation health state.
func (r *Reconciler) GetHealth() Health {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.health
}

// Reconcile reads all registered domains and regenerates dnsmasq DNS entries
// and HAProxy routes to match. Serialized by mutex — concurrent calls wait
// rather than racing on config files.
func (r *Reconciler) Reconcile() {
	r.mu.Lock()
	defer r.mu.Unlock()

	doms, err := r.Domains.List()
	if err != nil {
		slog.Error("failed to list domains", "component", "reconcile", "error", err)
		return
	}

	globalCfg, err := config.LoadState(r.StatePath)
	if err != nil {
		if os.IsNotExist(err) {
			globalCfg = &config.State{} // first run, no state yet
		} else {
			slog.Error("failed to load state (refusing to reconcile with empty config)", "component", "reconcile", "error", err)
			return
		}
	}

	l4Routes, httpRoutes := buildRoutes(doms)

	cleanZeroByteCerts("/etc/haproxy/certs/")

	// Only include L7 HTTP routes for domains that have a valid cert.
	var activeHTTPRoutes []haproxy.HTTPRoute
	for _, route := range httpRoutes {
		if hasCertForDomain(route.Domain) {
			activeHTTPRoutes = append(activeHTTPRoutes, route)
		} else {
			slog.Warn("skipping L7 route (no cert)", "component", "reconcile", "domain", route.Domain)
		}
	}

	previousHealth := r.health

	r.writeHAProxyConfig(l4Routes, activeHTTPRoutes, globalCfg)

	// Build and apply DNS config
	centralIP := globalCfg.Cloud.Dnsmasq.IP
	if centralIP == "" {
		centralIP, _ = network.GetWildCentralIP()
	}
	if centralIP == "" {
		centralIP = "127.0.0.1"
	}

	dnsEntries := buildDNSEntries(doms, centralIP)

	if globalCfg.Cloud.DNSFilter.Enabled {
		hostsPath := r.DNSFilter.HostsFilePath()
		if storage.FileExists(hostsPath) {
			r.DNS.SetFilterConfPath(hostsPath)
		}
	} else {
		r.DNS.SetFilterConfPath("")
	}

	dnsConfig := r.DNS.Generate(globalCfg, dnsEntries)
	if err := r.DNS.SafeApply(dnsConfig); err != nil {
		slog.Error("failed to update dnsmasq", "component", "reconcile", "error", err)
		r.health.DNS = SubsystemHealth{Status: "error", Message: err.Error()}
		r.broadcastEvent("dnsmasq:error", err.Error())
	} else {
		r.health.DNS = SubsystemHealth{Status: "ok"}
		r.broadcastDNSEvent("dnsmasq:config", "DNS config regenerated from registered domains")
	}

	if len(httpRoutes) > 0 {
		missingCerts := r.ensureTLSCerts(globalCfg, doms)
		r.health.MissingCerts = missingCerts
		if len(missingCerts) > 0 {
			r.health.TLS = SubsystemHealth{
				Status:  "degraded",
				Message: fmt.Sprintf("%d certs missing: %v", len(missingCerts), missingCerts),
			}
		} else {
			r.health.TLS = SubsystemHealth{Status: "ok"}
		}
	} else {
		r.health.TLS = SubsystemHealth{Status: "ok"}
		r.health.MissingCerts = nil
	}

	r.DDNS.Trigger()
	r.ensureCloudflareCNAMEs(doms)

	r.health.UpdatedAt = time.Now()

	// Detect recoveries
	if previousHealth.HAProxy.Status == "error" && r.health.HAProxy.Status == "ok" {
		r.broadcastEvent("haproxy:recovered", "HAProxy recovered")
	}
	if previousHealth.DNS.Status == "error" && r.health.DNS.Status == "ok" {
		r.broadcastEvent("dnsmasq:recovered", "DNS recovered")
	}
	if previousHealth.TLS.Status != "ok" && r.health.TLS.Status == "ok" {
		r.broadcastEvent("tls:recovered", "All TLS certificates provisioned")
	}

	slog.Info("networking updated", "component", "reconcile",
		"domains", len(doms),
		"l4Routes", len(l4Routes),
		"l7Routes", len(httpRoutes),
		"haproxy", r.health.HAProxy.Status,
		"dns", r.health.DNS.Status,
		"tls", r.health.TLS.Status,
	)
}

// buildRoutes converts registered domains into HAProxy L4 and L7 route lists.
func buildRoutes(doms []domains.Domain) ([]haproxy.L4Route, []haproxy.HTTPRoute) {
	var l4Routes []haproxy.L4Route
	var httpRoutes []haproxy.HTTPRoute

	for _, dom := range doms {
		switch dom.EffectiveBackendType() {
		case domains.BackendTCPPassthrough:
			l4Routes = append(l4Routes, haproxy.L4Route{
				Name:       dom.DomainName,
				Domain:     dom.DomainName,
				BackendIP:  extractHost(dom.EffectiveBackendAddress()),
				Subdomains: dom.Subdomains,
			})
		case domains.BackendDNSOnly:
			// dns-only: no gateway routing, just DNS resolution
		case domains.BackendHTTP:
			var routeBackends []haproxy.HTTPRouteBackend
			for _, route := range dom.EffectiveRoutes() {
				rb := haproxy.HTTPRouteBackend{
					Paths:      route.Paths,
					Backend:    route.Backend.Address,
					HealthPath: route.Backend.Health,
					Headers:    route.Headers,
					IPAllow:    route.IPAllow,
				}
				if dom.Auth != nil && dom.Auth.Enabled {
					rb.AuthEnabled = true
					rb.AuthPolicy = dom.Auth.Policy
				}
				routeBackends = append(routeBackends, rb)
			}
			httpRoutes = append(httpRoutes, haproxy.HTTPRoute{
				Name:       dom.DomainName,
				Domain:     dom.DomainName,
				Subdomains: dom.Subdomains,
				Routes:     routeBackends,
			})
		}
	}

	return l4Routes, httpRoutes
}

// buildDNSEntries converts domains to dnsmasq entries with appropriate IPs.
// HTTP domains point to centralIP (HAProxy terminates TLS).
// TCP passthrough and DNS-only domains point to their backend IP directly.
func buildDNSEntries(doms []domains.Domain, centralIP string) []dnsmasq.DNSEntry {
	var entries []dnsmasq.DNSEntry
	for _, dom := range doms {
		if dom.DomainName == "" {
			continue
		}

		dnsIP := centralIP
		bt := dom.EffectiveBackendType()
		if bt == domains.BackendTCPPassthrough || bt == domains.BackendDNSOnly {
			dnsIP = extractHost(dom.EffectiveBackendAddress())
		}

		entries = append(entries, dnsmasq.DNSEntry{
			Domain:   dom.DomainName,
			IP:       dnsIP,
			Wildcard: dom.Subdomains,
		})
	}
	return entries
}

// writeHAProxyConfig generates and applies the HAProxy config via SafeApply.
// On validation failure, identifies broken domains and retries without them.
func (r *Reconciler) writeHAProxyConfig(l4Routes []haproxy.L4Route, httpRoutes []haproxy.HTTPRoute, globalCfg *config.State) {
	genOpts := haproxy.GenerateOpts{
		HTTPRoutes: httpRoutes,
		CertsDir:   "/etc/haproxy/certs/",
	}

	if globalCfg.Cloud.Authelia.Enabled && globalCfg.Cloud.Authelia.Domain != "" {
		genOpts.AuthEnabled = true
		genOpts.AuthBackend = "127.0.0.1:9091"
		genOpts.AuthDomain = globalCfg.Cloud.Authelia.Domain
		genOpts.AuthSessionDomain = authelia.SessionDomainFromAuthDomain(globalCfg.Cloud.Authelia.Domain)

		if r.GenerateAutheliaConfig != nil {
			if err := r.GenerateAutheliaConfig(globalCfg); err != nil {
				slog.Warn("failed to regenerate authelia config", "component", "reconcile", "error", err)
			} else if r.Auth.UserCount() > 0 {
				_ = r.Auth.RestartService()
			}
		}
	}

	cfg := r.HAProxy.GenerateWithOpts(l4Routes, nil, genOpts)

	if err := r.HAProxy.SafeApply(cfg); err != nil {
		// SafeApply handles rollback internally. Try to identify broken domains and retry.
		brokenDomains := haproxy.FindBrokenServices(cfg, haproxy.ParseValidationErrors(err.Error()))
		if len(brokenDomains) > 0 {
			slog.Error("excluding broken domains and retrying",
				"component", "reconcile", "broken", brokenDomains, "error", err)

			exclude := map[string]bool{}
			for _, d := range brokenDomains {
				exclude[d] = true
			}

			var filteredL4 []haproxy.L4Route
			for _, route := range l4Routes {
				if !exclude[route.Domain] {
					filteredL4 = append(filteredL4, route)
				}
			}
			var filteredHTTP []haproxy.HTTPRoute
			for _, route := range httpRoutes {
				if !exclude[route.Domain] {
					filteredHTTP = append(filteredHTTP, route)
				}
			}

			genOpts.HTTPRoutes = filteredHTTP
			cfg = r.HAProxy.GenerateWithOpts(filteredL4, nil, genOpts)
			if err := r.HAProxy.SafeApply(cfg); err != nil {
				slog.Error("retry also failed", "component", "reconcile", "error", err)
				r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
				r.broadcastEvent("haproxy:error", err.Error())
			} else {
				r.health.HAProxy = SubsystemHealth{Status: "degraded", Message: fmt.Sprintf("excluded domains: %v", brokenDomains)}
				r.health.ExcludedDomains = brokenDomains
				r.broadcastEvent("haproxy:config", "HAProxy config regenerated (excluded broken domains)")
			}
		} else {
			slog.Error("failed to apply HAProxy config (no broken domains identified)", "component", "reconcile", "error", err)
			r.health.HAProxy = SubsystemHealth{Status: "error", Message: err.Error()}
			r.broadcastEvent("haproxy:error", err.Error())
		}
	} else {
		r.health.HAProxy = SubsystemHealth{Status: "ok"}
		r.health.ExcludedDomains = nil
		r.broadcastEvent("haproxy:config", "HAProxy config regenerated from registered domains")
	}
}

// cleanZeroByteCerts removes 0-byte cert files that would poison HAProxy validation.
func cleanZeroByteCerts(certsDir string) {
	entries, err := os.ReadDir(certsDir)
	if err != nil {
		return
	}
	for _, e := range entries {
		if strings.HasSuffix(e.Name(), ".pem") {
			if info, err := e.Info(); err == nil && info.Size() == 0 {
				slog.Warn("removing 0-byte cert file", "component", "reconcile", "file", e.Name())
				_ = os.Remove(filepath.Join(certsDir, e.Name()))
			}
		}
	}
}

// broadcastEvent sends a simple SSE event.
func (r *Reconciler) broadcastEvent(eventType, message string) {
	if r.SSE == nil {
		return
	}
	r.SSE.Broadcast(&sse.Event{
		ID:           fmt.Sprintf("%s-%d", strings.SplitN(eventType, ":", 2)[0], time.Now().UnixNano()),
		Type:         eventType,
		InstanceName: "global",
		Timestamp:    time.Now(),
		Data:         map[string]any{"message": message},
	})
}

// broadcastDNSEvent sends a dnsmasq SSE event including current status.
func (r *Reconciler) broadcastDNSEvent(eventType, message string) {
	if r.SSE == nil {
		return
	}
	status, err := r.DNS.GetStatus()
	if err != nil {
		status = nil
	}
	r.SSE.Broadcast(&sse.Event{
		ID:           fmt.Sprintf("dnsmasq-%d", time.Now().UnixNano()),
		Type:         eventType,
		InstanceName: "global",
		Timestamp:    time.Now(),
		Data: map[string]any{
			"message": message,
			"status":  status,
		},
	})
}

// ensureTLSCerts checks for missing certificates and auto-provisions them
// when Cloudflare credentials and operator email are available.
// Returns the list of domains still missing certs after provisioning attempts.
func (r *Reconciler) ensureTLSCerts(globalCfg *config.State, doms []domains.Domain) []string {
	gatewayDomain := ""
	if parts := strings.SplitN(globalCfg.Cloud.Central.Domain, ".", 2); len(parts) == 2 {
		gatewayDomain = parts[1]
	}

	// Collect certs that need provisioning: missing OR expiring within 30 days.
	// Uses hasCertForDomain for existence (checks individual + wildcard coverage)
	// and certExpiresWithin for expiry checking.
	const renewThreshold = 30 * 24 * time.Hour
	needsProvision := map[string]bool{}
	for _, dom := range doms {
		if dom.TLS != domains.TLSTerminate || dom.DomainName == "" {
			continue
		}
		if hasCertForDomain(dom.DomainName) {
			// Cert exists — check if it's expiring soon
			if path := certPathForDomain(dom.DomainName); path != "" && !certExpiresWithin(path, renewThreshold) {
				continue
			}
			slog.Info("cert expired or expiring soon, scheduling renewal", "component", "reconcile", "domain", dom.DomainName)
		}
		// Determine what cert to provision: wildcard for gateway subdomains, individual otherwise
		if gatewayDomain != "" && strings.HasSuffix(dom.DomainName, "."+gatewayDomain) {
			needsProvision["*."+gatewayDomain] = true
		} else {
			needsProvision[dom.DomainName] = true
		}
	}

	if len(needsProvision) == 0 {
		return nil
	}

	// Can we auto-provision?
	email := globalCfg.Operator.Email
	cfToken := ""
	if r.GetCloudflareToken != nil {
		cfToken = r.GetCloudflareToken()
	}

	if cfToken == "" || email == "" || r.Certs == nil {
		// Can't auto-provision — report what's needed
		var names []string
		for domain := range needsProvision {
			names = append(names, domain)
			slog.Warn("cert needs provisioning (auto-provision unavailable — configure Cloudflare token and operator email)",
				"component", "reconcile", "domain", domain)
		}
		return names
	}

	// Auto-provision missing/expiring certs
	if err := r.Certs.EnsureCredentials(cfToken); err != nil {
		slog.Error("failed to write certbot credentials", "component", "reconcile", "error", err)
		var names []string
		for domain := range needsProvision {
			names = append(names, domain)
		}
		return names
	}

	var stillMissing []string
	for domain := range needsProvision {
		provisionDomain := domain
		if strings.HasPrefix(domain, "*.") {
			provisionDomain = domain // certbot handles wildcard with -d *.example.com
		}

		slog.Info("auto-provisioning certificate", "component", "reconcile", "domain", provisionDomain)
		if err := r.Certs.Provision(provisionDomain, email); err != nil {
			slog.Error("cert auto-provision failed", "component", "reconcile", "domain", provisionDomain, "error", err)
			stillMissing = append(stillMissing, domain)
		} else {
			slog.Info("certificate provisioned", "component", "reconcile", "domain", provisionDomain)
			// Build HAProxy PEM for non-wildcard certs
			baseDomain := strings.TrimPrefix(domain, "*.")
			_ = r.Certs.BuildHAProxyCert(baseDomain)
		}
	}
	return stillMissing
}

// ensureCloudflareCNAMEs creates wildcard CNAME records in Cloudflare for
// public domains with subdomains enabled (e.g., *.cloud.payne.io → cloud.payne.io).
// This makes subdomains resolvable on the public internet without individual
// DNS records per app.
func (r *Reconciler) ensureCloudflareCNAMEs(doms []domains.Domain) {
	cfToken := ""
	if r.GetCloudflareToken != nil {
		cfToken = r.GetCloudflareToken()
	}
	if cfToken == "" {
		return
	}

	for _, dom := range doms {
		if !dom.Public || !dom.Subdomains || dom.DomainName == "" {
			continue
		}
		cname := "*." + dom.DomainName
		if err := ddns.EnsureCNAME(cfToken, cname, dom.DomainName); err != nil {
			slog.Warn("failed to ensure wildcard CNAME", "component", "reconcile",
				"cname", cname, "target", dom.DomainName, "error", err)
		}
	}
}

// hasCertForDomain checks if a valid (non-empty) cert exists for a domain —
// either an individual cert (<domain>.pem) or a wildcard cert that covers it.
func hasCertForDomain(domain string) bool {
	if isValidCertFile(certbot.HAProxyCertPath(domain)) {
		return true
	}
	parts := strings.SplitN(domain, ".", 2)
	if len(parts) == 2 {
		if isValidCertFile(certbot.HAProxyCertPath(parts[1])) {
			return true
		}
	}
	return false
}

func isValidCertFile(path string) bool {
	info, err := os.Stat(path)
	return err == nil && info.Size() > 0
}

// certPathForDomain returns the HAProxy PEM path covering a domain, or "" if none exists.
// Checks individual cert first, then wildcard.
func certPathForDomain(domain string) string {
	if path := certbot.HAProxyCertPath(domain); isValidCertFile(path) {
		return path
	}
	if parts := strings.SplitN(domain, ".", 2); len(parts) == 2 {
		if path := certbot.HAProxyCertPath(parts[1]); isValidCertFile(path) {
			return path
		}
	}
	return ""
}

// certExpiresWithin checks if the cert at path expires within the given threshold.
// Returns true if the cert is expired or will expire within the threshold.
func certExpiresWithin(path string, threshold time.Duration) bool {
	data, err := os.ReadFile(path)
	if err != nil {
		return false
	}
	block, _ := pem.Decode(data)
	if block == nil {
		return false
	}
	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return false
	}
	return time.Until(cert.NotAfter) < threshold
}

func extractHost(addr string) string {
	for i := len(addr) - 1; i >= 0; i-- {
		if addr[i] == ':' {
			return addr[:i]
		}
	}
	return addr
}