Add resiliency primitives: atomic writes, reconcile mutex, state backup

- Add storage.WriteFileAtomic (temp + rename) and storage.CopyFile helpers
- Convert all 8 production config writers to atomic writes: config/state.yaml,
  dnsmasq, nftables, domains, wireguard (config + secrets + peers + wg0.conf),
  tunnel/cloudflared
- Add sync.Mutex to Reconciler to serialize concurrent Reconcile() calls
  triggered by domain registration goroutines
- Add state.yaml backup (.bak) before every write; LoadState falls back to
  backup if primary is corrupted
- Reconciler refuses to use empty config on corruption (only on first run
  when no state file exists yet)

internal/config/config.go

@@ -3,11 +3,14 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v3"
+
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 // ExtraPort is a TCP or UDP port that the firewall should allow on the WAN interface.
@@ -155,24 +158,41 @@ func (s *State) RemoveDHCPStaticLease(mac string) bool {
 	return false
 }
 
-// LoadState loads state from the specified path
+// LoadState loads state from the specified path. If the primary file is
+// corrupted, falls back to the .bak backup if one exists.
 func LoadState(configPath string) (*State, error) {
-	data, err := os.ReadFile(configPath)
+	state, err := loadStateFrom(configPath)
-	if err != nil {
+	if err == nil {
+		return state, nil
+	}
+
+	// Primary failed — try backup
+	bakPath := configPath + ".bak"
+	bakState, bakErr := loadStateFrom(bakPath)
+	if bakErr == nil {
+		slog.Warn("loaded state from backup (primary corrupted)", "component", "config",
+			"path", configPath, "error", err)
+		return bakState, nil
+	}
+
 	return nil, fmt.Errorf("reading config file %s: %w", configPath, err)
 }
 
+func loadStateFrom(path string) (*State, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
 	config := &State{}
 	if err := yaml.Unmarshal(data, config); err != nil {
-		return nil, fmt.Errorf("parsing config file: %w", err)
+		return nil, fmt.Errorf("parsing %s: %w", path, err)
 	}
-
 	return config, nil
 }
 
-// SaveState saves the state to the specified path
+// SaveState saves the state to the specified path. Backs up the current
+// file to .bak before writing so LoadState can recover from corruption.
 func SaveState(config *State, configPath string) error {
-	// Ensure the directory exists
 	if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
 		return fmt.Errorf("creating config directory: %w", err)
 	}
@@ -182,5 +202,10 @@ func SaveState(config *State, configPath string) error {
 		return fmt.Errorf("marshaling config: %w", err)
 	}
 
-	return os.WriteFile(configPath, data, 0644)
+	// Backup current state before overwriting
+	if storage.FileExists(configPath) {
+		_ = storage.CopyFile(configPath, configPath+".bak")
+	}
+
+	return storage.WriteFileAtomic(configPath, data, 0644)
 }

internal/config/config_test.go

@@ -115,7 +115,7 @@ func TestLoadState_Errors(t *testing.T) {
 				}
 				return statePath
 			},
-			errContains: "parsing config file",
+			errContains: "yaml:",
 		},
 	}
 

internal/dnsmasq/config.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/wild-cloud/wild-central/internal/config"
 	"github.com/wild-cloud/wild-central/internal/network"
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 // DNSEntry represents a single domain-to-IP DNS mapping for dnsmasq.
@@ -220,7 +221,7 @@ func (g *Manager) UpdateConfig(cfg *config.State, entries []DNSEntry, restart bo
 
 	// Write config
 	slog.Info("writing dnsmasq config", "component", "dnsmasq", "path", g.configPath)
-	if err := os.WriteFile(g.configPath, []byte(configContent), 0644); err != nil {
+	if err := storage.WriteFileAtomic(g.configPath, []byte(configContent), 0644); err != nil {
 		return fmt.Errorf("writing dnsmasq config: %w", err)
 	}
 

internal/domains/manager.go

@@ -15,6 +15,8 @@ import (
 	"sync"
 
 	"gopkg.in/yaml.v3"
+
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 // BackendType describes how the gateway handles traffic for this domain.
@@ -215,7 +217,7 @@ func (m *Manager) Register(dom Domain) error {
 		return fmt.Errorf("marshaling domain: %w", err)
 	}
 
-	if err := os.WriteFile(m.domainPath(dom.DomainName), data, 0644); err != nil {
+	if err := storage.WriteFileAtomic(m.domainPath(dom.DomainName), data, 0644); err != nil {
 		return fmt.Errorf("writing domain file: %w", err)
 	}
 

internal/nftables/manager.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 const defaultRulesPath = "/etc/nftables.d/wild-cloud.nft"
@@ -144,7 +146,7 @@ func (m *Manager) WriteRules(content string) error {
 		return err
 	}
 
-	if err := os.WriteFile(m.rulesPath, []byte(content), 0644); err != nil {
+	if err := storage.WriteFileAtomic(m.rulesPath, []byte(content), 0644); err != nil {
 		return fmt.Errorf("writing rules file: %w", err)
 	}
 
@@ -171,7 +173,7 @@ func (m *Manager) WriteDisabledRules() error {
 		"# Managed by Wild Cloud Central API — do not edit manually\n\n" +
 		"table inet wild-cloud {}\n" +
 		"delete table inet wild-cloud\n"
-	if err := os.WriteFile(m.rulesPath, []byte(content), 0644); err != nil {
+	if err := storage.WriteFileAtomic(m.rulesPath, []byte(content), 0644); err != nil {
 		return fmt.Errorf("writing disabled rules file: %w", err)
 	}
 	slog.Info("nftables rules disabled", "component", "nftables", "path", m.rulesPath)

internal/reconcile/reconciler.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/wild-cloud/wild-central/internal/authelia"
@@ -69,6 +70,7 @@ type GenerateAutheliaConfigFn func(state *config.State) error
 
 // Reconciler orchestrates config regeneration when domains change.
 type Reconciler struct {
+	mu                     sync.Mutex // serializes concurrent Reconcile() calls
 	Domains                DomainManager
 	HAProxy                HAProxyManager
 	DNS                    DNSManager
@@ -81,8 +83,12 @@ type Reconciler struct {
 }
 
 // Reconcile reads all registered domains and regenerates dnsmasq DNS entries
-// and HAProxy routes to match.
+// and HAProxy routes to match. Serialized by mutex — concurrent calls wait
+// rather than racing on config files.
 func (r *Reconciler) Reconcile() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	doms, err := r.Domains.List()
 	if err != nil {
 		slog.Error("failed to list domains", "component", "reconcile", "error", err)
@@ -91,8 +97,12 @@ func (r *Reconciler) Reconcile() {
 
 	globalCfg, err := config.LoadState(r.StatePath)
 	if err != nil {
-		slog.Warn("failed to load state, using empty", "component", "reconcile", "error", err)
+		if os.IsNotExist(err) {
-		globalCfg = &config.State{}
+			globalCfg = &config.State{} // first run, no state yet
+		} else {
+			slog.Error("failed to load state (refusing to reconcile with empty config)", "component", "reconcile", "error", err)
+			return
+		}
 	}
 
 	l4Routes, httpRoutes := buildRoutes(doms)

internal/storage/storage.go

@@ -101,6 +101,50 @@ func WithLock(lockPath string, fn func() error) error {
 	return fn()
 }
 
+// WriteFileAtomic writes content to a file atomically via temp + rename.
+// If the process crashes mid-write, the original file is untouched.
+func WriteFileAtomic(path string, content []byte, perm os.FileMode) error {
+	dir := filepath.Dir(path)
+	if err := EnsureDir(dir, 0755); err != nil {
+		return err
+	}
+
+	tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp.*")
+	if err != nil {
+		return fmt.Errorf("creating temp file for %s: %w", path, err)
+	}
+	tmpPath := tmp.Name()
+
+	if _, err := tmp.Write(content); err != nil {
+		tmp.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("writing temp file %s: %w", tmpPath, err)
+	}
+	if err := tmp.Chmod(perm); err != nil {
+		tmp.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("setting permissions on %s: %w", tmpPath, err)
+	}
+	if err := tmp.Close(); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("closing temp file %s: %w", tmpPath, err)
+	}
+	if err := os.Rename(tmpPath, path); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("installing %s → %s: %w", tmpPath, path, err)
+	}
+	return nil
+}
+
+// CopyFile copies a file atomically. Used for creating .bak backups.
+func CopyFile(src, dst string) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return fmt.Errorf("reading %s: %w", src, err)
+	}
+	return WriteFileAtomic(dst, data, 0644)
+}
+
 // EnsureFilePermissions ensures a file has the correct permissions
 func EnsureFilePermissions(path string, perm os.FileMode) error {
 	if err := os.Chmod(path, perm); err != nil {

internal/tunnel/manager.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 
 	"gopkg.in/yaml.v3"
+
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 // Config holds tunnel configuration.
@@ -143,12 +145,7 @@ func (m *Manager) WriteConfig(cfg Config, services []PublicDomain) error {
 		return nil
 	}
 
-	dir := filepath.Dir(m.configPath())
+	if err := storage.WriteFileAtomic(m.configPath(), []byte(content), 0644); err != nil {
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return fmt.Errorf("creating tunnel config dir: %w", err)
-	}
-
-	if err := os.WriteFile(m.configPath(), []byte(content), 0644); err != nil {
 		return fmt.Errorf("writing tunnel config: %w", err)
 	}
 

internal/wireguard/manager.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/google/uuid"
 	"gopkg.in/yaml.v3"
+
+	"github.com/wild-cloud/wild-central/internal/storage"
 )
 
 // Config holds the WireGuard server interface configuration.
@@ -110,7 +112,7 @@ func (m *Manager) SaveConfig(cfg *Config) error {
 	if err != nil {
 		return fmt.Errorf("marshal vpn config: %w", err)
 	}
-	return os.WriteFile(m.configFilePath(), data, 0644)
+	return storage.WriteFileAtomic(m.configFilePath(), data, 0644)
 }
 
 // --- Keys ---
@@ -133,7 +135,7 @@ func (m *Manager) GenerateKeypair() error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(m.secretsFilePath(), data, 0600)
+	return storage.WriteFileAtomic(m.secretsFilePath(), data, 0600)
 }
 
 // GetPublicKey returns the server public key, or empty string if not yet generated.
@@ -261,7 +263,7 @@ func (m *Manager) savePeer(p *Peer) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(m.peersDir(), p.ID+".yaml"), data, 0600)
+	return storage.WriteFileAtomic(filepath.Join(m.peersDir(), p.ID+".yaml"), data, 0600)
 }
 
 // nextAvailableIP finds the next unused host IP in the given CIDR (skipping the network
@@ -409,7 +411,7 @@ func (m *Manager) Apply() error {
 	if err := os.MkdirAll(filepath.Dir(m.configPath), 0755); err != nil {
 		return fmt.Errorf("create wireguard config dir: %w", err)
 	}
-	if err := os.WriteFile(m.configPath, []byte(content), 0600); err != nil {
+	if err := storage.WriteFileAtomic(m.configPath, []byte(content), 0600); err != nil {
 		return fmt.Errorf("write wireguard config: %w", err)
 	}
 	upCmd := exec.Command("sudo", "wg-quick", "up", "wg0")