wild-cloud/wild-central
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add resiliency primitives: atomic writes, reconcile mutex, state backup

Browse Source 
- Add storage.WriteFileAtomic (temp + rename) and storage.CopyFile helpers
- Convert all 8 production config writers to atomic writes: config/state.yaml,
  dnsmasq, nftables, domains, wireguard (config + secrets + peers + wg0.conf),
  tunnel/cloudflared
- Add sync.Mutex to Reconciler to serialize concurrent Reconcile() calls
  triggered by domain registration goroutines
- Add state.yaml backup (.bak) before every write; LoadState falls back to
  backup if primary is corrupted
- Reconciler refuses to use empty config on corruption (only on first run
  when no state file exists yet)
This commit is contained in:
Paul Payne
2026-07-14 11:37:33 +00:00
parent 3172e56288
commit 88acd437a1
9 changed files with 110 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/config/config.go
View File

@@ -3,11 +3,14 @@ package config
import (
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
"github.com/wild-cloud/wild-central/internal/storage"
)
// ExtraPort is a TCP or UDP port that the firewall should allow on the WAN interface.
@@ -155,24 +158,41 @@ func (s *State) RemoveDHCPStaticLease(mac string) bool {
return false
}
// LoadState loads state from the specified path
// LoadState loads state from the specified path. If the primary file is
// corrupted, falls back to the .bak backup if one exists.
func LoadState(configPath string) (*State, error) {
data, err := os.ReadFile(configPath)
if err != nil {
state, err := loadStateFrom(configPath)
if err == nil {
return state, nil
}
// Primary failed — try backup
bakPath := configPath + ".bak"
bakState, bakErr := loadStateFrom(bakPath)
if bakErr == nil {
slog.Warn("loaded state from backup (primary corrupted)", "component", "config",
"path", configPath, "error", err)
return bakState, nil
}
return nil, fmt.Errorf("reading config file %s: %w", configPath, err)
}
func loadStateFrom(path string) (*State, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
config := &State{}
if err := yaml.Unmarshal(data, config); err != nil {
return nil, fmt.Errorf("parsing config file: %w", err)
return nil, fmt.Errorf("parsing %s: %w", path, err)
}
return config, nil
}
// SaveState saves the state to the specified path
// SaveState saves the state to the specified path. Backs up the current
// file to .bak before writing so LoadState can recover from corruption.
func SaveState(config *State, configPath string) error {
// Ensure the directory exists
if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
return fmt.Errorf("creating config directory: %w", err)
}
@@ -182,5 +202,10 @@ func SaveState(config *State, configPath string) error {
return fmt.Errorf("marshaling config: %w", err)
}
return os.WriteFile(configPath, data, 0644)
// Backup current state before overwriting
if storage.FileExists(configPath) {
_ = storage.CopyFile(configPath, configPath+".bak")
}
return storage.WriteFileAtomic(configPath, data, 0644)
}

2
internal/config/config_test.go
View File

@@ -115,7 +115,7 @@ func TestLoadState_Errors(t *testing.T) {
}
return statePath
},
errContains: "parsing config file",
errContains: "yaml:",
},
}

3
internal/dnsmasq/config.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/wild-cloud/wild-central/internal/config"
"github.com/wild-cloud/wild-central/internal/network"
"github.com/wild-cloud/wild-central/internal/storage"
)
// DNSEntry represents a single domain-to-IP DNS mapping for dnsmasq.
@@ -220,7 +221,7 @@ func (g *Manager) UpdateConfig(cfg *config.State, entries []DNSEntry, restart bo
// Write config
slog.Info("writing dnsmasq config", "component", "dnsmasq", "path", g.configPath)
if err := os.WriteFile(g.configPath, []byte(configContent), 0644); err != nil {
if err := storage.WriteFileAtomic(g.configPath, []byte(configContent), 0644); err != nil {
return fmt.Errorf("writing dnsmasq config: %w", err)
}

4
internal/domains/manager.go
View File

@@ -15,6 +15,8 @@ import (
"sync"
"gopkg.in/yaml.v3"
"github.com/wild-cloud/wild-central/internal/storage"
)
// BackendType describes how the gateway handles traffic for this domain.
@@ -215,7 +217,7 @@ func (m *Manager) Register(dom Domain) error {
return fmt.Errorf("marshaling domain: %w", err)
}
if err := os.WriteFile(m.domainPath(dom.DomainName), data, 0644); err != nil {
if err := storage.WriteFileAtomic(m.domainPath(dom.DomainName), data, 0644); err != nil {
return fmt.Errorf("writing domain file: %w", err)
}

6
internal/nftables/manager.go
View File

@@ -6,6 +6,8 @@ import (
"os"
"os/exec"
"strings"
"github.com/wild-cloud/wild-central/internal/storage"
)
const defaultRulesPath = "/etc/nftables.d/wild-cloud.nft"
@@ -144,7 +146,7 @@ func (m *Manager) WriteRules(content string) error {
return err
}
if err := os.WriteFile(m.rulesPath, []byte(content), 0644); err != nil {
if err := storage.WriteFileAtomic(m.rulesPath, []byte(content), 0644); err != nil {
return fmt.Errorf("writing rules file: %w", err)
}
@@ -171,7 +173,7 @@ func (m *Manager) WriteDisabledRules() error {
"# Managed by Wild Cloud Central API — do not edit manually\n\n" +
"table inet wild-cloud {}\n" +
"delete table inet wild-cloud\n"
if err := os.WriteFile(m.rulesPath, []byte(content), 0644); err != nil {
if err := storage.WriteFileAtomic(m.rulesPath, []byte(content), 0644); err != nil {
return fmt.Errorf("writing disabled rules file: %w", err)
}
slog.Info("nftables rules disabled", "component", "nftables", "path", m.rulesPath)

16
internal/reconcile/reconciler.go
View File

@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/wild-cloud/wild-central/internal/authelia"
@@ -69,6 +70,7 @@ type GenerateAutheliaConfigFn func(state *config.State) error
// Reconciler orchestrates config regeneration when domains change.
type Reconciler struct {
mu sync.Mutex // serializes concurrent Reconcile() calls
Domains DomainManager
HAProxy HAProxyManager
DNS DNSManager
@@ -81,8 +83,12 @@ type Reconciler struct {
}
// Reconcile reads all registered domains and regenerates dnsmasq DNS entries
// and HAProxy routes to match.
// and HAProxy routes to match. Serialized by mutex — concurrent calls wait
// rather than racing on config files.
func (r *Reconciler) Reconcile() {
r.mu.Lock()
defer r.mu.Unlock()
doms, err := r.Domains.List()
if err != nil {
slog.Error("failed to list domains", "component", "reconcile", "error", err)
@@ -91,8 +97,12 @@ func (r *Reconciler) Reconcile() {
globalCfg, err := config.LoadState(r.StatePath)
if err != nil {
slog.Warn("failed to load state, using empty", "component", "reconcile", "error", err)
globalCfg = &config.State{}
if os.IsNotExist(err) {
globalCfg = &config.State{} // first run, no state yet
} else {
slog.Error("failed to load state (refusing to reconcile with empty config)", "component", "reconcile", "error", err)
return
}
}
l4Routes, httpRoutes := buildRoutes(doms)

44
internal/storage/storage.go
View File

@@ -101,6 +101,50 @@ func WithLock(lockPath string, fn func() error) error {
return fn()
}
// WriteFileAtomic writes content to a file atomically via temp + rename.
// If the process crashes mid-write, the original file is untouched.
func WriteFileAtomic(path string, content []byte, perm os.FileMode) error {
dir := filepath.Dir(path)
if err := EnsureDir(dir, 0755); err != nil {
return err
}
tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp.*")
if err != nil {
return fmt.Errorf("creating temp file for %s: %w", path, err)
}
tmpPath := tmp.Name()
if _, err := tmp.Write(content); err != nil {
tmp.Close()
os.Remove(tmpPath)
return fmt.Errorf("writing temp file %s: %w", tmpPath, err)
}
if err := tmp.Chmod(perm); err != nil {
tmp.Close()
os.Remove(tmpPath)
return fmt.Errorf("setting permissions on %s: %w", tmpPath, err)
}
if err := tmp.Close(); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("closing temp file %s: %w", tmpPath, err)
}
if err := os.Rename(tmpPath, path); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("installing %s → %s: %w", tmpPath, path, err)
}
return nil
}
// CopyFile copies a file atomically. Used for creating .bak backups.
func CopyFile(src, dst string) error {
data, err := os.ReadFile(src)
if err != nil {
return fmt.Errorf("reading %s: %w", src, err)
}
return WriteFileAtomic(dst, data, 0644)
}
// EnsureFilePermissions ensures a file has the correct permissions
func EnsureFilePermissions(path string, perm os.FileMode) error {
if err := os.Chmod(path, perm); err != nil {

9
internal/tunnel/manager.go
View File

@@ -19,6 +19,8 @@ import (
"strings"
"gopkg.in/yaml.v3"
"github.com/wild-cloud/wild-central/internal/storage"
)
// Config holds tunnel configuration.
@@ -143,12 +145,7 @@ func (m *Manager) WriteConfig(cfg Config, services []PublicDomain) error {
return nil
}
dir := filepath.Dir(m.configPath())
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("creating tunnel config dir: %w", err)
}
if err := os.WriteFile(m.configPath(), []byte(content), 0644); err != nil {
if err := storage.WriteFileAtomic(m.configPath(), []byte(content), 0644); err != nil {
return fmt.Errorf("writing tunnel config: %w", err)
}

10
internal/wireguard/manager.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/google/uuid"
"gopkg.in/yaml.v3"
"github.com/wild-cloud/wild-central/internal/storage"
)
// Config holds the WireGuard server interface configuration.
@@ -110,7 +112,7 @@ func (m *Manager) SaveConfig(cfg *Config) error {
if err != nil {
return fmt.Errorf("marshal vpn config: %w", err)
}
return os.WriteFile(m.configFilePath(), data, 0644)
return storage.WriteFileAtomic(m.configFilePath(), data, 0644)
}
// --- Keys ---
@@ -133,7 +135,7 @@ func (m *Manager) GenerateKeypair() error {
if err != nil {
return err
}
return os.WriteFile(m.secretsFilePath(), data, 0600)
return storage.WriteFileAtomic(m.secretsFilePath(), data, 0600)
}
// GetPublicKey returns the server public key, or empty string if not yet generated.
@@ -261,7 +263,7 @@ func (m *Manager) savePeer(p *Peer) error {
if err != nil {
return err
}
return os.WriteFile(filepath.Join(m.peersDir(), p.ID+".yaml"), data, 0600)
return storage.WriteFileAtomic(filepath.Join(m.peersDir(), p.ID+".yaml"), data, 0600)
}
// nextAvailableIP finds the next unused host IP in the given CIDR (skipping the network
@@ -409,7 +411,7 @@ func (m *Manager) Apply() error {
if err := os.MkdirAll(filepath.Dir(m.configPath), 0755); err != nil {
return fmt.Errorf("create wireguard config dir: %w", err)
}
if err := os.WriteFile(m.configPath, []byte(content), 0600); err != nil {
if err := storage.WriteFileAtomic(m.configPath, []byte(content), 0600); err != nil {
return fmt.Errorf("write wireguard config: %w", err)
}
upCmd := exec.Command("sudo", "wg-quick", "up", "wg0")