wild-cloud/api/internal/backup/backup.go

// Package backup provides backup and restore operations for apps
package backup

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"log/slog"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/wild-cloud/wild-central/daemon/internal/apps"
	"github.com/wild-cloud/wild-central/daemon/internal/backup/destinations"
	"github.com/wild-cloud/wild-central/daemon/internal/backup/strategies"
	btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
	"github.com/wild-cloud/wild-central/daemon/internal/tools"
	"gopkg.in/yaml.v3"
)

type BackupInfo = btypes.BackupInfo
type ComponentBackup = btypes.ComponentBackup
type RestoreOptions = btypes.RestoreOptions
type Strategy = btypes.Strategy
type BackupDestination = btypes.BackupDestination
type BackupObject = btypes.BackupObject
type VerificationResult = btypes.VerificationResult
type ComponentVerification = btypes.ComponentVerification
type ProgressCallback = btypes.ProgressCallback
type BackupConfiguration = btypes.BackupConfiguration
type BackupSchedule = btypes.BackupSchedule
type ScheduleRetention = btypes.ScheduleRetention
type DestinationConfig = btypes.DestinationConfig
type S3Config = btypes.S3Config
type AzureConfig = btypes.AzureConfig
type NFSConfig = btypes.NFSConfig
type LocalConfig = btypes.LocalConfig
type RetentionPolicy = btypes.RetentionPolicy
type VerificationConfig = btypes.VerificationConfig
type RecoveryPlan = btypes.RecoveryPlan
type StrategyEntry = btypes.StrategyEntry
type PhaseTime = btypes.PhaseTime
type AppScheduleConfig = btypes.AppScheduleConfig
type ScheduleTierConfig = btypes.ScheduleTierConfig
type AppScheduleState = btypes.AppScheduleState
type TierState = btypes.TierState

// Manager handles backup and restore operations
type Manager struct {
	dataDir          string
	appsDir          string
	strategies       map[string]Strategy
	destination      BackupDestination // Will be loaded per-instance
	progressCallback ProgressCallback  // Optional callback for progress updates
}

// NewManager creates a new backup manager
func NewManager(dataDir string) *Manager {
	return &Manager{
		dataDir:    dataDir,
		appsDir:    os.Getenv("WILD_DIRECTORY"),
		strategies: initStrategies(dataDir),
	}
}

// NewManagerWithProgress creates a new backup manager with progress callback
func NewManagerWithProgress(dataDir string, progressCallback ProgressCallback) *Manager {
	return &Manager{
		dataDir:          dataDir,
		appsDir:          os.Getenv("WILD_DIRECTORY"),
		strategies:       initStrategies(dataDir),
		progressCallback: progressCallback,
	}
}

// reportProgress reports progress if a callback is set
func (m *Manager) reportProgress(progress int, message string) {
	if m.progressCallback != nil {
		m.progressCallback(progress, message)
	}
}

// initStrategies initializes all available backup strategies
func initStrategies(dataDir string) map[string]Strategy {
	strats := map[string]Strategy{
		"postgres": strategies.NewPostgreSQLStrategy(dataDir),
		"mysql":    strategies.NewMySQLStrategy(dataDir),
		"config":   strategies.NewConfigStrategy(dataDir),
	}

	longhornStrategy := strategies.NewLonghornNativeStrategy(dataDir)
	strats["pvc"] = longhornStrategy
	strats["longhorn-native"] = longhornStrategy

	return strats
}

// GetBackupDir returns the backup directory for an instance
func (m *Manager) GetBackupDir(instanceName string) string {
	return tools.GetInstanceBackupsPath(m.dataDir, instanceName)
}

// BackupApp creates a backup of an app's data, producing a RecoveryPlan.
func (m *Manager) BackupApp(instanceName, appName string) (*RecoveryPlan, error) {
	m.reportProgress(20, "Loading backup configuration")

	destination, err := m.loadDestination(instanceName)
	if err != nil {
		return nil, fmt.Errorf("failed to load backup destination: %w", err)
	}
	m.destination = destination

	m.reportProgress(30, "Loading app manifest")

	manifest, err := m.loadAppManifest(instanceName, appName)
	if err != nil {
		return nil, fmt.Errorf("failed to load app manifest: %w", err)
	}

	timestamp := time.Now().UTC().Format("20060102T150405Z")

	plan := &RecoveryPlan{
		App:       appName,
		Instance:  instanceName,
		Timestamp: timestamp,
		Version:   manifest.Version,
		Status:    "backing_up",
		Source: btypes.RecoverySource{
			Namespace: appName,
			AppDir:    filepath.Join("instances", instanceName, "apps", appName),
		},
		Strategies: []StrategyEntry{},
		Phases:     map[string]PhaseTime{},
	}

	now := time.Now()
	plan.Phases["backup"] = PhaseTime{StartedAt: &now}

	// Detect strategies and create entries in the plan
	detectedStrategies := m.detectStrategies(manifest)
	for _, s := range detectedStrategies {
		plan.Strategies = append(plan.Strategies, StrategyEntry{
			Name:   s.Name(),
			Status: "pending",
		})
	}

	m.reportProgress(40, fmt.Sprintf("Backing up %d components", len(detectedStrategies)))

	progressStart := 40
	progressEnd := 90
	progressPerStrategy := (progressEnd - progressStart) / max(len(detectedStrategies), 1)

	for i, strategy := range detectedStrategies {
		currentProgress := progressStart + (i * progressPerStrategy)
		m.reportProgress(currentProgress, fmt.Sprintf("Backing up %s", strategy.Name()))

		if err := strategy.Backup(plan, m.destination); err != nil {
			plan.Status = "failed"
			plan.Error = fmt.Sprintf("%s backup failed: %v", strategy.Name(), err)
			entry := plan.GetStrategyEntry(strategy.Name())
			if entry != nil {
				entry.Status = "failed"
				entry.Error = err.Error()
			}
			break
		}
	}

	if plan.Status != "failed" {
		plan.Status = "backed_up"
		completed := time.Now()
		phase := plan.Phases["backup"]
		phase.CompletedAt = &completed
		plan.Phases["backup"] = phase
		m.reportProgress(95, "Saving recovery plan")
	}

	// Save plan
	if err := m.savePlan(instanceName, appName, timestamp, plan); err != nil {
		return nil, fmt.Errorf("failed to save recovery plan: %w", err)
	}

	m.reportProgress(100, "Backup completed")
	return plan, nil
}

// RestoreApp restores an app from the latest backup using plan-driven coordination.
func (m *Manager) RestoreApp(instanceName, appName string, opts RestoreOptions) (*RecoveryPlan, error) {
	m.reportProgress(20, "Loading backup configuration")

	destination, err := m.loadDestination(instanceName)
	if err != nil {
		return nil, fmt.Errorf("failed to load backup destination: %w", err)
	}
	m.destination = destination

	m.reportProgress(30, "Finding latest recovery plan")

	// Find the latest plan
	plan, err := m.loadLatestPlan(instanceName, appName)
	if err != nil {
		return nil, fmt.Errorf("no recovery plan found for app %s: %w", appName, err)
	}

	if plan.Status != "backed_up" {
		return nil, fmt.Errorf("plan status is %s, expected backed_up", plan.Status)
	}

	// Always restore in-place: data goes into the source namespace
	plan.Status = "restoring"
	plan.Standby = btypes.RecoveryStandby{
		Namespace: plan.Source.Namespace,
		AppDir:    plan.Source.AppDir,
	}

	now := time.Now()
	plan.Phases["restore"] = PhaseTime{StartedAt: &now}

	m.reportProgress(40, fmt.Sprintf("Restoring to %s namespace", plan.Standby.Namespace))

	progressStart := 40
	progressEnd := 80
	strategiesToRestore := plan.Strategies
	if len(opts.Components) > 0 {
		var filtered []StrategyEntry
		for _, e := range plan.Strategies {
			if contains(opts.Components, e.Name) {
				filtered = append(filtered, e)
			}
		}
		strategiesToRestore = filtered
	}

	progressPerStrategy := (progressEnd - progressStart) / max(len(strategiesToRestore), 1)

	for i, entry := range strategiesToRestore {
		strategy, exists := m.strategies[entry.Name]
		if !exists {
			continue
		}

		currentProgress := progressStart + (i * progressPerStrategy)
		m.reportProgress(currentProgress, fmt.Sprintf("Restoring %s", entry.Name))

		if err := strategy.Restore(plan, m.destination); err != nil {
			plan.Status = "failed"
			plan.Error = fmt.Sprintf("%s restore failed: %v", entry.Name, err)
			_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
			return plan, fmt.Errorf("failed to restore %s: %w", entry.Name, err)
		}
	}

	plan.Status = "restored"
	completed := time.Now()
	phase := plan.Phases["restore"]
	phase.CompletedAt = &completed
	plan.Phases["restore"] = phase

	_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
	m.reportProgress(100, "Restore completed")
	return plan, nil
}

// SwitchApp performs an in-place switch: scales down the app, runs strategy switches,
// then always scales back up (even on failure, so the app is visible rather than stuck at zero).
func (m *Manager) SwitchApp(instanceName, appName string) (*RecoveryPlan, error) {
	m.reportProgress(20, "Loading recovery plan")

	plan, err := m.loadLatestPlan(instanceName, appName)
	if err != nil {
		return nil, fmt.Errorf("no recovery plan found: %w", err)
	}

	if plan.Status != "restored" {
		return nil, fmt.Errorf("plan status is %s, expected restored", plan.Status)
	}

	plan.Status = "switching"
	now := time.Now()
	plan.Phases["switch"] = PhaseTime{StartedAt: &now}

	kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
	namespace := plan.Source.Namespace

	// Scale down so strategies can safely swap data
	m.reportProgress(30, "Scaling down app")
	if scaleErr := scaleNamespace(kubeconfigPath, namespace, 0); scaleErr != nil {
		return nil, fmt.Errorf("failed to scale down namespace before switch: %w", scaleErr)
	}
	scaledDown := true
	if waitErr := waitForPodsGone(kubeconfigPath, namespace); waitErr != nil {
		slog.Warn("pods did not fully terminate before switch", "namespace", namespace, "error", waitErr)
	}

	// Always scale back up when we're done, even on failure
	defer func() {
		if scaledDown {
			m.reportProgress(90, "Scaling app back up")
			if upErr := scaleNamespace(kubeconfigPath, namespace, 1); upErr != nil {
				slog.Error("failed to scale up namespace after switch", "namespace", namespace, "error", upErr)
			}
		}
	}()

	m.reportProgress(40, "Switching strategies")

	// Call Switch on each strategy
	for _, entry := range plan.Strategies {
		strategy, exists := m.strategies[entry.Name]
		if !exists {
			continue
		}

		m.reportProgress(50, fmt.Sprintf("Switching %s", entry.Name))
		if err := strategy.Switch(plan); err != nil {
			plan.Status = "failed"
			plan.Error = fmt.Sprintf("%s switch failed: %v", entry.Name, err)
			_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
			return plan, fmt.Errorf("failed to switch %s: %w", entry.Name, err)
		}
	}

	plan.Status = "switched"
	completed := time.Now()
	phase := plan.Phases["switch"]
	phase.CompletedAt = &completed
	plan.Phases["switch"] = phase

	_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
	m.reportProgress(85, "Switch completed")
	return plan, nil
}

// CleanupApp removes the previous active resources after a successful switch.
func (m *Manager) CleanupApp(instanceName, appName string) (*RecoveryPlan, error) {
	m.reportProgress(20, "Loading recovery plan")

	plan, err := m.loadLatestPlan(instanceName, appName)
	if err != nil {
		return nil, fmt.Errorf("no recovery plan found: %w", err)
	}

	if plan.Status != "switched" {
		return nil, fmt.Errorf("plan status is %s, expected switched", plan.Status)
	}

	plan.Status = "cleaning_up"
	now := time.Now()
	plan.Phases["cleanup"] = PhaseTime{StartedAt: &now}

	m.reportProgress(40, "Cleaning up strategies")

	// Call Cleanup on each strategy
	for _, entry := range plan.Strategies {
		strategy, exists := m.strategies[entry.Name]
		if !exists {
			continue
		}

		m.reportProgress(50, fmt.Sprintf("Cleaning up %s", entry.Name))
		if err := strategy.Cleanup(plan); err != nil {
			plan.Status = "failed"
			plan.Error = fmt.Sprintf("%s cleanup failed: %v", entry.Name, err)
			_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
			return plan, fmt.Errorf("failed to cleanup %s: %w", entry.Name, err)
		}
	}

	// In-place restore: the Longhorn Switch phase already cleaned up the old PVC/PV/volume.
	// The namespace is the same throughout, so nothing to delete here.

	plan.Status = "cleaned_up"
	completed := time.Now()
	phase := plan.Phases["cleanup"]
	phase.CompletedAt = &completed
	plan.Phases["cleanup"] = phase

	_ = m.savePlan(instanceName, appName, plan.Timestamp, plan)
	m.reportProgress(100, "Cleanup completed")
	return plan, nil
}

// RestoreAndSwitch performs a full restore cycle: restore → switch → cleanup.
// Progress is reported across all three phases as a single operation.
// If any phase fails, the process stops and the system is left in a resumable state.
func (m *Manager) RestoreAndSwitch(instanceName, appName string, opts RestoreOptions) (*RecoveryPlan, error) {
	// Phase 1: Restore (0-50%)
	originalCb := m.progressCallback
	m.progressCallback = func(progress int, message string) {
		// Scale restore progress (0-100) to (0-50)
		scaled := progress / 2
		if originalCb != nil {
			originalCb(scaled, message)
		}
	}

	plan, err := m.RestoreApp(instanceName, appName, opts)
	if err != nil {
		m.progressCallback = originalCb
		return plan, fmt.Errorf("restore phase failed: %w", err)
	}

	// Phase 2: Switch (50-80%)
	m.progressCallback = func(progress int, message string) {
		// Scale switch progress (0-100) to (50-80)
		scaled := 50 + (progress * 30 / 100)
		if originalCb != nil {
			originalCb(scaled, message)
		}
	}

	plan, err = m.SwitchApp(instanceName, appName)
	if err != nil {
		m.progressCallback = originalCb
		return plan, fmt.Errorf("switch phase failed (restore succeeded, run switch manually): %w", err)
	}

	// Phase 3: Cleanup (80-100%)
	m.progressCallback = func(progress int, message string) {
		// Scale cleanup progress (0-100) to (80-100)
		scaled := 80 + (progress * 20 / 100)
		if originalCb != nil {
			originalCb(scaled, message)
		}
	}

	plan, err = m.CleanupApp(instanceName, appName)
	m.progressCallback = originalCb
	if err != nil {
		return plan, fmt.Errorf("cleanup phase failed (switch succeeded, run cleanup manually): %w", err)
	}

	return plan, nil
}

// GetRecoveryPlan returns the latest recovery plan for an app
func (m *Manager) GetRecoveryPlan(instanceName, appName string) (*RecoveryPlan, error) {
	return m.loadLatestPlan(instanceName, appName)
}

// scaleNamespace sets replicas on all Deployments and StatefulSets in a namespace.
func scaleNamespace(kubeconfigPath, namespace string, replicas int) error {
	for _, kind := range []string{"deployment", "statefulset"} {
		listCmd := exec.Command("kubectl", "get", kind, "-n", namespace, "-o", "name")
		tools.WithKubeconfig(listCmd, kubeconfigPath)
		out, _ := listCmd.Output()
		names := strings.Fields(string(out))
		for _, name := range names {
			cmd := exec.Command("kubectl", "scale", name, "-n", namespace,
				fmt.Sprintf("--replicas=%d", replicas))
			tools.WithKubeconfig(cmd, kubeconfigPath)
			if output, err := cmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to scale %s in %s: %w, output: %s", name, namespace, err, output)
			}
		}
	}
	return nil
}

// waitForPodsGone waits until all pods in a namespace have terminated.
func waitForPodsGone(kubeconfigPath, namespace string) error {
	cmd := exec.Command("kubectl", "wait", "--for=delete", "pod", "--all",
		"-n", namespace, "--timeout=120s")
	tools.WithKubeconfig(cmd, kubeconfigPath)
	if output, err := cmd.CombinedOutput(); err != nil {
		if strings.Contains(string(output), "no matching resources found") {
			return nil
		}
		return fmt.Errorf("waiting for pods to terminate in %s: %w, output: %s", namespace, err, output)
	}
	return nil
}

// ListBackups returns all recovery plans for an app
func (m *Manager) ListBackups(instanceName, appName string) ([]*RecoveryPlan, error) {
	backupDir := filepath.Join(m.GetBackupDir(instanceName), appName)

	if _, err := os.Stat(backupDir); os.IsNotExist(err) {
		return []*RecoveryPlan{}, nil
	}

	entries, err := os.ReadDir(backupDir)
	if err != nil {
		return nil, fmt.Errorf("failed to read backup directory: %w", err)
	}

	plans := []*RecoveryPlan{}
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}

		planFile := filepath.Join(backupDir, entry.Name(), "recovery-plan.yaml")
		if plan, err := m.loadPlan(planFile); err == nil {
			plans = append(plans, plan)
		}
	}

	// Sort by timestamp (newest first)
	sort.Slice(plans, func(i, j int) bool {
		return plans[i].Timestamp > plans[j].Timestamp
	})

	return plans, nil
}

// DeleteAppBackup deletes a specific app backup by timestamp
func (m *Manager) DeleteAppBackup(instanceName, appName, timestamp string) error {
	backupDir := filepath.Join(m.GetBackupDir(instanceName), appName, timestamp)

	if _, err := os.Stat(backupDir); os.IsNotExist(err) {
		return nil // Already deleted, nothing to do
	}

	// Load plan to get strategy locations
	planFile := filepath.Join(backupDir, "recovery-plan.yaml")
	plan, err := m.loadPlan(planFile)

	// Load destination and clean up remote files (best-effort)
	destination, err2 := m.loadDestination(instanceName)
	if err2 != nil {
		slog.Error("could not load backup destination, remote files may be orphaned", "component", "backup", "error", err2)
	} else if err == nil && plan != nil {
		for _, entry := range plan.Strategies {
			if location, ok := entry.Backup["location"].(string); ok && location != "" {
				if delErr := destination.Delete(location); delErr != nil {
					slog.Error("failed to delete backup from destination", "component", "backup", "location", location, "error", delErr)
				}
			}
		}
	}

	// Delete local metadata
	if err := os.RemoveAll(backupDir); err != nil {
		return fmt.Errorf("failed to delete backup metadata: %w", err)
	}

	return nil
}

// VerifyBackup verifies that a backup can be restored
func (m *Manager) VerifyBackup(instanceName, appName, timestamp string) (*VerificationResult, error) {
	destination, err := m.loadDestination(instanceName)
	if err != nil {
		return nil, fmt.Errorf("failed to load backup destination: %w", err)
	}

	// Load recovery plan
	backupDir := filepath.Join(m.GetBackupDir(instanceName), appName, timestamp)
	planFile := filepath.Join(backupDir, "recovery-plan.yaml")
	plan, err := m.loadPlan(planFile)
	if err != nil {
		return nil, fmt.Errorf("failed to load recovery plan: %w", err)
	}

	result := &VerificationResult{
		Success:    true,
		TestedAt:   time.Now(),
		Components: []ComponentVerification{},
	}

	for _, entry := range plan.Strategies {
		cv := ComponentVerification{
			Type: entry.Name,
		}

		strategy, exists := m.strategies[entry.Name]
		if !exists {
			cv.Success = false
			cv.Error = "Strategy not found"
		} else if err := strategy.Verify(plan, destination); err != nil {
			cv.Success = false
			cv.Error = err.Error()
		} else {
			cv.Success = true
		}

		result.Components = append(result.Components, cv)
		if !cv.Success {
			result.Success = false
		}
	}

	return result, nil
}

// detectStrategies determines which backup strategies to use based on the app manifest
func (m *Manager) detectStrategies(manifest *apps.AppManifest) []Strategy {
	var strats []Strategy

	// Always backup config
	if configStrategy, exists := m.strategies["config"]; exists {
		strats = append(strats, configStrategy)
	}

	// Check dependencies for database strategies
	for _, dep := range manifest.Requires {
		switch dep.Name {
		case "postgres", "postgresql":
			if s, exists := m.strategies["postgres"]; exists {
				strats = append(strats, s)
			}
		case "mysql", "mariadb":
			if s, exists := m.strategies["mysql"]; exists {
				strats = append(strats, s)
			}
		}
	}

	// Check for PVCs
	if pvcStrategy, exists := m.strategies["pvc"]; exists {
		strats = append(strats, pvcStrategy)
	}

	return strats
}

// loadAppManifest loads the app manifest from the instance directory
func (m *Manager) loadAppManifest(instanceName, appName string) (*apps.AppManifest, error) {
	manifestPath := filepath.Join(tools.GetAppPackagePath(m.dataDir, instanceName, appName), "manifest.yaml")

	data, err := os.ReadFile(manifestPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest apps.AppManifest
	if err := yaml.Unmarshal(data, &manifest); err != nil {
		return nil, fmt.Errorf("failed to parse manifest: %w", err)
	}

	return &manifest, nil
}

// loadDestination loads the backup destination configuration for an instance
func (m *Manager) loadDestination(instanceName string) (BackupDestination, error) {
	config, err := LoadInstanceBackupConfig(m.dataDir, instanceName)
	if err != nil {
		return nil, fmt.Errorf("failed to load backup config: %w", err)
	}

	switch config.Destination.Type {
	case "s3":
		if config.Destination.S3 == nil {
			return nil, fmt.Errorf("S3 configuration missing")
		}
		return destinations.NewS3Destination(config.Destination.S3)

	case "azure":
		if config.Destination.Azure == nil {
			return nil, fmt.Errorf("Azure configuration missing")
		}
		return destinations.NewAzureDestination(config.Destination.Azure)

	case "nfs":
		if config.Destination.NFS == nil {
			return nil, fmt.Errorf("NFS configuration missing")
		}
		return destinations.NewNFSDestination(config.Destination.NFS)

	case "local":
		if config.Destination.Local == nil {
			config.Destination.Local = &LocalConfig{
				Path: filepath.Join(m.dataDir, "instances", instanceName, "backups"),
			}
		}
		return destinations.NewLocalDestination(config.Destination.Local)

	default:
		return nil, fmt.Errorf("unknown backup destination type: %s", config.Destination.Type)
	}
}

// BackupClusterConfig creates a backup of cluster-level configuration files for disaster recovery.
// This backs up kubeconfig, talosconfig, config.yaml, secrets.yaml, and talos generated configs.
func (m *Manager) BackupClusterConfig(instanceName string) (*RecoveryPlan, error) {
	m.reportProgress(20, "Loading backup configuration")

	destination, err := m.loadDestination(instanceName)
	if err != nil {
		return nil, fmt.Errorf("failed to load backup destination: %w", err)
	}
	m.destination = destination

	instancePath := tools.GetInstancePath(m.dataDir, instanceName)

	// Collect files to back up (skip missing gracefully).
	// Helpers are format-aware: they return new paths after migration, legacy paths before.
	filePaths := []string{
		tools.GetKubeconfigPath(m.dataDir, instanceName),
		tools.GetInstanceConfigPath(m.dataDir, instanceName),
		tools.GetInstanceSecretsPath(m.dataDir, instanceName),
		tools.GetTalosconfigPath(m.dataDir, instanceName),
		tools.GetControlplaneConfigPath(m.dataDir, instanceName),
		tools.GetWorkerConfigPath(m.dataDir, instanceName),
		tools.GetTalosPkiPath(m.dataDir, instanceName),
	}

	var existingFiles []string
	for _, f := range filePaths {
		if _, err := os.Stat(f); err == nil {
			existingFiles = append(existingFiles, f)
		}
	}

	if len(existingFiles) == 0 {
		return nil, fmt.Errorf("no cluster config files found for instance %s", instanceName)
	}

	m.reportProgress(40, fmt.Sprintf("Archiving %d cluster config files", len(existingFiles)))

	timestamp := time.Now().UTC().Format("20060102T150405Z")
	key := fmt.Sprintf("cluster-config/%s/%s.tar.gz", instanceName, timestamp)

	// Create tar.gz archive in memory
	var buf bytes.Buffer
	gzWriter := gzip.NewWriter(&buf)
	tarWriter := tar.NewWriter(gzWriter)

	totalSize := int64(0)
	for _, filePath := range existingFiles {
		file, err := os.Open(filePath)
		if err != nil {
			tarWriter.Close()
			gzWriter.Close()
			return nil, fmt.Errorf("failed to open %s: %w", filePath, err)
		}

		stat, err := file.Stat()
		if err != nil {
			file.Close()
			tarWriter.Close()
			gzWriter.Close()
			return nil, fmt.Errorf("failed to stat %s: %w", filePath, err)
		}

		header, err := tar.FileInfoHeader(stat, "")
		if err != nil {
			file.Close()
			tarWriter.Close()
			gzWriter.Close()
			return nil, fmt.Errorf("failed to create tar header for %s: %w", filePath, err)
		}

		// Use relative path from instance directory
		relPath, _ := filepath.Rel(instancePath, filePath)
		header.Name = relPath

		if err := tarWriter.WriteHeader(header); err != nil {
			file.Close()
			tarWriter.Close()
			gzWriter.Close()
			return nil, fmt.Errorf("failed to write tar header for %s: %w", filePath, err)
		}

		if _, err := io.Copy(tarWriter, file); err != nil {
			file.Close()
			tarWriter.Close()
			gzWriter.Close()
			return nil, fmt.Errorf("failed to write file %s to archive: %w", filePath, err)
		}

		totalSize += stat.Size()
		file.Close()
	}

	if err := tarWriter.Close(); err != nil {
		gzWriter.Close()
		return nil, fmt.Errorf("failed to close tar: %w", err)
	}
	if err := gzWriter.Close(); err != nil {
		return nil, fmt.Errorf("failed to close gzip: %w", err)
	}

	m.reportProgress(70, "Uploading cluster config backup")

	reader := bytes.NewReader(buf.Bytes())
	size, err := destination.Put(key, reader)
	if err != nil {
		return nil, fmt.Errorf("failed to upload cluster config backup: %w", err)
	}

	m.reportProgress(90, "Saving recovery plan")

	now := time.Now()
	completed := time.Now()
	plan := &RecoveryPlan{
		App:       "_cluster",
		Instance:  instanceName,
		Timestamp: timestamp,
		Status:    "backed_up",
		Strategies: []StrategyEntry{
			{
				Name:   "cluster-config",
				Status: "backed_up",
				Backup: map[string]interface{}{
					"location":  key,
					"size":      size,
					"files":     len(existingFiles),
					"format":    "tar.gz",
					"totalSize": totalSize,
				},
			},
		},
		Phases: map[string]PhaseTime{
			"backup": {StartedAt: &now, CompletedAt: &completed},
		},
	}

	if err := m.savePlan(instanceName, "_cluster", timestamp, plan); err != nil {
		return nil, fmt.Errorf("failed to save recovery plan: %w", err)
	}

	m.reportProgress(100, "Cluster config backup completed")
	return plan, nil
}

// savePlan saves a RecoveryPlan to YAML file
func (m *Manager) savePlan(instanceName, appName, timestamp string, plan *RecoveryPlan) error {
	backupDir := filepath.Join(m.GetBackupDir(instanceName), appName, timestamp)
	if err := os.MkdirAll(backupDir, 0755); err != nil {
		return fmt.Errorf("failed to create backup directory: %w", err)
	}

	planFile := filepath.Join(backupDir, "recovery-plan.yaml")
	data, err := yaml.Marshal(plan)
	if err != nil {
		return err
	}

	return os.WriteFile(planFile, data, 0600)
}

// loadPlan loads a RecoveryPlan from YAML file
func (m *Manager) loadPlan(path string) (*RecoveryPlan, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var plan RecoveryPlan
	if err := yaml.Unmarshal(data, &plan); err != nil {
		return nil, err
	}

	return &plan, nil
}

// loadLatestPlan loads the most recent RecoveryPlan for an app
func (m *Manager) loadLatestPlan(instanceName, appName string) (*RecoveryPlan, error) {
	plans, err := m.ListBackups(instanceName, appName)
	if err != nil {
		return nil, err
	}
	if len(plans) == 0 {
		return nil, fmt.Errorf("no recovery plans found")
	}
	return plans[0], nil
}

// contains checks if a string slice contains a value
func contains(slice []string, value string) bool {
	for _, s := range slice {
		if s == value {
			return true
		}
	}
	return false
}