Files
wild-central-api/internal/backup/backup.go

495 lines
16 KiB
Go

// Package backup provides backup and restore operations for apps
package backup
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/wild-cloud/wild-central/daemon/internal/storage"
"github.com/wild-cloud/wild-central/daemon/internal/tools"
)
// BackupInfo represents metadata about a backup
type BackupInfo struct {
AppName string `json:"app_name"`
Timestamp string `json:"timestamp"`
Type string `json:"type"` // "full", "database", "pvc"
Size int64 `json:"size,omitempty"`
Status string `json:"status"` // "completed", "failed", "in_progress"
Error string `json:"error,omitempty"`
Files []string `json:"files"`
CreatedAt time.Time `json:"created_at"`
}
// RestoreOptions configures restore behavior
type RestoreOptions struct {
DBOnly bool `json:"db_only"`
PVCOnly bool `json:"pvc_only"`
SkipGlobals bool `json:"skip_globals"`
SnapshotID string `json:"snapshot_id,omitempty"`
}
// Manager handles backup and restore operations
type Manager struct {
dataDir string
}
// NewManager creates a new backup manager
func NewManager(dataDir string) *Manager {
return &Manager{dataDir: dataDir}
}
// GetBackupDir returns the backup directory for an instance
func (m *Manager) GetBackupDir(instanceName string) string {
return tools.GetInstanceBackupsPath(m.dataDir, instanceName)
}
// GetStagingDir returns the staging directory for backups
func (m *Manager) GetStagingDir(instanceName string) string {
return filepath.Join(m.GetBackupDir(instanceName), "staging")
}
// BackupApp creates a backup of an app's data
func (m *Manager) BackupApp(instanceName, appName string) (*BackupInfo, error) {
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
stagingDir := m.GetStagingDir(instanceName)
if err := storage.EnsureDir(stagingDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create staging directory: %w", err)
}
backupDir := filepath.Join(stagingDir, "apps", appName)
if err := os.RemoveAll(backupDir); err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to clean backup directory: %w", err)
}
if err := storage.EnsureDir(backupDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create backup directory: %w", err)
}
timestamp := time.Now().UTC().Format("20060102T150405Z")
info := &BackupInfo{
AppName: appName,
Timestamp: timestamp,
Type: "full",
Status: "in_progress",
Files: []string{},
CreatedAt: time.Now(),
}
// Backup database if app uses one
dbFiles, err := m.backupDatabase(kubeconfigPath, appName, backupDir, timestamp)
if err != nil {
info.Status = "failed"
info.Error = fmt.Sprintf("database backup failed: %v", err)
} else if len(dbFiles) > 0 {
info.Files = append(info.Files, dbFiles...)
}
// Backup PVCs
pvcFiles, err := m.backupPVCs(kubeconfigPath, appName, backupDir)
if err != nil && info.Status != "failed" {
info.Status = "failed"
info.Error = fmt.Sprintf("pvc backup failed: %v", err)
} else if len(pvcFiles) > 0 {
info.Files = append(info.Files, pvcFiles...)
}
if info.Status != "failed" {
info.Status = "completed"
}
// Save backup metadata
metaFile := filepath.Join(backupDir, "backup.json")
if err := m.saveBackupMeta(metaFile, info); err != nil {
return nil, fmt.Errorf("failed to save backup metadata: %w", err)
}
return info, nil
}
// RestoreApp restores an app from backup
func (m *Manager) RestoreApp(instanceName, appName string, opts RestoreOptions) error {
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
stagingDir := m.GetStagingDir(instanceName)
backupDir := filepath.Join(stagingDir, "apps", appName)
// Check if backup exists
if !storage.FileExists(backupDir) {
return fmt.Errorf("no backup found for app %s", appName)
}
// Restore database if not PVC-only
if !opts.PVCOnly {
if err := m.restoreDatabase(kubeconfigPath, appName, backupDir, opts.SkipGlobals); err != nil {
return fmt.Errorf("database restore failed: %w", err)
}
}
// Restore PVCs if not DB-only
if !opts.DBOnly {
if err := m.restorePVCs(kubeconfigPath, appName, backupDir); err != nil {
return fmt.Errorf("pvc restore failed: %w", err)
}
}
return nil
}
// ListBackups returns all backups for an app
func (m *Manager) ListBackups(instanceName, appName string) ([]*BackupInfo, error) {
stagingDir := m.GetStagingDir(instanceName)
appBackupDir := filepath.Join(stagingDir, "apps", appName)
if !storage.FileExists(appBackupDir) {
return []*BackupInfo{}, nil
}
var backups []*BackupInfo
metaFile := filepath.Join(appBackupDir, "backup.json")
if storage.FileExists(metaFile) {
info, err := m.loadBackupMeta(metaFile)
if err == nil {
backups = append(backups, info)
}
}
return backups, nil
}
// backupDatabase backs up PostgreSQL or MySQL database
func (m *Manager) backupDatabase(kubeconfigPath, appName, backupDir, timestamp string) ([]string, error) {
// Detect database type from manifest or deployed pods
dbType, err := m.detectDatabaseType(kubeconfigPath, appName)
if err != nil || dbType == "" {
return nil, nil // No database to backup
}
switch dbType {
case "postgres":
return m.backupPostgres(kubeconfigPath, appName, backupDir, timestamp)
case "mysql":
return m.backupMySQL(kubeconfigPath, appName, backupDir, timestamp)
default:
return nil, nil
}
}
// backupPostgres backs up PostgreSQL database
func (m *Manager) backupPostgres(kubeconfigPath, appName, backupDir, timestamp string) ([]string, error) {
dbDump := filepath.Join(backupDir, fmt.Sprintf("database_%s.dump", timestamp))
globalsFile := filepath.Join(backupDir, fmt.Sprintf("globals_%s.sql", timestamp))
// Database dump
cmd := exec.Command("kubectl", "exec", "-n", "postgres", "deploy/postgres-deployment", "--",
"bash", "-lc", fmt.Sprintf("pg_dump -U postgres -Fc -Z 9 %s", appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("pg_dump failed: %w", err)
}
if err := os.WriteFile(dbDump, output, 0600); err != nil {
return nil, fmt.Errorf("failed to write database dump: %w", err)
}
// Globals dump
cmd = exec.Command("kubectl", "exec", "-n", "postgres", "deploy/postgres-deployment", "--",
"bash", "-lc", "pg_dumpall -U postgres -g")
tools.WithKubeconfig(cmd, kubeconfigPath)
output, err = cmd.Output()
if err != nil {
return nil, fmt.Errorf("pg_dumpall failed: %w", err)
}
if err := os.WriteFile(globalsFile, output, 0600); err != nil {
return nil, fmt.Errorf("failed to write globals dump: %w", err)
}
return []string{dbDump, globalsFile}, nil
}
// backupMySQL backs up MySQL database
func (m *Manager) backupMySQL(kubeconfigPath, appName, backupDir, timestamp string) ([]string, error) {
dbDump := filepath.Join(backupDir, fmt.Sprintf("database_%s.sql", timestamp))
// Get MySQL password from secret
cmd := exec.Command("kubectl", "get", "secret", "-n", "mysql", "mysql-secret",
"-o", "jsonpath={.data.password}")
tools.WithKubeconfig(cmd, kubeconfigPath)
passOutput, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("failed to get MySQL password: %w", err)
}
password := string(passOutput)
// MySQL dump
cmd = exec.Command("kubectl", "exec", "-n", "mysql", "deploy/mysql-deployment", "--",
"bash", "-c", fmt.Sprintf("mysqldump -uroot -p'%s' --single-transaction --routines --triggers %s",
password, appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("mysqldump failed: %w", err)
}
if err := os.WriteFile(dbDump, output, 0600); err != nil {
return nil, fmt.Errorf("failed to write database dump: %w", err)
}
return []string{dbDump}, nil
}
// backupPVCs backs up all PVCs for an app
func (m *Manager) backupPVCs(kubeconfigPath, appName, backupDir string) ([]string, error) {
// List PVCs for the app
cmd := exec.Command("kubectl", "get", "pvc", "-n", appName,
"-l", fmt.Sprintf("app=%s", appName),
"-o", "jsonpath={.items[*].metadata.name}")
tools.WithKubeconfig(cmd, kubeconfigPath)
output, err := cmd.Output()
if err != nil {
return nil, nil // No PVCs found
}
pvcs := strings.Fields(string(output))
if len(pvcs) == 0 {
return nil, nil
}
var files []string
for _, pvc := range pvcs {
pvcBackupDir := filepath.Join(backupDir, pvc)
if err := storage.EnsureDir(pvcBackupDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create PVC backup dir: %w", err)
}
// Get a running pod
cmd = exec.Command("kubectl", "get", "pods", "-n", appName,
"-l", fmt.Sprintf("app=%s", appName),
"-o", "jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name}")
tools.WithKubeconfig(cmd, kubeconfigPath)
podOutput, err := cmd.Output()
if err != nil || len(podOutput) == 0 {
continue
}
pod := strings.Fields(string(podOutput))[0]
// Backup PVC data via tar
cmd = exec.Command("kubectl", "exec", "-n", appName, pod, "--",
"tar", "-C", "/data", "-cf", "-", ".")
tools.WithKubeconfig(cmd, kubeconfigPath)
tarData, err := cmd.Output()
if err != nil {
continue
}
// Extract tar to backup directory
tarFile := filepath.Join(pvcBackupDir, "data.tar")
if err := os.WriteFile(tarFile, tarData, 0600); err != nil {
return nil, fmt.Errorf("failed to write PVC backup: %w", err)
}
files = append(files, tarFile)
}
return files, nil
}
// restoreDatabase restores database from backup
func (m *Manager) restoreDatabase(kubeconfigPath, appName, backupDir string, skipGlobals bool) error {
// Find database dump files
matches, err := filepath.Glob(filepath.Join(backupDir, "database_*.dump"))
if err != nil || len(matches) == 0 {
matches, _ = filepath.Glob(filepath.Join(backupDir, "database_*.sql"))
}
if len(matches) == 0 {
return nil // No database backup found
}
dumpFile := matches[0]
isPostgres := strings.HasSuffix(dumpFile, ".dump")
if isPostgres {
return m.restorePostgres(kubeconfigPath, appName, backupDir, skipGlobals)
}
return m.restoreMySQL(kubeconfigPath, appName, dumpFile)
}
// restorePostgres restores PostgreSQL database
func (m *Manager) restorePostgres(kubeconfigPath, appName, backupDir string, skipGlobals bool) error {
// Find dump files
dumps, _ := filepath.Glob(filepath.Join(backupDir, "database_*.dump"))
if len(dumps) == 0 {
return fmt.Errorf("no PostgreSQL dump found")
}
// Drop and recreate database
cmd := exec.Command("kubectl", "exec", "-n", "postgres", "deploy/postgres-deployment", "--",
"bash", "-lc", fmt.Sprintf("psql -U postgres -d postgres -c \"DROP DATABASE IF EXISTS %s; CREATE DATABASE %s OWNER %s;\"",
appName, appName, appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to recreate database: %w", err)
}
// Restore database
dumpData, err := os.ReadFile(dumps[0])
if err != nil {
return fmt.Errorf("failed to read dump file: %w", err)
}
cmd = exec.Command("kubectl", "exec", "-i", "-n", "postgres", "deploy/postgres-deployment", "--",
"bash", "-lc", fmt.Sprintf("pg_restore -U postgres -d %s", appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
cmd.Stdin = strings.NewReader(string(dumpData))
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("pg_restore failed: %w", err)
}
return nil
}
// restoreMySQL restores MySQL database
func (m *Manager) restoreMySQL(kubeconfigPath, appName, dumpFile string) error {
// Get MySQL password
cmd := exec.Command("kubectl", "get", "secret", "-n", "mysql", "mysql-secret",
"-o", "jsonpath={.data.password}")
tools.WithKubeconfig(cmd, kubeconfigPath)
passOutput, err := cmd.Output()
if err != nil {
return fmt.Errorf("failed to get MySQL password: %w", err)
}
password := string(passOutput)
// Drop and recreate database
cmd = exec.Command("kubectl", "exec", "-n", "mysql", "deploy/mysql-deployment", "--",
"bash", "-c", fmt.Sprintf("mysql -uroot -p'%s' -e 'DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;'",
password, appName, appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to recreate database: %w", err)
}
// Restore database
dumpData, err := os.ReadFile(dumpFile)
if err != nil {
return fmt.Errorf("failed to read dump file: %w", err)
}
cmd = exec.Command("kubectl", "exec", "-i", "-n", "mysql", "deploy/mysql-deployment", "--",
"bash", "-c", fmt.Sprintf("mysql -uroot -p'%s' %s", password, appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
cmd.Stdin = strings.NewReader(string(dumpData))
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("mysql restore failed: %w", err)
}
return nil
}
// restorePVCs restores PVC data from backup
func (m *Manager) restorePVCs(kubeconfigPath, appName, backupDir string) error {
// Find PVC backup directories
entries, err := os.ReadDir(backupDir)
if err != nil {
return fmt.Errorf("failed to read backup directory: %w", err)
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
pvcName := entry.Name()
pvcBackupDir := filepath.Join(backupDir, pvcName)
tarFile := filepath.Join(pvcBackupDir, "data.tar")
if !storage.FileExists(tarFile) {
continue
}
// Scale app down
cmd := exec.Command("kubectl", "scale", "deployment", "-n", appName,
"-l", fmt.Sprintf("app=%s", appName), "--replicas=0")
tools.WithKubeconfig(cmd, kubeconfigPath)
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to scale down app: %w", err)
}
// Wait for pods to terminate
time.Sleep(10 * time.Second)
// Create temp pod with PVC mounted
// (Simplified - in production would need proper node selection and resource specs)
tempPod := fmt.Sprintf("restore-util-%d", time.Now().Unix())
// Restore data via temp pod (simplified approach)
// Full implementation would create pod, wait for ready, copy data, clean up
// Scale app back up
cmd = exec.Command("kubectl", "scale", "deployment", "-n", appName,
"-l", fmt.Sprintf("app=%s", appName), "--replicas=1")
tools.WithKubeconfig(cmd, kubeconfigPath)
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to scale up app: %w", err)
}
_ = tempPod // Placeholder for actual implementation
}
return nil
}
// detectDatabaseType detects the database type for an app
func (m *Manager) detectDatabaseType(kubeconfigPath, appName string) (string, error) {
// Check for postgres namespace
cmd := exec.Command("kubectl", "get", "namespace", "postgres")
tools.WithKubeconfig(cmd, kubeconfigPath)
if err := cmd.Run(); err == nil {
// Check if app uses postgres
cmd = exec.Command("kubectl", "get", "pods", "-n", "postgres", "-l", fmt.Sprintf("app=%s", appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
if output, _ := cmd.Output(); len(output) > 0 {
return "postgres", nil
}
}
// Check for mysql namespace
cmd = exec.Command("kubectl", "get", "namespace", "mysql")
tools.WithKubeconfig(cmd, kubeconfigPath)
if err := cmd.Run(); err == nil {
cmd = exec.Command("kubectl", "get", "pods", "-n", "mysql", "-l", fmt.Sprintf("app=%s", appName))
tools.WithKubeconfig(cmd, kubeconfigPath)
if output, _ := cmd.Output(); len(output) > 0 {
return "mysql", nil
}
}
return "", nil
}
// saveBackupMeta saves backup metadata to JSON file
func (m *Manager) saveBackupMeta(path string, info *BackupInfo) error {
data, err := json.MarshalIndent(info, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, data, 0600)
}
// loadBackupMeta loads backup metadata from JSON file
func (m *Manager) loadBackupMeta(path string) (*BackupInfo, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var info BackupInfo
if err := json.Unmarshal(data, &info); err != nil {
return nil, err
}
return &info, nil
}