379 lines
12 KiB
Bash
Executable File
379 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
# wild-app-backup - Generic backup script for wild-cloud apps
|
|
# Usage: wild-app-backup <app-name> [--all]
|
|
|
|
# --- Initialize Wild Cloud environment ---------------------------------------
|
|
if [ -z "${WC_ROOT:-}" ]; then
|
|
echo "WC_ROOT is not set." >&2
|
|
exit 1
|
|
else
|
|
source "${WC_ROOT}/scripts/common.sh"
|
|
init_wild_env
|
|
fi
|
|
|
|
# --- Configuration ------------------------------------------------------------
|
|
get_staging_dir() {
|
|
if wild-config cloud.backup.staging --check; then
|
|
wild-config cloud.backup.staging
|
|
else
|
|
echo "Staging directory is not set. Configure 'cloud.backup.staging' in config.yaml." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# --- Helpers ------------------------------------------------------------------
|
|
require_k8s() {
|
|
if ! command -v kubectl >/dev/null 2>&1; then
|
|
echo "kubectl not found." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
require_yq() {
|
|
if ! command -v yq >/dev/null 2>&1; then
|
|
echo "yq not found. Required for parsing manifest.yaml files." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
get_timestamp() {
|
|
date -u +'%Y%m%dT%H%M%SZ'
|
|
}
|
|
|
|
# --- App Discovery ------------------------------------------------------------
|
|
discover_database_deps() {
|
|
local app_name="$1"
|
|
local manifest_file="${WC_HOME}/apps/${app_name}/manifest.yaml"
|
|
|
|
if [[ -f "$manifest_file" ]]; then
|
|
yq eval '.requires[].name' "$manifest_file" 2>/dev/null | grep -E '^(postgres|mysql|redis)$' || true
|
|
fi
|
|
}
|
|
|
|
discover_app_pvcs() {
|
|
local app_name="$1"
|
|
kubectl get pvc -n "$app_name" -l "app=$app_name" --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true
|
|
}
|
|
|
|
get_app_pods() {
|
|
local app_name="$1"
|
|
kubectl get pods -n "$app_name" -l "app=$app_name" \
|
|
-o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null | \
|
|
tr ' ' '\n' | head -1 || true
|
|
}
|
|
|
|
discover_pvc_mount_paths() {
|
|
local app_name="$1" pvc_name="$2"
|
|
|
|
# Find the volume name that uses this PVC
|
|
local volume_name
|
|
volume_name=$(kubectl get deploy -n "$app_name" -l "app=$app_name" \
|
|
-o jsonpath='{.items[*].spec.template.spec.volumes[?(@.persistentVolumeClaim.claimName=="'$pvc_name'")].name}' 2>/dev/null | awk 'NR==1{print; exit}')
|
|
|
|
if [[ -n "$volume_name" ]]; then
|
|
# Find the mount path for this volume (get first mount path)
|
|
local mount_path
|
|
mount_path=$(kubectl get deploy -n "$app_name" -l "app=$app_name" \
|
|
-o jsonpath='{.items[*].spec.template.spec.containers[*].volumeMounts[?(@.name=="'$volume_name'")].mountPath}' 2>/dev/null | \
|
|
tr ' ' '\n' | head -1)
|
|
|
|
if [[ -n "$mount_path" ]]; then
|
|
echo "$mount_path"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# No mount path found
|
|
return 1
|
|
}
|
|
|
|
# --- Database Backup Functions -----------------------------------------------
|
|
backup_postgres_database() {
|
|
local app_name="$1"
|
|
local backup_dir="$2"
|
|
local timestamp="$3"
|
|
local db_name="${app_name}"
|
|
|
|
local pg_ns="postgres"
|
|
local pg_deploy="postgres-deployment"
|
|
local db_superuser="postgres"
|
|
|
|
echo "Backing up PostgreSQL database '$db_name'..." >&2
|
|
|
|
# Check if postgres is available
|
|
if ! kubectl get pods -n "$pg_ns" >/dev/null 2>&1; then
|
|
echo "PostgreSQL namespace '$pg_ns' not accessible. Skipping database backup." >&2
|
|
return 1
|
|
fi
|
|
|
|
local db_dump="${backup_dir}/database_${timestamp}.dump"
|
|
local db_globals="${backup_dir}/globals_${timestamp}.sql"
|
|
|
|
# Database dump (custom format, compressed)
|
|
if ! kubectl exec -n "$pg_ns" deploy/"$pg_deploy" -- bash -lc \
|
|
"pg_dump -U ${db_superuser} -Fc -Z 9 ${db_name}" > "$db_dump"
|
|
then
|
|
echo "Database dump failed for '$app_name'." >&2
|
|
return 1
|
|
fi
|
|
|
|
# Verify dump integrity
|
|
# if ! kubectl exec -i -n "$pg_ns" deploy/"$pg_deploy" -- bash -lc "pg_restore -l >/dev/null" < "$db_dump"; then
|
|
# echo "Database dump integrity check failed for '$app_name'." >&2
|
|
# return 1
|
|
# fi
|
|
|
|
# Dump globals (roles, permissions)
|
|
if ! kubectl exec -n "$pg_ns" deploy/"$pg_deploy" -- bash -lc \
|
|
"pg_dumpall -U ${db_superuser} -g" > "$db_globals"
|
|
then
|
|
echo "Globals dump failed for '$app_name'." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo " Database dump: $db_dump" >&2
|
|
echo " Globals dump: $db_globals" >&2
|
|
|
|
# Return paths for manifest generation
|
|
echo "$db_dump $db_globals"
|
|
}
|
|
|
|
backup_mysql_database() {
|
|
local app_name="$1"
|
|
local backup_dir="$2"
|
|
local timestamp="$3"
|
|
local db_name="${app_name}"
|
|
|
|
local mysql_ns="mysql"
|
|
local mysql_deploy="mysql-deployment"
|
|
local mysql_user="root"
|
|
|
|
echo "Backing up MySQL database '$db_name'..." >&2
|
|
|
|
if ! kubectl get pods -n "$mysql_ns" >/dev/null 2>&1; then
|
|
echo "MySQL namespace '$mysql_ns' not accessible. Skipping database backup." >&2
|
|
return 1
|
|
fi
|
|
|
|
local db_dump="${backup_dir}/database_${timestamp}.sql"
|
|
|
|
# Get MySQL root password from secret
|
|
local mysql_password
|
|
if mysql_password=$(kubectl get secret -n "$mysql_ns" mysql-secret -o jsonpath='{.data.password}' 2>/dev/null | base64 -d); then
|
|
# MySQL dump with password
|
|
if ! kubectl exec -n "$mysql_ns" deploy/"$mysql_deploy" -- bash -c \
|
|
"mysqldump -u${mysql_user} -p'${mysql_password}' --single-transaction --routines --triggers ${db_name}" > "$db_dump"
|
|
then
|
|
echo "MySQL dump failed for '$app_name'." >&2
|
|
return 1
|
|
fi
|
|
else
|
|
echo "Could not retrieve MySQL password. Skipping database backup." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo " Database dump: $db_dump" >&2
|
|
echo "$db_dump"
|
|
}
|
|
|
|
# --- PVC Backup Functions ----------------------------------------------------
|
|
backup_pvc() {
|
|
local app_name="$1"
|
|
local pvc_name="$2"
|
|
local backup_dir="$3"
|
|
local timestamp="$4"
|
|
|
|
echo "Backing up PVC '$pvc_name' from namespace '$app_name'..." >&2
|
|
|
|
# Get a running pod that actually uses this specific PVC
|
|
local app_pod
|
|
# First try to find a pod that has this exact PVC volume mounted
|
|
local pvc_volume_id=$(kubectl get pvc -n "$app_name" "$pvc_name" -o jsonpath='{.spec.volumeName}' 2>/dev/null)
|
|
if [[ -n "$pvc_volume_id" ]]; then
|
|
# Look for a pod that has a mount from this specific volume
|
|
app_pod=$(kubectl get pods -n "$app_name" -l "app=$app_name" -o json 2>/dev/null | \
|
|
jq -r '.items[] | select(.status.phase=="Running") | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="'$pvc_name'") | .metadata.name' | head -1)
|
|
fi
|
|
|
|
# Fallback to any running pod
|
|
if [[ -z "$app_pod" ]]; then
|
|
app_pod=$(get_app_pods "$app_name")
|
|
fi
|
|
|
|
if [[ -z "$app_pod" ]]; then
|
|
echo "No running pods found for app '$app_name'. Skipping PVC backup." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "Using pod '$app_pod' for PVC backup" >&2
|
|
|
|
# Discover mount path for this PVC
|
|
local mount_path
|
|
mount_path=$(discover_pvc_mount_paths "$app_name" "$pvc_name" | awk 'NR==1{print; exit}')
|
|
|
|
if [[ -z "$mount_path" ]]; then
|
|
echo "Could not determine mount path for PVC '$pvc_name'. Trying to detect..." >&2
|
|
# Try to find any volume mount that might be the PVC by looking at df output
|
|
mount_path=$(kubectl exec -n "$app_name" "$app_pod" -- sh -c "df | grep longhorn | awk '{print \$6}' | head -1" 2>/dev/null)
|
|
if [[ -z "$mount_path" ]]; then
|
|
mount_path="/data" # Final fallback
|
|
fi
|
|
echo "Using detected/fallback mount path: $mount_path" >&2
|
|
fi
|
|
|
|
local pvc_backup_dir="${backup_dir}/${pvc_name}"
|
|
mkdir -p "$pvc_backup_dir"
|
|
|
|
# Stream tar directly from pod to staging directory for restic deduplication
|
|
local parent_dir=$(dirname "$mount_path")
|
|
local dir_name=$(basename "$mount_path")
|
|
|
|
echo " Streaming PVC data directly to staging..." >&2
|
|
if kubectl exec -n "$app_name" "$app_pod" -- tar -C "$parent_dir" -cf - "$dir_name" | tar -xf - -C "$pvc_backup_dir" 2>/dev/null; then
|
|
echo " PVC data streamed successfully" >&2
|
|
else
|
|
echo "PVC backup failed for '$pvc_name' in '$app_name'." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo " PVC backup directory: $pvc_backup_dir" >&2
|
|
echo "$pvc_backup_dir"
|
|
}
|
|
|
|
# --- Main Backup Function ----------------------------------------------------
|
|
backup_app() {
|
|
local app_name="$1"
|
|
local staging_dir="$2"
|
|
|
|
echo "=========================================="
|
|
echo "Starting backup of app: $app_name"
|
|
echo "=========================================="
|
|
|
|
local timestamp
|
|
timestamp=$(get_timestamp)
|
|
|
|
local backup_dir="${staging_dir}/apps/${app_name}"
|
|
|
|
# Clean up any existing backup files for this app
|
|
if [[ -d "$backup_dir" ]]; then
|
|
echo "Cleaning up existing backup files for '$app_name'..." >&2
|
|
rm -rf "$backup_dir"
|
|
fi
|
|
mkdir -p "$backup_dir"
|
|
|
|
local backup_files=()
|
|
|
|
# Check if app has custom backup script first
|
|
local custom_backup_script="${WC_HOME}/apps/${app_name}/backup.sh"
|
|
if [[ -x "$custom_backup_script" ]]; then
|
|
echo "Found custom backup script for '$app_name'. Running..."
|
|
"$custom_backup_script"
|
|
echo "Custom backup completed for '$app_name'."
|
|
return 0
|
|
fi
|
|
|
|
# Generic backup based on manifest discovery
|
|
local database_deps
|
|
database_deps=$(discover_database_deps "$app_name")
|
|
|
|
local pvcs
|
|
pvcs=$(discover_app_pvcs "$app_name")
|
|
|
|
if [[ -z "$database_deps" && -z "$pvcs" ]]; then
|
|
echo "No databases or PVCs found for app '$app_name'. Nothing to backup." >&2
|
|
return 0
|
|
fi
|
|
|
|
# Backup databases
|
|
for db_type in $database_deps; do
|
|
case "$db_type" in
|
|
postgres)
|
|
if db_files=$(backup_postgres_database "$app_name" "$backup_dir" "$timestamp"); then
|
|
read -ra db_file_array <<< "$db_files"
|
|
backup_files+=("${db_file_array[@]}")
|
|
fi
|
|
;;
|
|
mysql)
|
|
if db_files=$(backup_mysql_database "$app_name" "$backup_dir" "$timestamp"); then
|
|
backup_files+=("$db_files")
|
|
fi
|
|
;;
|
|
redis)
|
|
echo "Redis backup not implemented yet. Skipping."
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Backup PVCs
|
|
for pvc in $pvcs; do
|
|
if pvc_file=$(backup_pvc "$app_name" "$pvc" "$backup_dir" "$timestamp"); then
|
|
backup_files+=("$pvc_file")
|
|
fi
|
|
done
|
|
|
|
# Summary
|
|
if [[ ${#backup_files[@]} -gt 0 ]]; then
|
|
echo "----------------------------------------"
|
|
echo "Backup completed for '$app_name'"
|
|
echo "Files backed up:"
|
|
printf ' - %s\n' "${backup_files[@]}"
|
|
echo "----------------------------------------"
|
|
else
|
|
echo "No files were successfully backed up for '$app_name'." >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# --- Main Script Logic -------------------------------------------------------
|
|
main() {
|
|
|
|
if [[ $# -eq 0 || "$1" == "--help" || "$1" == "-h" ]]; then
|
|
echo "Usage: $0 <app-name> [app-name2...] | --all"
|
|
echo " $0 --list # List available apps"
|
|
exit 1
|
|
fi
|
|
|
|
require_k8s
|
|
require_yq
|
|
|
|
local staging_dir
|
|
staging_dir=$(get_staging_dir)
|
|
mkdir -p "$staging_dir"
|
|
echo "Staging backups at: $staging_dir"
|
|
|
|
if [[ "$1" == "--list" ]]; then
|
|
echo "Available apps:"
|
|
find "${WC_HOME}/apps" -maxdepth 1 -type d -not -path "${WC_HOME}/apps" -exec basename {} \; | sort
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$1" == "--all" ]]; then
|
|
echo "Backing up all apps..."
|
|
local apps
|
|
mapfile -t apps < <(find "${WC_HOME}/apps" -maxdepth 1 -type d -not -path "${WC_HOME}/apps" -exec basename {} \;)
|
|
for app in "${apps[@]}"; do
|
|
if ! backup_app "$app" "$staging_dir"; then
|
|
echo "Backup failed for '$app', continuing with next app..." >&2
|
|
fi
|
|
done
|
|
else
|
|
# Backup specific apps
|
|
local failed_apps=()
|
|
for app in "$@"; do
|
|
if ! backup_app "$app" "$staging_dir"; then
|
|
failed_apps+=("$app")
|
|
fi
|
|
done
|
|
|
|
if [[ ${#failed_apps[@]} -gt 0 ]]; then
|
|
echo "The following app backups failed: ${failed_apps[*]}" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "All backups completed successfully."
|
|
}
|
|
|
|
main "$@" |