Paul Payne
6d0332e32b
- Introduced `docx-extractor` for extracting content and metadata from Word .docx files. - Added `docx2md` for converting Word .docx files to Markdown format. - Implemented `gpt` for generating text using OpenAI's GPT models. - Created `html2text` for converting HTML content to plain text. - Developed `mbox2eml` for converting MBOX mailbox files to individual .eml files. - Added `md2pdf` for converting Markdown files to PDF format. - Introduced `mdscraper` for combining text files into a single markdown document. - Created `pdf-extractor` for extracting content and metadata from PDF files. - Developed `pdf2md` for converting PDF files to Markdown format. - Implemented `protonmail` for managing ProtonMail emails via Bridge. - Added `schedule` for managing systemd timers and services. - Introduced `search` for managing searchable collections of files. - Added `text-extractor` for extracting content and metadata from text files. - Removed outdated recommendations document.
290 lines
9.8 KiB
Python
Executable File
290 lines
9.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
backup-collect: Collect files from various sources into backup directory
|
|
|
|
Incrementally backs up specified directories to the central backup location
|
|
using rsync. Creates hardlinks to unchanged files from previous backups to
|
|
save disk space.
|
|
|
|
Usage: backup-collect [options]
|
|
|
|
Examples:
|
|
backup-collect # Backup using default configuration
|
|
backup-collect -c custom.json # Use custom configuration file
|
|
backup-collect --full # Perform a full backup instead of incremental
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import json
|
|
import argparse
|
|
import subprocess
|
|
import datetime
|
|
import logging
|
|
|
|
# Constants
|
|
DEFAULT_CONFIG_DIR = os.path.expanduser("~/.config/toolkit")
|
|
DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_CONFIG_DIR, "backup-config.json")
|
|
DEFAULT_BACKUP_DIR = os.environ.get("DBACKUP", "/data/backup")
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
logger = logging.getLogger("backup-collect")
|
|
|
|
|
|
def create_default_config(config_path):
|
|
"""Create a default configuration file if none exists."""
|
|
default_config = {
|
|
"sources": [
|
|
{
|
|
"name": "config",
|
|
"paths": [
|
|
os.path.expanduser("~/.config"),
|
|
],
|
|
"exclusions": [
|
|
os.path.expanduser("~/.config/Code/Cache/*"),
|
|
"*/Cache/*",
|
|
"*/CacheStorage/*",
|
|
],
|
|
}
|
|
],
|
|
"retention": {"local": 7},
|
|
"schedule": {"incremental": "daily", "full": "weekly"},
|
|
}
|
|
|
|
# Create config directory if it doesn't exist
|
|
os.makedirs(os.path.dirname(config_path), exist_ok=True)
|
|
|
|
with open(config_path, "w") as f:
|
|
json.dump(default_config, f, indent=2)
|
|
|
|
logger.info(f"Created default configuration at {config_path}")
|
|
return default_config
|
|
|
|
|
|
def load_config(config_path):
|
|
"""Load the configuration file or create default if it doesn't exist."""
|
|
if not os.path.exists(config_path):
|
|
return create_default_config(config_path)
|
|
|
|
try:
|
|
with open(config_path, "r") as f:
|
|
config = json.load(f)
|
|
logger.info(f"Loaded configuration from {config_path}")
|
|
return config
|
|
except Exception as e:
|
|
logger.error(f"Error loading configuration: {e}")
|
|
logger.info("Using default configuration")
|
|
return create_default_config(config_path + ".default")
|
|
|
|
|
|
def run_backup(source, backup_dir, date_str, incremental=True):
|
|
"""Run rsync backup for a given source configuration."""
|
|
source_name = source["name"]
|
|
source_paths = source["paths"]
|
|
exclusions = source.get("exclusions", [])
|
|
|
|
# Create dated and latest directory names
|
|
dated_dir = os.path.join(backup_dir, f"{source_name}-{date_str}")
|
|
latest_dir = os.path.join(backup_dir, f"{source_name}-latest")
|
|
previous_dir = os.path.join(backup_dir, f"{source_name}-previous")
|
|
|
|
logger.info(f"Backing up {source_name} to {dated_dir}")
|
|
|
|
# Create backup directory if it doesn't exist
|
|
os.makedirs(dated_dir, exist_ok=True)
|
|
|
|
for source_path in source_paths:
|
|
# Expand user paths like ~/.config
|
|
expanded_path = os.path.expanduser(source_path)
|
|
|
|
if not os.path.exists(expanded_path):
|
|
logger.warning(f"Source path does not exist: {expanded_path}")
|
|
continue
|
|
|
|
# Prepare rsync command
|
|
rsync_cmd = ["rsync", "-av", "--delete", "--block-size=131072"]
|
|
|
|
# Add exclusions
|
|
for exclusion in exclusions:
|
|
rsync_cmd.extend(["--exclude", exclusion])
|
|
|
|
# For incremental backups, use hardlinks to previous backup
|
|
if incremental and os.path.exists(latest_dir):
|
|
rsync_cmd.extend(["--link-dest", latest_dir])
|
|
|
|
# Add source and destination with path preservation
|
|
if os.path.isdir(expanded_path):
|
|
# For directories, preserve the trailing slash behavior
|
|
source_with_slash = (
|
|
expanded_path if expanded_path.endswith("/") else expanded_path + "/"
|
|
)
|
|
rsync_cmd.extend(["-R", source_with_slash]) # -R preserves relative path
|
|
rsync_cmd.append(dated_dir + "/")
|
|
else:
|
|
# For files, preserve the full path structure
|
|
rsync_cmd.extend(["-R", expanded_path]) # -R preserves relative path
|
|
rsync_cmd.append(dated_dir + "/")
|
|
|
|
# Execute rsync
|
|
logger.info(f"Running: {' '.join(rsync_cmd)}")
|
|
try:
|
|
# Use check=False to continue even if there are errors
|
|
result = subprocess.run(
|
|
rsync_cmd, check=False, capture_output=True, text=True
|
|
)
|
|
if result.returncode == 0:
|
|
logger.info(f"Backup of {expanded_path} completed successfully")
|
|
else:
|
|
logger.error(f"Backup failed for {expanded_path}: {result.stderr}")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"Backup failed for {expanded_path}: {e}")
|
|
return False
|
|
|
|
# Update symlinks
|
|
try:
|
|
# Move previous latest to previous if it exists
|
|
if os.path.exists(latest_dir) and os.path.islink(latest_dir):
|
|
# Remove existing previous symlink first
|
|
if os.path.exists(previous_dir) or os.path.islink(previous_dir):
|
|
os.unlink(previous_dir)
|
|
os.symlink(os.readlink(latest_dir), previous_dir)
|
|
|
|
# Update latest symlink
|
|
if os.path.exists(latest_dir) or os.path.islink(latest_dir):
|
|
os.unlink(latest_dir)
|
|
os.symlink(os.path.basename(dated_dir), latest_dir)
|
|
logger.info(f"Updated symlinks for {source_name}")
|
|
except Exception as e:
|
|
logger.error(f"Failed to update symlinks: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def cleanup_old_backups(source, backup_dir, retention):
|
|
"""Remove old backups beyond the retention period."""
|
|
source_name = source["name"]
|
|
logger.info(f"Cleaning up old backups for {source_name}")
|
|
|
|
try:
|
|
# Find all dated backups for this source
|
|
backups = sorted(
|
|
[
|
|
d
|
|
for d in os.listdir(backup_dir)
|
|
if os.path.isdir(os.path.join(backup_dir, d))
|
|
and d.startswith(f"{source_name}-")
|
|
and not os.path.islink(os.path.join(backup_dir, d))
|
|
]
|
|
)
|
|
|
|
# Keep only the most recent 'retention' number of backups
|
|
if len(backups) > retention:
|
|
to_delete = backups[:-retention]
|
|
for backup in to_delete:
|
|
backup_path = os.path.join(backup_dir, backup)
|
|
logger.info(f"Removing old backup: {backup_path}")
|
|
# Use subprocess to remove directories that might have read-only files
|
|
subprocess.run(["rm", "-rf", backup_path], check=True)
|
|
|
|
logger.info(f"Cleanup completed, kept {min(len(backups), retention)} backups")
|
|
except Exception as e:
|
|
logger.error(f"Error during cleanup: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Collect files from various sources into backup directory",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=__doc__.split("\n\n", 1)[1]
|
|
if __doc__
|
|
else "", # Use the docstring as extended help
|
|
)
|
|
parser.add_argument(
|
|
"-c", "--config", help=f"Path to config file (default: {DEFAULT_CONFIG_FILE})"
|
|
)
|
|
parser.add_argument(
|
|
"-b",
|
|
"--backup-dir",
|
|
help=f"Backup directory (default: $DBACKUP or {DEFAULT_BACKUP_DIR})",
|
|
)
|
|
parser.add_argument(
|
|
"-f",
|
|
"--full",
|
|
action="store_true",
|
|
help="Perform a full backup instead of incremental",
|
|
)
|
|
parser.add_argument(
|
|
"-v", "--verbose", action="store_true", help="Enable verbose output"
|
|
)
|
|
parser.add_argument(
|
|
"-d", "--debug", action="store_true", help="Enable debug output"
|
|
)
|
|
parser.add_argument("--version", action="version", version="backup-collect 1.0.0")
|
|
args = parser.parse_args()
|
|
|
|
# Set verbosity
|
|
if args.verbose or args.debug:
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
# Extra debug info
|
|
if args.debug:
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
|
|
# Load configuration
|
|
config_path = args.config or DEFAULT_CONFIG_FILE
|
|
config = load_config(config_path)
|
|
|
|
# Determine backup directory
|
|
backup_dir = args.backup_dir or DEFAULT_BACKUP_DIR
|
|
os.makedirs(backup_dir, exist_ok=True)
|
|
logger.info(f"Using backup directory: {backup_dir}")
|
|
|
|
# Current date for backup naming
|
|
date_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
|
|
# Determine if using incremental backup
|
|
incremental = not args.full
|
|
logger.info(f"Performing {'incremental' if incremental else 'full'} backup")
|
|
|
|
success = True
|
|
|
|
# Debug output of all config
|
|
if args.debug:
|
|
logger.debug(f"Using config: {json.dumps(config, indent=2)}")
|
|
logger.debug(f"Backup dir: {backup_dir}")
|
|
logger.debug(f"Sources to backup: {len(config.get('sources', []))}")
|
|
for source in config.get("sources", []):
|
|
logger.debug(f"Source: {source['name']}, Paths: {source['paths']}")
|
|
|
|
# Process each source
|
|
for source in config.get("sources", []):
|
|
logger.info(f"Processing source: {source['name']}")
|
|
if not run_backup(source, backup_dir, date_str, incremental):
|
|
logger.error(f"Backup failed for {source['name']}")
|
|
success = False
|
|
|
|
# Cleanup old backups
|
|
retention = config.get("retention", {}).get("local", 7)
|
|
cleanup_old_backups(source, backup_dir, retention)
|
|
|
|
if success:
|
|
logger.info("✅ All backups completed successfully")
|
|
return 0
|
|
else:
|
|
logger.error("One or more backups failed")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|