CSTF/wild-cloud
1
1
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects 1 Releases 1 Wiki Activity

Compare commits

base: CSTF:golang
Branches Tags
CSTF:main CSTF:golang CSTF:apps/jellyfin CSTF:apps/homepage CSTF:apps/homeassistant CSTF:apps/codimd
CSTF:POC
...
compare: CSTF:61460b63a32b007b2ff6b37f7ae3f8f5959caa92
Branches Tags
CSTF:main CSTF:golang CSTF:apps/jellyfin CSTF:apps/homepage CSTF:apps/homeassistant CSTF:apps/codimd
CSTF:POC

10 Commits
golang ... 61460b63a3

Author SHA1 Message Date
Paul Payne
61460b63a3 Adds more to wild-backup. 2025-08-31 15:02:35 -07:00
Paul Payne
9f302e0e29 Update backup/restore docs. 2025-08-31 15:00:00 -07:00
Paul Payne
f83763b070 Fix immich secret key ref. 2025-08-31 14:49:35 -07:00
Paul Payne
c9ab5a31d8 Discourse app fixes. 2025-08-31 14:46:42 -07:00
Paul Payne
1aa9f1050d Update docs. 2025-08-31 14:30:09 -07:00
Paul Payne
3b8b6de338 Removes PXE booting from dnsmasq setup. 2025-08-31 12:44:53 -07:00
Paul Payne
fd1ba7fbe0 Provide disk size for select of disks during setup of cluster nodes. Setup NEXT worker #. 2025-08-31 11:56:26 -07:00
Paul Payne
5edf14695f Sets password in REDIS app. 2025-08-31 11:54:18 -07:00
Paul Payne
9a7b2fec72 Merge branch 'main' of https://git.civilsociety.dev/CSTF/wild-cloud 2025-08-23 06:15:08 -07:00
Paul Payne
3a9bd7c6b3 Add link to Wild Cloud forum to README. 2025-08-21 14:04:55 +00:00
36 changed files with 1143 additions and 1266 deletions
Expand all files Collapse all files

2
README.md
View File

@@ -2,7 +2,7 @@
Welcome! So excited you're here!
_This project is massively in progress. It's not ready to be used yet (even though I am using it as I develop it). This is published publicly for transparency. If you want to help out, please get in touch._
_This project is massively in progress. It's not ready to be used yet (even though I am using it as I develop it). This is published publicly for transparency. If you want to help out, please [get in touch](https://forum.civilsociety.dev/c/wild-cloud/5)._
## Why Build Your Own Cloud?

26
apps/discourse/configmap.yaml
View File

@@ -11,33 +11,21 @@ data:
DISCOURSE_SITE_NAME: "{{ .apps.discourse.siteName }}"
DISCOURSE_USERNAME: "{{ .apps.discourse.adminUsername }}"
DISCOURSE_EMAIL: "{{ .apps.discourse.adminEmail }}"
DISCOURSE_REDIS_HOST: "{{ .apps.discourse.redisHostname }}"
DISCOURSE_REDIS_PORT_NUMBER: "6379"
DISCOURSE_DATABASE_HOST: "{{ .apps.discourse.dbHostname }}"
DISCOURSE_DATABASE_PORT_NUMBER: "5432"
DISCOURSE_DATABASE_NAME: "{{ .apps.discourse.dbName }}"
DISCOURSE_DATABASE_USER: "{{ .apps.discourse.dbUsername }}"
# DISCOURSE_SMTP_ADDRESS: "{{ .apps.discourse.smtp.host }}"
# DISCOURSE_SMTP_PORT: "{{ .apps.discourse.smtp.port }}"
# DISCOURSE_SMTP_USER_NAME: "{{ .apps.discourse.smtp.user }}"
# DISCOURSE_SMTP_ENABLE_START_TLS: "{{ .apps.discourse.smtp.startTls }}"
# DISCOURSE_SMTP_AUTHENTICATION: "login"
# Bitnami specific environment variables (diverges from the original)
# https://techdocs.broadcom.com/us/en/vmware-tanzu/bitnami-secure-images/bitnami-secure-images/services/bsi-app-doc/apps-containers-discourse-index.html
DISCOURSE_SMTP_HOST: "{{ .apps.discourse.smtp.host }}"
DISCOURSE_SMTP_PORT_NUMBER: "{{ .apps.discourse.smtp.port }}"
DISCOURSE_SMTP_PORT: "{{ .apps.discourse.smtp.port }}"
DISCOURSE_SMTP_USER: "{{ .apps.discourse.smtp.user }}"
DISCOURSE_SMTP_ENABLE_START_TLS: "{{ .apps.discourse.smtp.startTls }}"
DISCOURSE_SMTP_AUTH: "login"
DISCOURSE_SMTP_PROTOCOL: "tls"
DISCOURSE_SMTP_AUTH: "login"
DISCOURSE_PRECOMPILE_ASSETS: "false"
# SMTP_HOST: "{{ .apps.discourse.smtp.host }}"
# SMTP_PORT: "{{ .apps.discourse.smtp.port }}"
# SMTP_USER_NAME: "{{ .apps.discourse.smtp.user }}"
# SMTP_TLS: "{{ .apps.discourse.smtp.tls }}"
# SMTP_ENABLE_START_TLS: "{{ .apps.discourse.smtp.startTls }}"
# SMTP_AUTHENTICATION: "login"
# DISCOURSE_PRECOMPILE_ASSETS: "false"
# DISCOURSE_SKIP_INSTALL: "no"
# DISCOURSE_SKIP_BOOTSTRAP: "yes"

8
apps/discourse/deployment.yaml
View File

@@ -37,7 +37,7 @@ spec:
initContainers:
containers:
- name: discourse
image: { { .apps.discourse.image } }
image: docker.io/bitnami/discourse:3.4.7-debian-12-r0
imagePullPolicy: "IfNotPresent"
securityContext:
allowPrivilegeEscalation: false
@@ -85,7 +85,7 @@ spec:
valueFrom:
secretKeyRef:
name: discourse-secrets
key: apps.discourse.redisPassword
key: apps.redis.password
- name: DISCOURSE_SECRET_KEY_BASE
valueFrom:
secretKeyRef:
@@ -139,7 +139,7 @@ spec:
mountPath: /bitnami/discourse
subPath: discourse
- name: sidekiq
image: { { .apps.discourse.sidekiqImage } }
image: docker.io/bitnami/discourse:3.4.7-debian-12-r0
imagePullPolicy: "IfNotPresent"
securityContext:
allowPrivilegeEscalation: false
@@ -182,7 +182,7 @@ spec:
valueFrom:
secretKeyRef:
name: discourse-secrets
key: apps.discourse.redisPassword
key: apps.redis.password
- name: DISCOURSE_SECRET_KEY_BASE
valueFrom:
secretKeyRef:

4
apps/discourse/manifest.yaml
View File

@@ -6,8 +6,6 @@ requires:
- name: postgres
- name: redis
defaultConfig:
image: docker.io/bitnami/discourse:3.4.7-debian-12-r0
sidekiqImage: docker.io/bitnami/discourse:3.4.7-debian-12-r0
timezone: UTC
port: 8080
storage: 10Gi
@@ -32,7 +30,7 @@ requiredSecrets:
- apps.discourse.adminPassword
- apps.discourse.dbPassword
- apps.discourse.dbUrl
- apps.discourse.redisPassword
- apps.redis.password
- apps.discourse.secretKeyBase
- apps.discourse.smtpPassword
- apps.postgres.password

2
apps/immich/db-init-job.yaml
View File

@@ -52,7 +52,7 @@ spec:
- name: POSTGRES_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secrets
name: immich-secrets
key: apps.postgres.password
- name: DB_HOSTNAME
value: "{{ .apps.immich.dbHostname }}"

9
apps/redis/deployment.yaml
View File

@@ -21,4 +21,13 @@ spec:
env:
- name: TZ
value: "{{ .apps.redis.timezone }}"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-secrets
key: apps.redis.password
command:
- redis-server
- --requirepass
- $(REDIS_PASSWORD)
restartPolicy: Always

207
bin/wild-backup
View File

@@ -4,6 +4,72 @@
set -e
set -o pipefail
# Parse command line flags
BACKUP_HOME=true
BACKUP_APPS=true
BACKUP_CLUSTER=true
show_help() {
echo "Usage: $0 [OPTIONS]"
echo "Backup components of your wild-cloud infrastructure"
echo ""
echo "Options:"
echo " --home-only Backup only WC_HOME (wild-cloud configuration)"
echo " --apps-only Backup only applications (databases and PVCs)"
echo " --cluster-only Backup only Kubernetes cluster resources"
echo " --no-home Skip WC_HOME backup"
echo " --no-apps Skip application backups"
echo " --no-cluster Skip cluster resource backup"
echo " -h, --help Show this help message"
echo ""
echo "Default: Backup all components (home, apps, cluster)"
}
# Process command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
--home-only)
BACKUP_HOME=true
BACKUP_APPS=false
BACKUP_CLUSTER=false
shift
;;
--apps-only)
BACKUP_HOME=false
BACKUP_APPS=true
BACKUP_CLUSTER=false
shift
;;
--cluster-only)
BACKUP_HOME=false
BACKUP_APPS=false
BACKUP_CLUSTER=true
shift
;;
--no-home)
BACKUP_HOME=false
shift
;;
--no-apps)
BACKUP_APPS=false
shift
;;
--no-cluster)
BACKUP_CLUSTER=false
shift
;;
-h|--help)
show_help
exit 0
;;
*)
echo "Unknown option: $1"
show_help
exit 1
;;
esac
done
# Initialize Wild Cloud environment
if [ -z "${WC_ROOT}" ]; then
echo "WC_ROOT is not set."
@@ -46,33 +112,134 @@ else
echo "Repository initialized successfully."
fi
# Backup entire WC_HOME.
restic --verbose --tag wild-cloud --tag wc-home --tag "$(date +%Y-%m-%d)" backup $WC_HOME
# TODO: Ignore wild cloud cache?
# Backup entire WC_HOME
if [ "$BACKUP_HOME" = true ]; then
echo "Backing up WC_HOME..."
restic --verbose --tag wild-cloud --tag wc-home --tag "$(date +%Y-%m-%d)" backup $WC_HOME
echo "WC_HOME backup completed."
# TODO: Ignore wild cloud cache?
else
echo "Skipping WC_HOME backup."
fi
mkdir -p "$STAGING_DIR"
# Run backup for all apps at once
echo "Running backup for all apps..."
wild-app-backup --all
if [ "$BACKUP_APPS" = true ]; then
echo "Running backup for all apps..."
wild-app-backup --all
# Upload each app's backup to restic individually
for app_dir in "$STAGING_DIR"/apps/*; do
if [ ! -d "$app_dir" ]; then
continue
# Upload each app's backup to restic individually
for app_dir in "$STAGING_DIR"/apps/*; do
if [ ! -d "$app_dir" ]; then
continue
fi
app="$(basename "$app_dir")"
echo "Uploading backup for app: $app"
restic --verbose --tag wild-cloud --tag "$app" --tag "$(date +%Y-%m-%d)" backup "$app_dir"
echo "Backup for app '$app' completed."
done
else
echo "Skipping application backups."
fi
# --- etcd Backup Function ----------------------------------------------------
backup_etcd() {
local cluster_backup_dir="$1"
local etcd_backup_file="$cluster_backup_dir/etcd-snapshot.db"
echo "Creating etcd snapshot..."
# For Talos, we use talosctl to create etcd snapshots
if command -v talosctl >/dev/null 2>&1; then
# Try to get etcd snapshot via talosctl (works for Talos clusters)
local control_plane_nodes
control_plane_nodes=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}' | tr ' ' '\n' | head -1)
if [[ -n "$control_plane_nodes" ]]; then
echo "Using talosctl to backup etcd from control plane node: $control_plane_nodes"
if talosctl --nodes "$control_plane_nodes" etcd snapshot "$etcd_backup_file"; then
echo " etcd backup created: $etcd_backup_file"
return 0
else
echo " talosctl etcd snapshot failed, trying alternative method..."
fi
else
echo " No control plane nodes found for talosctl method"
fi
fi
app="$(basename "$app_dir")"
echo "Uploading backup for app: $app"
restic --verbose --tag wild-cloud --tag "$app" --tag "$(date +%Y-%m-%d)" backup "$app_dir"
echo "Backup for app '$app' completed."
done
# Alternative: Try to backup via etcd pod if available
local etcd_pod
etcd_pod=$(kubectl get pods -n kube-system -l component=etcd -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$etcd_pod" ]]; then
echo "Using etcd pod: $etcd_pod"
# Create snapshot using etcdctl inside the etcd pod
if kubectl exec -n kube-system "$etcd_pod" -- etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot save /tmp/etcd-snapshot.db; then
# Copy snapshot out of pod
kubectl cp -n kube-system "$etcd_pod:/tmp/etcd-snapshot.db" "$etcd_backup_file"
# Clean up temporary file in pod
kubectl exec -n kube-system "$etcd_pod" -- rm -f /tmp/etcd-snapshot.db
echo " etcd backup created: $etcd_backup_file"
return 0
else
echo " etcd pod snapshot failed"
fi
else
echo " No etcd pod found in kube-system namespace"
fi
# Final fallback: Try direct etcdctl if available on local system
if command -v etcdctl >/dev/null 2>&1; then
echo "Attempting local etcdctl backup..."
# This would need proper certificates and endpoints configured
echo " Local etcdctl backup not implemented (requires certificate configuration)"
fi
echo " Warning: Could not create etcd backup - no working method found"
echo " Consider installing talosctl or ensuring etcd pods are accessible"
return 1
}
# Back up Kubernetes resources
# kubectl get all -A -o yaml > "$BACKUP_DIR/all-resources.yaml"
# kubectl get secrets -A -o yaml > "$BACKUP_DIR/secrets.yaml"
# kubectl get configmaps -A -o yaml > "$BACKUP_DIR/configmaps.yaml"
# Back up Kubernetes cluster resources
if [ "$BACKUP_CLUSTER" = true ]; then
echo "Backing up Kubernetes cluster resources..."
CLUSTER_BACKUP_DIR="$STAGING_DIR/cluster"
# Clean up any existing cluster backup files
if [[ -d "$CLUSTER_BACKUP_DIR" ]]; then
echo "Cleaning up existing cluster backup files..."
rm -rf "$CLUSTER_BACKUP_DIR"
fi
mkdir -p "$CLUSTER_BACKUP_DIR"
# Back up persistent volumes
# TODO: Add logic to back up persistent volume data
kubectl get all -A -o yaml > "$CLUSTER_BACKUP_DIR/all-resources.yaml"
kubectl get secrets -A -o yaml > "$CLUSTER_BACKUP_DIR/secrets.yaml"
kubectl get configmaps -A -o yaml > "$CLUSTER_BACKUP_DIR/configmaps.yaml"
kubectl get persistentvolumes -o yaml > "$CLUSTER_BACKUP_DIR/persistentvolumes.yaml"
kubectl get persistentvolumeclaims -A -o yaml > "$CLUSTER_BACKUP_DIR/persistentvolumeclaims.yaml"
kubectl get storageclasses -o yaml > "$CLUSTER_BACKUP_DIR/storageclasses.yaml"
echo "Backing up etcd..."
backup_etcd "$CLUSTER_BACKUP_DIR"
echo "Cluster resources backed up to $CLUSTER_BACKUP_DIR"
# Upload cluster backup to restic
echo "Uploading cluster backup to restic..."
restic --verbose --tag wild-cloud --tag cluster --tag "$(date +%Y-%m-%d)" backup "$CLUSTER_BACKUP_DIR"
echo "Cluster backup completed."
else
echo "Skipping cluster backup."
fi
echo "Backup completed: $BACKUP_DIR"

141
bin/wild-dnsmasq-install
View File

@@ -68,85 +68,92 @@ fi
# Create setup bundle.
# Copy iPXE bootloader to ipxe-web from cached assets.
echo "Copying Talos PXE assets from cache..."
PXE_WEB_ROOT="${BUNDLE_DIR}/ipxe-web"
mkdir -p "${PXE_WEB_ROOT}/amd64"
cp "${DNSMASQ_SETUP_DIR}/boot.ipxe" "${PXE_WEB_ROOT}/boot.ipxe"
# The following was a completely fine process for making your dnsmasq server
# also serve PXE boot assets for the cluster. However, after using it for a bit,
# it seems to be more complexity for no additional benefit when the operators
# can just use USB keys.
# Get schematic ID from override or config
if [ -n "$SCHEMATIC_ID_OVERRIDE" ]; then
SCHEMATIC_ID="$SCHEMATIC_ID_OVERRIDE"
echo "Using schematic ID from command line: $SCHEMATIC_ID"
else
SCHEMATIC_ID=$(wild-config cluster.nodes.talos.schematicId)
if [ -z "$SCHEMATIC_ID" ] || [ "$SCHEMATIC_ID" = "null" ]; then
echo "Error: No schematic ID found in config"
echo "Please run 'wild-setup' first to configure your cluster"
echo "Or specify one with --schematic-id option"
exit 1
fi
echo "Using schematic ID from config: $SCHEMATIC_ID"
fi
## Setup PXE boot assets
# Define cache directories using new structure
CACHE_DIR="${WC_HOME}/.wildcloud"
SCHEMATIC_CACHE_DIR="${CACHE_DIR}/node-boot-assets/${SCHEMATIC_ID}"
PXE_CACHE_DIR="${SCHEMATIC_CACHE_DIR}/pxe"
IPXE_CACHE_DIR="${SCHEMATIC_CACHE_DIR}/ipxe"
# # Copy iPXE bootloader to ipxe-web from cached assets.
# echo "Copying Talos PXE assets from cache..."
# PXE_WEB_ROOT="${BUNDLE_DIR}/ipxe-web"
# mkdir -p "${PXE_WEB_ROOT}/amd64"
# cp "${DNSMASQ_SETUP_DIR}/boot.ipxe" "${PXE_WEB_ROOT}/boot.ipxe"
# Check if cached assets exist
KERNEL_CACHE_PATH="${PXE_CACHE_DIR}/amd64/vmlinuz"
INITRAMFS_CACHE_PATH="${PXE_CACHE_DIR}/amd64/initramfs.xz"
# # Get schematic ID from override or config
# if [ -n "$SCHEMATIC_ID_OVERRIDE" ]; then
# SCHEMATIC_ID="$SCHEMATIC_ID_OVERRIDE"
# echo "Using schematic ID from command line: $SCHEMATIC_ID"
# else
# SCHEMATIC_ID=$(wild-config cluster.nodes.talos.schematicId)
# if [ -z "$SCHEMATIC_ID" ] || [ "$SCHEMATIC_ID" = "null" ]; then
# echo "Error: No schematic ID found in config"
# echo "Please run 'wild-setup' first to configure your cluster"
# echo "Or specify one with --schematic-id option"
# exit 1
# fi
# echo "Using schematic ID from config: $SCHEMATIC_ID"
# fi
if [ ! -f "${KERNEL_CACHE_PATH}" ] || [ ! -f "${INITRAMFS_CACHE_PATH}" ]; then
echo "Error: Talos PXE assets not found in cache for schematic ID: ${SCHEMATIC_ID}"
echo "Expected locations:"
echo " Kernel: ${KERNEL_CACHE_PATH}"
echo " Initramfs: ${INITRAMFS_CACHE_PATH}"
echo ""
echo "Please run 'wild-cluster-node-boot-assets-download' first to download and cache the assets."
exit 1
fi
# # Define cache directories using new structure
# CACHE_DIR="${WC_HOME}/.wildcloud"
# SCHEMATIC_CACHE_DIR="${CACHE_DIR}/node-boot-assets/${SCHEMATIC_ID}"
# PXE_CACHE_DIR="${SCHEMATIC_CACHE_DIR}/pxe"
# IPXE_CACHE_DIR="${SCHEMATIC_CACHE_DIR}/ipxe"
# Copy Talos PXE assets from cache
echo "Copying Talos kernel from cache..."
cp "${KERNEL_CACHE_PATH}" "${PXE_WEB_ROOT}/amd64/vmlinuz"
echo "✅ Talos kernel copied from cache"
# # Check if cached assets exist
# KERNEL_CACHE_PATH="${PXE_CACHE_DIR}/amd64/vmlinuz"
# INITRAMFS_CACHE_PATH="${PXE_CACHE_DIR}/amd64/initramfs.xz"
echo "Copying Talos initramfs from cache..."
cp "${INITRAMFS_CACHE_PATH}" "${PXE_WEB_ROOT}/amd64/initramfs.xz"
echo "✅ Talos initramfs copied from cache"
# if [ ! -f "${KERNEL_CACHE_PATH}" ] || [ ! -f "${INITRAMFS_CACHE_PATH}" ]; then
# echo "Error: Talos PXE assets not found in cache for schematic ID: ${SCHEMATIC_ID}"
# echo "Expected locations:"
# echo " Kernel: ${KERNEL_CACHE_PATH}"
# echo " Initramfs: ${INITRAMFS_CACHE_PATH}"
# echo ""
# echo "Please run 'wild-cluster-node-boot-assets-download' first to download and cache the assets."
# exit 1
# fi
# Copy iPXE bootloader files from cache
echo "Copying iPXE bootloader files from cache..."
FTPD_DIR="${BUNDLE_DIR}/pxe-ftpd"
mkdir -p "${FTPD_DIR}"
# # Copy Talos PXE assets from cache
# echo "Copying Talos kernel from cache..."
# cp "${KERNEL_CACHE_PATH}" "${PXE_WEB_ROOT}/amd64/vmlinuz"
# echo "✅ Talos kernel copied from cache"
# Check if iPXE assets exist in cache
IPXE_EFI_CACHE="${IPXE_CACHE_DIR}/ipxe.efi"
IPXE_BIOS_CACHE="${IPXE_CACHE_DIR}/undionly.kpxe"
IPXE_ARM64_CACHE="${IPXE_CACHE_DIR}/ipxe-arm64.efi"
# echo "Copying Talos initramfs from cache..."
# cp "${INITRAMFS_CACHE_PATH}" "${PXE_WEB_ROOT}/amd64/initramfs.xz"
# echo "✅ Talos initramfs copied from cache"
if [ ! -f "${IPXE_EFI_CACHE}" ] || [ ! -f "${IPXE_BIOS_CACHE}" ] || [ ! -f "${IPXE_ARM64_CACHE}" ]; then
echo "Error: iPXE bootloader assets not found in cache for schematic ID: ${SCHEMATIC_ID}"
echo "Expected locations:"
echo " iPXE EFI: ${IPXE_EFI_CACHE}"
echo " iPXE BIOS: ${IPXE_BIOS_CACHE}"
echo " iPXE ARM64: ${IPXE_ARM64_CACHE}"
echo ""
echo "Please run 'wild-cluster-node-boot-assets-download' first to download and cache the assets."
exit 1
fi
# # Copy iPXE bootloader files from cache
# echo "Copying iPXE bootloader files from cache..."
# FTPD_DIR="${BUNDLE_DIR}/pxe-ftpd"
# mkdir -p "${FTPD_DIR}"
# Copy iPXE assets from cache
cp "${IPXE_EFI_CACHE}" "${FTPD_DIR}/ipxe.efi"
cp "${IPXE_BIOS_CACHE}" "${FTPD_DIR}/undionly.kpxe"
cp "${IPXE_ARM64_CACHE}" "${FTPD_DIR}/ipxe-arm64.efi"
echo "✅ iPXE bootloader files copied from cache"
# # Check if iPXE assets exist in cache
# IPXE_EFI_CACHE="${IPXE_CACHE_DIR}/ipxe.efi"
# IPXE_BIOS_CACHE="${IPXE_CACHE_DIR}/undionly.kpxe"
# IPXE_ARM64_CACHE="${IPXE_CACHE_DIR}/ipxe-arm64.efi"
# if [ ! -f "${IPXE_EFI_CACHE}" ] || [ ! -f "${IPXE_BIOS_CACHE}" ] || [ ! -f "${IPXE_ARM64_CACHE}" ]; then
# echo "Error: iPXE bootloader assets not found in cache for schematic ID: ${SCHEMATIC_ID}"
# echo "Expected locations:"
# echo " iPXE EFI: ${IPXE_EFI_CACHE}"
# echo " iPXE BIOS: ${IPXE_BIOS_CACHE}"
# echo " iPXE ARM64: ${IPXE_ARM64_CACHE}"
# echo ""
# echo "Please run 'wild-cluster-node-boot-assets-download' first to download and cache the assets."
# exit 1
# fi
# # Copy iPXE assets from cache
# cp "${IPXE_EFI_CACHE}" "${FTPD_DIR}/ipxe.efi"
# cp "${IPXE_BIOS_CACHE}" "${FTPD_DIR}/undionly.kpxe"
# cp "${IPXE_ARM64_CACHE}" "${FTPD_DIR}/ipxe-arm64.efi"
# echo "✅ iPXE bootloader files copied from cache"
cp "${DNSMASQ_SETUP_DIR}/nginx.conf" "${BUNDLE_DIR}/nginx.conf"
# cp "${DNSMASQ_SETUP_DIR}/nginx.conf" "${BUNDLE_DIR}/nginx.conf"
cp "${DNSMASQ_SETUP_DIR}/dnsmasq.conf" "${BUNDLE_DIR}/dnsmasq.conf"
cp "${DNSMASQ_SETUP_DIR}/setup.sh" "${BUNDLE_DIR}/setup.sh"

18
bin/wild-node-detect
View File

@@ -124,14 +124,14 @@ fi
# Discover available disks
echo "Discovering available disks..." >&2
if [ "$TALOS_MODE" = "insecure" ]; then
AVAILABLE_DISKS_RAW=$(talosctl -n "$NODE_IP" get disks --insecure -o json 2>/dev/null | \
jq -s -r '.[] | select(.spec.size > 10000000000) | .metadata.id')
DISKS_JSON=$(talosctl -n "$NODE_IP" get disks --insecure -o json 2>/dev/null | \
jq -s '[.[] | select(.spec.size > 10000000000) | {path: ("/dev/" + .metadata.id), size: .spec.size}]')
else
AVAILABLE_DISKS_RAW=$(talosctl -n "$NODE_IP" get disks -o json 2>/dev/null | \
jq -s -r '.[] | select(.spec.size > 10000000000) | .metadata.id')
DISKS_JSON=$(talosctl -n "$NODE_IP" get disks -o json 2>/dev/null | \
jq -s '[.[] | select(.spec.size > 10000000000) | {path: ("/dev/" + .metadata.id), size: .spec.size}]')
fi
if [ -z "$AVAILABLE_DISKS_RAW" ]; then
if [ "$(echo "$DISKS_JSON" | jq 'length')" -eq 0 ]; then
echo "Error: No suitable disks found (must be >10GB)" >&2
echo "Available disks:" >&2
if [ "$TALOS_MODE" = "insecure" ]; then
@@ -142,11 +142,11 @@ if [ -z "$AVAILABLE_DISKS_RAW" ]; then
exit 1
fi
# Convert to JSON array
AVAILABLE_DISKS=$(echo "$AVAILABLE_DISKS_RAW" | jq -R -s 'split("\n") | map(select(length > 0)) | map("/dev/" + .)')
# Use the disks with size info directly
AVAILABLE_DISKS="$DISKS_JSON"
# Select the first disk as default (largest first)
SELECTED_DISK=$(echo "$AVAILABLE_DISKS" | jq -r '.[0]')
# Select the first disk as default
SELECTED_DISK=$(echo "$AVAILABLE_DISKS" | jq -r '.[0].path')
echo "✅ Discovered $(echo "$AVAILABLE_DISKS" | jq -r 'length') suitable disks" >&2
echo "✅ Selected disk: $SELECTED_DISK" >&2

17
bin/wild-setup-cluster
View File

@@ -260,7 +260,7 @@ if [ "${SKIP_HARDWARE}" = false ]; then
# Parse JSON response
INTERFACE=$(echo "$NODE_INFO" | jq -r '.interface')
SELECTED_DISK=$(echo "$NODE_INFO" | jq -r '.selected_disk')
AVAILABLE_DISKS=$(echo "$NODE_INFO" | jq -r '.disks | join(", ")')
AVAILABLE_DISKS=$(echo "$NODE_INFO" | jq -r '.disks[] | "\(.path) (\((.size / 1000000000) | floor)GB)"' | paste -sd, -)
print_success "Hardware detected:"
print_info " - Interface: $INTERFACE"
@@ -272,9 +272,9 @@ if [ "${SKIP_HARDWARE}" = false ]; then
read -p "Use selected disk '$SELECTED_DISK'? (Y/n): " -r use_disk
if [[ $use_disk =~ ^[Nn]$ ]]; then
echo "Available disks:"
echo "$NODE_INFO" | jq -r '.disks[]' | nl -w2 -s') '
echo "$NODE_INFO" | jq -r '.disks[] | "\(.path) (\((.size / 1000000000) | floor)GB)"' | nl -w2 -s') '
read -p "Enter disk number: " -r disk_num
SELECTED_DISK=$(echo "$NODE_INFO" | jq -r ".disks[$((disk_num-1))]")
SELECTED_DISK=$(echo "$NODE_INFO" | jq -r ".disks[$((disk_num-1))].path")
if [ "$SELECTED_DISK" = "null" ] || [ -z "$SELECTED_DISK" ]; then
print_error "Invalid disk selection"
continue
@@ -359,6 +359,11 @@ if [ "${SKIP_HARDWARE}" = false ]; then
read -p "Do you want to register a worker node? (y/N): " -r register_worker
if [[ $register_worker =~ ^[Yy]$ ]]; then
# Find first available worker number
while [ -n "$(wild-config "cluster.nodes.active.\"${HOSTNAME_PREFIX}worker-${WORKER_COUNT}\".role" 2>/dev/null)" ] && [ "$(wild-config "cluster.nodes.active.\"${HOSTNAME_PREFIX}worker-${WORKER_COUNT}\".role" 2>/dev/null)" != "null" ]; do
WORKER_COUNT=$((WORKER_COUNT + 1))
done
NODE_NAME="${HOSTNAME_PREFIX}worker-${WORKER_COUNT}"
read -p "Enter current IP for worker node $NODE_NAME: " -r WORKER_IP
@@ -388,7 +393,7 @@ if [ "${SKIP_HARDWARE}" = false ]; then
# Parse JSON response
INTERFACE=$(echo "$WORKER_INFO" | jq -r '.interface')
SELECTED_DISK=$(echo "$WORKER_INFO" | jq -r '.selected_disk')
AVAILABLE_DISKS=$(echo "$WORKER_INFO" | jq -r '.disks | join(", ")')
AVAILABLE_DISKS=$(echo "$WORKER_INFO" | jq -r '.disks[] | "\(.path) (\((.size / 1000000000) | floor)GB)"' | paste -sd, -)
print_success "Hardware detected for worker node $NODE_NAME:"
print_info " - Interface: $INTERFACE"
@@ -400,9 +405,9 @@ if [ "${SKIP_HARDWARE}" = false ]; then
read -p "Use selected disk '$SELECTED_DISK'? (Y/n): " -r use_disk
if [[ $use_disk =~ ^[Nn]$ ]]; then
echo "Available disks:"
echo "$WORKER_INFO" | jq -r '.disks[]' | nl -w2 -s') '
echo "$WORKER_INFO" | jq -r '.disks[] | "\(.path) (\((.size / 1000000000) | floor)GB)"' | nl -w2 -s') '
read -p "Enter disk number: " -r disk_num
SELECTED_DISK=$(echo "$WORKER_INFO" | jq -r ".disks[$((disk_num-1))]")
SELECTED_DISK=$(echo "$WORKER_INFO" | jq -r ".disks[$((disk_num-1))].path")
if [ "$SELECTED_DISK" = "null" ] || [ -z "$SELECTED_DISK" ]; then
print_error "Invalid disk selection"
continue

341
docs/MAINTENANCE.md
View File

@@ -1,328 +1,23 @@
# Maintenance Guide
This guide covers essential maintenance tasks for your personal cloud infrastructure, including troubleshooting, backups, updates, and security best practices.
Keep your wild cloud running smoothly.
- [Security Best Practices](./guides/security.md)
- [Monitoring](./guides/monitoring.md)
- [Making backups](./guides/making-backups.md)
- [Restoring backups](./guides/restoring-backups.md)
## Upgrade
- [Upgrade applications](./guides/upgrade-applications.md)
- [Upgrade kubernetes](./guides/upgrade-kubernetes.md)
- [Upgrade Talos](./guides/upgrade-talos.md)
- [Upgrade Wild Cloud](./guides/upgrade-wild-cloud.md)
## Troubleshooting
### General Troubleshooting Steps
1. **Check Component Status**:
```bash
# Check all pods across all namespaces
kubectl get pods -A
# Look for pods that aren't Running or Ready
kubectl get pods -A | grep -v "Running\|Completed"
```
2. **View Detailed Pod Information**:
```bash
# Get detailed info about problematic pods
kubectl describe pod <pod-name> -n <namespace>
# Check pod logs
kubectl logs <pod-name> -n <namespace>
```
3. **Run Validation Script**:
```bash
./infrastructure_setup/validate_setup.sh
```
4. **Check Node Status**:
```bash
kubectl get nodes
kubectl describe node <node-name>
```
### Common Issues
#### Certificate Problems
If services show invalid certificates:
1. Check certificate status:
```bash
kubectl get certificates -A
```
2. Examine certificate details:
```bash
kubectl describe certificate <cert-name> -n <namespace>
```
3. Check for cert-manager issues:
```bash
kubectl get pods -n cert-manager
kubectl logs -l app=cert-manager -n cert-manager
```
4. Verify the Cloudflare API token is correctly set up:
```bash
kubectl get secret cloudflare-api-token -n internal
```
#### DNS Issues
If DNS resolution isn't working properly:
1. Check CoreDNS status:
```bash
kubectl get pods -n kube-system -l k8s-app=kube-dns
kubectl logs -l k8s-app=kube-dns -n kube-system
```
2. Verify CoreDNS configuration:
```bash
kubectl get configmap -n kube-system coredns -o yaml
```
3. Test DNS resolution from inside the cluster:
```bash
kubectl run -i --tty --rm debug --image=busybox --restart=Never -- nslookup kubernetes.default
```
#### Service Connectivity
If services can't communicate:
1. Check network policies:
```bash
kubectl get networkpolicies -A
```
2. Verify service endpoints:
```bash
kubectl get endpoints -n <namespace>
```
3. Test connectivity from within the cluster:
```bash
kubectl run -i --tty --rm debug --image=busybox --restart=Never -- wget -O- <service-name>.<namespace>
```
## Backup and Restore
### What to Back Up
1. **Persistent Data**:
- Database volumes
- Application storage
- Configuration files
2. **Kubernetes Resources**:
- Custom Resource Definitions (CRDs)
- Deployments, Services, Ingresses
- Secrets and ConfigMaps
### Backup Methods
#### Simple Backup Script
Create a backup script at `bin/backup.sh` (to be implemented):
```bash
#!/bin/bash
# Simple backup script for your personal cloud
# This is a placeholder for future implementation
BACKUP_DIR="/path/to/backups/$(date +%Y-%m-%d)"
mkdir -p "$BACKUP_DIR"
# Back up Kubernetes resources
kubectl get all -A -o yaml > "$BACKUP_DIR/all-resources.yaml"
kubectl get secrets -A -o yaml > "$BACKUP_DIR/secrets.yaml"
kubectl get configmaps -A -o yaml > "$BACKUP_DIR/configmaps.yaml"
# Back up persistent volumes
# TODO: Add logic to back up persistent volume data
echo "Backup completed: $BACKUP_DIR"
```
#### Using Velero (Recommended for Future)
[Velero](https://velero.io/) is a powerful backup solution for Kubernetes:
```bash
# Install Velero (future implementation)
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
helm install velero vmware-tanzu/velero --namespace velero --create-namespace
# Create a backup
velero backup create my-backup --include-namespaces default,internal
# Restore from backup
velero restore create --from-backup my-backup
```
### Database Backups
For database services, set up regular dumps:
```bash
# PostgreSQL backup (placeholder)
kubectl exec <postgres-pod> -n <namespace> -- pg_dump -U <username> <database> > backup.sql
# MariaDB/MySQL backup (placeholder)
kubectl exec <mariadb-pod> -n <namespace> -- mysqldump -u root -p<password> <database> > backup.sql
```
## Updates
### Updating Kubernetes (K3s)
1. Check current version:
```bash
k3s --version
```
2. Update K3s:
```bash
curl -sfL https://get.k3s.io | sh -
```
3. Verify the update:
```bash
k3s --version
kubectl get nodes
```
### Updating Infrastructure Components
1. Update the repository:
```bash
git pull
```
2. Re-run the setup script:
```bash
./infrastructure_setup/setup-all.sh
```
3. Or update specific components:
```bash
./infrastructure_setup/setup-cert-manager.sh
./infrastructure_setup/setup-dashboard.sh
# etc.
```
### Updating Applications
For Helm chart applications:
```bash
# Update Helm repositories
helm repo update
# Upgrade a specific application
./bin/helm-install <chart-name> --upgrade
```
For services deployed with `deploy-service`:
```bash
# Edit the service YAML
nano services/<service-name>/service.yaml
# Apply changes
kubectl apply -f services/<service-name>/service.yaml
```
## Security
### Best Practices
1. **Keep Everything Updated**:
- Regularly update K3s
- Update all infrastructure components
- Keep application images up to date
2. **Network Security**:
- Use internal services whenever possible
- Limit exposed services to only what's necessary
- Configure your home router's firewall properly
3. **Access Control**:
- Use strong passwords for all services
- Implement a secrets management strategy
- Rotate API tokens and keys regularly
4. **Regular Audits**:
- Review running services periodically
- Check for unused or outdated deployments
- Monitor resource usage for anomalies
### Security Scanning (Future Implementation)
Tools to consider implementing:
1. **Trivy** for image scanning:
```bash
# Example Trivy usage (placeholder)
trivy image <your-image>
```
2. **kube-bench** for Kubernetes security checks:
```bash
# Example kube-bench usage (placeholder)
kubectl apply -f https://raw.githubusercontent.com/aquasecurity/kube-bench/main/job.yaml
```
3. **Falco** for runtime security monitoring:
```bash
# Example Falco installation (placeholder)
helm repo add falcosecurity https://falcosecurity.github.io/charts
helm install falco falcosecurity/falco --namespace falco --create-namespace
```
## System Health Monitoring
### Basic Monitoring
Check system health with:
```bash
# Node resource usage
kubectl top nodes
# Pod resource usage
kubectl top pods -A
# Persistent volume claims
kubectl get pvc -A
```
### Advanced Monitoring (Future Implementation)
Consider implementing:
1. **Prometheus + Grafana** for comprehensive monitoring:
```bash
# Placeholder for future implementation
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace
```
2. **Loki** for log aggregation:
```bash
# Placeholder for future implementation
helm repo add grafana https://grafana.github.io/helm-charts
helm install loki grafana/loki-stack --namespace logging --create-namespace
```
## Additional Resources
This document will be expanded in the future with:
- Detailed backup and restore procedures
- Monitoring setup instructions
- Comprehensive security hardening guide
- Automated maintenance scripts
For now, refer to the following external resources:
- [K3s Documentation](https://docs.k3s.io/)
- [Kubernetes Troubleshooting Guide](https://kubernetes.io/docs/tasks/debug/)
- [Velero Backup Documentation](https://velero.io/docs/latest/)
- [Kubernetes Security Best Practices](https://kubernetes.io/docs/concepts/security/)
- [Cluster issues](./guides/troubleshoot-cluster.md)
- [DNS issues](./guides/troubleshoot-dns.md)
- [Service connectivity issues](./guides/troubleshoot-service-connectivity.md)
- [TLS certificate issues](./guides/troubleshoot-tls-certificates.md)
- [Visibility issues](./guides/troubleshoot-visibility.md)

22
docs/SETUP.md
View File

@@ -1,23 +1,3 @@
# Setting Up Your Wild Cloud
Install dependencies:
```bash
scripts/setup-utils.sh
```
Add the `bin` directory to your path.
Initialize a personal wild-cloud in any empty directory, for example:
```bash
cd ~
mkdir ~/my-wild-cloud
cd my-wild-cloud
```
Run:
```bash
wild-setup
```
Visit https://mywildcloud.org/get-started for full wild cloud setup instructions.

114
docs/SETUP_FULL.md
View File

@@ -1,114 +0,0 @@
# Wild Cloud Setup
## Hardware prerequisites
Procure the following before setup:
- Any machine for running setup and managing your cloud.
- One small machine for dnsmasq (running Ubuntu linux)
- Three machines for control nodes (2GB memory, 100GB hard drive).
- Any number of worker node machines.
- A network switch connecting all these machines to your router.
- A network router (e.g. Fluke 2) connected to the Internet.
- A domain of your choice registerd (or managed) on Cloudflare.
## Setup
Clone this repo (you probably already did this).
```bash
source env.sh
```
Initialize a personal wild-cloud in any empty directory, for example:
```bash
cd ~
mkdir ~/my-wild-cloud
cd my-wild-cloud
wild-setup-scaffold
```
## Download Cluster Node Boot Assets
We use Talos linux for node operating systems. Run this script to download the OS for use in the rest of the setup.
```bash
# Generate node boot assets (PXE, iPXE, ISO)
wild-cluster-node-boot-assets-download
```
## Dnsmasq
- Install a Linux machine on your LAN. Record it's IP address in your `config:cloud.dns.ip`.
- Ensure it is accessible with ssh.
```bash
# Install dnsmasq with PXE boot support
wild-dnsmasq-install --install
```
## Cluster Setup
### Cluster Infrastructure Setup
```bash
# Configure network, cluster settings, and register nodes
wild-setup-cluster
```
This interactive script will:
- Configure network settings (router IP, DNS, DHCP range)
- Configure cluster settings (Talos version, schematic ID, MetalLB pool)
- Help you register control plane and worker nodes by detecting their hardware
- Generate machine configurations for each node
- Apply machine configurations to nodes
- Bootstrap the cluster after the first node.
### Install Cluster Services
```bash
wild-setup-services
```
## Installing Wild Cloud Apps
```bash
# List available applications
wild-apps-list
# Deploy an application
wild-app-deploy <app-name>
# Check app status
wild-app-doctor <app-name>
# Remove an application
wild-app-delete <app-name>
```
## Individual Node Management
If you need to manage individual nodes:
```bash
# Generate patch for a specific node
wild-cluster-node-patch-generate <node-ip>
# Generate final machine config (uses existing patch)
wild-cluster-node-machine-config-generate <node-ip>
# Apply configuration with options
wild-cluster-node-up <node-ip> [--insecure] [--skip-patch] [--dry-run]
```
## Asset Management
```bash
# Download/cache boot assets (kernel, initramfs, ISO, iPXE)
wild-cluster-node-boot-assets-download
# Install dnsmasq with specific schematic
wild-dnsmasq-install --schematic-id <id> --install
```

15
docs/glossary.md
View File

@@ -1,15 +0,0 @@
# Cluster
- LAN
- cluster
## LAN
- router
## Cluster
- nameserver
- node
- master
- load balancer

2
docs/guides/app-workflow.md
View File

@@ -43,4 +43,4 @@ wild-app-deploy <app> # Deploys to Kubernetes
## App Directory Structure
Your wild-cloud apps are stored in the `apps/` directory. You can change them however you like. You should keep them all in git and make commits anytime you change something. Some `wild` commands will overwrite files in your app directory (like when you are updating apps, or updating your configuration) so you'll want to review any changes made to your files after using them using `git`.
Your wild-cloud apps are stored in the `apps/` directory. You can change them however you like. You should keep them all in git and make commits anytime you change something. Some `wild` commands will overwrite files in your app directory (like when you are updating apps, or updating your configuration) so you'll want to review any changes made to your files after using them using `git`.

265
docs/guides/making-backups.md Normal file
View File

@@ -0,0 +1,265 @@
# Making Backups
This guide covers how to create backups of your wild-cloud infrastructure using the integrated backup system.
## Overview
The wild-cloud backup system creates encrypted, deduplicated snapshots using restic. It backs up three main components:
- **Applications**: Database dumps and persistent volume data
- **Cluster**: Kubernetes resources and etcd state
- **Configuration**: Wild-cloud repository and settings
## Prerequisites
Before making backups, ensure you have:
1. **Environment configured**: Run `source env.sh` to load backup configuration
2. **Restic repository**: Backup repository configured in `config.yaml`
3. **Backup password**: Set in wild-cloud secrets
4. **Staging directory**: Configured path for temporary backup files
## Backup Components
### Applications (`wild-app-backup`)
Backs up individual applications including:
- **Database dumps**: PostgreSQL/MySQL databases in compressed custom format
- **PVC data**: Application files streamed directly for restic deduplication
- **Auto-discovery**: Finds databases and PVCs based on app manifest.yaml
### Cluster Resources (`wild-backup --cluster-only`)
Backs up cluster-wide resources:
- **Kubernetes resources**: All pods, services, deployments, secrets, configmaps
- **Storage definitions**: PersistentVolumes, PVCs, StorageClasses
- **etcd snapshot**: Complete cluster state for disaster recovery
### Configuration (`wild-backup --home-only`)
Backs up wild-cloud configuration:
- **Repository contents**: All app definitions, manifests, configurations
- **Settings**: Wild-cloud configuration files and customizations
## Making Backups
### Full System Backup (Recommended)
Create a complete backup of everything:
```bash
# Backup all components (apps + cluster + config)
wild-backup
```
This is equivalent to:
```bash
wild-backup --home --apps --cluster
```
### Selective Backups
#### Applications Only
```bash
# All applications
wild-backup --apps-only
# Single application
wild-app-backup discourse
# Multiple applications
wild-app-backup discourse gitea immich
```
#### Cluster Only
```bash
# Kubernetes resources + etcd
wild-backup --cluster-only
```
#### Configuration Only
```bash
# Wild-cloud repository
wild-backup --home-only
```
### Excluding Components
Skip specific components:
```bash
# Skip config, backup apps + cluster
wild-backup --no-home
# Skip applications, backup config + cluster
wild-backup --no-apps
# Skip cluster resources, backup config + apps
wild-backup --no-cluster
```
## Backup Process Details
### Application Backup Process
1. **Discovery**: Parses `manifest.yaml` to find database and PVC dependencies
2. **Database backup**: Creates compressed custom-format dumps
3. **PVC backup**: Streams files directly to staging for restic deduplication
4. **Staging**: Organizes files in clean directory structure
5. **Upload**: Creates individual restic snapshots per application
### Cluster Backup Process
1. **Resource export**: Exports all Kubernetes resources to YAML
2. **etcd snapshot**: Creates point-in-time etcd backup via talosctl
3. **Upload**: Creates single restic snapshot for cluster state
### Restic Snapshots
Each backup creates tagged restic snapshots:
```bash
# View all snapshots
restic snapshots
# Filter by component
restic snapshots --tag discourse # Specific app
restic snapshots --tag cluster # Cluster resources
restic snapshots --tag wc-home # Wild-cloud config
```
## Where Backup Files Are Staged
Before uploading to your restic repository, backup files are organized in a staging directory. This temporary area lets you see exactly what's being backed up and helps with deduplication.
Here's what the staging area looks like:
```
backup-staging/
├── apps/
│ ├── discourse/
│ │ ├── database_20250816T120000Z.dump
│ │ ├── globals_20250816T120000Z.sql
│ │ └── discourse/
│ │ └── data/ # All the actual files
│ ├── gitea/
│ │ ├── database_20250816T120000Z.dump
│ │ └── gitea-data/
│ │ └── data/ # Git repositories, etc.
│ └── immich/
│ ├── database_20250816T120000Z.dump
│ └── immich-data/
│ └── upload/ # Photos and videos
└── cluster/
├── all-resources.yaml # All running services
├── secrets.yaml # Passwords and certificates
├── configmaps.yaml # Configuration data
└── etcd-snapshot.db # Complete cluster state
```
This staging approach means you can examine backup contents before they're uploaded, and restic can efficiently deduplicate files that haven't changed.
## Advanced Usage
### Custom Backup Scripts
Applications can provide custom backup logic:
```bash
# Create apps/myapp/backup.sh for custom behavior
chmod +x apps/myapp/backup.sh
# wild-app-backup will use custom script if present
wild-app-backup myapp
```
### Monitoring Backup Status
```bash
# Check recent snapshots
restic snapshots | head -20
# Check specific app backups
restic snapshots --tag discourse
# Verify backup integrity
restic check
```
### Backup Automation
Set up automated backups with cron:
```bash
# Daily full backup at 2 AM
0 2 * * * cd /data/repos/payne-cloud && source env.sh && wild-backup
# Hourly app backups during business hours
0 9-17 * * * cd /data/repos/payne-cloud && source env.sh && wild-backup --apps-only
```
## Performance Considerations
### Large PVCs (like Immich photos)
The streaming backup approach provides:
- **First backup**: Full transfer time (all files processed)
- **Subsequent backups**: Only changed files processed (dramatically faster)
- **Storage efficiency**: Restic deduplication reduces storage usage
### Network Usage
- **Database dumps**: Compressed at source, efficient transfer
- **PVC data**: Uncompressed transfer, but restic handles deduplication
- **etcd snapshots**: Small files, minimal impact
## Troubleshooting
### Common Issues
**"No databases or PVCs found"**
- App has no `manifest.yaml` with database dependencies
- No PVCs with matching labels in app namespace
- Create custom `backup.sh` script for special cases
**"kubectl not found"**
- Ensure kubectl is installed and configured
- Check cluster connectivity with `kubectl get nodes`
**"Staging directory not set"**
- Configure `cloud.backup.staging` in `config.yaml`
- Ensure directory exists and is writable
**"Could not create etcd backup"**
- Ensure `talosctl` is installed for Talos clusters
- Check control plane node connectivity
- Verify etcd pods are accessible in kube-system namespace
### Backup Verification
Always verify backups periodically:
```bash
# Check restic repository integrity
restic check
# List recent snapshots
restic snapshots --compact
# Test restore to different directory
restic restore latest --target /tmp/restore-test
```
## Security Notes
- **Encryption**: All backups are encrypted with your backup password
- **Secrets**: Kubernetes secrets are included in cluster backups
- **Access control**: Secure your backup repository and passwords
- **Network**: Consider bandwidth usage for large initial backups
## Next Steps
- [Restoring Backups](restoring-backups.md) - Learn how to restore from backups
- Configure automated backup schedules
- Set up backup monitoring and alerting
- Test disaster recovery procedures

50
docs/guides/monitoring.md Normal file
View File

@@ -0,0 +1,50 @@
# System Health Monitoring
## Basic Monitoring
Check system health with:
```bash
# Node resource usage
kubectl top nodes
# Pod resource usage
kubectl top pods -A
# Persistent volume claims
kubectl get pvc -A
```
## Advanced Monitoring (Future Implementation)
Consider implementing:
1. **Prometheus + Grafana** for comprehensive monitoring:
```bash
# Placeholder for future implementation
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace
```
2. **Loki** for log aggregation:
```bash
# Placeholder for future implementation
helm repo add grafana https://grafana.github.io/helm-charts
helm install loki grafana/loki-stack --namespace logging --create-namespace
```
## Additional Resources
This document will be expanded in the future with:
- Detailed backup and restore procedures
- Monitoring setup instructions
- Comprehensive security hardening guide
- Automated maintenance scripts
For now, refer to the following external resources:
- [K3s Documentation](https://docs.k3s.io/)
- [Kubernetes Troubleshooting Guide](https://kubernetes.io/docs/tasks/debug/)
- [Velero Backup Documentation](https://velero.io/docs/latest/)
- [Kubernetes Security Best Practices](https://kubernetes.io/docs/concepts/security/)

246
docs/guides/node-setup.md
View File

@@ -1,246 +0,0 @@
# Node Setup Guide
This guide covers setting up Talos Linux nodes for your Kubernetes cluster using USB boot.
## Overview
There are two main approaches for booting Talos nodes:
1. **USB Boot** (covered here) - Boot from a custom USB drive with system extensions
2. **PXE Boot** - Network boot using dnsmasq setup (see `setup/dnsmasq/README.md`)
## USB Boot Setup
### Prerequisites
- Target hardware for Kubernetes nodes
- USB drive (8GB+ recommended)
- Admin access to create bootable USB drives
### Step 1: Upload Schematic and Download Custom Talos ISO
First, upload the system extensions schematic to Talos Image Factory, then download the custom ISO.
```bash
# Upload schematic configuration to get schematic ID
wild-talos-schema
# Download custom ISO with system extensions
wild-talos-iso
```
The custom ISO includes system extensions (iscsi-tools, util-linux-tools, intel-ucode, gvisor) needed for the cluster and is saved to `.wildcloud/iso/talos-v1.10.3-metal-amd64.iso`.
### Step 2: Create Bootable USB Drive
#### Linux (Recommended)
```bash
# Find your USB device (be careful to select the right device!)
lsblk
sudo dmesg | tail # Check for recently connected USB devices
# Create bootable USB (replace /dev/sdX with your USB device)
sudo dd if=.wildcloud/iso/talos-v1.10.3-metal-amd64.iso of=/dev/sdX bs=4M status=progress sync
# Verify the write completed
sync
```
**⚠️ Warning**: Double-check the device path (`/dev/sdX`). Writing to the wrong device will destroy data!
#### macOS
```bash
# Find your USB device
diskutil list
# Unmount the USB drive (replace diskX with your USB device)
diskutil unmountDisk /dev/diskX
# Create bootable USB
sudo dd if=.wildcloud/iso/talos-v1.10.3-metal-amd64.iso of=/dev/rdiskX bs=4m
# Eject when complete
diskutil eject /dev/diskX
```
#### Windows
Use one of these tools:
1. **Rufus** (Recommended)
- Download from https://rufus.ie/
- Select the Talos ISO file
- Choose your USB drive
- Use "DD Image" mode
- Click "START"
2. **Balena Etcher**
- Download from https://www.balena.io/etcher/
- Flash from file → Select Talos ISO
- Select target USB drive
- Flash!
3. **Command Line** (Windows 10/11)
```cmd
# List disks to find USB drive number
diskpart
list disk
exit
# Write ISO (replace X with your USB disk number)
dd if=.wildcloud\iso\talos-v1.10.3-metal-amd64.iso of=\\.\PhysicalDriveX bs=4M --progress
```
### Step 3: Boot Target Machine
1. **Insert USB** into target machine
2. **Boot from USB**:
- Restart machine and enter BIOS/UEFI (usually F2, F12, DEL, or ESC during startup)
- Change boot order to prioritize USB drive
- Or use one-time boot menu (usually F12)
3. **Talos will boot** in maintenance mode with a DHCP IP
### Step 4: Hardware Detection and Configuration
Once the machine boots, it will be in maintenance mode with a DHCP IP address.
```bash
# Find the node's maintenance IP (check your router/DHCP server)
# Then detect hardware and register the node
cd setup/cluster-nodes
./detect-node-hardware.sh <maintenance-ip> <node-number>
# Example: Node got DHCP IP 192.168.8.150, registering as node 1
./detect-node-hardware.sh 192.168.8.150 1
```
This script will:
- Discover network interface names (e.g., `enp4s0`)
- List available disks for installation
- Update `config.yaml` with node-specific hardware settings
### Step 5: Generate and Apply Configuration
```bash
# Generate machine configurations with detected hardware
./generate-machine-configs.sh
# Apply configuration (node will reboot with static IP)
talosctl apply-config --insecure -n <maintenance-ip> --file final/controlplane-node-<number>.yaml
# Example:
talosctl apply-config --insecure -n 192.168.8.150 --file final/controlplane-node-1.yaml
```
### Step 6: Verify Installation
After reboot, the node should come up with its assigned static IP:
```bash
# Check connectivity (node 1 should be at 192.168.8.31)
ping 192.168.8.31
# Verify system extensions are installed
talosctl -e 192.168.8.31 -n 192.168.8.31 get extensions
# Check for iscsi tools
talosctl -e 192.168.8.31 -n 192.168.8.31 list /usr/local/bin/ | grep iscsi
```
## Repeat for Additional Nodes
For each additional control plane node:
1. Boot with the same USB drive
2. Run hardware detection with the new maintenance IP and node number
3. Generate and apply configurations
4. Verify the node comes up at its static IP
Example for node 2:
```bash
./detect-node-hardware.sh 192.168.8.151 2
./generate-machine-configs.sh
talosctl apply-config --insecure -n 192.168.8.151 --file final/controlplane-node-2.yaml
```
## Cluster Bootstrap
Once all control plane nodes are configured:
```bash
# Bootstrap the cluster using the VIP
talosctl bootstrap -n 192.168.8.30
# Get kubeconfig
talosctl kubeconfig
# Verify cluster
kubectl get nodes
```
## Troubleshooting
### USB Boot Issues
- **Machine won't boot from USB**: Check BIOS boot order, disable Secure Boot if needed
- **Talos doesn't start**: Verify ISO was written correctly, try re-creating USB
- **Network issues**: Ensure DHCP is available on your network
### Hardware Detection Issues
- **Node not accessible**: Check IP assignment, firewall settings
- **Wrong interface detected**: Manual override in `config.yaml` if needed
- **Disk not found**: Verify disk size (must be >10GB), check disk health
### Installation Issues
- **Static IP not assigned**: Check network configuration in machine config
- **Extensions not installed**: Verify ISO includes extensions, check upgrade logs
- **Node won't join cluster**: Check certificates, network connectivity to VIP
### Checking Logs
```bash
# View system logs
talosctl -e <node-ip> -n <node-ip> logs machined
# Check kernel messages
talosctl -e <node-ip> -n <node-ip> dmesg
# Monitor services
talosctl -e <node-ip> -n <node-ip> get services
```
## System Extensions Included
The custom ISO includes these extensions:
- **siderolabs/iscsi-tools**: iSCSI initiator tools for persistent storage
- **siderolabs/util-linux-tools**: Utility tools including fstrim for storage
- **siderolabs/intel-ucode**: Intel CPU microcode updates (harmless on AMD)
- **siderolabs/gvisor**: Container runtime sandbox (optional security enhancement)
These extensions enable:
- Longhorn distributed storage
- Improved security isolation
- CPU microcode updates
- Storage optimization tools
## Next Steps
After all nodes are configured:
1. **Install CNI**: Deploy a Container Network Interface (Cilium, Calico, etc.)
2. **Install CSI**: Deploy Container Storage Interface (Longhorn for persistent storage)
3. **Deploy workloads**: Your applications and services
4. **Monitor cluster**: Set up monitoring and logging
See the main project documentation for application deployment guides.

294
docs/guides/restoring-backups.md Normal file
View File

@@ -0,0 +1,294 @@
# Restoring Backups
This guide will walk you through restoring your applications and cluster from wild-cloud backups. Hopefully you'll never need this, but when you do, it's critical that the process works smoothly.
## Understanding Restore Types
Your wild-cloud backup system can restore different types of data depending on what you need to recover:
**Application restores** bring back individual applications by restoring their database contents and file storage. This is what you'll use most often - maybe you accidentally deleted something in Discourse, or Gitea got corrupted, or you want to roll back Immich to before a bad update.
**Cluster restores** are for disaster recovery scenarios where you need to rebuild your entire Kubernetes cluster from scratch. This includes restoring all the cluster's configuration and even its internal state.
**Configuration restores** bring back your wild-cloud repository and settings, which contain all the "recipes" for how your infrastructure should be set up.
## Before You Start Restoring
Make sure you have everything needed to perform restores. You need to be in your wild-cloud directory with the environment loaded (`source env.sh`). Your backup repository and password should be configured and working - you can test this by running `restic snapshots` to see your available backups.
Most importantly, make sure you have kubectl access to your cluster, since restores involve creating temporary pods and manipulating storage.
## Restoring Applications
### Basic Application Restore
The most common restore scenario is bringing back a single application. To restore the latest backup of an app:
```bash
wild-app-restore discourse
```
This restores both the database and all file storage for the discourse app. The restore system automatically figures out what the app needs based on its manifest file and what was backed up.
If you want to restore from a specific backup instead of the latest:
```bash
wild-app-restore discourse abc123
```
Where `abc123` is the snapshot ID from `restic snapshots --tag discourse`.
### Partial Restores
Sometimes you only need to restore part of an application. Maybe the database is fine but the files got corrupted, or vice versa.
To restore only the database:
```bash
wild-app-restore discourse --db-only
```
To restore only the file storage:
```bash
wild-app-restore discourse --pvc-only
```
To restore without database roles and permissions (if they're causing conflicts):
```bash
wild-app-restore discourse --skip-globals
```
### Finding Available Backups
To see what backups are available for an app:
```bash
wild-app-restore discourse --list
```
This shows recent snapshots with their IDs, timestamps, and what was included.
## How Application Restores Work
Understanding what happens during a restore can help when things don't go as expected.
### Database Restoration
When restoring a database, the system first downloads the backup files from your restic repository. It then prepares the database by creating any needed roles, disconnecting existing users, and dropping/recreating the database to ensure a clean restore.
For PostgreSQL databases, it uses `pg_restore` with parallel processing to speed up large database imports. For MySQL, it uses standard mysql import commands. The system also handles database ownership and permissions automatically.
### File Storage Restoration
File storage (PVC) restoration is more complex because it involves safely replacing files that might be actively used by running applications.
First, the system creates a safety snapshot using Longhorn. This means if something goes wrong during the restore, you can get back to where you started. Then it scales your application down to zero replicas so no pods are using the storage.
Next, it creates a temporary utility pod with the PVC mounted and copies all the backup files into place, preserving file permissions and structure. Once the data is restored and verified, it removes the utility pod and scales your application back up.
If everything worked correctly, the safety snapshot is automatically deleted. If something went wrong, the safety snapshot is preserved so you can recover manually.
## Cluster Disaster Recovery
Cluster restoration is much less common but critical when you need to rebuild your entire infrastructure.
### Restoring Kubernetes Resources
To restore all cluster resources from a backup:
```bash
# Download cluster backup
restic restore --tag cluster latest --target ./restore/
# Apply all resources
kubectl apply -f restore/cluster/all-resources.yaml
```
You can also restore specific types of resources:
```bash
kubectl apply -f restore/cluster/secrets.yaml
kubectl apply -f restore/cluster/configmaps.yaml
```
### Restoring etcd State
**Warning: This is extremely dangerous and will affect your entire cluster.**
etcd restoration should only be done when rebuilding a cluster from scratch. For Talos clusters:
```bash
talosctl --nodes <control-plane-ip> etcd restore --from ./restore/cluster/etcd-snapshot.db
```
This command stops etcd, replaces its data with the backup, and restarts the cluster. Expect significant downtime while the cluster rebuilds itself.
## Common Disaster Recovery Scenarios
### Complete Application Loss
When an entire application is gone (namespace deleted, pods corrupted, etc.):
```bash
# Make sure the namespace exists
kubectl create namespace discourse --dry-run=client -o yaml | kubectl apply -f -
# Apply the application manifests if needed
kubectl apply -f apps/discourse/
# Restore the application data
wild-app-restore discourse
```
### Complete Cluster Rebuild
When rebuilding a cluster from scratch:
First, build your new cluster infrastructure and install wild-cloud components. Then configure backup access so you can reach your backup repository.
Restore cluster state:
```bash
restic restore --tag cluster latest --target ./restore/
# Apply etcd snapshot using appropriate method for your cluster type
```
Finally, restore all applications:
```bash
# See what applications are backed up
wild-app-restore --list
# Restore each application individually
wild-app-restore discourse
wild-app-restore gitea
wild-app-restore immich
```
### Rolling Back After Bad Changes
Sometimes you need to undo recent changes to an application:
```bash
# See available snapshots
wild-app-restore discourse --list
# Restore from before the problematic changes
wild-app-restore discourse abc123
```
## Cross-Cluster Migration
You can use backups to move applications between clusters:
On the source cluster, create a fresh backup:
```bash
wild-app-backup discourse
```
On the target cluster, deploy the application manifests:
```bash
kubectl apply -f apps/discourse/
```
Then restore the data:
```bash
wild-app-restore discourse
```
## Verifying Successful Restores
After any restore, verify that everything is working correctly.
For databases, check that you can connect and see expected data:
```bash
kubectl exec -n postgres deploy/postgres-deployment -- \
psql -U postgres -d discourse -c "SELECT count(*) FROM posts;"
```
For file storage, check that files exist and applications can start:
```bash
kubectl get pods -n discourse
kubectl logs -n discourse deployment/discourse
```
For web applications, test that you can access them:
```bash
curl -f https://discourse.example.com/latest.json
```
## When Things Go Wrong
### No Snapshots Found
If the restore system can't find backups for an application, check that snapshots exist:
```bash
restic snapshots --tag discourse
```
Make sure you're using the correct app name and that backups were actually created successfully.
### Database Restore Failures
Database restores can fail if the target database isn't accessible or if there are permission issues. Check that your postgres or mysql pods are running and that you can connect to them manually.
Review the restore error messages carefully - they usually indicate whether the problem is with the backup file, database connectivity, or permissions.
### PVC Restore Failures
If PVC restoration fails, check that you have sufficient disk space and that the PVC isn't being used by other pods. The error messages will usually indicate what went wrong.
Most importantly, remember that safety snapshots are preserved when PVC restores fail. You can see them with:
```bash
kubectl get snapshot.longhorn.io -n longhorn-system -l app=wild-app-restore
```
These snapshots let you recover to the pre-restore state if needed.
### Application Won't Start After Restore
If pods fail to start after restoration, check file permissions and ownership. Sometimes the restoration process doesn't perfectly preserve the exact permissions that the application expects.
You can also try scaling the application to zero and back to one, which sometimes resolves transient issues:
```bash
kubectl scale deployment/discourse -n discourse --replicas=0
kubectl scale deployment/discourse -n discourse --replicas=1
```
## Manual Recovery
When automated restore fails, you can always fall back to manual extraction and restoration:
```bash
# Extract backup files to local directory
restic restore --tag discourse latest --target ./manual-restore/
# Manually copy database dump to postgres pod
kubectl cp ./manual-restore/discourse/database_*.dump \
postgres/postgres-deployment-xxx:/tmp/
# Manually restore database
kubectl exec -n postgres deploy/postgres-deployment -- \
pg_restore -U postgres -d discourse /tmp/database_*.dump
```
For file restoration, you'd need to create a utility pod and manually copy files into the PVC.
## Best Practices
Test your restore procedures regularly in a non-production environment. It's much better to discover issues with your backup system during a planned test than during an actual emergency.
Always communicate with users before performing restores, especially if they involve downtime. Document any manual steps you had to take so you can improve the automated process.
After any significant restore, monitor your applications more closely than usual for a few days. Sometimes problems don't surface immediately.
## Security and Access Control
Restore operations are powerful and can be destructive. Make sure only trusted administrators can perform restores, and consider requiring approval or coordination before major restoration operations.
Be aware that cluster restores include all secrets, so they potentially expose passwords, API keys, and certificates. Ensure your backup repository is properly secured.
Remember that Longhorn safety snapshots are preserved when things go wrong. These snapshots may contain sensitive data, so clean them up appropriately once you've resolved any issues.
## What's Next
The best way to get comfortable with restore operations is to practice them in a safe environment. Set up a test cluster and practice restoring applications and data.
Consider creating runbooks for your most likely disaster scenarios, including the specific commands and verification steps for your infrastructure.
Read the [Making Backups](making-backups.md) guide to ensure you're creating the backups you'll need for successful recovery.

46
docs/guides/security.md Normal file
View File

@@ -0,0 +1,46 @@
# Security
## Best Practices
1. **Keep Everything Updated**:
- Regularly update K3s
- Update all infrastructure components
- Keep application images up to date
2. **Network Security**:
- Use internal services whenever possible
- Limit exposed services to only what's necessary
- Configure your home router's firewall properly
3. **Access Control**:
- Use strong passwords for all services
- Implement a secrets management strategy
- Rotate API tokens and keys regularly
4. **Regular Audits**:
- Review running services periodically
- Check for unused or outdated deployments
- Monitor resource usage for anomalies
## Security Scanning (Future Implementation)
Tools to consider implementing:
1. **Trivy** for image scanning:
```bash
# Example Trivy usage (placeholder)
trivy image <your-image>
```
2. **kube-bench** for Kubernetes security checks:
```bash
# Example kube-bench usage (placeholder)
kubectl apply -f https://raw.githubusercontent.com/aquasecurity/kube-bench/main/job.yaml
```
3. **Falco** for runtime security monitoring:
```bash
# Example Falco installation (placeholder)
helm repo add falcosecurity https://falcosecurity.github.io/charts
helm install falco falcosecurity/falco --namespace falco --create-namespace
```

18
docs/guides/taslos.md Normal file
View File

@@ -0,0 +1,18 @@
# Talos
## System Extensions Included
The custom ISO includes these extensions:
- **siderolabs/iscsi-tools**: iSCSI initiator tools for persistent storage
- **siderolabs/util-linux-tools**: Utility tools including fstrim for storage
- **siderolabs/intel-ucode**: Intel CPU microcode updates (harmless on AMD)
- **siderolabs/gvisor**: Container runtime sandbox (optional security enhancement)
These extensions enable:
- Longhorn distributed storage
- Improved security isolation
- CPU microcode updates
- Storage optimization tools

19
docs/guides/troubleshoot-cluster.md Normal file
View File

@@ -0,0 +1,19 @@
# Troubleshoot Wild Cloud Cluster issues
## General Troubleshooting Steps
1. **Check Node Status**:
```bash
kubectl get nodes
kubectl describe node <node-name>
```
1. **Check Component Status**:
```bash
# Check all pods across all namespaces
kubectl get pods -A
# Look for pods that aren't Running or Ready
kubectl get pods -A | grep -v "Running\|Completed"
```

20
docs/guides/troubleshoot-dns.md Normal file
View File

@@ -0,0 +1,20 @@
# Troubleshoot DNS
If DNS resolution isn't working properly:
1. Check CoreDNS status:
```bash
kubectl get pods -n kube-system -l k8s-app=kube-dns
kubectl logs -l k8s-app=kube-dns -n kube-system
```
2. Verify CoreDNS configuration:
```bash
kubectl get configmap -n kube-system coredns -o yaml
```
3. Test DNS resolution from inside the cluster:
```bash
kubectl run -i --tty --rm debug --image=busybox --restart=Never -- nslookup kubernetes.default
```

18
docs/guides/troubleshoot-service-connectivity.md Normal file
View File

@@ -0,0 +1,18 @@
# Troubleshoot Service Connectivity
If services can't communicate:
1. Check network policies:
```bash
kubectl get networkpolicies -A
```
2. Verify service endpoints:
```bash
kubectl get endpoints -n <namespace>
```
3. Test connectivity from within the cluster:
```bash
kubectl run -i --tty --rm debug --image=busybox --restart=Never -- wget -O- <service-name>.<namespace>
```

24
docs/guides/troubleshoot-tls-certificates.md Normal file
View File

@@ -0,0 +1,24 @@
# Troubleshoot TLS Certificates
If services show invalid certificates:
1. Check certificate status:
```bash
kubectl get certificates -A
```
2. Examine certificate details:
```bash
kubectl describe certificate <cert-name> -n <namespace>
```
3. Check for cert-manager issues:
```bash
kubectl get pods -n cert-manager
kubectl logs -l app=cert-manager -n cert-manager
```
4. Verify the Cloudflare API token is correctly set up:
```bash
kubectl get secret cloudflare-api-token -n internal
```

2
docs/troubleshooting/visibility.md → docs/guides/troubleshoot-visibility.md
View File

@@ -1,4 +1,4 @@
# Troubleshooting Service Visibility
# Troubleshoot Service Visibility
This guide covers common issues with accessing services from outside the cluster and how to diagnose and fix them.

3
docs/guides/upgrade-applications.md Normal file
View File

@@ -0,0 +1,3 @@
# Upgrade Applications
TBD

3
docs/guides/upgrade-kubernetes.md Normal file
View File

@@ -0,0 +1,3 @@
# Upgrade Kubernetes
TBD

3
docs/guides/upgrade-talos.md Normal file
View File

@@ -0,0 +1,3 @@
# Upgrade Talos
TBD

3
docs/guides/upgrade-wild-cloud.md Normal file
View File

@@ -0,0 +1,3 @@
# Upgrade Wild Cloud
TBD

12
docs/lan-routers/GL-iNet.md
View File

@@ -1,12 +0,0 @@
# GL-iNet LAN Router Setup
- Applications > Dynamic DNS > Enable DDNS
- Enable
- Use Host Name as your CNAME at Cloudflare.
- Network > LAN > Address Reservation
- Add all cluster nodes.
- Network > Port Forwarding
- Add TCP, port 22 to your bastion
- Add TCP/UDP, port 443 to your cluster load balancer.
- Network > DNS > DNS Server Settings
- Set to cluster DNS server IP

331
docs/learning/visibility.md
View File

@@ -1,331 +0,0 @@
# Understanding Network Visibility in Kubernetes
This guide explains how applications deployed on our Kubernetes cluster become accessible from both internal and external networks. Whether you're deploying a public-facing website or an internal admin panel, this document will help you understand the journey from deployment to accessibility.
## The Visibility Pipeline
When you deploy an application to the cluster, making it accessible involves several coordinated components working together:
1. **Kubernetes Services** - Direct traffic to your application pods
2. **Ingress Controllers** - Route external HTTP/HTTPS traffic to services
3. **Load Balancers** - Assign external IPs to services
4. **DNS Management** - Map domain names to IPs
5. **TLS Certificates** - Secure connections with HTTPS
Let's walk through how each part works and how they interconnect.
## From Deployment to Visibility
### 1. Application Deployment
Your journey begins with deploying your application on Kubernetes. This typically involves:
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-app
namespace: my-namespace
spec:
replicas: 1
selector:
matchLabels:
app: my-app
template:
metadata:
labels:
app: my-app
spec:
containers:
- name: my-app
image: myapp:latest
ports:
- containerPort: 80
```
This creates pods running your application, but they're not yet accessible outside their namespace.
### 2. Kubernetes Service: Internal Connectivity
A Kubernetes Service provides a stable endpoint to access your pods:
```yaml
apiVersion: v1
kind: Service
metadata:
name: my-app
namespace: my-namespace
spec:
selector:
app: my-app
ports:
- port: 80
targetPort: 80
type: ClusterIP
```
With this `ClusterIP` service, your application is accessible within the cluster at `my-app.my-namespace.svc.cluster.local`, but not from outside.
### 3. Ingress: Defining HTTP Routes
For HTTP/HTTPS traffic, an Ingress resource defines routing rules:
```yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: my-app
namespace: my-namespace
annotations:
kubernetes.io/ingress.class: "traefik"
external-dns.alpha.kubernetes.io/target: "CLOUD_DOMAIN"
external-dns.alpha.kubernetes.io/ttl: "60"
spec:
rules:
- host: my-app.CLOUD_DOMAIN
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: my-app
port:
number: 80
tls:
- hosts:
- my-app.CLOUD_DOMAIN
secretName: wildcard-wild-cloud-tls
```
This Ingress tells the cluster to route requests for `my-app.CLOUD_DOMAIN` to your service. The annotations provide hints to other systems like ExternalDNS.
### 4. Traefik: The Ingress Controller
Our cluster uses Traefik as the ingress controller. Traefik watches for Ingress resources and configures itself to handle the routing rules. It acts as a reverse proxy and edge router, handling:
- HTTP/HTTPS routing
- TLS termination
- Load balancing
- Path-based routing
- Host-based routing
Traefik runs as a service in the cluster with its own external IP (provided by MetalLB).
### 5. MetalLB: Assigning External IPs
Since we're running on-premises (not in a cloud that provides load balancers), we use MetalLB to assign external IPs to services. MetalLB manages a pool of IP addresses from our local network:
```yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: default
namespace: metallb-system
spec:
addresses:
- 192.168.8.240-192.168.8.250
```
This allows Traefik and any other LoadBalancer services to receive a real IP address from our network.
### 6. ExternalDNS: Automated DNS Management
ExternalDNS automatically creates and updates DNS records in our CloudFlare DNS zone:
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: external-dns
namespace: externaldns
spec:
# ...
template:
spec:
containers:
- name: external-dns
image: registry.k8s.io/external-dns/external-dns
args:
- --source=service
- --source=ingress
- --provider=cloudflare
- --txt-owner-id=wild-cloud
```
ExternalDNS watches Kubernetes Services and Ingresses with appropriate annotations, then creates corresponding DNS records in CloudFlare, making your applications discoverable by domain name.
### 7. Cert-Manager: TLS Certificate Automation
To secure connections with HTTPS, we use cert-manager to automatically obtain and renew TLS certificates:
```yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: wildcard-wild-cloud-io
namespace: default
spec:
secretName: wildcard-wild-cloud-tls
dnsNames:
- "*.CLOUD_DOMAIN"
- "CLOUD_DOMAIN"
issuerRef:
name: letsencrypt-prod
kind: ClusterIssuer
```
Cert-manager handles:
- Certificate request and issuance
- DNS validation (for wildcard certificates)
- Automatic renewal
- Secret storage of certificates
## The Two Visibility Paths
In our infrastructure, we support two primary visibility paths:
### Public Services (External Access)
Public services are those meant to be accessible from the public internet:
1. **Service**: Kubernetes ClusterIP service (internal)
2. **Ingress**: Defines routing with hostname like `service-name.CLOUD_DOMAIN`
3. **DNS**: ExternalDNS creates a CNAME record pointing to `CLOUD_DOMAIN`
4. **TLS**: Uses wildcard certificate for `*.CLOUD_DOMAIN`
5. **IP Addressing**: Traffic reaches the MetalLB-assigned IP for Traefik
6. **Network**: Traffic flows from external internet → router → MetalLB IP → Traefik → Kubernetes Service → Application Pods
**Deploy a public service with:**
```bash
./bin/deploy-service --type public --name myservice
```
### Internal Services (Private Access)
Internal services are restricted to the internal network:
1. **Service**: Kubernetes ClusterIP service (internal)
2. **Ingress**: Defines routing with hostname like `service-name.internal.CLOUD_DOMAIN`
3. **DNS**: ExternalDNS creates an A record pointing to the internal load balancer IP
4. **TLS**: Uses wildcard certificate for `*.internal.CLOUD_DOMAIN`
5. **IP Addressing**: Traffic reaches the MetalLB-assigned IP for Traefik
6. **Network**: Traffic flows from internal network → MetalLB IP → Traefik → Service → Pods
7. **Security**: Traefik middleware restricts access to internal network IPs
**Deploy an internal service with:**
```bash
./bin/deploy-service --type internal --name adminpanel
```
## How It All Works Together
1. **You deploy** an application using our deploy-service script
2. **Kubernetes** schedules and runs your application pods
3. **Services** provide a stable endpoint for your pods
4. **Traefik** configures routing based on Ingress definitions
5. **MetalLB** assigns real network IPs to LoadBalancer services
6. **ExternalDNS** creates DNS records for your services
7. **Cert-Manager** ensures valid TLS certificates for HTTPS
### Network Flow Diagram
```mermaid
flowchart TD
subgraph Internet["Internet"]
User("User Browser")
CloudDNS("CloudFlare DNS")
end
subgraph Cluster["Cluster"]
Router("Router")
MetalLB("MetalLB")
Traefik("Traefik Ingress")
IngSvc("Service")
IngPods("Application Pods")
Ingress("Ingress")
CertManager("cert-manager")
WildcardCert("Wildcard Certificate")
ExtDNS("ExternalDNS")
end
User -- "1\. DNS Query" --> CloudDNS
CloudDNS -- "2\. IP Address" --> User
User -- "3\. HTTPS Request" --> Router
Router -- "4\. Forward" --> MetalLB
MetalLB -- "5\. Route" --> Traefik
Traefik -- "6\. Route" --> Ingress
Ingress -- "7\. Forward" --> IngSvc
IngSvc -- "8\. Balance" --> IngPods
ExtDNS -- "A. Update DNS" --> CloudDNS
Ingress -- "B. Configure" --> ExtDNS
CertManager -- "C. Issue Cert" --> WildcardCert
Ingress -- "D. Use" --> WildcardCert
User:::internet
CloudDNS:::internet
Router:::cluster
MetalLB:::cluster
Traefik:::cluster
IngSvc:::cluster
IngPods:::cluster
Ingress:::cluster
CertManager:::cluster
WildcardCert:::cluster
ExtDNS:::cluster
classDef internet fill:#fcfcfc,stroke:#333
classDef cluster fill:#a6f3ff,stroke:#333
style User fill:#C8E6C9
style CloudDNS fill:#C8E6C9
style Router fill:#C8E6C9
style MetalLB fill:#C8E6C9
style Traefik fill:#C8E6C9
style IngSvc fill:#C8E6C9
style IngPods fill:#C8E6C9
style Ingress fill:#C8E6C9
style CertManager fill:#C8E6C9
style WildcardCert fill:#C8E6C9
style ExtDNS fill:#C8E6C9
```
A successful deployment creates a chain of connections:
```
Internet → DNS (domain name) → External IP → Traefik → Kubernetes Service → Application Pod
```
## Behind the Scenes: The Technical Magic
When you use our `deploy-service` script, several things happen:
1. **Template Processing**: The script processes a YAML template for your service type, using environment variables to customize it
2. **Namespace Management**: Creates or uses your service's namespace
3. **Resource Application**: Applies the generated YAML to create/update all Kubernetes resources
4. **DNS Configuration**: ExternalDNS detects the new resources and creates DNS records
5. **Certificate Management**: Cert-manager ensures TLS certificates exist or creates new ones
6. **Secret Distribution**: For internal services, certificates are copied to the appropriate namespaces
## Troubleshooting Visibility Issues
When services aren't accessible, the issue usually lies in one of these areas:
1. **DNS Resolution**: Domain not resolving to the correct IP
2. **Certificate Problems**: Invalid, expired, or missing TLS certificates
3. **Ingress Configuration**: Incorrect routing rules or annotations
4. **Network Issues**: Firewall rules or internal/external network segregation
Our [Visibility Troubleshooting Guide](/docs/troubleshooting/VISIBILITY.md) provides detailed steps for diagnosing these issues.
## Conclusion
The visibility layer in our infrastructure represents a sophisticated interplay of multiple systems working together. While complex under the hood, it provides a streamlined experience for developers to deploy applications with proper networking, DNS, and security.
By understanding these components and their relationships, you'll be better equipped to deploy applications and diagnose any visibility issues that arise.
## Further Reading
- [Traefik Documentation](https://doc.traefik.io/traefik/)
- [ExternalDNS Project](https://github.com/kubernetes-sigs/external-dns)
- [Cert-Manager Documentation](https://cert-manager.io/docs/)
- [MetalLB Project](https://metallb.universe.tf/)

19
docs/tutorial/README.md
View File

@@ -1,19 +0,0 @@
# Welcome to the Wild Cloud tutorial!
## Hi! I'm Paul.
Welcome! I am SO excited you're here!
Why am I so excited?? When I was an eight year old kid, I had a computer named the Commodore64. One of the coolest things about it was that it came with a User Manual that told you all about how to not just use that computer, but to actually _use computers_. It taught me how to write my own programs and run them! That experience of wonder, that I could write something and have it do something, is the single biggest reason why I have spent the last 40 years working with computers.
When I was 12, I found out I could plug a cartridge into the back of my Commodore, plug a telephone line into it (maybe some of you don't even know what that is anymore!), and _actually call_ other people's computers in my city. We developed such a sense of community, connecting our computers together and leaving each other messages about the things we were thinking. It was a tiny taste of the early Internet.
I had a similar experience when I was 19 and installed something called the "World Wide Web" on the computers I managed in a computer lab at college. My heart skipped a beat when I clicked on a few "links" and actually saw an image from a computer in Israel just magically appear on my screen! It felt like I was teleported to the other side of the world. Pretty amazing for a kid who had rarely been out of Nebraska!
Everything in those days was basically free. My Commodore cost $200, people connected to each other out of pure curiosity. If you wanted to be a presence on the Internet, you could just connect your computer to it and people around the world could visit you! _All_ of the early websites were entirely non-commercial. No ads! No sign-ups! No monthly subscription fees! It felt like the whole world was coming together to build something amazing for everyone.
Of course, as we all know, it didn't stay that way. After college, I had to figure out ways to pay for Internet connections myself. At some point search engines decided to make money by selling ads on their pages... and then providing ad services to other pages--"monetize" they called it. Then commercial companies found out about it and wanted to sell books and shoes to other people, and the government decided they wanted to capture that tax money. Instead of making the free and open software better, and the open communities stronger, and encouraging people to participate by running their own computers and software, companies started offering people to connect _inside_ their controlled computers. "Hey! You don't have to do all that stuff" they would say, "You can just jump on our servers for free!".
So people stopped being curious about what we could do with our computers together, and they got a login name, and they couldn't do their own things on their own computers anymore, and their data became the property of the company whose computer they were using, and those companies started working together to make it faster to go to their own computers, and to make it go very, very, slow if you wanted to let people come to your computer, or even to forbid having people come to your computer entirely. So now, we are _safe_ and _simple_ and _secure_ and we get whatever the companies want to give us, which seems to usually be ads (so many ads) or monthly fee increases, and they really, really, love getting our attention and putting it where they want it. Mostly, it's just all so... boring. So boring.
So, why am I excited you're here? Because with this project, this Wild Cloud project, I think I just might be able to pass on some of that sense of wonder that captured me so many years ago!

24
scripts/common.sh
View File

@@ -10,16 +10,16 @@
# #!/bin/bash
# set -e
# set -o pipefail
#
#
# # Source common utilities
# source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/wild-common.sh"
#
#
# # Initialize Wild Cloud environment
# init_wild_env
#
# AVAILABLE FUNCTIONS:
# - Print functions: print_header, print_info, print_warning, print_success, print_error
# - Config functions: prompt_with_default
# - Config functions: prompt_with_default
# - Config helpers: prompt_if_unset_config, prompt_if_unset_secret
# - Validation: check_wild_directory
# - Utilities: command_exists, file_readable, dir_writable, generate_random_string
@@ -72,7 +72,7 @@ prompt_with_default() {
local default="$2"
local current_value="$3"
local result
if [ -n "${current_value}" ] && [ "${current_value}" != "null" ]; then
printf "%s [current: %s]: " "${prompt}" "${current_value}" >&2
read -r result
@@ -99,7 +99,7 @@ prompt_with_default() {
read -r result
done
fi
echo "${result}"
}
@@ -108,10 +108,10 @@ prompt_if_unset_config() {
local config_path="$1"
local prompt="$2"
local default="$3"
local current_value
current_value=$(wild-config "${config_path}")
if [ -z "${current_value}" ] || [ "${current_value}" = "null" ]; then
local new_value
new_value=$(prompt_with_default "${prompt}" "${default}" "")
@@ -127,10 +127,10 @@ prompt_if_unset_secret() {
local secret_path="$1"
local prompt="$2"
local default="$3"
local current_value
current_value=$(wild-secret "${secret_path}")
if [ -z "${current_value}" ] || [ "${current_value}" = "null" ]; then
local new_value
new_value=$(prompt_with_default "${prompt}" "${default}" "")
@@ -149,7 +149,7 @@ prompt_if_unset_secret() {
# Returns the path to the project root, or empty string if not found
find_wc_home() {
local current_dir="$(pwd)"
while [ "$current_dir" != "/" ]; do
if [ -d "$current_dir/.wildcloud" ]; then
echo "$current_dir"
@@ -157,7 +157,7 @@ find_wc_home() {
fi
current_dir="$(dirname "$current_dir")"
done
# Not found
return 1
}
@@ -169,7 +169,7 @@ init_wild_env() {
echo "ERROR: WC_ROOT is not set."
exit 1
else
# Check if WC_ROOT is a valid directory
if [ ! -d "${WC_ROOT}" ]; then
echo "ERROR: WC_ROOT directory does not exist! Did you install the wild-cloud root?"

27
setup/dnsmasq/dnsmasq.conf
View File

@@ -16,24 +16,23 @@ server=1.1.1.1
server=8.8.8.8
# --- DHCP Settings ---
dhcp-range={{ .cloud.dhcpRange }},12h
dhcp-option=3,{{ .cloud.router.ip }} # gateway to assign
dhcp-option=6,{{ .cloud.dns.ip }} # dns to assign
# dhcp-range={{ .cloud.dhcpRange }},12h
# dhcp-option=3,{{ .cloud.router.ip }} # gateway to assign
# dhcp-option=6,{{ .cloud.dns.ip }} # dns to assign
# --- PXE Booting ---
enable-tftp
tftp-root=/var/ftpd
# enable-tftp
# tftp-root=/var/ftpd
dhcp-match=set:efi-x86_64,option:client-arch,7
dhcp-boot=tag:efi-x86_64,ipxe.efi
dhcp-boot=tag:!efi-x86_64,undionly.kpxe
# dhcp-match=set:efi-x86_64,option:client-arch,7
# dhcp-boot=tag:efi-x86_64,ipxe.efi
# dhcp-boot=tag:!efi-x86_64,undionly.kpxe
dhcp-match=set:efi-arm64,option:client-arch,11
dhcp-boot=tag:efi-arm64,ipxe-arm64.efi
# dhcp-match=set:efi-arm64,option:client-arch,11
# dhcp-boot=tag:efi-arm64,ipxe-arm64.efi
dhcp-userclass=set:ipxe,iPXE
dhcp-boot=tag:ipxe,http://{{ .cloud.dns.ip }}/boot.ipxe
# dhcp-userclass=set:ipxe,iPXE
# dhcp-boot=tag:ipxe,http://{{ .cloud.dns.ip }}/boot.ipxe
log-queries
log-dhcp
# log-dhcp

54
setup/dnsmasq/setup.sh
View File

@@ -9,43 +9,43 @@ echo "Installing dnsmasq and nginx."
sudo apt install -y dnsmasq nginx
DNSMASQ_SETUP_DIR="."
PXE_FTPD_DIR="${DNSMASQ_SETUP_DIR}/pxe-ftpd"
PXE_WEB_ROOT="${DNSMASQ_SETUP_DIR}/ipxe-web"
# PXE_FTPD_DIR="${DNSMASQ_SETUP_DIR}/pxe-ftpd"
# PXE_WEB_ROOT="${DNSMASQ_SETUP_DIR}/ipxe-web"
# Configure nginx.
echo "Configuring nginx."
sudo cp "${DNSMASQ_SETUP_DIR}/nginx.conf" /etc/nginx/sites-available/talos
sudo chown www-data:www-data /etc/nginx/sites-available/talos
sudo chmod -R 755 /etc/nginx/sites-available/talos
# echo "Configuring nginx."
# sudo cp "${DNSMASQ_SETUP_DIR}/nginx.conf" /etc/nginx/sites-available/talos
# sudo chown www-data:www-data /etc/nginx/sites-available/talos
# sudo chmod -R 755 /etc/nginx/sites-available/talos
# Copy assets to nginx web root
echo "Copying Talos PXE boot assets to nginx web root."
TALOS_PXE_WEB_ROOT="/var/www/html/talos"
sudo mkdir -p "${TALOS_PXE_WEB_ROOT}"
sudo rm -rf ${TALOS_PXE_WEB_ROOT}/* # Clean the web root directory
sudo cp -r ${PXE_WEB_ROOT}/* "${TALOS_PXE_WEB_ROOT}"
sudo chown -R www-data:www-data "${TALOS_PXE_WEB_ROOT}"
sudo chmod -R 755 "${TALOS_PXE_WEB_ROOT}"
# echo "Copying Talos PXE boot assets to nginx web root."
# TALOS_PXE_WEB_ROOT="/var/www/html/talos"
# sudo mkdir -p "${TALOS_PXE_WEB_ROOT}"
# sudo rm -rf ${TALOS_PXE_WEB_ROOT}/* # Clean the web root directory
# sudo cp -r ${PXE_WEB_ROOT}/* "${TALOS_PXE_WEB_ROOT}"
# sudo chown -R www-data:www-data "${TALOS_PXE_WEB_ROOT}"
# sudo chmod -R 755 "${TALOS_PXE_WEB_ROOT}"
# Start nginx service to serve the iPXE script and images
echo "Starting nginx service."
sudo ln -s /etc/nginx/sites-available/talos /etc/nginx/sites-enabled/talos > /dev/null 2>&1 || true
sudo rm -f /etc/nginx/sites-enabled/default
sudo systemctl reload nginx
# echo "Starting nginx service."
# sudo ln -s /etc/nginx/sites-available/talos /etc/nginx/sites-enabled/talos > /dev/null 2>&1 || true
# sudo rm -f /etc/nginx/sites-enabled/default
# sudo systemctl reload nginx
# Stop and disable systemd-resolved if it is running
if systemctl is-active --quiet systemd-resolved; then
echo "Stopping and disabling systemd-resolved..."
sudo systemctl disable systemd-resolved
sudo systemctl stop systemd-resolved
# sudo rm -f /etc/resolv.conf
echo "systemd-resolved stopped and disabled"
fi
# if systemctl is-active --quiet systemd-resolved; then
# echo "Stopping and disabling systemd-resolved..."
# sudo systemctl disable systemd-resolved
# sudo systemctl stop systemd-resolved
# # sudo rm -f /etc/resolv.conf
# echo "systemd-resolved stopped and disabled"
# fi
# Update PXE's iPXE bootloader files.
echo "Updating iPXE ftpd bootloader files."
sudo mkdir -p /var/ftpd
sudo cp ${PXE_FTPD_DIR}/* /var/ftpd/
# echo "Updating iPXE ftpd bootloader files."
# sudo mkdir -p /var/ftpd
# sudo cp ${PXE_FTPD_DIR}/* /var/ftpd/
# Finally, install and configure DNSMasq.
echo "Configuring and starting DNSMasq."