Enhance dnsmasq setup with new scripts and documentation

- Add custom dictionary words for spell checking. - Refactor wild-central-generate-setup script for improved error handling and structure. - Create README.md for central dnsmasq setup with detailed instructions. - Implement create-setup-bundle.sh and setup.sh scripts for setting up dnsmasq and PXE booting. - Add transfer-setup-bundle.sh for transferring setup files to the server. - Update SETUP.md with clearer instructions for initial setup and configuration. - Introduce .gitignore for dnsmasq setup bundle.
2025-06-21 13:01:19 -07:00
parent 0f2e73e54c
commit 1fcec15853
9 changed files with 219 additions and 95 deletions
--- a/.cspell/custom-dictionary-workspace.txt
+++ b/.cspell/custom-dictionary-workspace.txt
@@ -1,5 +1,6 @@
 # Custom Dictionary Words
 apiextensions
+Bootloading
 certificaterequests
 certmanager
 containo
@@ -8,9 +9,11 @@ crds
 dnsmasq
 envsubst
 externaldns
+ftpd
 glddns
 gomplate
 IMMICH
+ipxe
 Jellyfin
 keepalives
 KUBECONFIG
--- a/bin/wild-central-generate-setup
+++ b/bin/wild-central-generate-setup
@@ -5,6 +5,27 @@
 set -e
 set -o pipefail

+# Initialize wildcloud environment.
+
+if [ ! -d ".wildcloud" ]; then
+    echo "Error: You must run this script from a wild-cloud directory"
+    exit 1
+fi
+
+WILDCLOUD_CONFIG_FILE="./config.yaml"
+if [ ! -f ${WILDCLOUD_CONFIG_FILE} ]; then
+    echo "Error: ${WILDCLOUD_CONFIG_FILE} not found"
+    exit 1
+fi
+
+WILDCLOUD_ROOT=$(yq eval '.wildcloud.root' ${WILDCLOUD_CONFIG_FILE})
+if [ -z "${WILDCLOUD_ROOT}" ] || [ "${WILDCLOUD_ROOT}" = "null" ]; then
+    echo "Error: wildcloud.root not found in ${WILDCLOUD_CONFIG_FILE}"
+    exit 1
+fi
+
+# ---
+
 # Function to process a file with gomplate.
 process_file() {
    local src_file="$1"
@@ -12,39 +33,14 @@ process_file() {
    
    if [[ "${src_file}" == *.yaml ]] || [[ "${src_file}" == *.ipxe ]] || [[ "${src_file}" == *.conf ]]; then
        echo "Processing YAML file: ${dest_file}"
-        gomplate -d config=.wildcloud/config.yaml -f "${src_file}" > "${dest_file}"
+        gomplate -d config=./config.yaml -f "${src_file}" > "${dest_file}"
    else
        cp "${src_file}" "${dest_file}"
    fi
 }

-
-# Initialize wildcloud environment.
-
-# Ensure we have a .wildcloud directory.
-if [ ! -d ".wildcloud" ]; then
-    echo "Error: .wildcloud directory not found in current directory"
-    echo "This script must be run from a directory that contains a .wildcloud directory"
-    exit 1
-fi
-
-# Ensure we have a config file.
-if [ ! -f ".wildcloud/config.yaml" ]; then
-    echo "Error: .wildcloud/config.yaml not found"
-    exit 1
-fi
-
-WILDCLOUD_CACHE_DIR=".wildcloud/cache"
-
-# Find the wildcloud repository path from the config file.
-WILDCLOUD_REPO=$(yq eval '.wildcloud.repository' .wildcloud/config.yaml)
-if [ -z "${WILDCLOUD_REPO}" ] || [ "${WILDCLOUD_REPO}" = "null" ]; then
-    echo "Error: wildcloud.repository not found in .wildcloud/config.yaml"
-    exit 1
-fi
-
-# The source templates for asq setup.
-DNSMASQ_TEMPLATE_DIR="${WILDCLOUD_REPO}/central-setup/dnsmasq"
+# The source templates for dnsmasq setup.
+DNSMASQ_TEMPLATE_DIR="${WILDCLOUD_ROOT}/central-setup/dnsmasq"
 if [ ! -d "${DNSMASQ_TEMPLATE_DIR}" ]; then
    echo "Error: DNSMasq setup directory not found at ${DNSMASQ_TEMPLATE_DIR}"
    exit 1
@@ -52,7 +48,6 @@ fi

 # Where to put the processed DNSMasq files.
 DNSMASQ_SETUP_DIR="cluster/dnsmasq"
-mkdir -p $DNSMASQ_SETUP_DIR

 # Optionally remove the setup directory if it already exists.
 if [ -d "${DNSMASQ_SETUP_DIR}" ]; then
@@ -66,6 +61,8 @@ if [ -d "${DNSMASQ_SETUP_DIR}" ]; then
    rm -rf "${DNSMASQ_SETUP_DIR}"
 fi

+mkdir -p $DNSMASQ_SETUP_DIR
+
 # Compile templates to setup directory.
 find "${DNSMASQ_TEMPLATE_DIR}" -type d | while read -r src_dir; do
    rel_path="${src_dir#${DNSMASQ_TEMPLATE_DIR}}"
@@ -88,47 +85,3 @@ find "${DNSMASQ_TEMPLATE_DIR}" -type f | while read -r src_file; do
 done

 echo "Successfully created dnsmasq setup files from templates."
-
-# Create Talos bare metal boot assets. 
-echo "Creating Talos bare metal boot assets..."
-TALOS_ID=$(curl -X POST --data-binary @${DNSMASQ_TEMPLATE_DIR}/bare-metal.yaml https://factory.talos.dev/schematics | jq -r '.id')
-if [ -z "${TALOS_ID}" ] || [ "${TALOS_ID}" = "null" ]; then
-    echo "Error: Failed to create Talos bare metal boot assets"
-    exit 1
-fi
-echo "Successfully created Talos bare metal boot assets with ID: ${TALOS_ID}"
-
-# Download Talos kernel and initramfs.
-echo "Downloading Talos kernel and initramfs for PXE boot..."
-NODE_IMAGES_DIR="${WILDCLOUD_CACHE_DIR}/pxe-web-root"
-mkdir -p "${NODE_IMAGES_DIR}"
-cp "${DNSMASQ_SETUP_DIR}/boot.ipxe" "${NODE_IMAGES_DIR}/boot.ipxe"
-mkdir -p "${NODE_IMAGES_DIR}/amd64"
-
-# Get Talos version from config
-TALOS_VERSION=$(yq eval '.cluster.nodes.talos.version' .wildcloud/config.yaml)
-if [ -z "${TALOS_VERSION}" ] || [ "${TALOS_VERSION}" = "null" ]; then
-    echo "Error: .cluster.nodes.talos.version not found in .wildcloud/config.yaml"
-    exit 1
-fi
-
-# Download kernel if not already exists
-if [ ! -f "${NODE_IMAGES_DIR}/amd64/vmlinuz" ]; then
-    echo "Downloading Talos kernel..."
-    wget -O "${NODE_IMAGES_DIR}/amd64/vmlinuz" "https://pxe.factory.talos.dev/image/${TALOS_ID}/${TALOS_VERSION}/kernel-amd64"
-else
-    echo "Talos kernel already exists, skipping download"
-fi
-
-# Download initramfs if not already exists
-if [ ! -f "${NODE_IMAGES_DIR}/amd64/initramfs.xz" ]; then
-    echo "Downloading Talos initramfs..."
-    wget -O "${NODE_IMAGES_DIR}/amd64/initramfs.xz" "https://pxe.factory.talos.dev/image/${TALOS_ID}/${TALOS_VERSION}/initramfs-amd64.xz"
-else
-    echo "Talos initramfs already exists, skipping download"
-fi
-
-# Copy files to dnsmasq server.
-echo "Copying DNSMasq setup files to dnsmasq server..."
-scp -r "${DNSMASQ_SETUP_DIR}"/* root@192.168.8.50:/tmp/dnsmasq-setup/
-scp -r "${NODE_IMAGES_DIR}"/* root@192.168.8.50:/tmp/dnsmasq-setup/pxe-web-root/
--- a/central-setup/README.md
+++ b/central-setup/README.md
@@ -1,23 +1,17 @@
 # Central setup

-"Central" is a Wild-cloud concept for a network appliance use for cloud utilities.
+**Central** is a separate machine on your network that provides core wild-cloud services.

 Right now, this is entirely `dnsmasq` to provide:

 - LAN DNS w/ forwarding of internal and external cloud domains to the cluster.
 - PXE for setting up cluster nodes.

-## Setup
+Read the [dnsmasq README.md](./dnsmasq/README.md) for how we set things up right now.

-The setup is going through architecture design right now.
+## _Future_ setup

- Initially, the process used to bootstrap a node was:
-  - Run `bin/install-dnsmasq` in your personal wildcloud dir to create a set of install files in `cluster/dnsmasq`.
-  - Copy this dir to a configured debian machine to have the services set up correctly as your Central.
-
-## Future setup
-
-To provide a better user experience, we have been creating a debian apt package that also does this while providing a UI.
+We _may_ follow a Central network appliance in the future, where one could install an apt package and use Central like a LAN router.

 Development repo: https://github.com/civil-society-dev/wild-central

--- a/central-setup/dnsmasq/.gitignore
+++ b/central-setup/dnsmasq/.gitignore
@@ -0,0 +1 @@
+setup-bundle/
--- a/central-setup/dnsmasq/README.md
+++ b/central-setup/dnsmasq/README.md
@@ -0,0 +1,68 @@
+# Central dnsmasq setup
+
+## Overview
+
+dnsmasq solves two problems for us. It provides:
+
+- LAN DNS w/ forwarding of internal and external cloud domains to the cluster.
+- PXE for setting up cluster nodes.
+
+### PXE Bootloading
+
+A "PXE client" is any machine that is booting using PXE. This is a great way to set up a new cluster node.
+
+- PXE client broadcasts a request for help across the LAN.
+- Dnsmasq's DHCP service responds with an IP address and TFTP server info.
+- PXE client downloads PXE's iPEXE bootloader files:
+  - `ipxe.efi`
+  - `undionly.kpxe`
+  - `ipxe-arm64.efi`
+    (`pxelinux.0`) via TFTP.
+- PXE client reads the bootloader config for the correct web address and fetches the boot files:
+  - The kernel, `vmlinuz`.
+  - The initial RAM disk, `initrd`.
+  - The Talos image,
+
+## Setup
+
+- Install a Linux machine on your LAN. Record it's IP address in your `config:cloud.dns.ip`.
+- Ensure it is accessible with ssh.
+- From your wild-cloud directory, run `wild-central-generate-setup`.
+- Run `cluster/dnsmasq/bin/create-setup-bundle.sh`
+- Run `cluster/dnsmasq/bin/transfer-setup-bundle.sh`
+
+Now ssh into your dnsmasq machine and do the following:
+
+```bash
+sudo -i
+cd dnsmasq-setup
+./setup.sh
+```
+
+## Future setup
+
+To provide a better user experience, we have been creating a debian apt package that also does this while providing a UI.
+
+Development repo: https://github.com/civil-society-dev/wild-central
+
+The setup will look something like this:
+
+```bash
+# Download and install GPG key
+curl -fsSL https://mywildcloud.org/apt/wild-cloud-central.gpg | sudo tee /usr/share/keyrings/wild-cloud-central-archive-keyring.gpg > /dev/null
+
+# Add repository (modern .sources format)
+sudo tee /etc/apt/sources.list.d/wild-cloud-central.sources << 'EOF'
+Types: deb
+URIs: https://mywildcloud.org/apt
+Suites: stable
+Components: main
+Signed-By: /usr/share/keyrings/wild-cloud-central-archive-keyring.gpg
+EOF
+
+# Update and install
+sudo apt update
+sudo apt install wild-cloud-central
+```
+
+browse to `http://localhost:5050`!
--- a/central-setup/dnsmasq/bin/create-setup-bundle.sh
+++ b/central-setup/dnsmasq/bin/create-setup-bundle.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Set up 
+
+# Initialize wildcloud environment.
+
+if [ ! -d ".wildcloud" ]; then
+    echo "Error: You must run this script from a wild-cloud directory"
+    exit 1
+fi
+
+WILDCLOUD_CONFIG_FILE="./config.yaml"
+if [ ! -f ${WILDCLOUD_CONFIG_FILE} ]; then
+    echo "Error: ${WILDCLOUD_CONFIG_FILE} not found"
+    exit 1
+fi
+
+
+WILDCLOUD_ROOT=$(yq eval '.wildcloud.root' ${WILDCLOUD_CONFIG_FILE})
+if [ -z "${WILDCLOUD_ROOT}" ] || [ "${WILDCLOUD_ROOT}" = "null" ]; then
+    echo "Error: wildcloud.root not found in ${WILDCLOUD_CONFIG_FILE}"
+    exit 1
+fi
+
+# ---
+
+DNSMASQ_SETUP_DIR="./cluster/dnsmasq"
+BUNDLE_DIR="${DNSMASQ_SETUP_DIR}/setup-bundle"
+mkdir -p "${BUNDLE_DIR}"
+
+
+# Copy iPXE bootloader to ipxe-web.
+echo "Copying Talos kernel and initramfs for PXE boot..."
+PXE_WEB_ROOT="${BUNDLE_DIR}/ipxe-web"
+mkdir -p "${PXE_WEB_ROOT}/amd64"
+cp "${DNSMASQ_SETUP_DIR}/boot.ipxe" "${PXE_WEB_ROOT}/boot.ipxe"
+
+# Create Talos bare metal boot assets.
+# This uses the Talos factory API to create boot assets for bare metal nodes.
+# These assets include the kernel and initramfs needed for PXE booting Talos on bare metal.
+echo "Creating Talos bare metal boot assets..."
+TALOS_ID=$(curl -X POST --data-binary @${DNSMASQ_SETUP_DIR}/bare-metal.yaml https://factory.talos.dev/schematics | jq -r '.id')
+if [ -z "${TALOS_ID}" ] || [ "${TALOS_ID}" = "null" ]; then
+    echo "Error: Failed to create Talos bare metal boot assets"
+    exit 1
+fi
+echo "Successfully created Talos bare metal boot assets with ID: ${TALOS_ID}"
+
+# Download kernel to ipxe-web if it's not already there.
+TALOS_VERSION=$(wild-config .cluster.nodes.talos.version) || exit 1
+if [ ! -f "${PXE_WEB_ROOT}/amd64/vmlinuz" ]; then
+    echo "Downloading Talos kernel..."
+    wget -O "${PXE_WEB_ROOT}/amd64/vmlinuz" "https://pxe.factory.talos.dev/image/${TALOS_ID}/${TALOS_VERSION}/kernel-amd64"
+else
+    echo "Talos kernel already exists, skipping download"
+fi
+
+# Download initramfs to ipxe-web if it's not already there.
+if [ ! -f "${PXE_WEB_ROOT}/amd64/initramfs.xz" ]; then
+    echo "Downloading Talos initramfs..."
+    wget -O "${PXE_WEB_ROOT}/amd64/initramfs.xz" "https://pxe.factory.talos.dev/image/${TALOS_ID}/${TALOS_VERSION}/initramfs-amd64.xz"
+else
+    echo "Talos initramfs already exists, skipping download"
+fi
+
+# Update PXE's iPXE bootloader files.
+# TODO: Put download to cache first.
+echo "Updating iPXE ftpd bootloader files."
+FTPD_DIR="${BUNDLE_DIR}/pxe-ftpd"
+mkdir -p $FTPD_DIR
+wget http://boot.ipxe.org/ipxe.efi -O ${FTPD_DIR}/ipxe.efi
+wget http://boot.ipxe.org/undionly.kpxe -O ${FTPD_DIR}/undionly.kpxe
+wget http://boot.ipxe.org/arm64-efi/ipxe.efi -O ${FTPD_DIR}/ipxe-arm64.efi
+
+
+cp "${DNSMASQ_SETUP_DIR}/nginx.conf" "${BUNDLE_DIR}/nginx.conf"
+cp "${DNSMASQ_SETUP_DIR}/dnsmasq.conf" "${BUNDLE_DIR}/dnsmasq.conf"
+cp "${DNSMASQ_SETUP_DIR}/bin/setup.sh" "${BUNDLE_DIR}/setup.sh"
--- a/central-setup/dnsmasq/bin/setup.sh
+++ b/central-setup/dnsmasq/bin/setup.sh
@@ -9,7 +9,8 @@ echo "Installing dnsmasq and nginx."
 sudo apt install -y dnsmasq nginx

 DNSMASQ_SETUP_DIR="/tmp/dnsmasq-setup"
-NODE_IMAGES_DIR="${DNSMASQ_SETUP_DIR}/pxe-web-root"
+PXE_FTPD_DIR="${DNSMASQ_SETUP_DIR}/pxe-ftpd"
+PXE_WEB_ROOT="${DNSMASQ_SETUP_DIR}/pxe-web"

 # Configure nginx.
 echo "Configuring nginx."
@@ -22,7 +23,7 @@ echo "Copying Talos PXE boot assets to nginx web root."
 TALOS_PXE_WEB_ROOT="/var/www/html/talos"
 sudo mkdir -p "${TALOS_PXE_WEB_ROOT}"
 sudo rm -rf ${TALOS_PXE_WEB_ROOT}/* # Clean the web root directory
-sudo cp -r ${NODE_IMAGES_DIR}/* "${TALOS_PXE_WEB_ROOT}"
+sudo cp -r ${PXE_WEB_ROOT}/* "${TALOS_PXE_WEB_ROOT}"
 sudo chown -R www-data:www-data "${TALOS_PXE_WEB_ROOT}"
 sudo chmod -R 755 "${TALOS_PXE_WEB_ROOT}"

@@ -42,13 +43,9 @@ if systemctl is-active --quiet systemd-resolved; then
 fi

 # Update PXE's iPXE bootloader files.
-# TODO: Put download to cache first.
 echo "Updating iPXE ftpd bootloader files."
 sudo mkdir -p /var/ftpd
-sudo wget http://boot.ipxe.org/ipxe.efi -O /var/ftpd/ipxe.efi
-sudo wget http://boot.ipxe.org/undionly.kpxe -O /var/ftpd/undionly.kpxe
-sudo wget http://boot.ipxe.org/arm64-efi/ipxe.efi -O /var/ftpd/ipxe-arm64.efi
-
+sudo cp ${PXE_FTPD_DIR}/* /var/ftpd/

 # Finally, install and configure DNSMasq.
 echo "Configuring and starting DNSMasq."
--- a/central-setup/dnsmasq/bin/transfer-setup-bundle.sh
+++ b/central-setup/dnsmasq/bin/transfer-setup-bundle.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ ! -d ".wildcloud" ]; then
+    echo "Error: You must run this script from a wild-cloud directory"
+    exit 1
+fi
+
+SERVER_HOST=$(wild-config cloud.dns.ip2) || exit 1
+SETUP_DIR="./cluster/dnsmasq/setup-bundle"
+DESTINATION_DIR="~/dnsmasq-setup"
+
+echo "Copying DNSMasq setup files to ${SERVER_HOST}:${DESTINATION_DIR}..."
+scp -r ${SETUP_DIR}/* root@${SERVER_HOST}:${DESTINATION_DIR}
--- a/docs/SETUP.md
+++ b/docs/SETUP.md
@@ -1,14 +1,31 @@
-# Setting Up Your Wild Cloud
+# Setting Up Your Wild-cloud
+
+## Initial setup
+
+- Add `bin` directory to your path.

 ## Set up your personal cloud operations directory

- Create a directory somewhere. We recommend you use an Ubuntu machine.
- Inside it, run `wild-init`. This will scaffold your cloud directory.
- In your cloud directory, update `.wildcloud/config.yaml`. Use the same values in this dir in a `.env`
+```bash
+cd ~
+mkdir ~/my-wild-cloud
+cd my-wild-cloud
+wild-init
+cp config.example.yaml config.yaml
+cp secrets.example.yaml secrets.yaml
+```
+
+## Configuring your wild-cloud
+
+Now, update your config.yaml and secrets.yaml.
+
+Instructions TBD.

 ## Set up your Cloud Central

-See [Central Setup](../central-setup/README.md).
+```bash
+bin/wild-central-generate-setup
+```

 ## Set up Control Nodes