Add NFS support: implement setup scripts, Kubernetes integration, and configuration files

This commit is contained in:
2025-05-27 17:20:57 -07:00
parent 32d51b71ba
commit ccd7f4ba15
9 changed files with 652 additions and 2 deletions

View File

@@ -23,7 +23,10 @@ Internet → External DNS → MetalLB LoadBalancer → Traefik → Kubernetes Se
- **Traefik** - Handles ingress traffic, TLS termination, and routing - **Traefik** - Handles ingress traffic, TLS termination, and routing
- **cert-manager** - Manages TLS certificates - **cert-manager** - Manages TLS certificates
- **CoreDNS** - Provides DNS resolution for services - **CoreDNS** - Provides DNS resolution for services
- **Longhorn** - Distributed storage system for persistent volumes
- **NFS** - Network file system for shared media storage (optional)
- **Kubernetes Dashboard** - Web UI for cluster management (accessible via https://dashboard.internal.${DOMAIN}) - **Kubernetes Dashboard** - Web UI for cluster management (accessible via https://dashboard.internal.${DOMAIN})
- **Docker Registry** - Private container registry for custom images
## Configuration Approach ## Configuration Approach
@@ -44,3 +47,55 @@ All setup scripts are designed to be idempotent:
- Changes to configuration will be properly applied on subsequent runs - Changes to configuration will be properly applied on subsequent runs
This idempotent approach ensures consistent, reliable infrastructure setup and allows for incremental changes without requiring a complete teardown and rebuild. This idempotent approach ensures consistent, reliable infrastructure setup and allows for incremental changes without requiring a complete teardown and rebuild.
## NFS Setup (Optional)
The infrastructure supports optional NFS (Network File System) for shared media storage across the cluster:
### Host Setup
First, set up the NFS server on your chosen host:
```bash
# Set required environment variables
export NFS_HOST=box-01 # Hostname or IP of NFS server
export NFS_MEDIA_PATH=/data/media # Path to media directory
export NFS_STORAGE_CAPACITY=1Ti # Optional: PV size (default: 250Gi)
# Run host setup script on the NFS server
./infrastructure_setup/setup-nfs-host.sh
```
### Cluster Integration
Then integrate NFS with your Kubernetes cluster:
```bash
# Run cluster setup (part of setup-all.sh or standalone)
./infrastructure_setup/setup-nfs.sh
```
### Features
- **Automatic IP detection** - Uses network IP even when hostname resolves to localhost
- **Cluster-wide access** - Any pod can mount the NFS share regardless of node placement
- **Configurable capacity** - Set PersistentVolume size via `NFS_STORAGE_CAPACITY`
- **ReadWriteMany** - Multiple pods can simultaneously access the same storage
### Usage
Applications can use NFS storage by setting `storageClassName: nfs` in their PVCs:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: media-pvc
spec:
accessModes:
- ReadWriteMany
storageClassName: nfs
resources:
requests:
storage: 100Gi
```

View File

@@ -25,7 +25,7 @@ spec:
- --source=ingress - --source=ingress
- --txt-owner-id=${OWNER_ID} - --txt-owner-id=${OWNER_ID}
- --provider=cloudflare - --provider=cloudflare
- --domain-filter=${DOMAIN} - --domain-filter=payne.io
#- --exclude-domains=internal.${DOMAIN} #- --exclude-domains=internal.${DOMAIN}
- --cloudflare-dns-records-per-page=5000 - --cloudflare-dns-records-per-page=5000
- --publish-internal-services - --publish-internal-services

View File

@@ -0,0 +1,53 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- persistent-volume.yaml
- storage-class.yaml
replacements:
- source:
kind: ConfigMap
name: nfs-config
fieldPath: data.NFS_HOST_IP
targets:
- select:
kind: PersistentVolume
name: nfs-media-pv
fieldPaths:
- spec.nfs.server
- select:
kind: StorageClass
name: nfs
fieldPaths:
- parameters.server
- source:
kind: ConfigMap
name: nfs-config
fieldPath: data.NFS_MEDIA_PATH
targets:
- select:
kind: PersistentVolume
name: nfs-media-pv
fieldPaths:
- spec.nfs.path
- select:
kind: StorageClass
name: nfs
fieldPaths:
- parameters.path
- source:
kind: ConfigMap
name: nfs-config
fieldPath: data.NFS_STORAGE_CAPACITY
targets:
- select:
kind: PersistentVolume
name: nfs-media-pv
fieldPaths:
- spec.capacity.storage
configMapGenerator:
- name: nfs-config
envs:
- config/config.env

View File

@@ -0,0 +1,23 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: nfs-media-pv
labels:
storage: nfs-media
spec:
capacity:
storage: REPLACE_ME
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs
nfs:
server: REPLACE_ME
path: REPLACE_ME
mountOptions:
- nfsvers=4.1
- rsize=1048576
- wsize=1048576
- hard
- intr
- timeo=600

View File

@@ -0,0 +1,10 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs
provisioner: nfs
parameters:
server: REPLACE_ME
path: REPLACE_ME
reclaimPolicy: Retain
allowVolumeExpansion: true

View File

@@ -35,9 +35,11 @@ chmod +x *.sh
# Setup Kubernetes Dashboard # Setup Kubernetes Dashboard
./setup-dashboard.sh ./setup-dashboard.sh
# Setup NFS Kubernetes integration (optional)
./setup-nfs.sh
# Setup Docker Registry # Setup Docker Registry
./setup-registry.sh ./setup-registry.sh
kubectl apply -k docker-registry
echo "Infrastructure setup complete!" echo "Infrastructure setup complete!"
echo echo

View File

@@ -0,0 +1,257 @@
#!/bin/bash
set -e
set -o pipefail
# Navigate to script directory
SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")"
SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
# Source environment variables
source "${PROJECT_DIR}/load-env.sh"
echo "Setting up NFS server on this host..."
# Check if required NFS variables are configured
if [[ -z "${NFS_HOST}" ]]; then
echo "NFS_HOST not set. Please set NFS_HOST=<hostname> in your environment"
echo "Example: export NFS_HOST=box-01"
exit 1
fi
# Ensure NFS_MEDIA_PATH is explicitly set
if [[ -z "${NFS_MEDIA_PATH}" ]]; then
echo "Error: NFS_MEDIA_PATH not set. Please set it in your environment"
echo "Example: export NFS_MEDIA_PATH=/data/media"
exit 1
fi
# Set default for NFS_EXPORT_OPTIONS if not already set
if [[ -z "${NFS_EXPORT_OPTIONS}" ]]; then
export NFS_EXPORT_OPTIONS="*(rw,sync,no_subtree_check,no_root_squash)"
echo "Using default NFS_EXPORT_OPTIONS: ${NFS_EXPORT_OPTIONS}"
fi
echo "Target NFS host: ${NFS_HOST}"
echo "Media path: ${NFS_MEDIA_PATH}"
echo "Export options: ${NFS_EXPORT_OPTIONS}"
# Function to check if we're running on the correct host
check_host() {
local current_hostname=$(hostname)
if [[ "${current_hostname}" != "${NFS_HOST}" ]]; then
echo "Warning: Current host (${current_hostname}) differs from NFS_HOST (${NFS_HOST})"
echo "This script should be run on ${NFS_HOST}"
read -p "Continue anyway? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
fi
}
# Function to install NFS server and SMB/CIFS
install_nfs_server() {
echo "Installing NFS server and SMB/CIFS packages..."
# Detect package manager and install NFS server + Samba
if command -v apt-get >/dev/null 2>&1; then
# Debian/Ubuntu
sudo apt-get update
sudo apt-get install -y nfs-kernel-server nfs-common samba samba-common-bin
elif command -v yum >/dev/null 2>&1; then
# RHEL/CentOS
sudo yum install -y nfs-utils samba samba-client
elif command -v dnf >/dev/null 2>&1; then
# Fedora
sudo dnf install -y nfs-utils samba samba-client
else
echo "Error: Unable to detect package manager. Please install NFS server and Samba manually."
exit 1
fi
}
# Function to create media directory
create_media_directory() {
echo "Creating media directory: ${NFS_MEDIA_PATH}"
# Create directory if it doesn't exist
sudo mkdir -p "${NFS_MEDIA_PATH}"
# Set appropriate permissions
# Using 755 for directory, allowing read/execute for all, write for owner
sudo chmod 755 "${NFS_MEDIA_PATH}"
echo "Media directory created with appropriate permissions"
echo "Directory info:"
ls -la "${NFS_MEDIA_PATH}/"
}
# Function to configure NFS exports
configure_nfs_exports() {
echo "Configuring NFS exports..."
local export_line="${NFS_MEDIA_PATH} ${NFS_EXPORT_OPTIONS}"
local exports_file="/etc/exports"
# Backup existing exports file
sudo cp "${exports_file}" "${exports_file}.backup.$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
# Check if export already exists
if sudo grep -q "^${NFS_MEDIA_PATH}" "${exports_file}" 2>/dev/null; then
echo "Export for ${NFS_MEDIA_PATH} already exists, updating..."
sudo sed -i "s|^${NFS_MEDIA_PATH}.*|${export_line}|" "${exports_file}"
else
echo "Adding new export for ${NFS_MEDIA_PATH}..."
echo "${export_line}" | sudo tee -a "${exports_file}"
fi
# Export the filesystems
sudo exportfs -rav
echo "NFS exports configured:"
sudo exportfs -v
}
# Function to start and enable NFS services
start_nfs_services() {
echo "Starting NFS services..."
# Start and enable NFS server
sudo systemctl enable nfs-server
sudo systemctl start nfs-server
# Also enable related services
sudo systemctl enable rpcbind
sudo systemctl start rpcbind
echo "NFS services started and enabled"
# Show service status
sudo systemctl status nfs-server --no-pager --lines=5
}
# Function to configure SMB/CIFS sharing
configure_smb_sharing() {
echo "Configuring SMB/CIFS sharing..."
local smb_config="/etc/samba/smb.conf"
local share_name="media"
# Backup existing config
sudo cp "${smb_config}" "${smb_config}.backup.$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
# Check if share already exists
if sudo grep -q "^\[${share_name}\]" "${smb_config}" 2>/dev/null; then
echo "SMB share '${share_name}' already exists, updating..."
# Remove existing share section
sudo sed -i "/^\[${share_name}\]/,/^\[/{ /^\[${share_name}\]/d; /^\[/!d; }" "${smb_config}"
fi
# Add media share configuration
cat << EOF | sudo tee -a "${smb_config}"
[${share_name}]
comment = Media files for Jellyfin
path = ${NFS_MEDIA_PATH}
browseable = yes
read only = no
guest ok = yes
create mask = 0664
directory mask = 0775
force user = $(whoami)
force group = $(whoami)
EOF
echo "SMB share configuration added"
# Test configuration
if sudo testparm -s >/dev/null 2>&1; then
echo "✓ SMB configuration is valid"
else
echo "✗ SMB configuration has errors"
sudo testparm
exit 1
fi
}
# Function to start SMB services
start_smb_services() {
echo "Starting SMB services..."
# Enable and start Samba services
sudo systemctl enable smbd
sudo systemctl start smbd
sudo systemctl enable nmbd
sudo systemctl start nmbd
echo "SMB services started and enabled"
# Show service status
sudo systemctl status smbd --no-pager --lines=3
}
# Function to test NFS setup
test_nfs_setup() {
echo "Testing NFS setup..."
# Test if NFS is responding
if command -v showmount >/dev/null 2>&1; then
echo "Available NFS exports:"
showmount -e localhost || echo "Warning: showmount failed, but NFS may still be working"
fi
# Check if the export directory is accessible
if [[ -d "${NFS_MEDIA_PATH}" ]]; then
echo "✓ Media directory exists and is accessible"
else
echo "✗ Media directory not accessible"
exit 1
fi
}
# Function to show usage instructions
show_usage_instructions() {
echo
echo "=== NFS/SMB Host Setup Complete ==="
echo
echo "NFS and SMB servers are now running on this host with media directory: ${NFS_MEDIA_PATH}"
echo
echo "Access methods:"
echo "1. NFS (for Kubernetes): Use setup-nfs-k8s.sh to register with cluster"
echo "2. SMB/CIFS (for Windows): \\\\${NFS_HOST}\\media"
echo
echo "To add media files:"
echo "- Copy directly to: ${NFS_MEDIA_PATH}"
echo "- Or mount SMB share from Windows and copy there"
echo
echo "Windows SMB mount:"
echo "- Open File Explorer"
echo "- Map network drive to: \\\\${NFS_HOST}\\media"
echo "- Or use: \\\\$(hostname -I | awk '{print $1}')\\media"
echo
echo "To verify services:"
echo "- NFS: showmount -e ${NFS_HOST}"
echo "- SMB: smbclient -L ${NFS_HOST} -N"
echo "- Status: systemctl status nfs-server smbd"
echo
echo "Current NFS exports:"
sudo exportfs -v
echo
}
# Main execution
main() {
check_host
install_nfs_server
create_media_directory
configure_nfs_exports
start_nfs_services
configure_smb_sharing
start_smb_services
test_nfs_setup
show_usage_instructions
}
# Run main function
main "$@"

230
infrastructure_setup/setup-nfs.sh Executable file
View File

@@ -0,0 +1,230 @@
#!/bin/bash
set -e
set -o pipefail
# Navigate to script directory
SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")"
SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
# Source environment variables
source "${PROJECT_DIR}/load-env.sh"
echo "Registering NFS server with Kubernetes cluster..."
# Check if NFS_HOST is configured
if [[ -z "${NFS_HOST}" ]]; then
echo "NFS_HOST not set. Skipping NFS Kubernetes setup."
echo "To enable NFS media sharing:"
echo "1. Set NFS_HOST=<hostname> in your environment"
echo "2. Run setup-nfs-host.sh on the NFS host"
echo "3. Re-run this script"
exit 0
fi
# Set default for NFS_STORAGE_CAPACITY if not already set
if [[ -z "${NFS_STORAGE_CAPACITY}" ]]; then
export NFS_STORAGE_CAPACITY="250Gi"
echo "Using default NFS_STORAGE_CAPACITY: ${NFS_STORAGE_CAPACITY}"
fi
echo "NFS host: ${NFS_HOST}"
echo "Media path: ${NFS_MEDIA_PATH}"
echo "Storage capacity: ${NFS_STORAGE_CAPACITY}"
# Function to resolve NFS host to IP
resolve_nfs_host() {
if [[ "${NFS_HOST}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
# NFS_HOST is already an IP address
NFS_HOST_IP="${NFS_HOST}"
else
# Resolve hostname to IP
NFS_HOST_IP=$(getent hosts "${NFS_HOST}" | awk '{print $1}' | head -n1)
if [[ -z "${NFS_HOST_IP}" ]]; then
echo "Error: Unable to resolve hostname ${NFS_HOST} to IP address"
echo "Make sure ${NFS_HOST} is resolvable from this cluster"
exit 1
fi
# Check if resolved IP is localhost - auto-detect network IP instead
if [[ "${NFS_HOST_IP}" =~ ^127\. ]]; then
echo "Warning: ${NFS_HOST} resolves to localhost (${NFS_HOST_IP})"
echo "Auto-detecting network IP for cluster access..."
# Try to find the primary network interface IP (exclude docker/k8s networks)
local network_ip=$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' 2>/dev/null)
if [[ -n "${network_ip}" && ! "${network_ip}" =~ ^127\. ]]; then
echo "Using detected network IP: ${network_ip}"
NFS_HOST_IP="${network_ip}"
else
echo "Could not auto-detect network IP. Available IPs:"
ip addr show | grep "inet " | grep -v "127.0.0.1" | grep -v "10.42" | grep -v "172." | awk '{print " " $2}' | cut -d/ -f1
echo "Please set NFS_HOST to the correct IP address manually."
exit 1
fi
fi
fi
echo "NFS server IP: ${NFS_HOST_IP}"
export NFS_HOST_IP
}
# Function to test NFS accessibility
test_nfs_accessibility() {
echo "Testing NFS accessibility from cluster..."
# Check if showmount is available
if ! command -v showmount >/dev/null 2>&1; then
echo "Installing NFS client tools..."
if command -v apt-get >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y nfs-common
elif command -v yum >/dev/null 2>&1; then
sudo yum install -y nfs-utils
elif command -v dnf >/dev/null 2>&1; then
sudo dnf install -y nfs-utils
else
echo "Warning: Unable to install NFS client tools. Skipping accessibility test."
return 0
fi
fi
# Test if we can reach the NFS server
echo "Testing connection to NFS server..."
if timeout 10 showmount -e "${NFS_HOST_IP}" >/dev/null 2>&1; then
echo "✓ NFS server is accessible"
echo "Available exports:"
showmount -e "${NFS_HOST_IP}"
else
echo "✗ Cannot connect to NFS server at ${NFS_HOST_IP}"
echo "Make sure:"
echo "1. NFS server is running on ${NFS_HOST}"
echo "2. Network connectivity exists between cluster and NFS host"
echo "3. Firewall allows NFS traffic (port 2049)"
exit 1
fi
# Test specific export
if showmount -e "${NFS_HOST_IP}" | grep -q "${NFS_MEDIA_PATH}"; then
echo "✓ Media path ${NFS_MEDIA_PATH} is exported"
else
echo "✗ Media path ${NFS_MEDIA_PATH} is not found in exports"
echo "Available exports:"
showmount -e "${NFS_HOST_IP}"
echo
echo "Run setup-nfs-host.sh on ${NFS_HOST} to configure the export"
exit 1
fi
}
# Function to create test mount
test_nfs_mount() {
echo "Testing NFS mount functionality..."
local test_mount="/tmp/nfs-test-$$"
mkdir -p "${test_mount}"
# Try to mount the NFS export
if timeout 30 sudo mount -t nfs4 "${NFS_HOST_IP}:${NFS_MEDIA_PATH}" "${test_mount}"; then
echo "✓ NFS mount successful"
# Test read access
if ls "${test_mount}" >/dev/null 2>&1; then
echo "✓ NFS read access working"
else
echo "✗ NFS read access failed"
fi
# Unmount
sudo umount "${test_mount}" || echo "Warning: Failed to unmount test directory"
else
echo "✗ NFS mount failed"
echo "Check NFS server configuration and network connectivity"
exit 1
fi
# Clean up
rmdir "${test_mount}" 2>/dev/null || true
}
# Function to create Kubernetes resources
create_k8s_resources() {
echo "Creating Kubernetes NFS resources..."
# Generate config file with resolved variables
local nfs_dir="${SCRIPT_DIR}/nfs"
local env_file="${nfs_dir}/config/.env"
local config_file="${nfs_dir}/config/config.env"
echo "Generating NFS configuration..."
export NFS_HOST_IP
export NFS_MEDIA_PATH
export NFS_STORAGE_CAPACITY
envsubst < "${env_file}" > "${config_file}"
# Apply the NFS Kubernetes manifests using kustomize
echo "Applying NFS manifests from: ${nfs_dir}"
kubectl apply -k "${nfs_dir}"
echo "✓ NFS PersistentVolume and StorageClass created"
# Verify resources were created
echo "Verifying Kubernetes resources..."
if kubectl get storageclass nfs >/dev/null 2>&1; then
echo "✓ StorageClass 'nfs' created"
else
echo "✗ StorageClass 'nfs' not found"
exit 1
fi
if kubectl get pv nfs-media-pv >/dev/null 2>&1; then
echo "✓ PersistentVolume 'nfs-media-pv' created"
kubectl get pv nfs-media-pv
else
echo "✗ PersistentVolume 'nfs-media-pv' not found"
exit 1
fi
}
# Function to show usage instructions
show_usage_instructions() {
echo
echo "=== NFS Kubernetes Setup Complete ==="
echo
echo "NFS server ${NFS_HOST} (${NFS_HOST_IP}) has been registered with the cluster"
echo
echo "Kubernetes resources created:"
echo "- StorageClass: nfs"
echo "- PersistentVolume: nfs-media-pv (${NFS_STORAGE_CAPACITY}, ReadWriteMany)"
echo
echo "To use NFS storage in your applications:"
echo "1. Set storageClassName: nfs in your PVC"
echo "2. Use accessMode: ReadWriteMany for shared access"
echo
echo "Example PVC:"
echo "---"
echo "apiVersion: v1"
echo "kind: PersistentVolumeClaim"
echo "metadata:"
echo " name: my-nfs-pvc"
echo "spec:"
echo " accessModes:"
echo " - ReadWriteMany"
echo " storageClassName: nfs"
echo " resources:"
echo " requests:"
echo " storage: 10Gi"
echo
}
# Main execution
main() {
resolve_nfs_host
test_nfs_accessibility
test_nfs_mount
create_k8s_resources
show_usage_instructions
}
# Run main function
main "$@"

View File

@@ -0,0 +1,20 @@
#!/bin/bash
set -e
# Navigate to script directory
SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")"
SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
echo "Setting up Docker Registry..."
# Apply the docker registry manifests using kustomize
kubectl apply -k "${SCRIPT_DIR}/docker-registry"
echo "Waiting for Docker Registry to be ready..."
kubectl wait --for=condition=available --timeout=300s deployment/docker-registry -n docker-registry
echo "Docker Registry setup complete!"
# Show deployment status
kubectl get pods -n docker-registry
kubectl get services -n docker-registry