Enhance wild-app-config and wild-app-fetch scripts with update option and improved argument parsing. Fixes secret management

bin/wild-app-config

@@ -3,13 +3,48 @@
 set -e
 set -o pipefail
 
-if [ $# -ne 1 ]; then
+UPDATE=false
-    echo "Usage: $0 <app_name>"
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --update)
+            UPDATE=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 <app_name> [--update]"
+            echo ""
+            echo "Configure an app by applying templates and merging configuration."
+            echo ""
+            echo "Options:"
+            echo "  --update    Overwrite existing app files without confirmation"
+            echo "  -h, --help  Show this help message"
+            exit 0
+            ;;
+        -*)
+            echo "Unknown option $1"
+            echo "Usage: $0 <app_name> [--update]"
+            exit 1
+            ;;
+        *)
+            if [ -z "${APP_NAME}" ]; then
+                APP_NAME="$1"
+            else
+                echo "Too many arguments"
+                echo "Usage: $0 <app_name> [--update]"
+                exit 1
+            fi
+            shift
+            ;;
+    esac
+done
+
+if [ -z "${APP_NAME}" ]; then
+    echo "Usage: $0 <app_name> [--update]"
     exit 1
 fi
 
-APP_NAME="$1"
-
 if [ ! -d ".wildcloud" ]; then
     echo "Error: .wildcloud directory not found in current directory"
     echo "This script must be run from a directory that contains a .wildcloud directory"
@@ -26,25 +61,105 @@ CACHE_APP_DIR=".wildcloud/cache/apps/${APP_NAME}"
 # Check if app is cached, if not fetch it first
 if [ ! -d "${CACHE_APP_DIR}" ]; then
     echo "App '${APP_NAME}' not found in cache, fetching..."
-    ./bin/wild-app-fetch "${APP_NAME}"
+    if [ "${UPDATE}" = true ]; then
+        ./bin/wild-app-fetch "${APP_NAME}" --update
+    else
+        ./bin/wild-app-fetch "${APP_NAME}"
+    fi
 fi
 
 DEST_APP_DIR="apps/${APP_NAME}"
 mkdir -p "apps"
 
 if [ -d "${DEST_APP_DIR}" ]; then
-    echo "Warning: Destination directory ${DEST_APP_DIR} already exists"
+    if [ "${UPDATE}" = true ]; then
-    read -p "Do you want to overwrite it? (y/N): " -n 1 -r
+        echo "Updating app '${APP_NAME}'"
-    echo
+        rm -rf "${DEST_APP_DIR}"
-    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    else
-        echo "Pull cancelled"
+        echo "Warning: Destination directory ${DEST_APP_DIR} already exists"
-        exit 1
+        read -p "Do you want to overwrite it? (y/N): " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            echo "Configuration cancelled"
+            exit 1
+        fi
+        rm -rf "${DEST_APP_DIR}"
     fi
-    rm -rf "${DEST_APP_DIR}"
 fi
 
 echo "Pulling app '${APP_NAME}' from cache to ${DEST_APP_DIR}"
 
+# Merge defaultConfig from manifest.yaml into .wildcloud/config.yaml
+MANIFEST_FILE="${CACHE_APP_DIR}/manifest.yaml"
+if [ -f "${MANIFEST_FILE}" ]; then
+    echo "Merging defaultConfig from manifest.yaml into .wildcloud/config.yaml"
+    
+    # Check if the app section exists in config.yaml, if not create it
+    if ! yq eval ".apps.${APP_NAME}" .wildcloud/config.yaml >/dev/null 2>&1; then
+        yq eval ".apps.${APP_NAME} = {}" -i .wildcloud/config.yaml
+    fi
+    
+    # Extract defaultConfig from manifest.yaml and merge into config.yaml
+    if yq eval '.defaultConfig' "${MANIFEST_FILE}" | grep -q -v '^null$'; then
+        # Merge each key from defaultConfig into the app's config, only if not already set
+        yq eval '.defaultConfig | keys | .[]' "${MANIFEST_FILE}" | while read -r key; do
+            # Get the value from defaultConfig
+            value=$(yq eval ".defaultConfig.${key}" "${MANIFEST_FILE}")
+            
+            # Check if key exists and is not null in app config
+            current_value=$(yq eval ".apps.${APP_NAME}.${key} // \"null\"" .wildcloud/config.yaml)
+            
+            if [ "${current_value}" = "null" ]; then
+                if [[ "${value}" =~ ^[0-9]+$ ]] || [[ "${value}" =~ ^[0-9]+\.[0-9]+$ ]] || [ "${value}" = "true" ] || [ "${value}" = "false" ]; then
+                    # Numeric, boolean values don't need quotes
+                    yq eval ".apps.${APP_NAME}.${key} = ${value}" -i .wildcloud/config.yaml
+                else
+                    # String values need quotes
+                    yq eval ".apps.${APP_NAME}.${key} = \"${value}\"" -i .wildcloud/config.yaml
+                fi
+            fi
+        done
+        echo "Merged defaultConfig for app '${APP_NAME}'"
+    fi
+    
+    # Scaffold required secrets into .wildcloud/secrets.yaml if they don't exist
+    if yq eval '.requiredSecrets' "${MANIFEST_FILE}" | grep -q -v '^null$'; then
+        echo "Scaffolding required secrets for app '${APP_NAME}'"
+        
+        # Ensure .wildcloud/secrets.yaml exists
+        if [ ! -f ".wildcloud/secrets.yaml" ]; then
+            echo "# Wild-Cloud Secrets Configuration" > .wildcloud/secrets.yaml
+            echo "# This file contains sensitive data and should NOT be committed to git" >> .wildcloud/secrets.yaml
+            echo "# Add this file to your .gitignore" >> .wildcloud/secrets.yaml
+            echo "" >> .wildcloud/secrets.yaml
+        fi
+        
+        # Check if apps section exists, if not create it
+        if ! yq eval ".apps" .wildcloud/secrets.yaml >/dev/null 2>&1; then
+            yq eval ".apps = {}" -i .wildcloud/secrets.yaml
+        fi
+        
+        # Check if app section exists, if not create it
+        if ! yq eval ".apps.${APP_NAME}" .wildcloud/secrets.yaml >/dev/null 2>&1; then
+            yq eval ".apps.${APP_NAME} = {}" -i .wildcloud/secrets.yaml
+        fi
+        
+        # Add dummy values for each required secret if not already present
+        yq eval '.requiredSecrets[]' "${MANIFEST_FILE}" | while read -r secret_path; do
+            current_value=$(yq eval ".${secret_path} // \"null\"" .wildcloud/secrets.yaml)
+            
+            if [ "${current_value}" = "null" ]; then
+                echo "Adding dummy secret: ${secret_path}"
+                # Extract just the key name for the dummy value
+                secret_key=$(basename "${secret_path}")
+                yq eval ".${secret_path} = \"CHANGE_ME_${secret_key^^}\"" -i .wildcloud/secrets.yaml
+            fi
+        done
+        
+        echo "Required secrets scaffolded for app '${APP_NAME}'"
+    fi
+fi
+
 # Function to process a file with gomplate if it's a YAML file
 process_file() {
     local src_file="$1"
@@ -52,7 +167,17 @@ process_file() {
     
     if [[ "${src_file}" == *.yaml ]] || [[ "${src_file}" == *.yml ]]; then
         echo "Processing YAML file: ${dest_file}"
-        gomplate -d config=.wildcloud/config.yaml -f "${src_file}" > "${dest_file}"
+        
+        # Build gomplate command with config context (enables .config shorthand)
+        gomplate_cmd="gomplate -c config=.wildcloud/config.yaml"
+        
+        # Add secrets context if secrets.yaml exists (enables .secrets shorthand)
+        if [ -f ".wildcloud/secrets.yaml" ]; then
+            gomplate_cmd="${gomplate_cmd} -c secrets=.wildcloud/secrets.yaml"
+        fi
+        
+        # Execute gomplate with the file
+        ${gomplate_cmd} -f "${src_file}" > "${dest_file}"
     else
         cp "${src_file}" "${dest_file}"
     fi

bin/wild-app-deploy

@@ -43,6 +43,140 @@ if [ ! -d "apps/${APP_NAME}" ]; then
     exit 1
 fi
 
+# Function to deploy secrets for an app
+deploy_secrets() {
+    local app_name="$1"
+    local target_namespace="${2:-${app_name}}"  # Default to app name if not specified
+    
+    # Check if app has a manifest with requiredSecrets
+    local manifest_file="apps/${app_name}/manifest.yaml"
+    if [ ! -f "${manifest_file}" ]; then
+        return 0
+    fi
+    
+    # Check if there are required secrets defined
+    if ! yq eval '.requiredSecrets' "${manifest_file}" | grep -q -v '^null$'; then
+        return 0
+    fi
+    
+    # Check if secrets.yaml exists
+    if [ ! -f ".wildcloud/secrets.yaml" ]; then
+        echo "Warning: .wildcloud/secrets.yaml not found, skipping secret deployment for ${app_name}"
+        return 0
+    fi
+    
+    # Use the target namespace parameter
+    local namespace="${target_namespace}"
+    
+    echo "Deploying secrets for app '${app_name}' in namespace '${namespace}'"
+    
+    # Create secret data
+    local secret_data=""
+    while IFS= read -r secret_path; do
+        # Get the secret value using full path from .wildcloud/secrets.yaml
+        secret_value=$(yq eval ".${secret_path} // \"\"" .wildcloud/secrets.yaml)
+        
+        # Extract just the key name for the Kubernetes secret (handle dotted paths)
+        secret_key="${secret_path##*.}"
+        
+        if [ -n "${secret_value}" ] && [ "${secret_value}" != "null" ]; then
+            if [[ "${secret_value}" == CHANGE_ME_* ]]; then
+                echo "Warning: Secret '${secret_path}' for app '${app_name}' still has dummy value: ${secret_value}"
+            fi
+            secret_data="${secret_data} --from-literal=${secret_key}=${secret_value}"
+        else
+            echo "Error: Required secret '${secret_path}' not found in .wildcloud/secrets.yaml for app '${app_name}'"
+            exit 1
+        fi
+    done < <(yq eval '.requiredSecrets[]' "${manifest_file}")
+    
+    # Create the secret if we have data
+    if [ -n "${secret_data}" ]; then
+        echo "Creating/updating secret '${app_name}-secrets' in namespace '${namespace}'"
+        if [ "${DRY_RUN:-}" = "--dry-run=client" ]; then
+            echo "DRY RUN: kubectl create secret generic ${app_name}-secrets ${secret_data} --namespace=${namespace} --dry-run=client -o yaml"
+        else
+            # Delete existing secret if it exists, then create new one
+            kubectl delete secret "${app_name}-secrets" --namespace="${namespace}" --ignore-not-found=true
+            kubectl create secret generic "${app_name}-secrets" ${secret_data} --namespace="${namespace}"
+        fi
+    fi
+}
+
+# Step 1: Create namespaces first (dependencies and main app)
+echo "Creating namespaces..."
+MANIFEST_FILE="apps/${APP_NAME}/manifest.yaml"
+if [ -f "${MANIFEST_FILE}" ]; then
+    if yq eval '.requires' "${MANIFEST_FILE}" | grep -q -v '^null$'; then
+        yq eval '.requires[].name' "${MANIFEST_FILE}" | while read -r required_app; do
+            if [ -z "${required_app}" ] || [ "${required_app}" = "null" ]; then
+                echo "Warning: Empty or null dependency found, skipping"
+                continue
+            fi
+            
+            if [ ! -d "apps/${required_app}" ]; then
+                echo "Error: Required dependency '${required_app}' not found in apps/ directory"
+                exit 1
+            fi
+            
+            if [ -f "apps/${required_app}/namespace.yaml" ]; then
+                echo "Creating namespace for dependency: ${required_app}"
+                kubectl apply -f "apps/${required_app}/namespace.yaml" ${DRY_RUN:-}
+            else
+                echo "Warning: No namespace.yaml found for dependency: ${required_app}"
+            fi
+        done
+    fi
+fi
+
+# Create namespace for main app
+if [ -f "apps/${APP_NAME}/namespace.yaml" ]; then
+    echo "Creating namespace for app: ${APP_NAME}"
+    kubectl apply -f "apps/${APP_NAME}/namespace.yaml" ${DRY_RUN:-}
+fi
+
+# Step 2: Deploy secrets (dependencies and main app)
+echo "Deploying secrets..."
+if [ -f "${MANIFEST_FILE}" ]; then
+    if yq eval '.requires' "${MANIFEST_FILE}" | grep -q -v '^null$'; then
+        echo "Deploying secrets for required dependencies..."
+        yq eval '.requires[].name' "${MANIFEST_FILE}" | while read -r required_app; do
+            if [ -z "${required_app}" ] || [ "${required_app}" = "null" ]; then
+                echo "Warning: Empty or null dependency found, skipping"
+                continue
+            fi
+            
+            if [ ! -d "apps/${required_app}" ]; then
+                echo "Error: Required dependency '${required_app}' not found in apps/ directory"
+                exit 1
+            fi
+            
+            echo "Deploying secrets for dependency: ${required_app}"
+            # Deploy secrets in dependency's own namespace
+            deploy_secrets "${required_app}"
+            # Also deploy dependency secrets in consuming app's namespace
+            echo "Copying dependency secrets to app namespace: ${APP_NAME}"
+            deploy_secrets "${required_app}" "${APP_NAME}"
+        done
+    fi
+fi
+
+# Deploy secrets for this app
+deploy_secrets "${APP_NAME}"
+
+# Step 2.5: Handle idempotent jobs (delete and recreate)
+echo "Managing idempotent jobs..."
+if [ -f "apps/${APP_NAME}/db-init-job.yaml" ]; then
+    echo "Deleting and recreating db-init job for idempotent execution"
+    kubectl delete job immich-db-init --namespace="${APP_NAME}" --ignore-not-found=true ${DRY_RUN:-}
+    # Wait for job deletion to complete
+    if [ "${DRY_RUN:-}" != "--dry-run=client" ]; then
+        kubectl wait --for=delete job/immich-db-init --namespace="${APP_NAME}" --timeout=30s || true
+    fi
+fi
+
+# Step 3: Deploy the main application
+echo "Deploying application..."
 if [ "${FORCE}" = true ]; then
     echo "Force deploying app '${APP_NAME}'"
     kubectl replace --force -k "apps/${APP_NAME}" ${DRY_RUN:-}

bin/wild-app-fetch

@@ -3,13 +3,48 @@
 set -e
 set -o pipefail
 
-if [ $# -ne 1 ]; then
+UPDATE=false
-    echo "Usage: $0 <app_name>"
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --update)
+            UPDATE=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 <app_name> [--update]"
+            echo ""
+            echo "Fetch an app template from the Wild-Cloud repository to cache."
+            echo ""
+            echo "Options:"
+            echo "  --update    Overwrite existing cached files without confirmation"
+            echo "  -h, --help  Show this help message"
+            exit 0
+            ;;
+        -*)
+            echo "Unknown option $1"
+            echo "Usage: $0 <app_name> [--update]"
+            exit 1
+            ;;
+        *)
+            if [ -z "${APP_NAME}" ]; then
+                APP_NAME="$1"
+            else
+                echo "Too many arguments"
+                echo "Usage: $0 <app_name> [--update]"
+                exit 1
+            fi
+            shift
+            ;;
+    esac
+done
+
+if [ -z "${APP_NAME}" ]; then
+    echo "Usage: $0 <app_name> [--update]"
     exit 1
 fi
 
-APP_NAME="$1"
-
 if [ ! -d ".wildcloud" ]; then
     echo "Error: .wildcloud directory not found in current directory"
     echo "This script must be run from a directory that contains a .wildcloud directory"
@@ -38,14 +73,19 @@ CACHE_APP_DIR=".wildcloud/cache/apps/${APP_NAME}"
 mkdir -p ".wildcloud/cache/apps"
 
 if [ -d "${CACHE_APP_DIR}" ]; then
-    echo "Warning: Cache directory ${CACHE_APP_DIR} already exists"
+    if [ "${UPDATE}" = true ]; then
-    read -p "Do you want to overwrite it? (y/N): " -n 1 -r
+        echo "Updating cached app '${APP_NAME}'"
-    echo
+        rm -rf "${CACHE_APP_DIR}"
-    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    else
-        echo "Fetch cancelled"
+        echo "Warning: Cache directory ${CACHE_APP_DIR} already exists"
-        exit 1
+        read -p "Do you want to overwrite it? (y/N): " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            echo "Fetch cancelled"
+            exit 1
+        fi
+        rm -rf "${CACHE_APP_DIR}"
     fi
-    rm -rf "${CACHE_APP_DIR}"
 fi
 
 echo "Fetching app '${APP_NAME}' from ${SOURCE_APP_DIR} to ${CACHE_APP_DIR}"