wild-cloud/api/internal/apps/apps.go

package apps

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log/slog"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v3"

	wcconfig "github.com/wild-cloud/wild-central/daemon/internal/config"
	"github.com/wild-cloud/wild-central/daemon/internal/instance"
	"github.com/wild-cloud/wild-central/daemon/internal/operations"
	"github.com/wild-cloud/wild-central/daemon/internal/secrets"
	"github.com/wild-cloud/wild-central/daemon/internal/storage"
	"github.com/wild-cloud/wild-central/daemon/internal/tools"
	"github.com/wild-cloud/wild-central/daemon/internal/utilities"
)

// Manager handles application lifecycle operations
type Manager struct {
	dataDir string
	appsDir string // Path to apps directory in repo
}

// NewManager creates a new apps manager
func NewManager(dataDir, appsDir string) *Manager {
	return &Manager{
		dataDir: dataDir,
		appsDir: appsDir,
	}
}

// getAppPackageDir returns the directory for a compiled app package.
func (m *Manager) getAppPackageDir(instanceName, appName string) string {
	return tools.GetAppPackagePath(m.dataDir, instanceName, appName)
}

// getAppsListDir returns the parent directory that contains all compiled app packages.
func (m *Manager) getAppsListDir(instanceName string) string {
	return tools.GetAppsPackageDirPath(m.dataDir, instanceName)
}

// getAppConfigFile returns the config file path for an app.
func (m *Manager) getAppConfigFile(instanceName, appName string) string {
	return tools.GetAppConfigPath(m.dataDir, instanceName, appName)
}

// getAppSecretsFile returns the secrets file path for an app.
func (m *Manager) getAppSecretsFile(instanceName, appName string) string {
	return tools.GetAppSecretsPath(m.dataDir, instanceName, appName)
}

// appConfigKeyPrefix returns the yq key prefix for app config access.
func (m *Manager) appConfigKeyPrefix(instanceName, appName string) string {
	return "."
}

// resolveAppDir returns the filesystem path containing the app's files to use.
// New-style: reads app.yaml, returns versions/{latest}/ directory.
// Old-style (fallback): returns the app root if manifest.yaml exists there.
// If version is non-empty, returns versions/{version}/ (new) or .versions/{version}/ (old).
func (m *Manager) resolveAppDir(appName, version string) (string, *AppMeta, error) {
	appRoot := filepath.Join(m.appsDir, appName)
	appYamlPath := filepath.Join(appRoot, "app.yaml")

	// New-style: app.yaml exists
	if storage.FileExists(appYamlPath) {
		data, err := os.ReadFile(appYamlPath)
		if err != nil {
			return "", nil, fmt.Errorf("failed to read app.yaml for %s: %w", appName, err)
		}
		var meta AppMeta
		if err := yaml.Unmarshal(data, &meta); err != nil {
			return "", nil, fmt.Errorf("failed to parse app.yaml for %s: %w", appName, err)
		}
		if version == "" {
			// No version requested — use latest slot
			versionDir := filepath.Join(appRoot, "versions", meta.Latest)
			if !storage.FileExists(filepath.Join(versionDir, "manifest.yaml")) {
				return "", nil, fmt.Errorf("latest slot %q not found at %s", meta.Latest, versionDir)
			}
			return versionDir, &meta, nil
		}
		// Version requested — try as slot name first, then scan by version string
		versionDir := filepath.Join(appRoot, "versions", version)
		if storage.FileExists(filepath.Join(versionDir, "manifest.yaml")) {
			return versionDir, &meta, nil
		}
		// Scan version directories for a manifest with the matching version string
		versionsDir := filepath.Join(appRoot, "versions")
		entries, err := os.ReadDir(versionsDir)
		if err == nil {
			for _, entry := range entries {
				if !entry.IsDir() {
					continue
				}
				manifest := readManifestFile(filepath.Join(versionsDir, entry.Name()))
				if manifest != nil && manifest.Version == version {
					return filepath.Join(versionsDir, entry.Name()), &meta, nil
				}
			}
		}
		return "", nil, fmt.Errorf("version %s not found at %s", version, versionDir)
	}

	// Old-style fallback: manifest.yaml at root
	if version != "" {
		versionDir := filepath.Join(appRoot, ".versions", version)
		if storage.FileExists(filepath.Join(versionDir, "manifest.yaml")) {
			return versionDir, nil, nil
		}
		return "", nil, fmt.Errorf("version %s not found for app %s", version, appName)
	}
	manifestPath := filepath.Join(appRoot, "manifest.yaml")
	if storage.FileExists(manifestPath) {
		return appRoot, nil, nil
	}
	return "", nil, fmt.Errorf("app %s not found", appName)
}

// applyMeta overlays app-level fields from AppMeta onto a manifest parsed from a version directory.
func applyMeta(manifest *AppManifest, meta *AppMeta) {
	if meta == nil {
		return
	}
	manifest.Name = meta.Name
	manifest.Is = meta.Is
	manifest.Description = meta.Description
	manifest.Icon = meta.Icon
	manifest.Category = meta.Category
}

// ResolveNamespace determines the Kubernetes namespace for an app.
// Priority: app config (apps.<appName>.namespace or per-app namespace) > manifest namespace field > appName fallback.
func (m *Manager) ResolveNamespace(inst *instance.Instance, appName string) string {
	// 1. Check app config for namespace
	configFile := m.getAppConfigFile(inst.Name, appName)
	prefix := m.appConfigKeyPrefix(inst.Name, appName)
	yq := tools.NewYQ()
	if ns, err := yq.Get(configFile, prefix+"namespace"); err == nil && ns != "" && ns != "null" {
		return strings.TrimSpace(ns)
	}

	// 2. Check manifest namespace field (legacy)
	manifestPath := filepath.Join(m.getAppPackageDir(inst.Name, appName), "manifest.yaml")
	if storage.FileExists(manifestPath) {
		data, _ := os.ReadFile(manifestPath)
		var manifest struct {
			Namespace string `yaml:"namespace"`
		}
		if yaml.Unmarshal(data, &manifest) == nil && manifest.Namespace != "" {
			return manifest.Namespace
		}
	}

	// 3. Fallback: appName
	return appName
}

// App represents an application
type App struct {
	Name           string                 `json:"name" yaml:"name"`
	Description    string                 `json:"description" yaml:"description"`
	Version        string                 `json:"version" yaml:"version"`
	Category       string                 `json:"category,omitempty" yaml:"category,omitempty"`
	Icon           string                 `json:"icon,omitempty" yaml:"icon,omitempty"`
	Requires       []AppDependency        `json:"requires,omitempty" yaml:"requires,omitempty"`
	Dependencies   []string               `json:"dependencies" yaml:"dependencies"` // Deprecated: kept for backward compatibility
	Config         map[string]string      `json:"config,omitempty" yaml:"config,omitempty"`
	DefaultConfig  map[string]any `json:"defaultConfig,omitempty" yaml:"defaultConfig,omitempty"`
	DefaultSecrets []string               `json:"defaultSecrets,omitempty" yaml:"defaultSecrets,omitempty"`
}

// DeployedApp represents a deployed application instance
type DeployedApp struct {
	Name       string `json:"name"`
	Is         string `json:"is,omitempty"` // The original app type
	Status     string `json:"status"`
	Version    string `json:"version"`
	Category   string `json:"category,omitempty"`
	Namespace  string `json:"namespace"`
	URL        string `json:"url,omitempty"`
	Icon       string `json:"icon,omitempty"`
	ConfigOnly bool   `json:"configOnly,omitempty"`
}

// ListAvailable lists all available apps from the apps directory
// If category is non-empty, only apps matching that category are returned.
func (m *Manager) ListAvailable(category string) ([]App, error) {
	if m.appsDir == "" {
		return []App{}, fmt.Errorf("apps directory not configured")
	}

	// Read apps directory
	entries, err := os.ReadDir(m.appsDir)
	if err != nil {
		return []App{}, fmt.Errorf("failed to read apps directory: %w", err)
	}

	apps := []App{}
	for _, entry := range entries {
		if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
			continue
		}

		// Resolve app directory (handles both app.yaml + versions/ and legacy root manifest)
		appDir, meta, err := m.resolveAppDir(entry.Name(), "")
		if err != nil {
			continue
		}

		// Parse version manifest
		data, err := os.ReadFile(filepath.Join(appDir, "manifest.yaml"))
		if err != nil {
			continue
		}

		var manifest AppManifest
		if err := yaml.Unmarshal(data, &manifest); err != nil {
			continue
		}

		// Apply app-level fields from app.yaml if available
		applyMeta(&manifest, meta)

		// Convert manifest to App struct
		app := App{
			Name:          entry.Name(), // Use directory name as app name
			Description:   manifest.Description,
			Version:       manifest.Version,
			Category:      manifest.Category,
			Icon:          manifest.Icon,
			DefaultConfig: manifest.DefaultConfig,
			Requires:      manifest.Requires, // Include full dependency information
		}

		// Extract secret keys from DefaultSecrets
		if len(manifest.DefaultSecrets) > 0 {
			app.DefaultSecrets = make([]string, len(manifest.DefaultSecrets))
			for i, secret := range manifest.DefaultSecrets {
				app.DefaultSecrets[i] = secret.Key
			}
		}

		// Extract dependencies from Requires field (for backward compatibility)
		if len(manifest.Requires) > 0 {
			app.Dependencies = make([]string, len(manifest.Requires))
			for i, dep := range manifest.Requires {
				app.Dependencies[i] = dep.Name
			}
		}

		// Filter by category if specified
		if category != "" && manifest.Category != category {
			continue
		}

		apps = append(apps, app)
	}

	return apps, nil
}

// Get returns details for a specific available app
func (m *Manager) Get(appName string) (*App, error) {
	appDir, meta, err := m.resolveAppDir(appName, "")
	if err != nil {
		return nil, err
	}

	data, err := os.ReadFile(filepath.Join(appDir, "manifest.yaml"))
	if err != nil {
		return nil, fmt.Errorf("failed to read app file: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(data, &manifest); err != nil {
		return nil, fmt.Errorf("failed to parse app file: %w", err)
	}

	applyMeta(&manifest, meta)

	// Convert manifest to App struct
	app := &App{
		Name:          appName,
		Description:   manifest.Description,
		Version:       manifest.Version,
		Category:      manifest.Category,
		Icon:          manifest.Icon,
		DefaultConfig: manifest.DefaultConfig,
		Requires:      manifest.Requires, // Include full dependency information
	}

	// Extract secret keys from DefaultSecrets
	if len(manifest.DefaultSecrets) > 0 {
		app.DefaultSecrets = make([]string, len(manifest.DefaultSecrets))
		for i, secret := range manifest.DefaultSecrets {
			app.DefaultSecrets[i] = secret.Key
		}
	}

	// Extract dependencies from Requires field (for backward compatibility)
	if len(manifest.Requires) > 0 {
		app.Dependencies = make([]string, len(manifest.Requires))
		for i, dep := range manifest.Requires {
			app.Dependencies[i] = dep.Name
		}
	}

	return app, nil
}

// ListDeployed lists deployed apps for an instance
// If category is provided and non-empty, only apps matching that category are returned.
func (m *Manager) ListDeployed(inst *instance.Instance, category ...string) ([]DeployedApp, error) {
	kubeconfigPath := inst.KubeconfigPath()
	appsDir := m.getAppsListDir(inst.Name)

	apps := []DeployedApp{}

	// Check if apps directory exists
	if !storage.FileExists(appsDir) {
		return apps, nil
	}

	// List all app directories
	entries, err := os.ReadDir(appsDir)
	if err != nil {
		return apps, fmt.Errorf("failed to read apps directory: %w", err)
	}

	// Batch fetch cluster state: 3 kubectl calls total instead of ~3 per app
	activeNamespaces := fetchActiveNamespaces(kubeconfigPath)
	ingressRouteURLs := fetchIngressRouteURLs(kubeconfigPath)
	ingressURLs := fetchIngressURLs(kubeconfigPath)

	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}

		appName := entry.Name()

		app := DeployedApp{
			Name:      appName,
			Namespace: m.ResolveNamespace(inst, appName),
			Status:    "added",
		}

		// Try to get version, 'is', icon, and category from manifest
		appDir := filepath.Join(appsDir, appName)
		manifestPath := filepath.Join(appDir, "manifest.yaml")
		if storage.FileExists(manifestPath) {
			manifestData, _ := os.ReadFile(manifestPath)
			var manifest struct {
				Version  string `yaml:"version"`
				Is       string `yaml:"is"`
				Icon     string `yaml:"icon"`
				Category string `yaml:"category"`
			}
			if yaml.Unmarshal(manifestData, &manifest) == nil {
				app.Version = manifest.Version
				app.Is = manifest.Is
				app.Icon = manifest.Icon
				app.Category = manifest.Category

				if len(category) > 0 && category[0] != "" && manifest.Category != category[0] {
					continue
				}
			}
		}

		app.ConfigOnly = isConfigOnly(appDir)

		if app.ConfigOnly {
			// Config-only apps have no k8s resources; being added is equivalent to deployed
			app.Status = "deployed"
		} else if activeNamespaces[app.Namespace] {
			app.Status = "deployed"
			if url, ok := ingressRouteURLs[app.Namespace]; ok {
				app.URL = url
			} else if url, ok := ingressURLs[app.Namespace]; ok {
				app.URL = url
			}
		}

		apps = append(apps, app)
	}

	return apps, nil
}

// fetchActiveNamespaces returns a set of namespace names that are Active.
func fetchActiveNamespaces(kubeconfigPath string) map[string]bool {
	result := map[string]bool{}
	cmd := exec.Command("kubectl", "get", "namespaces", "-o", "json")
	tools.WithKubeconfig(cmd, kubeconfigPath)
	output, err := cmd.Output()
	if err != nil {
		return result
	}
	var nsList struct {
		Items []struct {
			Metadata struct {
				Name string `json:"name"`
			} `json:"metadata"`
			Status struct {
				Phase string `json:"phase"`
			} `json:"status"`
		} `json:"items"`
	}
	if json.Unmarshal(output, &nsList) == nil {
		for _, ns := range nsList.Items {
			if ns.Status.Phase == "Active" {
				result[ns.Metadata.Name] = true
			}
		}
	}
	return result
}

// fetchIngressRouteURLs returns a map of namespace -> URL from Traefik IngressRoutes.
func fetchIngressRouteURLs(kubeconfigPath string) map[string]string {
	result := map[string]string{}
	cmd := exec.Command("kubectl", "get", "ingressroute", "--all-namespaces", "-o", "json")
	tools.WithKubeconfig(cmd, kubeconfigPath)
	output, err := cmd.Output()
	if err != nil {
		return result
	}
	var irList struct {
		Items []struct {
			Metadata struct {
				Namespace string `json:"namespace"`
			} `json:"metadata"`
			Spec struct {
				Routes []struct {
					Match string `json:"match"`
				} `json:"routes"`
			} `json:"spec"`
		} `json:"items"`
	}
	if json.Unmarshal(output, &irList) == nil {
		for _, ir := range irList.Items {
			ns := ir.Metadata.Namespace
			if _, exists := result[ns]; exists {
				continue
			}
			if len(ir.Spec.Routes) > 0 {
				match := ir.Spec.Routes[0].Match
				if strings.Contains(match, "Host(`") {
					start := strings.Index(match, "Host(`") + 6
					end := strings.Index(match[start:], "`")
					if end > 0 {
						result[ns] = "https://" + match[start:start+end]
					}
				}
			}
		}
	}
	return result
}

// fetchIngressURLs returns a map of namespace -> URL from standard Kubernetes Ingresses.
func fetchIngressURLs(kubeconfigPath string) map[string]string {
	result := map[string]string{}
	cmd := exec.Command("kubectl", "get", "ingress", "--all-namespaces", "-o", "json")
	tools.WithKubeconfig(cmd, kubeconfigPath)
	output, err := cmd.Output()
	if err != nil {
		return result
	}
	var ingList struct {
		Items []struct {
			Metadata struct {
				Namespace string `json:"namespace"`
			} `json:"metadata"`
			Spec struct {
				Rules []struct {
					Host string `json:"host"`
				} `json:"rules"`
			} `json:"spec"`
		} `json:"items"`
	}
	if json.Unmarshal(output, &ingList) == nil {
		for _, ing := range ingList.Items {
			ns := ing.Metadata.Namespace
			if _, exists := result[ns]; exists {
				continue
			}
			if len(ing.Spec.Rules) > 0 && ing.Spec.Rules[0].Host != "" {
				result[ns] = "https://" + ing.Spec.Rules[0].Host
			}
		}
	}
	return result
}

// processSecretTemplate processes a gomplate template for secret defaults
// This function uses named contexts for config and secrets (e.g., {{ .config.apps.loomio.db.user }}, {{ .secrets.apps.loomio.dbPassword }})
func processSecretTemplate(template string, appName string, configMap map[string]any, secretsFile string, gomplate *tools.Gomplate) (string, error) {
	// Build context: start with full config, add app-specific keys
	context := make(map[string]any)
	for k, v := range configMap {
		context[k] = v
	}

	// Extract app-specific config and add it under "app" key
	if apps, ok := context["apps"].(map[string]any); ok {
		if appConfig, ok := apps[appName].(map[string]any); ok {
			context["app"] = appConfig
		}
	}

	// Load secrets and extract app-specific secrets under "secrets" key
	secretsData, err := os.ReadFile(secretsFile)
	if err != nil {
		return "", fmt.Errorf("failed to read secrets file: %w", err)
	}

	var allSecrets map[string]any
	if err := yaml.Unmarshal(secretsData, &allSecrets); err != nil {
		return "", fmt.Errorf("failed to parse secrets: %w", err)
	}

	if apps, ok := allSecrets["apps"].(map[string]any); ok {
		// Old monolithic format: apps.{appName}.*
		if appSecrets, ok := apps[appName].(map[string]any); ok {
			context["secrets"] = appSecrets
		}
	} else {
		// Per-app format: root-level keys
		context["secrets"] = allSecrets
	}

	contextYAML, err := yaml.Marshal(context)
	if err != nil {
		return "", fmt.Errorf("failed to marshal context: %w", err)
	}

	compiled, err := gomplate.RenderTemplate(template, string(contextYAML))
	if err != nil {
		return "", fmt.Errorf("failed to process template: %w", err)
	}

	return strings.TrimSpace(compiled), nil
}

// ensureDefaultSecrets generates any missing secrets defined in a manifest.
// Existing secrets are preserved. New secrets are either generated randomly
// or compiled from their default template.
func ensureDefaultSecrets(secretDefs []SecretDefinition, appName string, configMap map[string]any, secretsFile string) error {
	secretsMgr := secrets.NewManager()
	gomplate := tools.NewGomplate()

	for _, secretDef := range secretDefs {
		secretPath := secretDef.Key

		existingSecret, _ := secretsMgr.GetSecret(secretsFile, secretPath)
		if existingSecret != "" && existingSecret != "null" {
			continue
		}

		var secretValue string
		if secretDef.Default != "" {
			if strings.Contains(secretDef.Default, "{{") {
				compiled, err := processSecretTemplate(secretDef.Default, appName, configMap, secretsFile, gomplate)
				if err != nil {
					return fmt.Errorf("failed to compile secret template for %s: %w", secretDef.Key, err)
				}
				secretValue = compiled
			} else {
				secretValue = secretDef.Default
			}
		} else {
			var err error
			secretValue, err = secrets.GenerateSecret(secrets.DefaultSecretLength)
			if err != nil {
				return fmt.Errorf("failed to generate secret for %s: %w", secretDef.Key, err)
			}
		}

		if err := secretsMgr.SetSecret(secretsFile, secretPath, secretValue); err != nil {
			return fmt.Errorf("failed to set secret %s: %w", secretPath, err)
		}
	}

	return nil
}

// setNestedConfig recursively sets nested configuration values using yq
func setNestedConfig(yq *tools.YQ, configFile, basePath string, value any) error {
	switch v := value.(type) {
	case map[string]any:
		// Handle nested map - recursively set each field
		for k, nested := range v {
			nestedPath := fmt.Sprintf("%s.%s", basePath, k)
			if err := setNestedConfig(yq, configFile, nestedPath, nested); err != nil {
				return err
			}
		}
	case map[any]any:
		// YAML unmarshaling sometimes produces this type
		for k, nested := range v {
			key := fmt.Sprintf("%v", k)
			nestedPath := fmt.Sprintf("%s.%s", basePath, key)
			if err := setNestedConfig(yq, configFile, nestedPath, nested); err != nil {
				return err
			}
		}
	default:
		// Primitive value - set it directly
		return yq.Set(configFile, basePath, fmt.Sprintf("%v", value))
	}
	return nil
}

// Add adds an app to the instance configuration
func (m *Manager) Add(inst *instance.Instance, appName, version string, config map[string]any, requiredAppMappings map[string]string) error {
	slog.Info("adding app", "component", "apps", "instance", inst.Name, "app", appName, "version", version)

	// 1. Verify app exists, optionally at a specific version
	sourceAppDir, meta, err := m.resolveAppDir(appName, version)
	if err != nil {
		return err
	}

	appDestDir := m.getAppPackageDir(inst.Name, appName)
	appConfigFile := m.getAppConfigFile(inst.Name, appName)
	secretsFile := m.getAppSecretsFile(inst.Name, appName)
	keyPrefix := m.appConfigKeyPrefix(inst.Name, appName)

	// Ensure per-app config directory and files exist
	if err := os.MkdirAll(filepath.Dir(appConfigFile), 0755); err != nil {
		return fmt.Errorf("failed to create app config dir: %w", err)
	}
	if !storage.FileExists(appConfigFile) {
		if err := os.WriteFile(appConfigFile, []byte("{}\n"), 0644); err != nil {
			return fmt.Errorf("failed to create app config file: %w", err)
		}
	}
	if !storage.FileExists(secretsFile) {
		if err := os.WriteFile(secretsFile, []byte("{}\n"), 0600); err != nil {
			return fmt.Errorf("failed to create app secrets file: %w", err)
		}
	}

	// Load merged config (global + instance) for template resolution
	mergedConfig, err := wcconfig.LoadMergedInstanceConfig(m.dataDir, inst.Name)
	if err != nil {
		return fmt.Errorf("failed to load merged config: %w", err)
	}

	// Create app directory structure
	if err := storage.EnsureDir(appDestDir, 0755); err != nil {
		return fmt.Errorf("failed to create app directory: %w", err)
	}

	// Create .package directory for source preservation
	packageDir := filepath.Join(appDestDir, ".package")
	if err := storage.EnsureDir(packageDir, 0755); err != nil {
		return fmt.Errorf("failed to create package directory: %w", err)
	}

	// 2. Parse manifest
	manifestData, err := os.ReadFile(filepath.Join(sourceAppDir, "manifest.yaml"))
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	// Apply app-level fields from app.yaml if available
	applyMeta(&manifest, meta)

	// 3. Update configuration
	yq := tools.NewYQ()
	configLock := appConfigFile + ".lock"

	if err := storage.WithLock(configLock, func() error {
		// Process config in order from manifest YAML to handle {{ .app.X }} references correctly
		// Use the source manifest since the destination hasn't been copied yet
		sourceManifestPath := filepath.Join(sourceAppDir, "manifest.yaml")
		if err := processConfigInOrder(sourceManifestPath, appName, appConfigFile, keyPrefix, mergedConfig); err != nil {
			return fmt.Errorf("failed to process config in order: %w", err)
		}

		// Apply user-provided config overrides (process templates first)
		if len(config) > 0 {
			gomplate := tools.NewGomplate()
			processedConfig, err := processUserConfig(config, appName, mergedConfig, gomplate)
			if err != nil {
				return fmt.Errorf("failed to process user config: %w", err)
			}

			for key, value := range processedConfig {
				keyPath := keyPrefix + key
				// Use setNestedConfig to handle both simple values and nested objects
				if err := setNestedConfig(yq, appConfigFile, keyPath, value); err != nil {
					return fmt.Errorf("failed to set config %s: %w", key, err)
				}
			}
		}

		return nil
	}); err != nil {
		return err
	}

	// 4. Generate required secrets — reload merged config since processConfigInOrder wrote new keys
	mergedConfig, err = wcconfig.LoadMergedInstanceConfig(m.dataDir, inst.Name)
	if err != nil {
		return fmt.Errorf("failed to reload merged config: %w", err)
	}
	if err := ensureDefaultSecrets(manifest.DefaultSecrets, appName, mergedConfig, secretsFile); err != nil {
		return err
	}

	// 5. Copy source files to .package directory first
	entries, err := os.ReadDir(sourceAppDir)
	if err != nil {
		return fmt.Errorf("failed to read app directory: %w", err)
	}

	// Build set of source file names for cleanup
	sourceNames := make(map[string]bool, len(entries))
	for _, entry := range entries {
		sourceNames[entry.Name()] = true
	}

	// Remove stale files from .package and app directory that no longer exist in source
	if existingEntries, err := os.ReadDir(packageDir); err == nil {
		for _, entry := range existingEntries {
			if !sourceNames[entry.Name()] {
				os.RemoveAll(filepath.Join(packageDir, entry.Name()))
				os.RemoveAll(filepath.Join(appDestDir, entry.Name()))
			}
		}
	}

	// Copy all source files to .package directory
	for _, entry := range entries {
		sourcePath := filepath.Join(sourceAppDir, entry.Name())
		packagePath := filepath.Join(packageDir, entry.Name())

		if entry.IsDir() {
			subSrcDir := filepath.Join(sourceAppDir, entry.Name())
			subDstDir := filepath.Join(packageDir, entry.Name())
			if err := copyDir(subSrcDir, subDstDir); err != nil {
				return fmt.Errorf("failed to copy directory %s: %w", entry.Name(), err)
			}
			continue
		}

		// Copy file to .package
		data, err := os.ReadFile(sourcePath)
		if err != nil {
			return fmt.Errorf("failed to read %s: %w", entry.Name(), err)
		}
		if err := storage.WriteFile(packagePath, data, 0644); err != nil {
			return fmt.Errorf("failed to write to package %s: %w", entry.Name(), err)
		}
	}

	// Update manifest with source information and required app mappings
	manifestDestPath := filepath.Join(appDestDir, "manifest.yaml")
	if manifest.Source == "" {
		// Construct source URI based on appsDir location (always points to app root, not version dir)
		appRootDir := filepath.Join(m.appsDir, appName)
		absPath, err := filepath.Abs(appRootDir)
		if err != nil {
			absPath = appRootDir
		}
		manifest.Source = fmt.Sprintf("file://%s", absPath)
	}

	// Update Requires field with installedAs values from requiredAppMappings
	if len(requiredAppMappings) > 0 && len(manifest.Requires) > 0 {
		for i := range manifest.Requires {
			// Use alias as key, or name if no alias
			key := manifest.Requires[i].Name
			if manifest.Requires[i].Alias != "" {
				key = manifest.Requires[i].Alias
			}

			// Set installedAs from the mapping
			if installedAs, ok := requiredAppMappings[key]; ok {
				manifest.Requires[i].InstalledAs = installedAs
			}
		}
	}

	// Save updated manifest
	manifestYAML, err := yaml.Marshal(manifest)
	if err != nil {
		return fmt.Errorf("failed to marshal manifest: %w", err)
	}
	if err := storage.WriteFile(manifestDestPath, manifestYAML, 0644); err != nil {
		return fmt.Errorf("failed to write manifest: %w", err)
	}

	// 6. Compile app files from .package to app directory
	if err := m.compileFromPackage(appName, appDestDir, packageDir, appConfigFile, secretsFile, keyPrefix); err != nil {
		return fmt.Errorf("failed to compile app templates: %w", err)
	}

	slog.Info("app added", "component", "apps", "instance", inst.Name, "app", appName)
	return nil
}

// Deploy deploys an app to the cluster
func (m *Manager) Deploy(inst *instance.Instance, appName string, opID string, broadcaster *operations.Broadcaster) error {
	slog.Info("deploying app", "component", "apps", "instance", inst.Name, "app", appName)

	kubeconfigPath := inst.KubeconfigPath()
	secretsFile := m.getAppSecretsFile(inst.Name, appName)
	appConfigFile := m.getAppConfigFile(inst.Name, appName)
	keyPrefix := m.appConfigKeyPrefix(inst.Name, appName)

	// Get compiled app manifests from instance directory
	appDir := m.getAppPackageDir(inst.Name, appName)
	if !storage.FileExists(appDir) {
		return fmt.Errorf("app %s not found in instance (run 'wild app add %s' first)", appName, appName)
	}

	// Auto-recompile if .package exists to ensure manifests reflect current config
	packageDir := filepath.Join(appDir, ".package")
	if storage.FileExists(packageDir) {
		if err := m.compileFromPackage(appName, appDir, packageDir, appConfigFile, secretsFile, keyPrefix); err != nil {
			return fmt.Errorf("failed to recompile before deploy: %w", err)
		}
	}

	// Load app manifest
	manifestPath := filepath.Join(appDir, "manifest.yaml")
	var manifest AppManifest
	if storage.FileExists(manifestPath) {
		manifestData, err := os.ReadFile(manifestPath)
		if err == nil {
			if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
				slog.Error("failed to parse manifest", "component", "apps", "path", manifestPath, "error", err)
			}
		}
	}

	// Check deploy config prerequisites
	if manifest.Deploy != nil && manifest.Deploy.RequireWorkerNodes {
		cmd := exec.Command("kubectl", "get", "nodes", "--selector=!node-role.kubernetes.io/control-plane", "-o", "name")
		tools.WithKubeconfig(cmd, kubeconfigPath)
		output, err := cmd.CombinedOutput()
		if err != nil || strings.TrimSpace(string(output)) == "" {
			return fmt.Errorf("no worker nodes found in cluster; %s requires worker nodes", appName)
		}
	}

	// Apply CRDs before namespace and resources
	if manifest.Deploy != nil {
		for _, crd := range manifest.Deploy.CRDs {
			cmd := exec.Command("kubectl", "apply", "-f", crd.URL)
			tools.WithKubeconfig(cmd, kubeconfigPath)
			if output, err := cmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to apply CRDs from %s: %w\nOutput: %s", crd.URL, err, string(output))
			}
			for _, crdName := range crd.WaitFor {
				waitCmd := exec.Command("kubectl", "wait", "--for=condition=established", fmt.Sprintf("crd/%s", crdName), "--timeout=60s")
				tools.WithKubeconfig(waitCmd, kubeconfigPath)
				if output, err := waitCmd.CombinedOutput(); err != nil {
					return fmt.Errorf("CRD %s not established: %w\nOutput: %s", crdName, err, string(output))
				}
			}
		}
	}

	namespace := m.ResolveNamespace(inst, appName)

	// Create namespace if it doesn't exist
	namespaceCmd := exec.Command("kubectl", "create", "namespace", namespace, "--dry-run=client", "-o", "yaml")
	tools.WithKubeconfig(namespaceCmd, kubeconfigPath)
	namespaceYaml, _ := namespaceCmd.CombinedOutput()

	applyNsCmd := exec.Command("kubectl", "apply", "-f", "-")
	applyNsCmd.Stdin = bytes.NewReader(namespaceYaml)
	tools.WithKubeconfig(applyNsCmd, kubeconfigPath)
	_, _ = applyNsCmd.CombinedOutput()

	// Check if this app needs postgres-secrets (for db-init jobs)
	dbInitJobPath := filepath.Join(appDir, "db-init-job.yaml")
	if storage.FileExists(dbInitJobPath) {
		dbInitContent, _ := os.ReadFile(dbInitJobPath)
		if bytes.Contains(dbInitContent, []byte("postgres-secrets")) {
			getSecretCmd := exec.Command("kubectl", "get", "secret", "postgres-secrets",
				"-n", "postgres", "-o", "yaml")
			tools.WithKubeconfig(getSecretCmd, kubeconfigPath)
			secretYaml, err := getSecretCmd.CombinedOutput()
			if err == nil {
				secretYaml = bytes.ReplaceAll(secretYaml, []byte("namespace: postgres"),
					[]byte(fmt.Sprintf("namespace: %s", namespace)))
				applySecretCmd := exec.Command("kubectl", "apply", "-f", "-")
				applySecretCmd.Stdin = bytes.NewReader(secretYaml)
				tools.WithKubeconfig(applySecretCmd, kubeconfigPath)
				_, _ = applySecretCmd.CombinedOutput()
			}
		}
	}

	// Copy wildcard TLS certificates from cert-manager namespace
	ingressPath := filepath.Join(appDir, "ingress.yaml")
	if storage.FileExists(ingressPath) {
		ingressContent, _ := os.ReadFile(ingressPath)
		wildcardSecrets := []string{"wildcard-wild-cloud-tls", "wildcard-internal-wild-cloud-tls"}
		for _, secretName := range wildcardSecrets {
			if bytes.Contains(ingressContent, []byte(secretName)) {
				if err := utilities.CopySecretBetweenNamespaces(kubeconfigPath, secretName, "cert-manager", namespace); err != nil {
					slog.Error("failed to copy TLS secret", "component", "apps", "secret", secretName, "error", err)
				}
			}
		}
	}

	// Create Kubernetes secrets from secrets.yaml
	if storage.FileExists(secretsFile) {
		secretsMgr := secrets.NewManager()
		instanceSecretsFile := inst.SecretsPath

		// Create app-secrets (from defaultSecrets + requiredSecrets)
		deleteCmd := exec.Command("kubectl", "delete", "secret", fmt.Sprintf("%s-secrets", appName), "-n", namespace, "--ignore-not-found")
		tools.WithKubeconfig(deleteCmd, kubeconfigPath)
		_, _ = deleteCmd.CombinedOutput()

		createSecretCmd := exec.Command("kubectl", "create", "secret", "generic", fmt.Sprintf("%s-secrets", appName), "-n", namespace)

		if len(manifest.DefaultSecrets) > 0 {
			for _, secretDef := range manifest.DefaultSecrets {
				// Per-app secrets file has root-level keys
				secretValue, err := secretsMgr.GetSecret(secretsFile, secretDef.Key)
				if err == nil && secretValue != "" && secretValue != "null" {
					createSecretCmd.Args = append(createSecretCmd.Args, fmt.Sprintf("--from-literal=%s=%s", secretDef.Key, secretValue))
				}
			}
		}

		if len(manifest.RequiredSecrets) > 0 {
			for _, requiredSecret := range manifest.RequiredSecrets {
				// Cross-app secrets live in the monolithic instance secrets file
				secretPath := fmt.Sprintf("apps.%s", requiredSecret)
				secretValue, err := secretsMgr.GetSecret(instanceSecretsFile, secretPath)
				if err != nil || secretValue == "" || secretValue == "null" {
					secretValue, _ = secretsMgr.GetSecret(instanceSecretsFile, requiredSecret)
				}
				if secretValue != "" && secretValue != "null" {
					createSecretCmd.Args = append(createSecretCmd.Args,
						fmt.Sprintf("--from-literal=%s=%s", requiredSecret, secretValue))
				}
			}
		}

		if len(createSecretCmd.Args) > 5 { // base command has 5 args
			tools.WithKubeconfig(createSecretCmd, kubeconfigPath)
			if output, err := createSecretCmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to create secret: %w\nOutput: %s", err, string(output))
			}
		}

		// Create additional secrets from deploy config
		if manifest.Deploy != nil {
			for _, cs := range manifest.Deploy.CreateSecrets {
				targetNs := namespace
				if cs.Namespace != "" {
					targetNs = cs.Namespace
				}

				delCmd := exec.Command("kubectl", "delete", "secret", cs.Name, "-n", targetNs, "--ignore-not-found")
				tools.WithKubeconfig(delCmd, kubeconfigPath)
				_, _ = delCmd.CombinedOutput()

				createCmd := exec.Command("kubectl", "create", "secret", "generic", cs.Name, "-n", targetNs)
				for k8sKey, secretRef := range cs.Entries {
					// Try per-app secrets file first (root-level key)
					secretValue, err := secretsMgr.GetSecret(secretsFile, secretRef)
					if err != nil || secretValue == "" || secretValue == "null" {
						// Fall back to monolithic instance secrets file
						secretPath := fmt.Sprintf("apps.%s", secretRef)
						secretValue, _ = secretsMgr.GetSecret(instanceSecretsFile, secretPath)
					}
					if secretValue != "" && secretValue != "null" {
						createCmd.Args = append(createCmd.Args, fmt.Sprintf("--from-literal=%s=%s", k8sKey, secretValue))
					}
				}
				if len(createCmd.Args) > 5 {
					tools.WithKubeconfig(createCmd, kubeconfigPath)
					if output, err := createCmd.CombinedOutput(); err != nil {
						return fmt.Errorf("failed to create secret %s: %w\nOutput: %s", cs.Name, err, string(output))
					}
				}
			}
		}
	}

	// Apply manifests
	if manifest.Deploy != nil && len(manifest.Deploy.Phases) > 0 {
		// Multi-phase deployment
		for _, phase := range manifest.Deploy.Phases {
			phaseDir := filepath.Join(appDir, phase.Path)
			cmd := exec.Command("kubectl", "apply", "-k", phaseDir)
			tools.WithKubeconfig(cmd, kubeconfigPath)
			output, err := cmd.CombinedOutput()
			if err != nil {
				return fmt.Errorf("failed to deploy phase %s: %w\nOutput: %s", phase.Path, err, string(output))
			}
			if hasConfigMapChanges(output) {
				slog.Info("ConfigMap updated, restarting workloads", "component", "apps", "namespace", namespace)
				if err := m.restartNamespaceWorkloads(kubeconfigPath, namespace); err != nil {
					return err
				}
			}

			if phase.WaitFor != nil {
				if err := m.waitForRollout(kubeconfigPath, namespace, phase.WaitFor); err != nil {
					return err
				}
			}
		}
	} else {
		cmd := exec.Command("kubectl", "apply", "-k", appDir)
		tools.WithKubeconfig(cmd, kubeconfigPath)
		output, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("failed to deploy app: %w\nOutput: %s", err, string(output))
		}
		if hasConfigMapChanges(output) {
			slog.Info("ConfigMap updated, restarting workloads", "component", "apps", "namespace", namespace)
			if err := m.restartNamespaceWorkloads(kubeconfigPath, namespace); err != nil {
				return err
			}
		}
	}

	// Post-deploy: restart deployments
	if manifest.Deploy != nil {
		for _, deploymentName := range manifest.Deploy.RestartDeployments {
			cmd := exec.Command("kubectl", "rollout", "restart", fmt.Sprintf("deployment/%s", deploymentName), "-n", namespace)
			tools.WithKubeconfig(cmd, kubeconfigPath)
			if output, err := cmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to restart deployment %s: %w\nOutput: %s", deploymentName, err, string(output))
			}
		}
	}

	// Post-deploy: wait for rollout
	if manifest.Deploy != nil && manifest.Deploy.WaitForRollout != nil {
		if err := m.waitForRollout(kubeconfigPath, namespace, manifest.Deploy.WaitForRollout); err != nil {
			return err
		}
	}

	slog.Info("app deployed", "component", "apps", "instance", inst.Name, "app", appName, "namespace", namespace)
	return nil
}

// waitForRollout waits for a rollout to complete
func (m *Manager) waitForRollout(kubeconfigPath, namespace string, wait *RolloutWait) error {
	kind := wait.Kind
	if kind == "" {
		kind = "deployment"
	}
	timeout := wait.Timeout
	if timeout == "" {
		timeout = "120s"
	}
	cmd := exec.Command("kubectl", "rollout", "status", fmt.Sprintf("%s/%s", kind, wait.Name), "-n", namespace, fmt.Sprintf("--timeout=%s", timeout))
	tools.WithKubeconfig(cmd, kubeconfigPath)
	if output, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("failed waiting for %s/%s rollout: %w\nOutput: %s", kind, wait.Name, err, string(output))
	}
	return nil
}

// Restart performs a rolling restart of all deployments and statefulsets in an app's namespace
func (m *Manager) Restart(inst *instance.Instance, appName string) error {
	slog.Info("restarting app", "component", "apps", "instance", inst.Name, "app", appName)
	kubeconfigPath := inst.KubeconfigPath()
	namespace := m.ResolveNamespace(inst, appName)
	return m.restartNamespaceWorkloads(kubeconfigPath, namespace)
}

// restartNamespaceWorkloads restarts all deployments and statefulsets in the namespace.
func (m *Manager) restartNamespaceWorkloads(kubeconfigPath, namespace string) error {
	deployCmd := exec.Command("kubectl", "rollout", "restart", "deployment", "-n", namespace)
	tools.WithKubeconfig(deployCmd, kubeconfigPath)
	if output, err := deployCmd.CombinedOutput(); err != nil {
		if !strings.Contains(string(output), "no resources found") {
			return fmt.Errorf("failed to restart deployments: %w\nOutput: %s", err, string(output))
		}
	}

	stsCmd := exec.Command("kubectl", "rollout", "restart", "statefulset", "-n", namespace)
	tools.WithKubeconfig(stsCmd, kubeconfigPath)
	if output, err := stsCmd.CombinedOutput(); err != nil {
		if !strings.Contains(string(output), "no resources found") {
			return fmt.Errorf("failed to restart statefulsets: %w\nOutput: %s", err, string(output))
		}
	}

	return nil
}

// hasConfigMapChanges reports whether kubectl apply output indicates any ConfigMaps were updated.
// On first deploy all resources are "created", not "configured", so this correctly returns false
// and avoids a redundant restart before any pods exist.
func hasConfigMapChanges(output []byte) bool {
	for _, line := range strings.Split(string(output), "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "configmap/") && strings.HasSuffix(line, " configured") {
			return true
		}
	}
	return false
}

// protectedNamespaces that must never be deleted
var protectedNamespaces = map[string]bool{
	"kube-system": true, "kube-public": true,
	"kube-node-lease": true, "default": true,
}

// namespaceSharedByOtherApp checks if any other deployed app uses the same namespace
func (m *Manager) namespaceSharedByOtherApp(inst *instance.Instance, appName, namespace string) bool {
	deployed, err := m.ListDeployed(inst)
	if err != nil {
		return false
	}
	for _, app := range deployed {
		if app.Name != appName && app.Namespace == namespace {
			return true
		}
	}
	return false
}

// Delete removes an app from the cluster and configuration.
// Local filesystem cleanup always runs, even if k8s deletion fails.
func (m *Manager) Delete(inst *instance.Instance, appName string) error {
	slog.Info("deleting app", "component", "apps", "instance", inst.Name, "app", appName)

	kubeconfigPath := inst.KubeconfigPath()
	appDir := m.getAppPackageDir(inst.Name, appName)
	appConfigDir := tools.GetAppConfigDirPath(m.dataDir, inst.Name, appName)

	if !storage.FileExists(appDir) {
		return fmt.Errorf("app %s not found in instance", appName)
	}

	namespace := m.ResolveNamespace(inst, appName)

	// Attempt k8s cleanup; track error but don't return early so local files are always cleaned up.
	var k8sErr error
	if protectedNamespaces[namespace] || m.namespaceSharedByOtherApp(inst, appName, namespace) {
		deleteCmd := exec.Command("kubectl", "delete", "-k", appDir, "--ignore-not-found")
		tools.WithKubeconfig(deleteCmd, kubeconfigPath)
		if output, err := deleteCmd.CombinedOutput(); err != nil {
			k8sErr = fmt.Errorf("failed to delete app resources: %w\nOutput: %s", err, string(output))
			slog.Error("k8s resource deletion failed", "component", "apps", "instance", inst.Name, "app", appName, "error", k8sErr)
		}
	} else {
		// App owns this namespace exclusively - delete the whole namespace
		deleteNsCmd := exec.Command("kubectl", "delete", "namespace", namespace, "--ignore-not-found")
		tools.WithKubeconfig(deleteNsCmd, kubeconfigPath)
		if output, err := deleteNsCmd.CombinedOutput(); err != nil {
			k8sErr = fmt.Errorf("failed to delete namespace: %w\nOutput: %s", err, string(output))
			slog.Error("namespace deletion failed", "component", "apps", "instance", inst.Name, "app", appName, "error", k8sErr)
		} else {
			// Wait for namespace deletion to complete (timeout after 60s)
			waitCmd := exec.Command("kubectl", "wait", "--for=delete", "namespace", namespace, "--timeout=60s")
			tools.WithKubeconfig(waitCmd, kubeconfigPath)
			_, _ = waitCmd.CombinedOutput() // Ignore errors - namespace might not exist
		}
	}

	// Always clean up local files regardless of k8s outcome.
	if err := os.RemoveAll(appDir); err != nil {
		return fmt.Errorf("failed to delete local app directory: %w", err)
	}

	// Remove entire app config directory (config.yaml, secrets.yaml, schedule.yaml, etc.)
	if err := os.RemoveAll(appConfigDir); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to remove app config directory: %w", err)
	}

	slog.Info("app deleted", "component", "apps", "instance", inst.Name, "app", appName)
	return k8sErr
}

// GetStatus returns the status of a deployed app
func (m *Manager) GetStatus(inst *instance.Instance, appName string) (*DeployedApp, error) {
	kubeconfigPath := inst.KubeconfigPath()
	appDir := m.getAppPackageDir(inst.Name, appName)

	namespace := m.ResolveNamespace(inst, appName)

	app := &DeployedApp{
		Name:      appName,
		Status:    "not-added",
		Namespace: namespace,
	}

	// Check if app was added to instance
	if !storage.FileExists(appDir) {
		return app, nil
	}

	app.Status = "added"

	// Load manifest for version and deployment info
	manifestPath := filepath.Join(appDir, "manifest.yaml")
	var manifest AppManifest
	if storage.FileExists(manifestPath) {
		manifestData, err := os.ReadFile(manifestPath)
		if err == nil {
			if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
				slog.Error("failed to parse manifest", "component", "apps", "path", manifestPath, "error", err)
			}
		}
		app.Version = manifest.Version
	}

	// Config-only apps have no kustomization.yaml or install.sh — nothing to deploy.
	// "added" is their healthy terminal state (e.g., SMTP provides config for other apps).
	app.ConfigOnly = isConfigOnly(appDir)
	if app.ConfigOnly {
		return app, nil
	}

	// Check if namespace exists
	checkNsCmd := exec.Command("kubectl", "get", "namespace", namespace, "-o", "json")
	tools.WithKubeconfig(checkNsCmd, kubeconfigPath)
	nsOutput, err := checkNsCmd.CombinedOutput()
	if err != nil {
		// Namespace doesn't exist — check if cluster-scoped resources were applied.
		// Apps like NFS create StorageClasses/PVs without a namespace.
		// "kubectl get -k" succeeds if all resources in the kustomization exist.
		getCmd := exec.Command("kubectl", "get", "-k", appDir)
		tools.WithKubeconfig(getCmd, kubeconfigPath)
		if getCmd.Run() == nil {
			app.Status = "deployed"
		}
		return app, nil
	}

	// Parse namespace to check if it's active
	var ns struct {
		Status struct {
			Phase string `json:"phase"`
		} `json:"status"`
	}
	if err := yaml.Unmarshal(nsOutput, &ns); err != nil || ns.Status.Phase != "Active" {
		return app, nil
	}

	// Determine how to check status
	deployName, deployKind := resolveDeploymentResource(&manifest)

	if deployName != "" {
		// Check specific deployment/daemonset for apps with a known workload
		app.Status = m.getWorkloadStatus(kubeconfigPath, namespace, deployKind, deployName)
	} else {
		// Fall back: check all pods in namespace (for apps that own their namespace)
		app.Status = m.getNamespacePodStatus(kubeconfigPath, namespace)
	}

	return app, nil
}

// resolveDeploymentResource returns the workload name and kind from the manifest.
// Returns ("", "") if no deployment resource is specified.
func resolveDeploymentResource(manifest *AppManifest) (name, kind string) {
	kind = "deployment"

	if manifest == nil {
		return "", kind
	}

	if manifest.DeploymentName != "" {
		name = manifest.DeploymentName
	} else if manifest.Deploy != nil && manifest.Deploy.WaitForRollout != nil {
		name = manifest.Deploy.WaitForRollout.Name
	}

	if manifest.Deploy != nil && manifest.Deploy.WaitForRollout != nil && manifest.Deploy.WaitForRollout.Kind != "" {
		kind = manifest.Deploy.WaitForRollout.Kind
	}

	return name, kind
}

// getWorkloadStatus checks the status of a specific deployment or daemonset
func (m *Manager) getWorkloadStatus(kubeconfigPath, namespace, kind, name string) string {
	cmd := exec.Command("kubectl", "get", fmt.Sprintf("%s/%s", kind, name), "-n", namespace, "-o", "json")
	tools.WithKubeconfig(cmd, kubeconfigPath)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return "no-pods"
	}

	if kind == "daemonset" {
		var ds struct {
			Status struct {
				DesiredNumberScheduled int `json:"desiredNumberScheduled"`
				NumberReady            int `json:"numberReady"`
			} `json:"status"`
		}
		if err := json.Unmarshal(output, &ds); err != nil {
			return "error"
		}
		// DaemonSet with 0 desired is valid (e.g., no matching nodes for nvidia GPU plugin)
		if ds.Status.DesiredNumberScheduled == 0 {
			return "running"
		}
		if ds.Status.NumberReady >= ds.Status.DesiredNumberScheduled {
			return "running"
		}
		if ds.Status.NumberReady > 0 {
			return "starting"
		}
		return "unhealthy"
	}

	// Deployment (or statefulset - same status shape)
	var deploy struct {
		Spec struct {
			Replicas int `json:"replicas"`
		} `json:"spec"`
		Status struct {
			ReadyReplicas int `json:"readyReplicas"`
		} `json:"status"`
	}
	if err := json.Unmarshal(output, &deploy); err != nil {
		return "error"
	}

	desired := deploy.Spec.Replicas
	if desired == 0 {
		return "running"
	}
	if deploy.Status.ReadyReplicas >= desired {
		return "running"
	}
	if deploy.Status.ReadyReplicas > 0 {
		return "starting"
	}
	return "unhealthy"
}

// getNamespacePodStatus checks status by examining all pods in a namespace.
// Used for apps that own their namespace (where all pods belong to the app).
func (m *Manager) getNamespacePodStatus(kubeconfigPath, namespace string) string {
	podsCmd := exec.Command("kubectl", "get", "pods", "-n", namespace, "-o", "json")
	tools.WithKubeconfig(podsCmd, kubeconfigPath)
	podsOutput, err := podsCmd.CombinedOutput()
	if err != nil {
		return "error"
	}

	var podList struct {
		Items []struct {
			Status struct {
				Phase             string `json:"phase"`
				ContainerStatuses []struct {
					Ready bool `json:"ready"`
				} `json:"containerStatuses"`
			} `json:"status"`
		} `json:"items"`
	}
	if err := json.Unmarshal(podsOutput, &podList); err != nil {
		return "error"
	}

	if len(podList.Items) == 0 {
		return "no-pods"
	}

	allRunning := true
	allReady := true
	activePods := 0
	for _, pod := range podList.Items {
		if pod.Status.Phase == "Succeeded" {
			continue
		}
		activePods++
		if pod.Status.Phase != "Running" {
			allRunning = false
		}
		for _, cs := range pod.Status.ContainerStatuses {
			if !cs.Ready {
				allReady = false
			}
		}
	}

	if activePods == 0 {
		return "no-pods"
	}

	if allRunning && allReady {
		return "running"
	} else if allRunning {
		return "starting"
	}
	return "unhealthy"
}

// podListStatus derives an app status string from a list of pods.
// Succeeded (completed Job) pods are ignored. Returns "running", "starting", or "unhealthy".
func podListStatus(pods []tools.PodInfo) string {
	allRunning := true
	allReady := true
	activePods := 0
	for _, pod := range pods {
		if pod.Status == "Succeeded" {
			continue // skip completed Job pods
		}
		activePods++
		if pod.Status != "Running" {
			allRunning = false
		}
		parts := strings.Split(pod.Ready, "/")
		if len(parts) == 2 && parts[0] != parts[1] {
			allReady = false
		}
	}
	if activePods == 0 {
		allRunning = false
	}
	if allRunning && allReady {
		return "running"
	}
	if allRunning {
		return "starting"
	}
	return "unhealthy"
}

// GetEnhanced returns enhanced app information with runtime status
func (m *Manager) GetEnhanced(inst *instance.Instance, appName string) (*EnhancedApp, error) {
	kubeconfigPath := inst.KubeconfigPath()
	appDir := m.getAppPackageDir(inst.Name, appName)
	appConfigFile := m.getAppConfigFile(inst.Name, appName)
	prefix := m.appConfigKeyPrefix(inst.Name, appName)

	namespace := m.ResolveNamespace(inst, appName)

	enhanced := &EnhancedApp{
		Name:      appName,
		Status:    "not-added",
		Namespace: namespace,
	}

	// Check if app was added to instance
	if !storage.FileExists(appDir) {
		return enhanced, nil
	}

	enhanced.Status = "added"

	// Load manifest
	manifestPath := filepath.Join(appDir, "manifest.yaml")
	if storage.FileExists(manifestPath) {
		manifestData, _ := os.ReadFile(manifestPath)
		var manifest AppManifest
		if yaml.Unmarshal(manifestData, &manifest) == nil {
			enhanced.Version = manifest.Version
			enhanced.Description = manifest.Description
			enhanced.Icon = manifest.Icon
			enhanced.Manifest = &manifest
		}
	}

	// Note: README content is now served via dedicated /readme endpoint
	// No need to populate readme/documentation fields here

	// Load config
	yq := tools.NewYQ()
	configJSON, err := yq.Get(appConfigFile, prefix+" | @json")
	if err == nil && configJSON != "" && configJSON != "null" {
		var config map[string]any
		if json.Unmarshal([]byte(configJSON), &config) == nil {
			enhanced.Config = config
		}
	}

	// Check if namespace exists
	checkNsCmd := exec.Command("kubectl", "get", "namespace", namespace, "-o", "json")
	tools.WithKubeconfig(checkNsCmd, kubeconfigPath)
	nsOutput, err := checkNsCmd.CombinedOutput()
	if err != nil {
		// Namespace doesn't exist - not deployed
		return enhanced, nil
	}

	// Parse namespace to check if it's active
	var ns struct {
		Status struct {
			Phase string `json:"phase"`
		} `json:"status"`
	}
	if err := json.Unmarshal(nsOutput, &ns); err != nil || ns.Status.Phase != "Active" {
		return enhanced, nil
	}

	enhanced.Status = "deployed"

	// Get URL (ingress)
	enhanced.URL = m.getAppURL(kubeconfigPath, namespace)

	// Determine status from specific workload if available
	deployName, deployKind := resolveDeploymentResource(enhanced.Manifest)
	if deployName != "" {
		enhanced.Status = m.getWorkloadStatus(kubeconfigPath, namespace, deployKind, deployName)
	}

	// Get runtime status (for UI pod display)
	runtime, err := m.getRuntimeStatus(kubeconfigPath, namespace)
	if err == nil {
		enhanced.Runtime = runtime

		// If no specific workload was checked, derive status from pods
		if deployName == "" && len(runtime.Pods) > 0 {
			enhanced.Status = podListStatus(runtime.Pods)
		}
	}

	// Compute drift state
	enhanced.Drift = m.computeDrift(inst, appName, appDir, kubeconfigPath, enhanced.Status, enhanced.Manifest)

	return enhanced, nil
}

// GetEnhancedStatus returns just the runtime status for an app
func (m *Manager) GetEnhancedStatus(inst *instance.Instance, appName string) (*RuntimeStatus, error) {
	kubeconfigPath := inst.KubeconfigPath()
	namespace := m.ResolveNamespace(inst, appName)

	// Check if namespace exists
	checkNsCmd := exec.Command("kubectl", "get", "namespace", namespace, "-o", "json")
	tools.WithKubeconfig(checkNsCmd, kubeconfigPath)
	if err := checkNsCmd.Run(); err != nil {
		return nil, fmt.Errorf("namespace not found or not deployed")
	}

	return m.getRuntimeStatus(kubeconfigPath, namespace)
}

// GetCatalogReadme returns the README.md for a catalog app, read live from the wild-directory
// version slot (resolveAppDir → versions/{slot}/README.md). Changes to the file are reflected
// immediately without any re-add. Used for the available-apps catalog view only — not for
// deployed apps, which have their README copied into the instance directory by Add().
func (m *Manager) GetCatalogReadme(appName string) ([]byte, error) {
	if m.appsDir == "" {
		return nil, fmt.Errorf("apps directory not configured")
	}

	appDir, _, err := m.resolveAppDir(appName, "")
	if err != nil {
		return nil, err
	}

	return os.ReadFile(filepath.Join(appDir, "README.md"))
}

// GetAppManifest reads and parses the manifest.yaml for an app from the apps directory
func (m *Manager) GetAppManifest(appName string) (*AppManifest, error) {
	if m.appsDir == "" {
		return nil, fmt.Errorf("apps directory not configured")
	}

	appDir, meta, err := m.resolveAppDir(appName, "")
	if err != nil {
		return nil, err
	}

	data, err := os.ReadFile(filepath.Join(appDir, "manifest.yaml"))
	if err != nil {
		return nil, fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(data, &manifest); err != nil {
		return nil, fmt.Errorf("failed to parse manifest: %w", err)
	}

	applyMeta(&manifest, meta)

	return &manifest, nil
}

// getRuntimeStatus fetches runtime information from kubernetes
func (m *Manager) getRuntimeStatus(kubeconfigPath, namespace string) (*RuntimeStatus, error) {
	kubectl := tools.NewKubectl(kubeconfigPath)

	runtime := &RuntimeStatus{}

	// Get pods (with detailed info for app status display)
	pods, err := kubectl.GetPods(namespace, true)
	if err == nil {
		runtime.Pods = pods
	}

	// Get replicas
	replicas, err := kubectl.GetReplicas(namespace)
	if err == nil && (replicas.Desired > 0 || replicas.Current > 0) {
		runtime.Replicas = replicas
	}

	// Get resources
	resources, err := kubectl.GetResources(namespace)
	if err == nil {
		runtime.Resources = resources
	}

	// Get recent events (last 10)
	events, err := kubectl.GetRecentEvents(namespace, 10)
	if err == nil {
		runtime.RecentEvents = events
	}

	return runtime, nil
}

// Update updates an app from its source package
func (m *Manager) Update(inst *instance.Instance, appName string) error {
	appDestDir := m.getAppPackageDir(inst.Name, appName)
	packageDir := filepath.Join(appDestDir, ".package")

	// Check if .package exists (if not, it's a custom app)
	if !storage.FileExists(packageDir) {
		return fmt.Errorf("app %s is custom or not installed (no package source)", appName)
	}

	// Read manifest to get source
	manifestPath := filepath.Join(appDestDir, "manifest.yaml")
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	// Parse source URI (e.g., "file:///path/to/wild-directory/postgres")
	sourceParts := strings.Split(manifest.Source, "://")
	if len(sourceParts) != 2 {
		return fmt.Errorf("invalid source format: %s", manifest.Source)
	}

	switch sourceParts[0] {
	case "file":
		// Resolve through app.yaml if available, otherwise use source dir directly
		sourceAppDir, _, err := m.resolveAppDir(appName, "")
		if err != nil {
			// Fallback: use source URI directly (old-style or external source)
			sourceAppDir = sourceParts[1]
		}
		return m.updateFromSource(inst, appName, sourceAppDir, manifest.Source)
	case "git+https", "git+http", "git+ssh":
		return fmt.Errorf("git source not yet supported: %s", manifest.Source)
	default:
		return fmt.Errorf("unsupported source protocol: %s", sourceParts[0])
	}
}

// updateFromSource performs the core update logic: copies files from sourceDir
// to .package, merges config, preserves instance-specific manifest fields, and recompiles.
// preserveSource is the source URI to write to the manifest (always the top-level app dir, not waypoints).
func (m *Manager) updateFromSource(inst *instance.Instance, appName, sourceDir, preserveSource string) error {
	appDestDir := m.getAppPackageDir(inst.Name, appName)
	packageDir := filepath.Join(appDestDir, ".package")
	manifestPath := filepath.Join(appDestDir, "manifest.yaml")

	// Read current manifest for preserving instance-specific fields
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}
	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	// Copy new version to temp directory
	tempDir := filepath.Join(appDestDir, ".package.new")
	if err := storage.EnsureDir(tempDir, 0755); err != nil {
		return fmt.Errorf("failed to create temp directory: %w", err)
	}
	defer os.RemoveAll(tempDir)

	entries, err := os.ReadDir(sourceDir)
	if err != nil {
		return fmt.Errorf("failed to read source directory: %w", err)
	}

	for _, entry := range entries {
		if entry.IsDir() {
			if err := copyDir(filepath.Join(sourceDir, entry.Name()), filepath.Join(tempDir, entry.Name())); err != nil {
				return fmt.Errorf("failed to copy directory %s: %w", entry.Name(), err)
			}
			continue
		}

		data, err := os.ReadFile(filepath.Join(sourceDir, entry.Name()))
		if err != nil {
			return fmt.Errorf("failed to read %s: %w", entry.Name(), err)
		}
		if err := storage.WriteFile(filepath.Join(tempDir, entry.Name()), data, 0644); err != nil {
			return fmt.Errorf("failed to write %s: %w", entry.Name(), err)
		}
	}

	// Replace .package with new version, keeping old as backup
	oldPackageDir := packageDir + ".old"
	os.RemoveAll(oldPackageDir)
	if err := os.Rename(packageDir, oldPackageDir); err != nil {
		return fmt.Errorf("failed to backup old package: %w", err)
	}
	if err := os.Rename(tempDir, packageDir); err != nil {
		_ = os.Rename(oldPackageDir, packageDir)
		return fmt.Errorf("failed to update package: %w", err)
	}

	appConfigFile := m.getAppConfigFile(inst.Name, appName)
	secretsFile := m.getAppSecretsFile(inst.Name, appName)
	keyPrefix := m.appConfigKeyPrefix(inst.Name, appName)

	if !storage.FileExists(secretsFile) {
		if err := os.WriteFile(secretsFile, []byte("{}\n"), 0600); err != nil {
			return fmt.Errorf("failed to create app secrets file: %w", err)
		}
	}

	mergedConfig, err := wcconfig.LoadMergedInstanceConfig(m.dataDir, inst.Name)
	if err != nil {
		return fmt.Errorf("failed to load merged config: %w", err)
	}

	rollback := func() {
		os.RemoveAll(packageDir)
		_ = os.Rename(oldPackageDir, packageDir)
	}

	// Read the new manifest
	newManifestPath := filepath.Join(packageDir, "manifest.yaml")
	newManifestData, err := os.ReadFile(newManifestPath)
	if err != nil {
		rollback()
		return fmt.Errorf("failed to read new manifest: %w", err)
	}

	var newManifest AppManifest
	if err := yaml.Unmarshal(newManifestData, &newManifest); err != nil {
		rollback()
		return fmt.Errorf("failed to parse new manifest: %w", err)
	}

	// Merge new defaultConfig keys (skips existing values)
	configLock := appConfigFile + ".lock"
	if err := storage.WithLock(configLock, func() error {
		return processConfigInOrder(newManifestPath, appName, appConfigFile, keyPrefix, mergedConfig)
	}); err != nil {
		rollback()
		return fmt.Errorf("failed to merge new config: %w", err)
	}

	// Generate any new defaultSecrets — reload merged config since processConfigInOrder wrote new keys
	mergedConfig, err = wcconfig.LoadMergedInstanceConfig(m.dataDir, inst.Name)
	if err != nil {
		rollback()
		return fmt.Errorf("failed to reload merged config: %w", err)
	}
	if err := ensureDefaultSecrets(newManifest.DefaultSecrets, appName, mergedConfig, secretsFile); err != nil {
		rollback()
		return err
	}

	// Update local manifest with new version while preserving source and installedAs
	newManifest.Source = preserveSource
	if len(manifest.Requires) > 0 {
		installedAsMap := make(map[string]string)
		for _, req := range manifest.Requires {
			key := req.Name
			if req.Alias != "" {
				key = req.Alias
			}
			if req.InstalledAs != "" {
				installedAsMap[key] = req.InstalledAs
			}
		}
		for i := range newManifest.Requires {
			key := newManifest.Requires[i].Name
			if newManifest.Requires[i].Alias != "" {
				key = newManifest.Requires[i].Alias
			}
			if installedAs, ok := installedAsMap[key]; ok {
				newManifest.Requires[i].InstalledAs = installedAs
			}
		}
	}

	manifestYAML, err := yaml.Marshal(newManifest)
	if err != nil {
		rollback()
		return fmt.Errorf("failed to marshal updated manifest: %w", err)
	}
	if err := storage.WriteFile(manifestPath, manifestYAML, 0644); err != nil {
		rollback()
		return fmt.Errorf("failed to write updated manifest: %w", err)
	}

	// Re-compile from .package to app dir
	if err := m.compileFromPackage(appName, appDestDir, packageDir, appConfigFile, secretsFile, keyPrefix); err != nil {
		rollback()
		return fmt.Errorf("failed to recompile app templates: %w", err)
	}

	// Success - clean up backup
	os.RemoveAll(oldPackageDir)

	return nil
}

// Upgrade performs an upgrade-aware update: computes the upgrade plan, then executes
// each step (update + deploy) in sequence. Falls back to a single-step direct update
// for apps without upgrade metadata.
func (m *Manager) Upgrade(inst *instance.Instance, appName, opID string, broadcaster *operations.Broadcaster) error {
	appDestDir := m.getAppPackageDir(inst.Name, appName)
	manifestPath := filepath.Join(appDestDir, "manifest.yaml")

	// Read installed manifest
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}
	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	if manifest.Source == "" {
		return fmt.Errorf("app %s has no source (ejected or custom)", appName)
	}

	// Resolve the apps directory from the source URI
	// Source is like "file:///path/to/wild-directory/appname" — we need the parent
	sourceDir := parseSourceDir(manifest.Source)
	if sourceDir == "" {
		return fmt.Errorf("unsupported source format: %s", manifest.Source)
	}
	appsDir := filepath.Dir(sourceDir)

	plan, err := ComputeUpgradePlan(manifest.Version, appName, appsDir)
	if err != nil {
		return fmt.Errorf("failed to compute upgrade plan: %w", err)
	}

	if plan.Blocked {
		return fmt.Errorf("upgrade blocked: %s", plan.Notes)
	}

	if len(plan.Steps) == 0 {
		broadcaster.Publish(opID, []byte("Already up to date"))
		return nil
	}

	if plan.BackupRequired {
		return fmt.Errorf("backup required before upgrading %s (%s -> %s) — run 'wild app backup %s' first, then retry",
			appName, manifest.Version, plan.Steps[len(plan.Steps)-1].ToVersion, appName)
	}

	totalSteps := len(plan.Steps)
	for i, step := range plan.Steps {
		broadcaster.Publish(opID, []byte(fmt.Sprintf("Step %d/%d: upgrading %s to %s\n", i+1, totalSteps, appName, step.ToVersion)))

		// Apply config migrations if present
		if step.Manifest != nil && step.Manifest.Upgrade != nil && len(step.Manifest.Upgrade.ConfigMigrations) > 0 {
			if err := m.applyConfigMigrations(inst, appName, step.Manifest.Upgrade.ConfigMigrations); err != nil {
				return fmt.Errorf("step %d/%d config migration failed: %w", i+1, totalSteps, err)
			}
		}

		// Run pre-migration jobs
		if step.Manifest != nil && step.Manifest.Upgrade != nil && step.Manifest.Upgrade.Migrations != nil {
			if len(step.Manifest.Upgrade.Migrations.Pre) > 0 {
				broadcaster.Publish(opID, []byte(fmt.Sprintf("Step %d/%d: running pre-migration jobs\n", i+1, totalSteps)))
				if err := m.runMigrationJobs(inst, appName, step.Manifest.Upgrade.Migrations.Pre, step.SourceDir); err != nil {
					return fmt.Errorf("step %d/%d pre-migration failed: %w", i+1, totalSteps, err)
				}
			}
		}

		// Update from source (preserveSource stays the same across all steps)
		if err := m.updateFromSource(inst, appName, step.SourceDir, manifest.Source); err != nil {
			return fmt.Errorf("step %d/%d update failed: %w", i+1, totalSteps, err)
		}

		// Deploy to cluster
		broadcaster.Publish(opID, []byte(fmt.Sprintf("Step %d/%d: deploying %s\n", i+1, totalSteps, step.ToVersion)))
		if err := m.Deploy(inst, appName, opID, broadcaster); err != nil {
			return fmt.Errorf("step %d/%d deploy failed: %w", i+1, totalSteps, err)
		}

		// Run post-migration jobs
		if step.Manifest != nil && step.Manifest.Upgrade != nil && step.Manifest.Upgrade.Migrations != nil {
			if len(step.Manifest.Upgrade.Migrations.Post) > 0 {
				broadcaster.Publish(opID, []byte(fmt.Sprintf("Step %d/%d: running post-migration jobs\n", i+1, totalSteps)))
				if err := m.runMigrationJobs(inst, appName, step.Manifest.Upgrade.Migrations.Post, step.SourceDir); err != nil {
					return fmt.Errorf("step %d/%d post-migration failed: %w", i+1, totalSteps, err)
				}
			}
		}
	}

	broadcaster.Publish(opID, []byte(fmt.Sprintf("Upgrade complete: %s is now at %s\n", appName, plan.Steps[totalSteps-1].ToVersion)))
	return nil
}

// GetUpgradePlan reads the installed version and computes the upgrade plan from the Wild Directory
func (m *Manager) GetUpgradePlan(inst *instance.Instance, appName string) (*UpgradePlan, error) {
	manifestPath := filepath.Join(m.getAppPackageDir(inst.Name, appName), "manifest.yaml")

	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return nil, fmt.Errorf("failed to parse manifest: %w", err)
	}

	if manifest.Source == "" {
		return &UpgradePlan{}, nil // No source, nothing to upgrade
	}

	sourceDir := parseSourceDir(manifest.Source)
	if sourceDir == "" {
		return nil, fmt.Errorf("unsupported source format: %s", manifest.Source)
	}

	return ComputeUpgradePlan(manifest.Version, appName, filepath.Dir(sourceDir))
}

// applyConfigMigrations renames config keys for an app (format-aware)
func (m *Manager) applyConfigMigrations(inst *instance.Instance, appName string, migrations map[string]string) error {
	configFile := m.getAppConfigFile(inst.Name, appName)
	prefix := m.appConfigKeyPrefix(inst.Name, appName)
	yq := tools.NewYQ()

	configLock := configFile + ".lock"
	return storage.WithLock(configLock, func() error {
		for oldKey, newKey := range migrations {
			oldPath := prefix + oldKey
			newPath := prefix + newKey

			value, err := yq.Get(configFile, oldPath)
			if err != nil || value == "" || value == "null" {
				continue // Key doesn't exist, skip
			}

			if err := yq.Set(configFile, newPath, value); err != nil {
				return fmt.Errorf("failed to set %s: %w", newKey, err)
			}
			if err := yq.Delete(configFile, oldPath); err != nil {
				return fmt.Errorf("failed to delete %s: %w", oldKey, err)
			}
		}
		return nil
	})
}

// runMigrationJobs applies K8s Job manifests, waits for completion, then cleans up
func (m *Manager) runMigrationJobs(inst *instance.Instance, appName string, jobPaths []string, sourceDir string) error {
	kubeconfigPath := inst.KubeconfigPath()
	namespace := m.ResolveNamespace(inst, appName)

	for _, jobPath := range jobPaths {
		jobFile := filepath.Join(sourceDir, jobPath)
		if !storage.FileExists(jobFile) {
			return fmt.Errorf("migration job not found: %s", jobPath)
		}

		// Apply the job
		cmd := exec.Command("kubectl", "apply", "-f", jobFile, "-n", namespace)
		tools.WithKubeconfig(cmd, kubeconfigPath)
		if output, err := cmd.CombinedOutput(); err != nil {
			return fmt.Errorf("failed to apply migration job %s: %s: %w", jobPath, string(output), err)
		}

		// Wait for job completion
		cmd = exec.Command("kubectl", "wait", "--for=condition=complete", "job", "--all",
			"-n", namespace, "--timeout=300s")
		tools.WithKubeconfig(cmd, kubeconfigPath)
		if output, err := cmd.CombinedOutput(); err != nil {
			return fmt.Errorf("migration job %s did not complete: %s: %w", jobPath, string(output), err)
		}

		// Clean up the job
		cmd = exec.Command("kubectl", "delete", "-f", jobFile, "-n", namespace, "--ignore-not-found")
		tools.WithKubeconfig(cmd, kubeconfigPath)
		_, _ = cmd.CombinedOutput() // Best effort cleanup
	}

	return nil
}

// Eject converts an app from package-managed to custom
func (m *Manager) Eject(inst *instance.Instance, appName string) error {
	slog.Info("ejecting app to custom management", "component", "apps", "instance", inst.Name, "app", appName)

	appDestDir := m.getAppPackageDir(inst.Name, appName)
	packageDir := filepath.Join(appDestDir, ".package")

	// Remove .package directory
	if err := os.RemoveAll(packageDir); err != nil {
		return fmt.Errorf("failed to remove package directory: %w", err)
	}

	// Update manifest to remove source
	manifestPath := filepath.Join(appDestDir, "manifest.yaml")
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	// Remove source field
	manifest.Source = ""

	// Save updated manifest
	manifestYAML, err := yaml.Marshal(manifest)
	if err != nil {
		return fmt.Errorf("failed to marshal manifest: %w", err)
	}
	if err := storage.WriteFile(manifestPath, manifestYAML, 0644); err != nil {
		return fmt.Errorf("failed to write manifest: %w", err)
	}

	return nil
}

// UpdateConfig updates an app's configuration.
// Does not recompile templates — Deploy() handles recompilation to ensure convergence.
func (m *Manager) UpdateConfig(inst *instance.Instance, appName string, config map[string]any) error {
	configFile := m.getAppConfigFile(inst.Name, appName)
	prefix := m.appConfigKeyPrefix(inst.Name, appName)

	yq := tools.NewYQ()
	configLock := configFile + ".lock"

	if err := storage.WithLock(configLock, func() error {
		for key, value := range config {
			keyPath := prefix + key
			if err := setNestedConfig(yq, configFile, keyPath, value); err != nil {
				return fmt.Errorf("failed to set config %s: %w", key, err)
			}
		}
		return nil
	}); err != nil {
		return err
	}

	return nil
}

// copyDir recursively copies a directory and its contents
func copyDir(src, dst string) error {
	if err := storage.EnsureDir(dst, 0755); err != nil {
		return fmt.Errorf("failed to create directory %s: %w", dst, err)
	}

	entries, err := os.ReadDir(src)
	if err != nil {
		return fmt.Errorf("failed to read directory %s: %w", src, err)
	}

	for _, entry := range entries {
		srcPath := filepath.Join(src, entry.Name())
		dstPath := filepath.Join(dst, entry.Name())

		if entry.IsDir() {
			if err := copyDir(srcPath, dstPath); err != nil {
				return err
			}
			continue
		}

		data, err := os.ReadFile(srcPath)
		if err != nil {
			return fmt.Errorf("failed to read %s: %w", srcPath, err)
		}
		if err := storage.WriteFile(dstPath, data, 0644); err != nil {
			return fmt.Errorf("failed to write %s: %w", dstPath, err)
		}
	}
	return nil
}

// compileFromPackage compiles templates from .package directory to the app directory
func (m *Manager) compileFromPackage(appName, appDestDir, packageDir, configFile, secretsFile, configKeyPrefix string) error {
	gomplate := tools.NewGomplate()
	yq := tools.NewYQ()

	tempConfigFile := filepath.Join(appDestDir, ".config.tmp.yaml")
	appConfigYAML, _ := yq.Get(configFile, configKeyPrefix)
	if appConfigYAML != "" && appConfigYAML != "null" {
		if err := storage.WriteFile(tempConfigFile, []byte(appConfigYAML), 0644); err != nil {
			return fmt.Errorf("failed to write temp config: %w", err)
		}
		defer os.Remove(tempConfigFile)
	}

	tempSecretsFile := filepath.Join(appDestDir, ".secrets.tmp.yaml")
	appSecretsYAML, err := yq.Get(secretsFile, configKeyPrefix)
	if err == nil && appSecretsYAML != "" && appSecretsYAML != "null" {
		if err := storage.WriteFile(tempSecretsFile, []byte(appSecretsYAML), 0644); err != nil {
			return fmt.Errorf("failed to write temp secrets: %w", err)
		}
		defer os.Remove(tempSecretsFile)
	} else {
		if err := storage.WriteFile(tempSecretsFile, []byte("apps: {}"), 0644); err != nil {
			return fmt.Errorf("failed to write empty temp secrets: %w", err)
		}
		defer os.Remove(tempSecretsFile)
	}

	context := map[string]string{
		".":       tempConfigFile,
		"secrets": tempSecretsFile,
	}

	packageEntries, err := os.ReadDir(packageDir)
	if err != nil {
		return fmt.Errorf("failed to read package directory: %w", err)
	}

	for _, entry := range packageEntries {
		if entry.Name() == "manifest.yaml" || strings.HasPrefix(entry.Name(), ".") {
			continue
		}

		sourcePath := filepath.Join(packageDir, entry.Name())
		destPath := filepath.Join(appDestDir, entry.Name())

		if entry.IsDir() {
			if err := compileDir(sourcePath, destPath, gomplate, context); err != nil {
				return fmt.Errorf("failed to compile directory %s: %w", entry.Name(), err)
			}
			continue
		}

		if err := gomplate.RenderWithContext(sourcePath, destPath, context); err != nil {
			return fmt.Errorf("failed to compile %s: %w", entry.Name(), err)
		}
	}

	return nil
}

// compileDir recursively renders gomplate templates in a directory
func compileDir(srcDir, dstDir string, gomplate *tools.Gomplate, context map[string]string) error {
	if err := storage.EnsureDir(dstDir, 0755); err != nil {
		return fmt.Errorf("failed to create directory %s: %w", dstDir, err)
	}

	entries, err := os.ReadDir(srcDir)
	if err != nil {
		return fmt.Errorf("failed to read directory %s: %w", srcDir, err)
	}

	for _, entry := range entries {
		srcPath := filepath.Join(srcDir, entry.Name())
		dstPath := filepath.Join(dstDir, entry.Name())

		if entry.IsDir() {
			if err := compileDir(srcPath, dstPath, gomplate, context); err != nil {
				return err
			}
			continue
		}

		if err := gomplate.RenderWithContext(srcPath, dstPath, context); err != nil {
			return fmt.Errorf("failed to compile %s: %w", entry.Name(), err)
		}
	}

	return nil
}

// Compile recompiles an app's templates from .package to the app directory
func (m *Manager) Compile(inst *instance.Instance, appName string) error {
	configFile := m.getAppConfigFile(inst.Name, appName)
	secretsFile := m.getAppSecretsFile(inst.Name, appName)
	appDestDir := m.getAppPackageDir(inst.Name, appName)
	packageDir := filepath.Join(appDestDir, ".package")
	keyPrefix := m.appConfigKeyPrefix(inst.Name, appName)

	if !storage.FileExists(packageDir) {
		return fmt.Errorf("app %s has no package source (custom or not installed)", appName)
	}

	slog.Info("compiling app templates", "component", "apps", "instance", inst.Name, "app", appName)
	return m.compileFromPackage(appName, appDestDir, packageDir, configFile, secretsFile, keyPrefix)
}

// Fetch re-fetches an app's files from the source directory
func (m *Manager) Fetch(inst *instance.Instance, appName string) error {
	appDestDir := m.getAppPackageDir(inst.Name, appName)
	manifestPath := filepath.Join(appDestDir, "manifest.yaml")

	if !storage.FileExists(manifestPath) {
		return fmt.Errorf("app %s not found in instance", appName)
	}

	// Read manifest to get source
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	if manifest.Source == "" {
		return fmt.Errorf("app %s has no source (ejected or custom)", appName)
	}

	// Parse source URI
	sourceParts := strings.Split(manifest.Source, "://")
	if len(sourceParts) != 2 || sourceParts[0] != "file" {
		return fmt.Errorf("unsupported source: %s", manifest.Source)
	}

	// Resolve to latest version directory (handles both app.yaml + versions/ and old-style)
	sourceDir, meta, err := m.resolveAppDir(appName, "")
	if err != nil {
		// Fallback: use source URI directly
		sourceDir = sourceParts[1]
	}

	// Update .package directory with fresh files from source
	packageDir := filepath.Join(appDestDir, ".package")
	if err := storage.EnsureDir(packageDir, 0755); err != nil {
		return fmt.Errorf("failed to create package directory: %w", err)
	}

	entries, err := os.ReadDir(sourceDir)
	if err != nil {
		return fmt.Errorf("failed to read source directory: %w", err)
	}

	for _, entry := range entries {
		sourcePath := filepath.Join(sourceDir, entry.Name())
		packagePath := filepath.Join(packageDir, entry.Name())

		if entry.IsDir() {
			if err := copyDir(sourcePath, packagePath); err != nil {
				return fmt.Errorf("failed to copy directory %s: %w", entry.Name(), err)
			}
			continue
		}

		data, err := os.ReadFile(sourcePath)
		if err != nil {
			return fmt.Errorf("failed to read %s: %w", entry.Name(), err)
		}
		if err := storage.WriteFile(packagePath, data, 0644); err != nil {
			return fmt.Errorf("failed to write %s: %w", entry.Name(), err)
		}
	}

	// Update the compiled manifest with metadata from the source manifest
	// Preserves instance-specific fields (source, installedAs) while updating
	// package metadata (version, scripts, deploy, description, icon, upgrade)
	sourceManifestPath := filepath.Join(packageDir, "manifest.yaml")
	if sourceData, err := os.ReadFile(sourceManifestPath); err == nil {
		var sourceManifest AppManifest
		if err := yaml.Unmarshal(sourceData, &sourceManifest); err == nil {
			manifest.Version = sourceManifest.Version
			manifest.Scripts = sourceManifest.Scripts
			manifest.Deploy = sourceManifest.Deploy
			manifest.Upgrade = sourceManifest.Upgrade

			// Apply app-level fields from app.yaml if available, otherwise from source manifest
			if meta != nil {
				applyMeta(&manifest, meta)
			} else {
				manifest.Description = sourceManifest.Description
				manifest.Icon = sourceManifest.Icon
			}

			manifestYAML, err := yaml.Marshal(manifest)
			if err == nil {
				if err := storage.WriteFile(manifestPath, manifestYAML, 0644); err != nil {
					slog.Error("failed to write manifest", "component", "apps", "path", manifestPath, "error", err)
				}
			}
		}
	}

	// Note: Does not auto-compile. Deploy() handles recompilation to ensure convergence.
	return nil
}

// Install orchestrates add, compile, and deploy in sequence (for service-style installs)
func (m *Manager) Install(inst *instance.Instance, appName string, fetch, deploy bool, config map[string]any, requiredAppMappings map[string]string, opID string, broadcaster *operations.Broadcaster) error {
	appDestDir := m.getAppPackageDir(inst.Name, appName)

	if fetch || !storage.FileExists(appDestDir) {
		// Add (which includes fetch + compile)
		if err := m.Add(inst, appName, "", config, requiredAppMappings); err != nil {
			return fmt.Errorf("add failed: %w", err)
		}
	} else {
		// Just recompile
		packageDir := filepath.Join(appDestDir, ".package")
		if storage.FileExists(packageDir) {
			configFile := m.getAppConfigFile(inst.Name, appName)
			secretsFile := m.getAppSecretsFile(inst.Name, appName)
			keyPrefix := m.appConfigKeyPrefix(inst.Name, appName)
			if err := m.compileFromPackage(appName, appDestDir, packageDir, configFile, secretsFile, keyPrefix); err != nil {
				return fmt.Errorf("compile failed: %w", err)
			}
		}
	}

	// Deploy if requested
	if deploy {
		if err := m.Deploy(inst, appName, opID, broadcaster); err != nil {
			return fmt.Errorf("deploy failed: %w", err)
		}
	}

	return nil
}

// getAppURL extracts the ingress URL for an app
func (m *Manager) getAppURL(kubeconfigPath, namespace string) string {
	// Try Traefik IngressRoute first
	ingressCmd := exec.Command("kubectl", "get", "ingressroute", "-n", namespace, "-o", "json")
	tools.WithKubeconfig(ingressCmd, kubeconfigPath)
	ingressOutput, err := ingressCmd.CombinedOutput()

	if err == nil {
		var ingressList struct {
			Items []struct {
				Spec struct {
					Routes []struct {
						Match string `json:"match"`
					} `json:"routes"`
				} `json:"spec"`
			} `json:"items"`
		}
		if json.Unmarshal(ingressOutput, &ingressList) == nil && len(ingressList.Items) > 0 {
			if len(ingressList.Items[0].Spec.Routes) > 0 {
				match := ingressList.Items[0].Spec.Routes[0].Match
				// Parse Host(`domain.com`) format
				if strings.Contains(match, "Host(`") {
					start := strings.Index(match, "Host(`") + 6
					end := strings.Index(match[start:], "`")
					if end > 0 {
						host := match[start : start+end]
						return "https://" + host
					}
				}
			}
		}
	}

	// If no IngressRoute, try standard Ingress
	ingressCmd = exec.Command("kubectl", "get", "ingress", "-n", namespace, "-o", "json")
	tools.WithKubeconfig(ingressCmd, kubeconfigPath)
	ingressOutput, err = ingressCmd.CombinedOutput()

	if err == nil {
		var ingressList struct {
			Items []struct {
				Spec struct {
					Rules []struct {
						Host string `json:"host"`
					} `json:"rules"`
				} `json:"spec"`
			} `json:"items"`
		}
		if json.Unmarshal(ingressOutput, &ingressList) == nil && len(ingressList.Items) > 0 {
			if len(ingressList.Items[0].Spec.Rules) > 0 {
				host := ingressList.Items[0].Spec.Rules[0].Host
				if host != "" {
					return "https://" + host
				}
			}
		}
	}

	return ""
}

// processUserConfig processes user-provided config values, compiling any templates
// Reuses existing processValueNode logic by converting to YAML and back
func processUserConfig(userConfig map[string]any, appName string, configMap map[string]any, gomplate *tools.Gomplate) (map[string]any, error) {
	configYAML, err := yaml.Marshal(userConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal config: %w", err)
	}

	var node yaml.Node
	if err := yaml.Unmarshal(configYAML, &node); err != nil {
		return nil, fmt.Errorf("failed to parse config: %w", err)
	}

	if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
		return userConfig, nil
	}

	processed, err := processValueNode(node.Content[0], appName, configMap, nil, gomplate)
	if err != nil {
		return nil, err
	}

	result, ok := processed.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("unexpected result type from processValueNode: %T", processed)
	}

	return result, nil
}

// processValueNode recursively processes a yaml.Node value, compiling templates in all scalar (leaf) nodes
func processValueNode(node *yaml.Node, appName string, configMap map[string]any, appContext map[string]any, gomplate *tools.Gomplate) (any, error) {
	switch node.Kind {
	case yaml.ScalarNode:
		value := node.Value
		if strings.Contains(value, "{{") {
			// Build context: full config + app-specific under "app" key
			context := make(map[string]any)
			for k, v := range configMap {
				context[k] = v
			}
			context["app"] = appContext

			contextYAML, err := yaml.Marshal(context)
			if err != nil {
				return nil, fmt.Errorf("failed to marshal context: %w", err)
			}

			compiled, err := gomplate.RenderTemplate(value, string(contextYAML))
			if err != nil {
				return nil, fmt.Errorf("failed to compile template: %w", err)
			}
			return strings.TrimSpace(compiled), nil
		}
		return value, nil

	case yaml.SequenceNode:
		var arr []any
		for _, item := range node.Content {
			processed, err := processValueNode(item, appName, configMap, appContext, gomplate)
			if err != nil {
				return nil, err
			}
			arr = append(arr, processed)
		}
		return arr, nil

	case yaml.MappingNode:
		result := make(map[string]any)
		for i := 0; i < len(node.Content); i += 2 {
			keyNode := node.Content[i]
			valueNode := node.Content[i+1]

			key := keyNode.Value
			processed, err := processValueNode(valueNode, appName, configMap, appContext, gomplate)
			if err != nil {
				return nil, fmt.Errorf("failed to process nested key %s: %w", key, err)
			}
			result[key] = processed
		}
		return result, nil

	default:
		return node.Value, nil
	}
}

func processConfigInOrder(manifestPath string, appName string, configFile string, keyPrefix string, configMap map[string]any) error {
	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var node yaml.Node
	if err := yaml.Unmarshal(manifestData, &node); err != nil {
		return fmt.Errorf("failed to parse manifest YAML: %w", err)
	}

	// Find defaultConfig node
	var defaultConfigNode *yaml.Node
	if len(node.Content) > 0 && node.Content[0].Kind == yaml.MappingNode {
		for i := 0; i < len(node.Content[0].Content); i += 2 {
			if node.Content[0].Content[i].Value == "defaultConfig" {
				defaultConfigNode = node.Content[0].Content[i+1]
				break
			}
		}
	}

	if defaultConfigNode == nil || defaultConfigNode.Kind != yaml.MappingNode {
		return nil // No defaultConfig to process
	}

	yq := tools.NewYQ()
	gomplate := tools.NewGomplate()

	// Build up app context as we process values
	appContext := make(map[string]any)

	for i := 0; i < len(defaultConfigNode.Content); i += 2 {
		keyNode := defaultConfigNode.Content[i]
		valueNode := defaultConfigNode.Content[i+1]

		key := keyNode.Value
		keyPath := keyPrefix + key

		// Check if already exists in the config file (yq reads from actual file)
		existing, _ := yq.Get(configFile, keyPath)
		if existing != "" && existing != "null" {
			var existingValue any
			if err := yaml.Unmarshal([]byte(existing), &existingValue); err == nil {
				appContext[key] = existingValue
			} else {
				appContext[key] = existing
			}
			continue
		}

		// Template resolution uses merged config map (in-memory, no temp files)
		value, err := processValueNode(valueNode, appName, configMap, appContext, gomplate)
		if err != nil {
			return fmt.Errorf("failed to process config key %s: %w", key, err)
		}

		appContext[key] = value

		// Write to actual config file via yq
		if err := setNestedConfig(yq, configFile, keyPath, value); err != nil {
			return fmt.Errorf("failed to set config %s: %w", key, err)
		}
	}

	return nil
}

// RunScript executes a named script defined in the app's manifest
func (m *Manager) RunScript(inst *instance.Instance, appName, scriptName string, params map[string]string, opID string, broadcaster *operations.Broadcaster) error {
	appDir := m.getAppPackageDir(inst.Name, appName)
	manifestPath := filepath.Join(appDir, "manifest.yaml")

	manifestData, err := os.ReadFile(manifestPath)
	if err != nil {
		return fmt.Errorf("failed to read manifest: %w", err)
	}

	var manifest AppManifest
	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
		return fmt.Errorf("failed to parse manifest: %w", err)
	}

	// Find the script
	var script *Script
	for _, s := range manifest.Scripts {
		if s.Name == scriptName {
			script = &s
			break
		}
	}
	if script == nil {
		return fmt.Errorf("script '%s' not found in manifest", scriptName)
	}

	scriptPath := filepath.Join(appDir, script.Path)
	if !storage.FileExists(scriptPath) {
		return fmt.Errorf("script file not found: %s", script.Path)
	}

	kubeconfigPath := inst.KubeconfigPath()

	env := os.Environ()
	env = append(env,
		fmt.Sprintf("WILD_INSTANCE=%s", inst.Name),
		fmt.Sprintf("WILD_API_DATA_DIR=%s", m.dataDir),
		fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath),
	)
	for k, v := range params {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}

	var outputWriter *broadcastWriter
	if opID != "" {
		instanceDir := inst.Path
		logDir := filepath.Join(instanceDir, "operations", opID)
		if err := os.MkdirAll(logDir, 0755); err != nil {
			return fmt.Errorf("failed to create log directory: %w", err)
		}
		logFile, err := os.Create(filepath.Join(logDir, "output.log"))
		if err != nil {
			return fmt.Errorf("failed to create log file: %w", err)
		}
		defer logFile.Close()
		outputWriter = newBroadcastWriter(logFile, broadcaster, opID)
		if broadcaster != nil {
			broadcaster.Publish(opID, []byte(fmt.Sprintf("Running script '%s' for %s...\n", scriptName, appName)))
		}
	}

	cmd := exec.Command("/bin/bash", scriptPath)
	cmd.Dir = appDir
	cmd.Env = env

	if outputWriter != nil {
		cmd.Stdout = outputWriter
		cmd.Stderr = outputWriter
		err := cmd.Run()
		if broadcaster != nil {
			outputWriter.Flush()
			broadcaster.Close(opID)
		}
		return err
	}

	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("script failed: %w\nOutput: %s", err, string(output))
	}
	return nil
}

// isConfigOnly returns true if the app directory has no kustomization.yaml,
// meaning the app only provides configuration for other apps (e.g., SMTP).
func isConfigOnly(appDir string) bool {
	return !storage.FileExists(filepath.Join(appDir, "kustomization.yaml"))
}