wild-cloud/EXTERNAL_DEPENDENCIES.md

External Dependencies Strategy for Wild CLI

Overview

The Wild CLI needs to interface with multiple external tools that the current bash scripts depend on. This document outlines the strategy for managing these dependencies in the Go implementation.

Current External Dependencies

Primary Tools

1. kubectl - Kubernetes cluster management
2. yq - YAML processing and manipulation
3. gomplate - Template processing
4. kustomize - Kubernetes manifest processing
5. talosctl - Talos Linux node management
6. restic - Backup and restore operations
7. helm - Helm chart operations (limited use)

System Tools

• openssl - Random string generation (fallback to /dev/urandom)

Go Integration Strategies

1. Tool Abstraction Layer

Create interface-based abstractions for all external tools to enable:

• Testing with mocks
• Fallback strategies
• Version compatibility handling
• Platform-specific executable resolution

// pkg/external/interfaces.go
type ExternalTool interface {
    Name() string
    IsInstalled() bool
    Version() (string, error)
    Execute(ctx context.Context, args ...string) ([]byte, error)
}

type KubectlClient interface {
    ExternalTool
    Apply(ctx context.Context, manifests []string, namespace string, dryRun bool) error
    Delete(ctx context.Context, resource, name, namespace string) error
    CreateSecret(ctx context.Context, name, namespace string, data map[string]string) error
    GetResource(ctx context.Context, resource, name, namespace string) ([]byte, error)
}

type YqClient interface {
    ExternalTool
    Query(ctx context.Context, path, file string) (string, error)
    Set(ctx context.Context, path, value, file string) error
    Exists(ctx context.Context, path, file string) bool
}

type GomplateClient interface {
    ExternalTool
    Process(ctx context.Context, template string, contexts map[string]string) (string, error)
    ProcessFile(ctx context.Context, templateFile string, contexts map[string]string) (string, error)
}

2. Native Go Implementations (Preferred)

Where possible, replace external tools with native Go implementations:

YAML Processing (Replace yq)

// internal/config/yaml.go
import (
    "gopkg.in/yaml.v3"
    "github.com/mikefarah/yq/v4/pkg/yqlib"
)

type YAMLManager struct {
    configPath  string
    secretsPath string
}

func (y *YAMLManager) Get(path string) (interface{}, error) {
    // Use yq Go library directly instead of external binary
    return yqlib.NewYamlDecoder().Process(y.configPath, path)
}

func (y *YAMLManager) Set(path string, value interface{}) error {
    // Direct YAML manipulation using Go libraries
    return y.updateYAMLFile(path, value)
}

Template Processing (Replace gomplate)

// internal/config/template.go
import (
    "text/template"
    "github.com/Masterminds/sprig/v3"
)

type TemplateEngine struct {
    configData  map[string]interface{}
    secretsData map[string]interface{}
}

func (t *TemplateEngine) Process(templateContent string) (string, error) {
    tmpl := template.New("wild").Funcs(sprig.TxtFuncMap())
    
    // Add custom functions like gomplate
    tmpl = tmpl.Funcs(template.FuncMap{
        "config": func(path string) interface{} {
            return t.getValueByPath(t.configData, path)
        },
        "secret": func(path string) interface{} {
            return t.getValueByPath(t.secretsData, path)
        },
    })
    
    parsed, err := tmpl.Parse(templateContent)
    if err != nil {
        return "", err
    }
    
    var buf bytes.Buffer
    err = parsed.Execute(&buf, map[string]interface{}{
        "config":  t.configData,
        "secrets": t.secretsData,
    })
    
    return buf.String(), err
}

Kubernetes Client (Native Go)

// internal/kubernetes/client.go
import (
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    "k8s.io/apimachinery/pkg/util/yaml"
)

type Client struct {
    clientset kubernetes.Interface
    config    *rest.Config
}

func (c *Client) ApplyManifest(ctx context.Context, manifest string, namespace string) error {
    // Parse YAML into unstructured objects
    decoder := yaml.NewYAMLToJSONDecoder(strings.NewReader(manifest))
    
    for {
        var obj unstructured.Unstructured
        if err := decoder.Decode(&obj); err != nil {
            if err == io.EOF {
                break
            }
            return err
        }
        
        // Apply using dynamic client
        err := c.applyUnstructured(ctx, &obj, namespace)
        if err != nil {
            return err
        }
    }
    
    return nil
}

3. External Tool Wrappers (When Native Not Available)

For tools where native Go implementations aren't practical:

// internal/external/base.go
type ToolExecutor struct {
    name       string
    binaryPath string
    timeout    time.Duration
}

func (t *ToolExecutor) Execute(ctx context.Context, args ...string) ([]byte, error) {
    cmd := exec.CommandContext(ctx, t.binaryPath, args...)
    
    var stdout, stderr bytes.Buffer
    cmd.Stdout = &stdout
    cmd.Stderr = &stderr
    
    err := cmd.Run()
    if err != nil {
        return nil, fmt.Errorf("executing %s: %w\nstderr: %s", t.name, err, stderr.String())
    }
    
    return stdout.Bytes(), nil
}

// internal/external/kubectl.go
type KubectlWrapper struct {
    *ToolExecutor
    kubeconfig string
}

func (k *KubectlWrapper) Apply(ctx context.Context, manifest string, namespace string, dryRun bool) error {
    args := []string{"apply", "-f", "-"}
    
    if namespace != "" {
        args = append(args, "--namespace", namespace)
    }
    
    if dryRun {
        args = append(args, "--dry-run=client")
    }
    
    if k.kubeconfig != "" {
        args = append([]string{"--kubeconfig", k.kubeconfig}, args...)
    }
    
    cmd := exec.CommandContext(ctx, k.binaryPath, args...)
    cmd.Stdin = strings.NewReader(manifest)
    
    return cmd.Run()
}

4. Tool Discovery and Installation

// internal/external/discovery.go
type ToolManager struct {
    tools map[string]*ToolInfo
}

type ToolInfo struct {
    Name            string
    BinaryName      string
    MinVersion      string
    InstallURL      string
    CheckCommand    []string
    IsRequired      bool
    NativeAvailable bool
}

func (tm *ToolManager) DiscoverTools() error {
    for name, tool := range tm.tools {
        path, err := exec.LookPath(tool.BinaryName)
        if err != nil {
            if tool.IsRequired && !tool.NativeAvailable {
                return fmt.Errorf("required tool %s not found: %w", name, err)
            }
            continue
        }
        
        tool.BinaryPath = path
        
        // Check version compatibility
        version, err := tm.getVersion(tool)
        if err != nil {
            return fmt.Errorf("checking version of %s: %w", name, err)
        }
        
        if !tm.isVersionCompatible(version, tool.MinVersion) {
            return fmt.Errorf("tool %s version %s is not compatible (minimum: %s)", 
                name, version, tool.MinVersion)
        }
    }
    
    return nil
}

func (tm *ToolManager) PreferNative(toolName string) bool {
    tool, exists := tm.tools[toolName]
    return exists && tool.NativeAvailable
}

Implementation Priority

Phase 1: Core Native Implementations

1. YAML processing - Replace yq with native Go YAML libraries
2. Template processing - Replace gomplate with text/template + sprig
3. Configuration management - Native config/secrets handling
4. Kubernetes client - Use client-go instead of kubectl where possible

Phase 2: External Tool Wrappers

1. kubectl wrapper - For operations not covered by client-go
2. talosctl wrapper - For Talos-specific operations
3. restic wrapper - For backup/restore operations
4. kustomize wrapper - For manifest processing

Phase 3: Enhanced Features

1. Automatic tool installation - Download missing tools automatically
2. Version management - Handle multiple tool versions
3. Container fallbacks - Use containerized tools when local ones unavailable
4. Parallel execution - Run independent tool operations concurrently

Tool-Specific Strategies

kubectl

• Primary: Native client-go for most operations
• Fallback: kubectl binary for edge cases (port-forward, proxy, etc.)
• Kustomize integration: Use sigs.k8s.io/kustomize/api

yq

• Primary: Native YAML processing with gopkg.in/yaml.v3
• Advanced queries: Use github.com/mikefarah/yq/v4 Go library
• No external binary needed

gomplate

• Primary: Native template processing with text/template
• Functions: Use github.com/Masterminds/sprig for template functions
• Custom functions: Implement config/secret accessors natively

talosctl

• Only option: External binary wrapper
• Strategy: Embed in releases or auto-download
• Platform handling: talosctl-linux-amd64, talosctl-darwin-amd64, etc.

restic

• Only option: External binary wrapper
• Strategy: Auto-download appropriate version
• Configuration: Handle repository initialization and config

Error Handling and Recovery

// internal/external/manager.go
func (tm *ToolManager) ExecuteWithFallback(ctx context.Context, operation Operation) error {
    // Try native implementation first
    if tm.hasNativeImplementation(operation.Tool) {
        err := tm.executeNative(ctx, operation)
        if err == nil {
            return nil
        }
        
        // Log native failure, try external tool
        log.Warnf("Native %s implementation failed: %v, trying external tool", 
            operation.Tool, err)
    }
    
    // Use external tool wrapper
    return tm.executeExternal(ctx, operation)
}

Testing Strategy

// internal/external/mock.go  
type MockKubectl struct {
    ApplyCalls []ApplyCall
    ApplyError error
}

func (m *MockKubectl) Apply(ctx context.Context, manifest, namespace string, dryRun bool) error {
    m.ApplyCalls = append(m.ApplyCalls, ApplyCall{
        Manifest:  manifest,
        Namespace: namespace,
        DryRun:    dryRun,
    })
    return m.ApplyError
}

// Test usage
func TestAppDeploy(t *testing.T) {
    mockKubectl := &MockKubectl{}
    
    deployer := &AppDeployer{
        kubectl: mockKubectl,
    }
    
    err := deployer.Deploy(context.Background(), "test-app", false)
    
    assert.NoError(t, err)
    assert.Len(t, mockKubectl.ApplyCalls, 1)
    assert.Equal(t, "test-app", mockKubectl.ApplyCalls[0].Namespace)
}

Platform Compatibility

// internal/external/platform.go
func getPlatformBinary(toolName string) string {
    base := toolName
    
    if runtime.GOOS == "windows" {
        base += ".exe"
    }
    
    return base
}

func getToolDownloadURL(toolName, version string) string {
    switch toolName {
    case "talosctl":
        return fmt.Sprintf("https://github.com/siderolabs/talos/releases/download/%s/talosctl-%s-%s", 
            version, runtime.GOOS, runtime.GOARCH)
    case "restic":
        return fmt.Sprintf("https://github.com/restic/restic/releases/download/%s/restic_%s_%s_%s.bz2",
            version, strings.TrimPrefix(version, "v"), runtime.GOOS, runtime.GOARCH)
    }
    
    return ""
}

This strategy provides a robust foundation for managing external dependencies while maximizing the use of native Go implementations for better performance, testing, and cross-platform compatibility.