wild-cloud/wild-cloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat(config): Implement config value extraction and tracking for service compilation

Browse Source
This commit is contained in:
Paul Payne
2026-02-28 07:58:44 +00:00
parent 395b740d78
commit aa528a2b01
2 changed files with 631 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
api/internal/services/services.go
View File

@@ -1,6 +1,7 @@
package services
import (
"encoding/json"
"fmt"
"io/fs"
"os"
@@ -11,6 +12,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/wild-cloud/wild-central/daemon/internal/config"
"github.com/wild-cloud/wild-central/daemon/internal/operations"
"github.com/wild-cloud/wild-central/daemon/internal/setup"
"github.com/wild-cloud/wild-central/daemon/internal/storage"
@@ -276,10 +278,7 @@ func (m *Manager) checkConfigurationState(instanceName, serviceName string) Conf
templateModTime := getDirectoryModTime(templateDir)
kustomizeModTime := getDirectoryModTime(kustomizeDir)
configPath := filepath.Join(tools.GetInstancePath(m.dataDir, instanceName), "config.yaml")
configModTime := getFileModTime(configPath)
// If templates or config changed after last compile, needs recompile
// If templates changed after last compile, needs recompile
if templateModTime.After(kustomizeModTime) {
lastCompiled := kustomizeModTime.Format(time.RFC3339)
return ConfigurationState{
@@ -289,13 +288,53 @@ func (m *Manager) checkConfigurationState(instanceName, serviceName string) Conf
}
}
if configModTime.After(kustomizeModTime) {
lastCompiled := kustomizeModTime.Format(time.RFC3339)
return ConfigurationState{
State: "needs_recompile",
Reason: "config_changed",
LastCompiled: &lastCompiled,
// Check if config values have changed (only for services that use config)
manifestPath := filepath.Join(instanceServiceDir, "wild-manifest.yaml")
if fileExists(manifestPath) {
manifest, err := LoadManifest(instanceServiceDir)
if err == nil {
configPaths := manifest.GetAllConfigPaths()
if len(configPaths) > 0 {
// This service uses config, check if values have changed
configPath := filepath.Join(tools.GetInstancePath(m.dataDir, instanceName), "config.yaml")
// Load previously saved config values
previousConfig, err := loadCompileConfig(instanceServiceDir)
if err != nil || previousConfig == nil {
// No previous config or error reading it - needs recompile
lastCompiled := kustomizeModTime.Format(time.RFC3339)
return ConfigurationState{
State: "needs_recompile",
Reason: "config_changed",
LastCompiled: &lastCompiled,
}
}
// Extract current config values
currentValues, err := extractConfigValues(configPath, configPaths)
if err != nil {
// Error extracting values - be safe and recompile
lastCompiled := kustomizeModTime.Format(time.RFC3339)
return ConfigurationState{
State: "needs_recompile",
Reason: "config_changed",
LastCompiled: &lastCompiled,
}
}
// Compare values
if configValuesChanged(previousConfig.Values, currentValues) {
lastCompiled := kustomizeModTime.Format(time.RFC3339)
return ConfigurationState{
State: "needs_recompile",
Reason: "config_changed",
LastCompiled: &lastCompiled,
}
}
}
// Service doesn't use config, so config changes don't matter
}
// Error loading manifest - fall through to "compiled" state
}
// Up to date
@@ -714,6 +753,94 @@ func dirExists(path string) bool {
return err == nil && info.IsDir()
}
// CompileConfig stores the configuration values used during compilation
type CompileConfig struct {
Timestamp time.Time `json:"timestamp"`
Values map[string]string `json:"values"`
}
// extractConfigValues extracts the values for the given config paths from config.yaml
func extractConfigValues(configPath string, paths []string) (map[string]string, error) {
if len(paths) == 0 {
return make(map[string]string), nil
}
values := make(map[string]string)
configMgr := config.NewManager()
for _, path := range paths {
value, err := configMgr.GetConfigValue(configPath, path)
if err != nil {
// Config value might not exist yet, that's OK
values[path] = ""
} else {
// yq returns "null" for missing values
if value == "null" {
values[path] = ""
} else {
values[path] = value
}
}
}
return values, nil
}
// saveCompileConfig saves the config values used during compilation
func saveCompileConfig(serviceDir string, values map[string]string) error {
configFile := filepath.Join(serviceDir, ".last-compile-config.json")
compileConfig := CompileConfig{
Timestamp: time.Now(),
Values: values,
}
data, err := json.MarshalIndent(compileConfig, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal compile config: %w", err)
}
return os.WriteFile(configFile, data, 0644)
}
// loadCompileConfig loads the previously saved compile config
func loadCompileConfig(serviceDir string) (*CompileConfig, error) {
configFile := filepath.Join(serviceDir, ".last-compile-config.json")
data, err := os.ReadFile(configFile)
if err != nil {
if os.IsNotExist(err) {
return nil, nil // No previous compile config
}
return nil, fmt.Errorf("failed to read compile config: %w", err)
}
var compileConfig CompileConfig
if err := json.Unmarshal(data, &compileConfig); err != nil {
return nil, fmt.Errorf("failed to unmarshal compile config: %w", err)
}
return &compileConfig, nil
}
// configValuesChanged checks if any of the config values have changed
func configValuesChanged(oldValues, newValues map[string]string) bool {
// If the number of keys is different, something changed
if len(oldValues) != len(newValues) {
return true
}
// Check each value
for key, oldValue := range oldValues {
newValue, exists := newValues[key]
if !exists || oldValue != newValue {
return true
}
}
return false
}
// extractFS extracts files from an fs.FS to a destination directory
func extractFS(fsys fs.FS, dst string) error {
return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
@@ -761,6 +888,26 @@ func (m *Manager) Compile(instanceName, serviceName string) error {
return fmt.Errorf("config.yaml not found for instance %s", instanceName)
}
// 2a. Extract and save config values used by this service
// Load the service manifest to get config paths
manifestPath := filepath.Join(serviceDir, "wild-manifest.yaml")
if fileExists(manifestPath) {
manifest, err := LoadManifest(serviceDir)
if err == nil {
// Get all config paths this service uses
configPaths := manifest.GetAllConfigPaths()
if len(configPaths) > 0 {
// Extract current values
values, err := extractConfigValues(configFile, configPaths)
if err == nil {
// Save them for future comparison
saveCompileConfig(serviceDir, values)
}
// Ignore errors - this is a nice-to-have feature
}
}
}
// 3. Create output directory
if err := os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)

474
api/internal/services/services_test.go
View File

@@ -776,3 +776,477 @@ func TestNewManager(t *testing.T) {
// This is environment-dependent
t.Logf("Loaded %d service manifests", len(m.manifests))
}
// TestExtractConfigValues tests extracting config values from manifest references
func TestExtractConfigValues(t *testing.T) {
tests := []struct {
name string
configContent string
paths []string
expectedValues map[string]string
expectError bool
}{
{
name: "extract simple config references",
configContent: `
cloud:
domain: example.com
cluster:
ipAddressPool: 192.168.1.10-192.168.1.20
`,
paths: []string{
"cloud.domain",
"cluster.ipAddressPool",
},
expectedValues: map[string]string{
"cloud.domain": "example.com",
"cluster.ipAddressPool": "192.168.1.10-192.168.1.20",
},
},
{
name: "extract nested service config",
configContent: `
cluster:
services:
test-service:
smtp:
host: mail.example.com
port: "587"
`,
paths: []string{
"cluster.services.test-service.smtp.host",
"cluster.services.test-service.smtp.port",
},
expectedValues: map[string]string{
"cluster.services.test-service.smtp.host": "mail.example.com",
"cluster.services.test-service.smtp.port": "587",
},
},
{
name: "handle missing config values",
configContent: `
cloud:
domain: example.com
`,
paths: []string{
"cloud.domain",
"cloud.missing.value",
},
expectedValues: map[string]string{
"cloud.domain": "example.com",
"cloud.missing.value": "",
},
},
{
name: "no config references",
configContent: `config: test`,
paths: []string{},
expectedValues: map[string]string{},
},
{
name: "convert non-string values",
configContent: `
cluster:
port: 8080
enabled: true
`,
paths: []string{
"cluster.port",
"cluster.enabled",
},
expectedValues: map[string]string{
"cluster.port": "8080",
"cluster.enabled": "true",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.yaml")
// Write config file
if err := os.WriteFile(configPath, []byte(tt.configContent), 0644); err != nil {
t.Fatalf("Failed to write config file: %v", err)
}
values, err := extractConfigValues(configPath, tt.paths)
if tt.expectError {
if err == nil {
t.Error("Expected error but got none")
}
return
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(values) != len(tt.expectedValues) {
t.Errorf("Got %d values, expected %d", len(values), len(tt.expectedValues))
}
for key, expectedValue := range tt.expectedValues {
if actualValue, exists := values[key]; !exists {
t.Errorf("Missing key %s", key)
} else if actualValue != expectedValue {
t.Errorf("Key %s: got %s, expected %s", key, actualValue, expectedValue)
}
}
})
}
}
// TestSaveAndLoadCompileConfig tests saving and loading compile config
func TestSaveAndLoadCompileConfig(t *testing.T) {
tmpDir := t.TempDir()
// Create service directory
serviceDir := filepath.Join(tmpDir, "test-service")
os.MkdirAll(serviceDir, 0755)
// Test saving config
configValues := map[string]string{
"cloud.domain": "example.com",
"cluster.ipAddressPool": "192.168.1.10-192.168.1.20",
"smtp.host": "mail.example.com",
}
err := saveCompileConfig(serviceDir, configValues)
if err != nil {
t.Fatalf("saveCompileConfig failed: %v", err)
}
// Verify file was created
configFile := filepath.Join(serviceDir, ".last-compile-config.json")
if _, err := os.Stat(configFile); err != nil {
t.Fatalf("Config file not created: %v", err)
}
// Test loading config
config, err := loadCompileConfig(serviceDir)
if err != nil {
t.Fatalf("loadCompileConfig failed: %v", err)
}
if config == nil {
t.Fatal("Loaded config is nil")
}
// Verify values match
if len(config.Values) != len(configValues) {
t.Errorf("Loaded %d values, expected %d", len(config.Values), len(configValues))
}
for key, expectedValue := range configValues {
if actualValue, exists := config.Values[key]; !exists {
t.Errorf("Missing key %s in loaded config", key)
} else if actualValue != expectedValue {
t.Errorf("Key %s: got %s, expected %s", key, actualValue, expectedValue)
}
}
// Test that timestamp is recent
if time.Since(config.Timestamp) > time.Second {
t.Errorf("Timestamp too old: %v", config.Timestamp)
}
}
// TestConfigValuesChanged tests config value comparison
func TestConfigValuesChanged(t *testing.T) {
tests := []struct {
name string
oldValues map[string]string
newValues map[string]string
expectChanged bool
}{
{
name: "no changes",
oldValues: map[string]string{
"cloud.domain": "example.com",
"smtp.host": "mail.example.com",
},
newValues: map[string]string{
"cloud.domain": "example.com",
"smtp.host": "mail.example.com",
},
expectChanged: false,
},
{
name: "value changed",
oldValues: map[string]string{
"cloud.domain": "example.com",
},
newValues: map[string]string{
"cloud.domain": "newexample.com",
},
expectChanged: true,
},
{
name: "new key added",
oldValues: map[string]string{
"cloud.domain": "example.com",
},
newValues: map[string]string{
"cloud.domain": "example.com",
"smtp.host": "mail.example.com",
},
expectChanged: true,
},
{
name: "key removed",
oldValues: map[string]string{
"cloud.domain": "example.com",
"smtp.host": "mail.example.com",
},
newValues: map[string]string{
"cloud.domain": "example.com",
},
expectChanged: true,
},
{
name: "both empty",
oldValues: map[string]string{},
newValues: map[string]string{},
expectChanged: false,
},
{
name: "old nil, new empty",
oldValues: nil,
newValues: map[string]string{},
expectChanged: false,
},
{
name: "old empty, new has values",
oldValues: map[string]string{},
newValues: map[string]string{
"cloud.domain": "example.com",
},
expectChanged: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
changed := configValuesChanged(tt.oldValues, tt.newValues)
if changed != tt.expectChanged {
t.Errorf("configValuesChanged = %v, expected %v", changed, tt.expectChanged)
}
})
}
}
// TestCheckConfigurationStateWithValueTracking tests config state with value tracking
func TestCheckConfigurationStateWithValueTracking(t *testing.T) {
tests := []struct {
name string
setup func(tmpDir string) (*Manager, string, string)
expectedState string
expectedReason string
}{
{
name: "service without config references - never needs recompile for config changes",
setup: func(tmpDir string) (*Manager, string, string) {
m := &Manager{
dataDir: tmpDir,
manifests: map[string]*ServiceManifest{
"test-service": {
Name: "test-service",
// No ConfigReferences or ServiceConfig
},
},
}
instancePath := filepath.Join(tmpDir, "instances", "test-instance")
serviceDir := filepath.Join(instancePath, "setup", "cluster-services", "test-service")
// Create template and kustomize dirs
templateDir := filepath.Join(serviceDir, "kustomize.template")
kustomizeDir := filepath.Join(serviceDir, "kustomize")
os.MkdirAll(templateDir, 0755)
os.MkdirAll(kustomizeDir, 0755)
os.WriteFile(filepath.Join(templateDir, "deployment.yaml"), []byte("template"), 0644)
os.WriteFile(filepath.Join(kustomizeDir, "kustomization.yaml"), []byte("compiled"), 0644)
// Create config file (newer than kustomize)
time.Sleep(10 * time.Millisecond)
os.WriteFile(filepath.Join(instancePath, "config.yaml"), []byte("config: updated"), 0644)
// Write manifest to instance
manifestData, _ := yaml.Marshal(m.manifests["test-service"])
os.WriteFile(filepath.Join(serviceDir, "wild-manifest.yaml"), manifestData, 0644)
return m, "test-instance", "test-service"
},
expectedState: "compiled",
// Service without config refs should not trigger recompile for config changes
},
{
name: "config values unchanged - remains compiled",
setup: func(tmpDir string) (*Manager, string, string) {
m := &Manager{
dataDir: tmpDir,
manifests: map[string]*ServiceManifest{
"test-service": {
Name: "test-service",
ConfigReferences: []string{
"cloud.domain",
},
},
},
}
instancePath := filepath.Join(tmpDir, "instances", "test-instance")
serviceDir := filepath.Join(instancePath, "setup", "cluster-services", "test-service")
// Create dirs
templateDir := filepath.Join(serviceDir, "kustomize.template")
kustomizeDir := filepath.Join(serviceDir, "kustomize")
os.MkdirAll(templateDir, 0755)
os.MkdirAll(kustomizeDir, 0755)
os.WriteFile(filepath.Join(templateDir, "deployment.yaml"), []byte("template"), 0644)
os.WriteFile(filepath.Join(kustomizeDir, "kustomization.yaml"), []byte("compiled"), 0644)
// Create config
config := map[string]interface{}{
"cloud": map[string]interface{}{
"domain": "example.com",
},
}
configData, _ := yaml.Marshal(config)
os.WriteFile(filepath.Join(instancePath, "config.yaml"), configData, 0644)
// Save tracking file with same values
saveCompileConfig(serviceDir, map[string]string{
"cloud.domain": "example.com",
})
// Write manifest
manifestData, _ := yaml.Marshal(m.manifests["test-service"])
os.WriteFile(filepath.Join(serviceDir, "wild-manifest.yaml"), manifestData, 0644)
return m, "test-instance", "test-service"
},
expectedState: "compiled",
},
{
name: "config values changed - needs recompile",
setup: func(tmpDir string) (*Manager, string, string) {
m := &Manager{
dataDir: tmpDir,
manifests: map[string]*ServiceManifest{
"test-service": {
Name: "test-service",
ConfigReferences: []string{
"cloud.domain",
},
},
},
}
instancePath := filepath.Join(tmpDir, "instances", "test-instance")
serviceDir := filepath.Join(instancePath, "setup", "cluster-services", "test-service")
// Create dirs
templateDir := filepath.Join(serviceDir, "kustomize.template")
kustomizeDir := filepath.Join(serviceDir, "kustomize")
os.MkdirAll(templateDir, 0755)
os.MkdirAll(kustomizeDir, 0755)
os.WriteFile(filepath.Join(templateDir, "deployment.yaml"), []byte("template"), 0644)
os.WriteFile(filepath.Join(kustomizeDir, "kustomization.yaml"), []byte("compiled"), 0644)
// Create config with new value
config := map[string]interface{}{
"cloud": map[string]interface{}{
"domain": "newexample.com", // Changed!
},
}
configData, _ := yaml.Marshal(config)
os.WriteFile(filepath.Join(instancePath, "config.yaml"), configData, 0644)
// Save tracking file with old value
saveCompileConfig(serviceDir, map[string]string{
"cloud.domain": "example.com", // Old value
})
// Write manifest
manifestData, _ := yaml.Marshal(m.manifests["test-service"])
os.WriteFile(filepath.Join(serviceDir, "wild-manifest.yaml"), manifestData, 0644)
return m, "test-instance", "test-service"
},
expectedState: "needs_recompile",
expectedReason: "config_changed",
},
{
name: "no tracking file - fallback to timestamp check",
setup: func(tmpDir string) (*Manager, string, string) {
m := &Manager{
dataDir: tmpDir,
manifests: map[string]*ServiceManifest{
"test-service": {
Name: "test-service",
ConfigReferences: []string{
"cloud.domain",
},
},
},
}
instancePath := filepath.Join(tmpDir, "instances", "test-instance")
serviceDir := filepath.Join(instancePath, "setup", "cluster-services", "test-service")
// Create old kustomize
kustomizeDir := filepath.Join(serviceDir, "kustomize")
os.MkdirAll(kustomizeDir, 0755)
os.WriteFile(filepath.Join(kustomizeDir, "kustomization.yaml"), []byte("compiled"), 0644)
time.Sleep(10 * time.Millisecond)
// Create newer template
templateDir := filepath.Join(serviceDir, "kustomize.template")
os.MkdirAll(templateDir, 0755)
os.WriteFile(filepath.Join(templateDir, "deployment.yaml"), []byte("template"), 0644)
time.Sleep(10 * time.Millisecond)
// Create newer config
config := map[string]interface{}{
"cloud": map[string]interface{}{
"domain": "example.com",
},
}
configData, _ := yaml.Marshal(config)
os.WriteFile(filepath.Join(instancePath, "config.yaml"), configData, 0644)
// No tracking file created
// Write manifest
manifestData, _ := yaml.Marshal(m.manifests["test-service"])
os.WriteFile(filepath.Join(serviceDir, "wild-manifest.yaml"), manifestData, 0644)
return m, "test-instance", "test-service"
},
expectedState: "needs_recompile",
expectedReason: "config_changed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := t.TempDir()
m, instanceName, serviceName := tt.setup(tmpDir)
result := m.checkConfigurationState(instanceName, serviceName)
if result.State != tt.expectedState {
t.Errorf("State = %s, want %s", result.State, tt.expectedState)
}
if tt.expectedReason != "" && result.Reason != tt.expectedReason {
t.Errorf("Reason = %s, want %s", result.Reason, tt.expectedReason)
}
})
}
}