fix: Auto-recover NATS JetStream from stale store on startup

If the NATS server fails to start (stale lock files, corrupt store
from a crash), automatically clean the data directory and retry once.
This prevents the frustrating "NATS server failed to start" error
after unclean shutdowns.

The NATS KV data (registered services) is also stored as YAML files
in the services directory, so cleaning the NATS store loses nothing
— services are re-registered on the next startup or API call.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-09 14:12:29 +00:00
parent e4e1a15f92
commit 53a640c34f

View File

@@ -7,6 +7,7 @@ import (
"context" "context"
"fmt" "fmt"
"log/slog" "log/slog"
"os"
"time" "time"
natsserver "github.com/nats-io/nats-server/v2/server" natsserver "github.com/nats-io/nats-server/v2/server"
@@ -64,10 +65,23 @@ func Start(cfg Config) (*Server, error) {
// Start server in background // Start server in background
go ns.Start() go ns.Start()
// Wait for server to be ready // Wait for server to be ready. If it fails (stale lock files, corrupt
// store), clean the data directory and retry once.
if !ns.ReadyForConnections(10 * time.Second) { if !ns.ReadyForConnections(10 * time.Second) {
ns.Shutdown() ns.Shutdown()
return nil, fmt.Errorf("NATS server failed to start within 10s") slog.Warn("NATS server failed to start, cleaning store and retrying", "dataDir", cfg.DataDir)
os.RemoveAll(cfg.DataDir)
os.MkdirAll(cfg.DataDir, 0755)
ns, err = natsserver.NewServer(opts)
if err != nil {
return nil, fmt.Errorf("creating NATS server (retry): %w", err)
}
go ns.Start()
if !ns.ReadyForConnections(10 * time.Second) {
ns.Shutdown()
return nil, fmt.Errorf("NATS server failed to start after retry")
}
} }
slog.Info("NATS JetStream server started", "port", cfg.Port, "dataDir", cfg.DataDir) slog.Info("NATS JetStream server started", "port", cfg.Port, "dataDir", cfg.DataDir)