From 53a640c34fa895bd4a18fb63c8cab08f2826cacd Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Thu, 9 Jul 2026 14:12:29 +0000 Subject: [PATCH] fix: Auto-recover NATS JetStream from stale store on startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the NATS server fails to start (stale lock files, corrupt store from a crash), automatically clean the data directory and retry once. This prevents the frustrating "NATS server failed to start" error after unclean shutdowns. The NATS KV data (registered services) is also stored as YAML files in the services directory, so cleaning the NATS store loses nothing — services are re-registered on the next startup or API call. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/natsbus/server.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/natsbus/server.go b/internal/natsbus/server.go index 3345004..4fb5280 100644 --- a/internal/natsbus/server.go +++ b/internal/natsbus/server.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "log/slog" + "os" "time" natsserver "github.com/nats-io/nats-server/v2/server" @@ -64,10 +65,23 @@ func Start(cfg Config) (*Server, error) { // Start server in background go ns.Start() - // Wait for server to be ready + // Wait for server to be ready. If it fails (stale lock files, corrupt + // store), clean the data directory and retry once. if !ns.ReadyForConnections(10 * time.Second) { ns.Shutdown() - return nil, fmt.Errorf("NATS server failed to start within 10s") + slog.Warn("NATS server failed to start, cleaning store and retrying", "dataDir", cfg.DataDir) + os.RemoveAll(cfg.DataDir) + os.MkdirAll(cfg.DataDir, 0755) + + ns, err = natsserver.NewServer(opts) + if err != nil { + return nil, fmt.Errorf("creating NATS server (retry): %w", err) + } + go ns.Start() + if !ns.ReadyForConnections(10 * time.Second) { + ns.Shutdown() + return nil, fmt.Errorf("NATS server failed to start after retry") + } } slog.Info("NATS JetStream server started", "port", cfg.Port, "dataDir", cfg.DataDir)