fix: Auto-recover NATS JetStream from stale store on startup
If the NATS server fails to start (stale lock files, corrupt store from a crash), automatically clean the data directory and retry once. This prevents the frustrating "NATS server failed to start" error after unclean shutdowns. The NATS KV data (registered services) is also stored as YAML files in the services directory, so cleaning the NATS store loses nothing — services are re-registered on the next startup or API call. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
natsserver "github.com/nats-io/nats-server/v2/server"
|
natsserver "github.com/nats-io/nats-server/v2/server"
|
||||||
@@ -64,10 +65,23 @@ func Start(cfg Config) (*Server, error) {
|
|||||||
// Start server in background
|
// Start server in background
|
||||||
go ns.Start()
|
go ns.Start()
|
||||||
|
|
||||||
// Wait for server to be ready
|
// Wait for server to be ready. If it fails (stale lock files, corrupt
|
||||||
|
// store), clean the data directory and retry once.
|
||||||
if !ns.ReadyForConnections(10 * time.Second) {
|
if !ns.ReadyForConnections(10 * time.Second) {
|
||||||
ns.Shutdown()
|
ns.Shutdown()
|
||||||
return nil, fmt.Errorf("NATS server failed to start within 10s")
|
slog.Warn("NATS server failed to start, cleaning store and retrying", "dataDir", cfg.DataDir)
|
||||||
|
os.RemoveAll(cfg.DataDir)
|
||||||
|
os.MkdirAll(cfg.DataDir, 0755)
|
||||||
|
|
||||||
|
ns, err = natsserver.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("creating NATS server (retry): %w", err)
|
||||||
|
}
|
||||||
|
go ns.Start()
|
||||||
|
if !ns.ReadyForConnections(10 * time.Second) {
|
||||||
|
ns.Shutdown()
|
||||||
|
return nil, fmt.Errorf("NATS server failed to start after retry")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
slog.Info("NATS JetStream server started", "port", cfg.Port, "dataDir", cfg.DataDir)
|
slog.Info("NATS JetStream server started", "port", cfg.Port, "dataDir", cfg.DataDir)
|
||||||
|
|||||||
Reference in New Issue
Block a user