core: back off and give up on crash-looping services
Generated service units took systemd's default start limit (5 starts in 10s), which can never trip when each failed attempt outlives the window. openclaw failed ~5s in and restarted every ~5.5s, so the counter reset on every cycle: 260k restarts at ~7.4s CPU each, over a core burned continuously for days, with the unit stuck in `activating (auto-restart)` rather than anything a status check would flag. Service units now get exponential restart backoff (RestartSteps=8, RestartMaxDelaySec=300) and a start limit sized against it (10 starts per hour, in [Unit] where systemd reads it). A broken service retries with a growing delay for ~17min — wide enough to ride out a boot-time dependency race — then lands in `failed` where it's visible and stops consuming CPU. All four values are SystemdSpec fields, overridable per deployment. Timer-driven oneshot jobs are unaffected; they never restart. systemd older than 254 ignores RestartSteps/RestartMaxDelaySec with a warning and keeps the flat RestartSec, so the start limit still applies there.
This commit is contained in:
@@ -122,6 +122,73 @@ class TestUnitFromDeployed:
|
||||
unit = generate_unit_from_deployed("test-svc", deployed)
|
||||
assert "Restart=on-failure" in unit
|
||||
|
||||
def test_restart_backoff_and_start_limit(self) -> None:
|
||||
"""A crash loop must back off and eventually give up.
|
||||
|
||||
systemd's defaults (5 starts / 10s) never trip when each attempt outlives
|
||||
the window: openclaw failed ~5s in, so the counter reset on every cycle and
|
||||
the unit restarted 260k times, burning a core for days."""
|
||||
deployed = Deployment(
|
||||
manager="systemd", launcher="python",
|
||||
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
|
||||
env={},
|
||||
description="Test service",
|
||||
)
|
||||
unit = generate_unit_from_deployed("test-svc", deployed)
|
||||
# Backoff grows the delay instead of hammering every RestartSec.
|
||||
assert "RestartSteps=8" in unit
|
||||
assert "RestartMaxDelaySec=300" in unit
|
||||
# The limit is wide enough that it actually trips (systemd only counts
|
||||
# starts inside the window).
|
||||
assert "StartLimitIntervalSec=3600" in unit
|
||||
assert "StartLimitBurst=10" in unit
|
||||
|
||||
def test_start_limit_in_unit_section(self) -> None:
|
||||
"""StartLimit* belong to [Unit]; systemd ignores them under [Service]."""
|
||||
deployed = Deployment(
|
||||
manager="systemd", launcher="python",
|
||||
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
|
||||
env={},
|
||||
description="Test service",
|
||||
)
|
||||
unit = generate_unit_from_deployed("test-svc", deployed)
|
||||
unit_section = unit.split("[Service]")[0]
|
||||
assert "StartLimitIntervalSec=" in unit_section
|
||||
assert "StartLimitBurst=" in unit_section
|
||||
|
||||
def test_job_has_no_restart_tuning(self) -> None:
|
||||
"""A timer-driven oneshot never restarts, so it gets no restart keys."""
|
||||
deployed = Deployment(
|
||||
manager="systemd", launcher="command",
|
||||
run_cmd=["/home/user/.local/bin/my-job"],
|
||||
env={},
|
||||
description="Nightly job",
|
||||
schedule="0 2 * * *",
|
||||
)
|
||||
unit = generate_unit_from_deployed("my-job", deployed)
|
||||
for key in ("RestartSteps=", "RestartMaxDelaySec=", "StartLimitBurst="):
|
||||
assert key not in unit
|
||||
|
||||
def test_restart_tuning_is_overridable(self) -> None:
|
||||
"""A service can widen or tighten the policy in its manifest."""
|
||||
from wildpc_core.manifest import SystemdSpec
|
||||
|
||||
deployed = Deployment(
|
||||
manager="systemd", launcher="python",
|
||||
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
|
||||
env={},
|
||||
description="Test service",
|
||||
)
|
||||
spec = SystemdSpec(
|
||||
restart_steps=3, restart_max_delay_sec=60,
|
||||
start_limit_burst=4, start_limit_interval_sec=120,
|
||||
)
|
||||
unit = generate_unit_from_deployed("test-svc", deployed, systemd_spec=spec)
|
||||
assert "RestartSteps=3" in unit
|
||||
assert "RestartMaxDelaySec=60" in unit
|
||||
assert "StartLimitIntervalSec=120" in unit
|
||||
assert "StartLimitBurst=4" in unit
|
||||
|
||||
def test_exec_start_from_run_cmd(self) -> None:
|
||||
"""Unit file ExecStart uses resolved run_cmd."""
|
||||
deployed = Deployment(
|
||||
|
||||
Reference in New Issue
Block a user