core: back off and give up on crash-looping services

Generated service units took systemd's default start limit (5 starts in
10s), which can never trip when each failed attempt outlives the window.
openclaw failed ~5s in and restarted every ~5.5s, so the counter reset on
every cycle: 260k restarts at ~7.4s CPU each, over a core burned
continuously for days, with the unit stuck in `activating (auto-restart)`
rather than anything a status check would flag.

Service units now get exponential restart backoff (RestartSteps=8,
RestartMaxDelaySec=300) and a start limit sized against it (10 starts per
hour, in [Unit] where systemd reads it). A broken service retries with a
growing delay for ~17min — wide enough to ride out a boot-time dependency
race — then lands in `failed` where it's visible and stops consuming CPU.
All four values are SystemdSpec fields, overridable per deployment.

Timer-driven oneshot jobs are unaffected; they never restart. systemd
older than 254 ignores RestartSteps/RestartMaxDelaySec with a warning and
keeps the flat RestartSec, so the start limit still applies there.
This commit is contained in:
2026-08-24 09:23:49 -07:00
parent 6cf25ad2c7
commit 8a9843658c
3 changed files with 97 additions and 1 deletions

View File

@@ -122,6 +122,73 @@ class TestUnitFromDeployed:
unit = generate_unit_from_deployed("test-svc", deployed)
assert "Restart=on-failure" in unit
def test_restart_backoff_and_start_limit(self) -> None:
"""A crash loop must back off and eventually give up.
systemd's defaults (5 starts / 10s) never trip when each attempt outlives
the window: openclaw failed ~5s in, so the counter reset on every cycle and
the unit restarted 260k times, burning a core for days."""
deployed = Deployment(
manager="systemd", launcher="python",
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
env={},
description="Test service",
)
unit = generate_unit_from_deployed("test-svc", deployed)
# Backoff grows the delay instead of hammering every RestartSec.
assert "RestartSteps=8" in unit
assert "RestartMaxDelaySec=300" in unit
# The limit is wide enough that it actually trips (systemd only counts
# starts inside the window).
assert "StartLimitIntervalSec=3600" in unit
assert "StartLimitBurst=10" in unit
def test_start_limit_in_unit_section(self) -> None:
"""StartLimit* belong to [Unit]; systemd ignores them under [Service]."""
deployed = Deployment(
manager="systemd", launcher="python",
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
env={},
description="Test service",
)
unit = generate_unit_from_deployed("test-svc", deployed)
unit_section = unit.split("[Service]")[0]
assert "StartLimitIntervalSec=" in unit_section
assert "StartLimitBurst=" in unit_section
def test_job_has_no_restart_tuning(self) -> None:
"""A timer-driven oneshot never restarts, so it gets no restart keys."""
deployed = Deployment(
manager="systemd", launcher="command",
run_cmd=["/home/user/.local/bin/my-job"],
env={},
description="Nightly job",
schedule="0 2 * * *",
)
unit = generate_unit_from_deployed("my-job", deployed)
for key in ("RestartSteps=", "RestartMaxDelaySec=", "StartLimitBurst="):
assert key not in unit
def test_restart_tuning_is_overridable(self) -> None:
"""A service can widen or tighten the policy in its manifest."""
from wildpc_core.manifest import SystemdSpec
deployed = Deployment(
manager="systemd", launcher="python",
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
env={},
description="Test service",
)
spec = SystemdSpec(
restart_steps=3, restart_max_delay_sec=60,
start_limit_burst=4, start_limit_interval_sec=120,
)
unit = generate_unit_from_deployed("test-svc", deployed, systemd_spec=spec)
assert "RestartSteps=3" in unit
assert "RestartMaxDelaySec=60" in unit
assert "StartLimitIntervalSec=120" in unit
assert "StartLimitBurst=4" in unit
def test_exec_start_from_run_cmd(self) -> None:
"""Unit file ExecStart uses resolved run_cmd."""
deployed = Deployment(