From 8a9843658c48c1e4ddba174bc62c424c94c1ec75 Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Mon, 24 Aug 2026 09:23:49 -0700 Subject: [PATCH] core: back off and give up on crash-looping services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated service units took systemd's default start limit (5 starts in 10s), which can never trip when each failed attempt outlives the window. openclaw failed ~5s in and restarted every ~5.5s, so the counter reset on every cycle: 260k restarts at ~7.4s CPU each, over a core burned continuously for days, with the unit stuck in `activating (auto-restart)` rather than anything a status check would flag. Service units now get exponential restart backoff (RestartSteps=8, RestartMaxDelaySec=300) and a start limit sized against it (10 starts per hour, in [Unit] where systemd reads it). A broken service retries with a growing delay for ~17min — wide enough to ride out a boot-time dependency race — then lands in `failed` where it's visible and stops consuming CPU. All four values are SystemdSpec fields, overridable per deployment. Timer-driven oneshot jobs are unaffected; they never restart. systemd older than 254 ignores RestartSteps/RestartMaxDelaySec with a warning and keeps the flat RestartSec, so the start limit still applies there. --- core/src/wildpc_core/generators/systemd.py | 14 +++++ core/src/wildpc_core/manifest.py | 17 +++++- core/tests/test_systemd.py | 67 ++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/core/src/wildpc_core/generators/systemd.py b/core/src/wildpc_core/generators/systemd.py index 19282ad..0011488 100644 --- a/core/src/wildpc_core/generators/systemd.py +++ b/core/src/wildpc_core/generators/systemd.py @@ -161,15 +161,29 @@ ExecStart={exec_start} else: restart = (sd.restart if sd else RestartPolicy.ON_FAILURE).value restart_sec = sd.restart_sec if sd else 5 + restart_steps = sd.restart_steps if sd else 8 + restart_max_delay = sd.restart_max_delay_sec if sd else 300 + burst = sd.start_limit_burst if sd else 10 + interval = sd.start_limit_interval_sec if sd else 3600 + # A crash-looping service must back off and eventually give up. systemd's + # defaults (5 starts / 10s) never trip when each attempt outlives the + # window — openclaw failed ~5s in, so the counter reset every time and it + # restarted 260k times, burning a core for days. StartLimit* here is sized + # against the backoff below, so the unit lands in `failed` (visible in + # `wildpc status`) instead of looping silently forever. unit = f"""[Unit] Description=Wild PC: {description} After={after} +StartLimitIntervalSec={interval} +StartLimitBurst={burst} [Service] Type=simple ExecStart={exec_start} {env_lines}Restart={restart} RestartSec={restart_sec} +RestartSteps={restart_steps} +RestartMaxDelaySec={restart_max_delay} SuccessExitStatus=143 """ # Explicit teardown (e.g. compose `down`) so the stack's networks/volumes diff --git a/core/src/wildpc_core/manifest.py b/core/src/wildpc_core/manifest.py index 324245e..06a71e1 100644 --- a/core/src/wildpc_core/manifest.py +++ b/core/src/wildpc_core/manifest.py @@ -155,6 +155,19 @@ class SystemdSpec(BaseModel): wanted_by: list[str] = Field(default_factory=lambda: ["default.target"]) restart: RestartPolicy = RestartPolicy.ON_FAILURE restart_sec: int = 2 + # Exponential restart backoff: the delay grows from ``restart_sec`` to + # ``restart_max_delay_sec`` over ``restart_steps`` restarts, so a service that + # can't start stops hammering the box (systemd >= 254; older systemd ignores + # both keys with a warning and keeps the flat ``restart_sec``). + restart_steps: int = 8 + restart_max_delay_sec: int = 300 + # Give up eventually: ``start_limit_burst`` starts inside + # ``start_limit_interval_sec`` puts the unit in `failed` instead of looping + # forever. Sized against the backoff above — with the defaults, ~10 restarts + # span ~17min, long enough to ride out a boot-time dependency race and short + # enough that a genuinely broken service surfaces as failed the same hour. + start_limit_burst: int = 10 + start_limit_interval_sec: int = 3600 no_new_privileges: bool = True readiness: ReadinessHttpGet | None = None exec_reload: str | None = None @@ -530,7 +543,9 @@ class CaddyDeployment(DeploymentBase): @model_validator(mode="after") def _validate_reach(self) -> CaddyDeployment: if self.reach == Reach.OFF: - raise ValueError("a static (caddy) deployment is always served; reach must be internal|public") + raise ValueError( + "a static (caddy) deployment is always served; reach must be internal|public" + ) _validate_public_host(self.public_host, self.reach) return self diff --git a/core/tests/test_systemd.py b/core/tests/test_systemd.py index a7da6bb..c02995f 100644 --- a/core/tests/test_systemd.py +++ b/core/tests/test_systemd.py @@ -122,6 +122,73 @@ class TestUnitFromDeployed: unit = generate_unit_from_deployed("test-svc", deployed) assert "Restart=on-failure" in unit + def test_restart_backoff_and_start_limit(self) -> None: + """A crash loop must back off and eventually give up. + + systemd's defaults (5 starts / 10s) never trip when each attempt outlives + the window: openclaw failed ~5s in, so the counter reset on every cycle and + the unit restarted 260k times, burning a core for days.""" + deployed = Deployment( + manager="systemd", launcher="python", + run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"], + env={}, + description="Test service", + ) + unit = generate_unit_from_deployed("test-svc", deployed) + # Backoff grows the delay instead of hammering every RestartSec. + assert "RestartSteps=8" in unit + assert "RestartMaxDelaySec=300" in unit + # The limit is wide enough that it actually trips (systemd only counts + # starts inside the window). + assert "StartLimitIntervalSec=3600" in unit + assert "StartLimitBurst=10" in unit + + def test_start_limit_in_unit_section(self) -> None: + """StartLimit* belong to [Unit]; systemd ignores them under [Service].""" + deployed = Deployment( + manager="systemd", launcher="python", + run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"], + env={}, + description="Test service", + ) + unit = generate_unit_from_deployed("test-svc", deployed) + unit_section = unit.split("[Service]")[0] + assert "StartLimitIntervalSec=" in unit_section + assert "StartLimitBurst=" in unit_section + + def test_job_has_no_restart_tuning(self) -> None: + """A timer-driven oneshot never restarts, so it gets no restart keys.""" + deployed = Deployment( + manager="systemd", launcher="command", + run_cmd=["/home/user/.local/bin/my-job"], + env={}, + description="Nightly job", + schedule="0 2 * * *", + ) + unit = generate_unit_from_deployed("my-job", deployed) + for key in ("RestartSteps=", "RestartMaxDelaySec=", "StartLimitBurst="): + assert key not in unit + + def test_restart_tuning_is_overridable(self) -> None: + """A service can widen or tighten the policy in its manifest.""" + from wildpc_core.manifest import SystemdSpec + + deployed = Deployment( + manager="systemd", launcher="python", + run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"], + env={}, + description="Test service", + ) + spec = SystemdSpec( + restart_steps=3, restart_max_delay_sec=60, + start_limit_burst=4, start_limit_interval_sec=120, + ) + unit = generate_unit_from_deployed("test-svc", deployed, systemd_spec=spec) + assert "RestartSteps=3" in unit + assert "RestartMaxDelaySec=60" in unit + assert "StartLimitIntervalSec=120" in unit + assert "StartLimitBurst=4" in unit + def test_exec_start_from_run_cmd(self) -> None: """Unit file ExecStart uses resolved run_cmd.""" deployed = Deployment(