core: back off and give up on crash-looping services

Generated service units took systemd's default start limit (5 starts in
10s), which can never trip when each failed attempt outlives the window.
openclaw failed ~5s in and restarted every ~5.5s, so the counter reset on
every cycle: 260k restarts at ~7.4s CPU each, over a core burned
continuously for days, with the unit stuck in `activating (auto-restart)`
rather than anything a status check would flag.

Service units now get exponential restart backoff (RestartSteps=8,
RestartMaxDelaySec=300) and a start limit sized against it (10 starts per
hour, in [Unit] where systemd reads it). A broken service retries with a
growing delay for ~17min — wide enough to ride out a boot-time dependency
race — then lands in `failed` where it's visible and stops consuming CPU.
All four values are SystemdSpec fields, overridable per deployment.

Timer-driven oneshot jobs are unaffected; they never restart. systemd
older than 254 ignores RestartSteps/RestartMaxDelaySec with a warning and
keeps the flat RestartSec, so the start limit still applies there.

core/src/wildpc_core/generators/systemd.py

@@ -161,15 +161,29 @@ ExecStart={exec_start}
     else:
         restart = (sd.restart if sd else RestartPolicy.ON_FAILURE).value
         restart_sec = sd.restart_sec if sd else 5
+        restart_steps = sd.restart_steps if sd else 8
+        restart_max_delay = sd.restart_max_delay_sec if sd else 300
+        burst = sd.start_limit_burst if sd else 10
+        interval = sd.start_limit_interval_sec if sd else 3600
+        # A crash-looping service must back off and eventually give up. systemd's
+        # defaults (5 starts / 10s) never trip when each attempt outlives the
+        # window — openclaw failed ~5s in, so the counter reset every time and it
+        # restarted 260k times, burning a core for days. StartLimit* here is sized
+        # against the backoff below, so the unit lands in `failed` (visible in
+        # `wildpc status`) instead of looping silently forever.
         unit = f"""[Unit]
 Description=Wild PC: {description}
 After={after}
+StartLimitIntervalSec={interval}
+StartLimitBurst={burst}
 
 [Service]
 Type=simple
 ExecStart={exec_start}
 {env_lines}Restart={restart}
 RestartSec={restart_sec}
+RestartSteps={restart_steps}
+RestartMaxDelaySec={restart_max_delay}
 SuccessExitStatus=143
 """
         # Explicit teardown (e.g. compose `down`) so the stack's networks/volumes

core/src/wildpc_core/manifest.py

@@ -155,6 +155,19 @@ class SystemdSpec(BaseModel):
     wanted_by: list[str] = Field(default_factory=lambda: ["default.target"])
     restart: RestartPolicy = RestartPolicy.ON_FAILURE
     restart_sec: int = 2
+    # Exponential restart backoff: the delay grows from ``restart_sec`` to
+    # ``restart_max_delay_sec`` over ``restart_steps`` restarts, so a service that
+    # can't start stops hammering the box (systemd >= 254; older systemd ignores
+    # both keys with a warning and keeps the flat ``restart_sec``).
+    restart_steps: int = 8
+    restart_max_delay_sec: int = 300
+    # Give up eventually: ``start_limit_burst`` starts inside
+    # ``start_limit_interval_sec`` puts the unit in `failed` instead of looping
+    # forever. Sized against the backoff above — with the defaults, ~10 restarts
+    # span ~17min, long enough to ride out a boot-time dependency race and short
+    # enough that a genuinely broken service surfaces as failed the same hour.
+    start_limit_burst: int = 10
+    start_limit_interval_sec: int = 3600
     no_new_privileges: bool = True
     readiness: ReadinessHttpGet | None = None
     exec_reload: str | None = None
@@ -530,7 +543,9 @@ class CaddyDeployment(DeploymentBase):
     @model_validator(mode="after")
     def _validate_reach(self) -> CaddyDeployment:
         if self.reach == Reach.OFF:
-            raise ValueError("a static (caddy) deployment is always served; reach must be internal|public")
+            raise ValueError(
+                "a static (caddy) deployment is always served; reach must be internal|public"
+            )
         _validate_public_host(self.public_host, self.reach)
         return self
 

core/tests/test_systemd.py

@@ -122,6 +122,73 @@ class TestUnitFromDeployed:
         unit = generate_unit_from_deployed("test-svc", deployed)
         assert "Restart=on-failure" in unit
 
+    def test_restart_backoff_and_start_limit(self) -> None:
+        """A crash loop must back off and eventually give up.
+
+        systemd's defaults (5 starts / 10s) never trip when each attempt outlives
+        the window: openclaw failed ~5s in, so the counter reset on every cycle and
+        the unit restarted 260k times, burning a core for days."""
+        deployed = Deployment(
+            manager="systemd", launcher="python",
+            run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
+            env={},
+            description="Test service",
+        )
+        unit = generate_unit_from_deployed("test-svc", deployed)
+        # Backoff grows the delay instead of hammering every RestartSec.
+        assert "RestartSteps=8" in unit
+        assert "RestartMaxDelaySec=300" in unit
+        # The limit is wide enough that it actually trips (systemd only counts
+        # starts inside the window).
+        assert "StartLimitIntervalSec=3600" in unit
+        assert "StartLimitBurst=10" in unit
+
+    def test_start_limit_in_unit_section(self) -> None:
+        """StartLimit* belong to [Unit]; systemd ignores them under [Service]."""
+        deployed = Deployment(
+            manager="systemd", launcher="python",
+            run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
+            env={},
+            description="Test service",
+        )
+        unit = generate_unit_from_deployed("test-svc", deployed)
+        unit_section = unit.split("[Service]")[0]
+        assert "StartLimitIntervalSec=" in unit_section
+        assert "StartLimitBurst=" in unit_section
+
+    def test_job_has_no_restart_tuning(self) -> None:
+        """A timer-driven oneshot never restarts, so it gets no restart keys."""
+        deployed = Deployment(
+            manager="systemd", launcher="command",
+            run_cmd=["/home/user/.local/bin/my-job"],
+            env={},
+            description="Nightly job",
+            schedule="0 2 * * *",
+        )
+        unit = generate_unit_from_deployed("my-job", deployed)
+        for key in ("RestartSteps=", "RestartMaxDelaySec=", "StartLimitBurst="):
+            assert key not in unit
+
+    def test_restart_tuning_is_overridable(self) -> None:
+        """A service can widen or tighten the policy in its manifest."""
+        from wildpc_core.manifest import SystemdSpec
+
+        deployed = Deployment(
+            manager="systemd", launcher="python",
+            run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
+            env={},
+            description="Test service",
+        )
+        spec = SystemdSpec(
+            restart_steps=3, restart_max_delay_sec=60,
+            start_limit_burst=4, start_limit_interval_sec=120,
+        )
+        unit = generate_unit_from_deployed("test-svc", deployed, systemd_spec=spec)
+        assert "RestartSteps=3" in unit
+        assert "RestartMaxDelaySec=60" in unit
+        assert "StartLimitIntervalSec=120" in unit
+        assert "StartLimitBurst=4" in unit
+
     def test_exec_start_from_run_cmd(self) -> None:
         """Unit file ExecStart uses resolved run_cmd."""
         deployed = Deployment(