refactor: Enhance health check logic to separate HTTP and systemd checks for clarity and maintainability

This commit is contained in:
2026-03-05 10:16:17 -08:00
parent 99e70f8543
commit 0c70559b9d

View File

@@ -13,23 +13,34 @@ from castle_api.models import HealthStatus
async def check_all_health(registry: NodeRegistry) -> list[HealthStatus]: async def check_all_health(registry: NodeRegistry) -> list[HealthStatus]:
"""Check health of all deployed components with a port and health_path.""" """Check health of all deployed components.
targets: list[tuple[str, str]] = []
Services with a port + health_path are checked via HTTP.
Managed services without an HTTP health endpoint fall back to systemd unit status.
"""
http_targets: list[tuple[str, str]] = []
systemd_targets: list[str] = []
for name, deployed in registry.deployed.items(): for name, deployed in registry.deployed.items():
if not deployed.port or not deployed.health_path: if deployed.port and deployed.health_path:
continue url = f"http://127.0.0.1:{deployed.port}{deployed.health_path}"
url = f"http://127.0.0.1:{deployed.port}{deployed.health_path}" http_targets.append((name, url))
targets.append((name, url)) elif deployed.managed and not deployed.schedule:
# Managed service with no HTTP health endpoint — use systemd
if not targets: systemd_targets.append(name)
return []
tasks: list[asyncio.Task[HealthStatus]] = []
async with httpx.AsyncClient(timeout=3.0) as client: async with httpx.AsyncClient(timeout=3.0) as client:
tasks = [_check_one(client, name, url) for name, url in targets] for name, url in http_targets:
return await asyncio.gather(*tasks) tasks.append(asyncio.ensure_future(_check_http(client, name, url)))
for name in systemd_targets:
tasks.append(asyncio.ensure_future(_check_systemd(name)))
if not tasks:
return []
return list(await asyncio.gather(*tasks))
async def _check_one(client: httpx.AsyncClient, name: str, url: str) -> HealthStatus: async def _check_http(client: httpx.AsyncClient, name: str, url: str) -> HealthStatus:
"""Check a single service's health endpoint.""" """Check a single service's health endpoint."""
start = time.monotonic() start = time.monotonic()
try: try:
@@ -41,3 +52,16 @@ async def _check_one(client: httpx.AsyncClient, name: str, url: str) -> HealthSt
except httpx.HTTPError: except httpx.HTTPError:
latency = int((time.monotonic() - start) * 1000) latency = int((time.monotonic() - start) * 1000)
return HealthStatus(id=name, status="down", latency_ms=latency) return HealthStatus(id=name, status="down", latency_ms=latency)
async def _check_systemd(name: str) -> HealthStatus:
"""Check a managed service's health via its systemd unit state."""
unit = f"castle-{name}.service"
proc = await asyncio.create_subprocess_exec(
"systemctl", "--user", "is-active", unit,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _ = await proc.communicate()
state = (stdout or b"").decode().strip()
return HealthStatus(id=name, status="up" if state == "active" else "down")