feat(mesh): cross-node routing + presence breaker (Phase 3)

- NodeConfig.address: routable host peers proxy to (wired registry + wire)
- compute_routes emits 'remote' routes for consumed peer services
  (local requires ref satisfied by an online peer -> <address>:<port>)
- _host_remote_block: fail-fast reverse_proxy (2s dial, passive health);
  presence expiry removes the route entirely = the primary circuit-breaker
- mesh_gateway.py: API re-renders + reloads the Caddyfile on mesh change,
  iff content changed (no-op until a cross-node service is consumed)
- tests: route emit / address fallback / breaker-absent / unconsumed

Logic verified hermetically + against primer's real registry (castle-api ->
primer:9020); live integration proven a no-op on civil.
This commit is contained in:
2026-07-07 05:38:15 -07:00
parent f94517887e
commit 526736f778
7 changed files with 255 additions and 2 deletions

View File

@@ -0,0 +1,55 @@
"""Regenerate the gateway with cross-node (remote) routes from the live mesh.
Local routes come from `castle apply` (static). Remote routes are dynamic — they
appear/vanish as peers join/leave — so the API owns them: on a mesh change it
re-renders the Caddyfile (same generator `apply` uses, plus remote routes for
online peers) and reloads the gateway iff the content changed.
Safety: with no cross-node `requires`, the output equals the local-only Caddyfile,
so this is a verified no-op until this node actually consumes a peer service.
"""
from __future__ import annotations
import asyncio
import logging
import subprocess
from castle_core.config import SPECS_DIR
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
from castle_api.config import get_registry
from castle_api.mesh import mesh_state
logger = logging.getLogger(__name__)
_GATEWAY_UNIT = "castle-castle-gateway.service"
def _regenerate(reload: bool) -> bool:
try:
reg = get_registry()
except Exception:
return False
remotes = {h: n.registry for h, n in mesh_state.all_nodes().items()}
try:
content = generate_caddyfile_from_registry(reg, remotes)
except Exception:
logger.exception("mesh gateway: Caddyfile generation failed")
return False
path = SPECS_DIR / "Caddyfile"
old = path.read_text() if path.exists() else ""
if content == old:
return False
path.write_text(content)
logger.info("mesh gateway: Caddyfile updated with cross-node routes")
if reload:
subprocess.run(
["systemctl", "--user", "reload", _GATEWAY_UNIT], check=False
)
return True
async def refresh_remote_routes(reload: bool = True) -> bool:
"""Async wrapper — runs the blocking regen off the event loop."""
return await asyncio.to_thread(_regenerate, reload)

View File

@@ -28,6 +28,8 @@ def registry_to_json(registry: NodeRegistry) -> str:
"gateway_domain": registry.node.gateway_domain,
# fleet role — so peers know which node is the config/secret authority.
"role": registry.node.role,
# routable host peers proxy to for this node's services.
"address": registry.node.address,
},
"deployed": {},
}
@@ -76,6 +78,7 @@ def json_to_registry(payload: str) -> NodeRegistry:
gateway_port=node_data.get("gateway_port", 9000),
gateway_domain=node_data.get("gateway_domain"),
role=node_data.get("role", "follower"),
address=node_data.get("address"),
)
deployed: dict[str, Deployment] = {}
for key, comp_data in data.get("deployed", {}).items():

View File

@@ -28,6 +28,7 @@ from nats.js.api import KeyValueConfig
from castle_core.registry import NodeRegistry
from castle_api.mesh import mesh_state
from castle_api.mesh_gateway import refresh_remote_routes
from castle_api.mesh_wire import json_to_registry, registry_to_json
from castle_api.stream import broadcast
@@ -93,6 +94,7 @@ class CastleNATSClient:
await self.publish_registry(self._local_registry)
await self._presence_kv.put(self._local_hostname, b"online")
await self._seed_existing()
await refresh_remote_routes() # establish any cross-node routes on startup
self._tasks = [
asyncio.create_task(self._watch_loop()),
@@ -182,6 +184,7 @@ class CastleNATSClient:
await broadcast(
"mesh", {"event": "node_updated", "hostname": key}
)
asyncio.create_task(refresh_remote_routes())
except Exception:
logger.exception("Error handling mesh entry for %s", key)
@@ -199,6 +202,7 @@ class CastleNATSClient:
self._online.discard(hostname)
self._last_json.pop(hostname, None)
await broadcast("mesh", {"event": "node_offline", "hostname": hostname})
asyncio.create_task(refresh_remote_routes())
async def _heartbeat_loop(self) -> None:
while True: