feat(mesh): cross-node routing + presence breaker (Phase 3)
- NodeConfig.address: routable host peers proxy to (wired registry + wire) - compute_routes emits 'remote' routes for consumed peer services (local requires ref satisfied by an online peer -> <address>:<port>) - _host_remote_block: fail-fast reverse_proxy (2s dial, passive health); presence expiry removes the route entirely = the primary circuit-breaker - mesh_gateway.py: API re-renders + reloads the Caddyfile on mesh change, iff content changed (no-op until a cross-node service is consumed) - tests: route emit / address fallback / breaker-absent / unconsumed Logic verified hermetically + against primer's real registry (castle-api -> primer:9020); live integration proven a no-op on civil.
This commit is contained in:
55
castle-api/src/castle_api/mesh_gateway.py
Normal file
55
castle-api/src/castle_api/mesh_gateway.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""Regenerate the gateway with cross-node (remote) routes from the live mesh.
|
||||
|
||||
Local routes come from `castle apply` (static). Remote routes are dynamic — they
|
||||
appear/vanish as peers join/leave — so the API owns them: on a mesh change it
|
||||
re-renders the Caddyfile (same generator `apply` uses, plus remote routes for
|
||||
online peers) and reloads the gateway iff the content changed.
|
||||
|
||||
Safety: with no cross-node `requires`, the output equals the local-only Caddyfile,
|
||||
so this is a verified no-op until this node actually consumes a peer service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from castle_core.config import SPECS_DIR
|
||||
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
||||
|
||||
from castle_api.config import get_registry
|
||||
from castle_api.mesh import mesh_state
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GATEWAY_UNIT = "castle-castle-gateway.service"
|
||||
|
||||
|
||||
def _regenerate(reload: bool) -> bool:
|
||||
try:
|
||||
reg = get_registry()
|
||||
except Exception:
|
||||
return False
|
||||
remotes = {h: n.registry for h, n in mesh_state.all_nodes().items()}
|
||||
try:
|
||||
content = generate_caddyfile_from_registry(reg, remotes)
|
||||
except Exception:
|
||||
logger.exception("mesh gateway: Caddyfile generation failed")
|
||||
return False
|
||||
path = SPECS_DIR / "Caddyfile"
|
||||
old = path.read_text() if path.exists() else ""
|
||||
if content == old:
|
||||
return False
|
||||
path.write_text(content)
|
||||
logger.info("mesh gateway: Caddyfile updated with cross-node routes")
|
||||
if reload:
|
||||
subprocess.run(
|
||||
["systemctl", "--user", "reload", _GATEWAY_UNIT], check=False
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def refresh_remote_routes(reload: bool = True) -> bool:
|
||||
"""Async wrapper — runs the blocking regen off the event loop."""
|
||||
return await asyncio.to_thread(_regenerate, reload)
|
||||
@@ -28,6 +28,8 @@ def registry_to_json(registry: NodeRegistry) -> str:
|
||||
"gateway_domain": registry.node.gateway_domain,
|
||||
# fleet role — so peers know which node is the config/secret authority.
|
||||
"role": registry.node.role,
|
||||
# routable host peers proxy to for this node's services.
|
||||
"address": registry.node.address,
|
||||
},
|
||||
"deployed": {},
|
||||
}
|
||||
@@ -76,6 +78,7 @@ def json_to_registry(payload: str) -> NodeRegistry:
|
||||
gateway_port=node_data.get("gateway_port", 9000),
|
||||
gateway_domain=node_data.get("gateway_domain"),
|
||||
role=node_data.get("role", "follower"),
|
||||
address=node_data.get("address"),
|
||||
)
|
||||
deployed: dict[str, Deployment] = {}
|
||||
for key, comp_data in data.get("deployed", {}).items():
|
||||
|
||||
@@ -28,6 +28,7 @@ from nats.js.api import KeyValueConfig
|
||||
from castle_core.registry import NodeRegistry
|
||||
|
||||
from castle_api.mesh import mesh_state
|
||||
from castle_api.mesh_gateway import refresh_remote_routes
|
||||
from castle_api.mesh_wire import json_to_registry, registry_to_json
|
||||
from castle_api.stream import broadcast
|
||||
|
||||
@@ -93,6 +94,7 @@ class CastleNATSClient:
|
||||
await self.publish_registry(self._local_registry)
|
||||
await self._presence_kv.put(self._local_hostname, b"online")
|
||||
await self._seed_existing()
|
||||
await refresh_remote_routes() # establish any cross-node routes on startup
|
||||
|
||||
self._tasks = [
|
||||
asyncio.create_task(self._watch_loop()),
|
||||
@@ -182,6 +184,7 @@ class CastleNATSClient:
|
||||
await broadcast(
|
||||
"mesh", {"event": "node_updated", "hostname": key}
|
||||
)
|
||||
asyncio.create_task(refresh_remote_routes())
|
||||
except Exception:
|
||||
logger.exception("Error handling mesh entry for %s", key)
|
||||
|
||||
@@ -199,6 +202,7 @@ class CastleNATSClient:
|
||||
self._online.discard(hostname)
|
||||
self._last_json.pop(hostname, None)
|
||||
await broadcast("mesh", {"event": "node_offline", "hostname": hostname})
|
||||
asyncio.create_task(refresh_remote_routes())
|
||||
|
||||
async def _heartbeat_loop(self) -> None:
|
||||
while True:
|
||||
|
||||
Reference in New Issue
Block a user