feat(mesh): cross-node routing + presence breaker (Phase 3)
- NodeConfig.address: routable host peers proxy to (wired registry + wire) - compute_routes emits 'remote' routes for consumed peer services (local requires ref satisfied by an online peer -> <address>:<port>) - _host_remote_block: fail-fast reverse_proxy (2s dial, passive health); presence expiry removes the route entirely = the primary circuit-breaker - mesh_gateway.py: API re-renders + reloads the Caddyfile on mesh change, iff content changed (no-op until a cross-node service is consumed) - tests: route emit / address fallback / breaker-absent / unconsumed Logic verified hermetically + against primer's real registry (castle-api -> primer:9020); live integration proven a no-op on civil.
This commit is contained in:
@@ -121,8 +121,10 @@ def compute_routes(
|
||||
"""Build the ordered list of gateway routes. Every route is a host route whose
|
||||
address is the service/frontend **name** (published at ``<name>.<domain>``);
|
||||
``proxy`` routes reverse-proxy a local port, ``static`` routes file-serve a
|
||||
frontend's dist. Path routes no longer exist. ``remote_registries`` is accepted
|
||||
for signature compatibility but cross-node routing is out of scope here."""
|
||||
frontend's dist. Path routes no longer exist. When ``remote_registries`` is
|
||||
given (online peers, keyed by hostname), ``remote`` routes are added for
|
||||
services this node **consumes** (a local ``requires`` ref satisfied by a peer)
|
||||
— so a consumed cross-node service is reachable at ``<ref>.<domain>``."""
|
||||
if config is None:
|
||||
try:
|
||||
from castle_core.config import load_config
|
||||
@@ -141,9 +143,50 @@ def compute_routes(
|
||||
for name, kind, target in _local_routes(config, registry):
|
||||
routes.append(GatewayRoute(name, kind, target, name, node))
|
||||
|
||||
if remote_registries:
|
||||
routes.extend(_remote_routes(config, registry, remote_registries))
|
||||
|
||||
return routes
|
||||
|
||||
|
||||
def _remote_routes(
|
||||
config: CastleConfig | None,
|
||||
registry: NodeRegistry,
|
||||
remote_registries: dict[str, NodeRegistry],
|
||||
) -> list[GatewayRoute]:
|
||||
"""Routes to services this node consumes from online peers.
|
||||
|
||||
A route is emitted for each local ``requires`` ref that (a) isn't satisfied
|
||||
locally and (b) is provided by an exposed service on some peer. The route is
|
||||
only present while the peer is (presence expiry removes the peer from
|
||||
``remote_registries``, which *is* the circuit-breaker: gone → no route)."""
|
||||
# Refs this node consumes.
|
||||
consumed: set[str] = set()
|
||||
local_names: set[str] = set()
|
||||
if config is not None:
|
||||
for _kind, name, dep in config.all_deployments():
|
||||
local_names.add(name)
|
||||
for req in getattr(dep, "requires", []) or []:
|
||||
ref = getattr(req, "ref", None)
|
||||
if ref and getattr(req, "kind", "deployment") == "deployment":
|
||||
consumed.add(ref)
|
||||
# Drop refs already satisfied locally.
|
||||
consumed -= local_names
|
||||
|
||||
out: list[GatewayRoute] = []
|
||||
for host, remote in sorted(remote_registries.items()):
|
||||
addr = remote.node.address or host
|
||||
for _kind, name, dep in remote.all():
|
||||
if name not in consumed:
|
||||
continue
|
||||
if dep.subdomain and dep.port:
|
||||
out.append(
|
||||
GatewayRoute(name, "remote", f"{addr}:{dep.port}", name, host)
|
||||
)
|
||||
consumed.discard(name) # first online provider wins
|
||||
return out
|
||||
|
||||
|
||||
def _host_matcher_block(label: str, host: str, target: str) -> list[str]:
|
||||
"""A `@host_X host <host> / handle @host_X { reverse_proxy <target> }` block.
|
||||
|
||||
@@ -159,6 +202,27 @@ def _host_matcher_block(label: str, host: str, target: str) -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def _host_remote_block(label: str, host: str, target: str) -> list[str]:
|
||||
"""A remote (cross-node) host route with a fail-fast breaker: a short dial
|
||||
timeout + passive health, so an unreachable peer 502s in ~2s instead of
|
||||
hanging. (Presence removal drops the route entirely — this guards the
|
||||
there-but-wedged case.)"""
|
||||
matcher = f"@host_{label.replace('-', '_').replace('.', '_')}"
|
||||
return [
|
||||
f" {matcher} host {host}",
|
||||
f" handle {matcher} {{",
|
||||
f" reverse_proxy {target} {{",
|
||||
" lb_try_duration 1s",
|
||||
" fail_duration 30s",
|
||||
" transport http {",
|
||||
" dial_timeout 2s",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def _host_static_block(label: str, host: str, serve_dir: str) -> list[str]:
|
||||
"""A host matcher that file-serves a frontend's dist (with SPA fallback)."""
|
||||
matcher = f"@host_{label.replace('-', '_').replace('.', '_')}"
|
||||
@@ -237,6 +301,8 @@ def generate_caddyfile_from_registry(
|
||||
host = f"{r.address}.{domain}"
|
||||
if r.kind == "static":
|
||||
lines += _host_static_block(r.name or r.address, host, r.target)
|
||||
elif r.kind == "remote":
|
||||
lines += _host_remote_block(r.name or r.address, host, r.target)
|
||||
else:
|
||||
lines += _host_matcher_block(r.name or r.address, host, r.target)
|
||||
lines.append("}")
|
||||
|
||||
@@ -32,6 +32,10 @@ class NodeConfig:
|
||||
tunnel_id: str | None = None
|
||||
# Emit the cert_obtained → `castle tls reconcile` hook (needs events-exec plugin).
|
||||
cert_hook: bool = False
|
||||
# Routable host peers use to reach this node's services (LAN IP/hostname).
|
||||
# Defaults to the hostname; set explicitly when the hostname isn't resolvable
|
||||
# cross-node. Used to build `remote` gateway routes to this node.
|
||||
address: str | None = None
|
||||
# Fleet role: "authority" may write shared config/secrets to the mesh;
|
||||
# "follower" reconciles from it. Static (no election) — the authority is
|
||||
# pinned in castle.yaml. When the authority is down, shared state is
|
||||
@@ -158,6 +162,7 @@ def load_registry(path: Path | None = None) -> NodeRegistry:
|
||||
tunnel_id=node_data.get("tunnel_id"),
|
||||
cert_hook=node_data.get("cert_hook", False),
|
||||
role=node_data.get("role", "follower"),
|
||||
address=node_data.get("address"),
|
||||
)
|
||||
|
||||
deployed: dict[str, Deployment] = {}
|
||||
@@ -249,6 +254,8 @@ def save_registry(registry: NodeRegistry, path: Path | None = None) -> None:
|
||||
data["node"]["cert_hook"] = registry.node.cert_hook
|
||||
if registry.node.role and registry.node.role != "follower":
|
||||
data["node"]["role"] = registry.node.role
|
||||
if registry.node.address:
|
||||
data["node"]["address"] = registry.node.address
|
||||
|
||||
for key, comp in registry.deployed.items():
|
||||
entry: dict = {
|
||||
|
||||
Reference in New Issue
Block a user