feat: Implement multi-node support with MQTT and mDNS for service discovery and coordination

castle-api/pyproject.toml

@@ -9,6 +9,8 @@ dependencies = [
     "pydantic-settings>=2.0.0",
     "httpx>=0.27.0",
     "castle-core",
+    "paho-mqtt>=2.0.0",
+    "zeroconf>=0.131.0",
 ]
 
 [tool.uv.sources]

castle-api/src/castle_api/config.py

@@ -14,6 +14,12 @@ class Settings(BaseSettings):
     host: str = "0.0.0.0"
     port: int = 9020
 
+    # Mesh coordination (all off by default — single-node works without them)
+    mqtt_enabled: bool = False
+    mqtt_host: str = "localhost"
+    mqtt_port: int = 1883
+    mdns_enabled: bool = False
+
     model_config = {
         "env_prefix": "CASTLE_API_",
         "env_file": ".env",

castle-api/src/castle_api/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 
@@ -11,7 +12,7 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from starlette.responses import StreamingResponse
 
-from castle_api.config import settings
+from castle_api.config import get_registry, settings
 from castle_api.config_editor import router as config_router
 from castle_api.logs import router as logs_router
 from castle_api.routes import router as dashboard_router
@@ -23,8 +24,11 @@ from castle_api.stream import (
     subscribe,
     unsubscribe,
 )
+from castle_api.nodes import router as nodes_router
 from castle_api.tools import router as tools_router
 
+logger = logging.getLogger(__name__)
+
 # Set by _watch_shutdown when uvicorn begins its shutdown sequence.
 _shutting_down = False
 
@@ -46,10 +50,53 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
 
     poll_task = asyncio.create_task(health_poll_loop())
 
+    # --- Mesh coordination (opt-in) ---
+    mqtt_client = None
+    mdns_service = None
+
+    if settings.mqtt_enabled:
+        try:
+            from castle_api.mqtt_client import CastleMQTTClient
+
+            registry = get_registry()
+            mqtt_client = CastleMQTTClient(
+                local_hostname=registry.node.hostname,
+                local_registry=registry,
+                broker_host=settings.mqtt_host,
+                broker_port=settings.mqtt_port,
+            )
+            await mqtt_client.start()
+            app.state.mqtt_client = mqtt_client
+        except Exception:
+            logger.exception("Failed to start MQTT client")
+            mqtt_client = None
+
+    if settings.mdns_enabled:
+        try:
+            from castle_api.mdns import CastleMDNS
+
+            registry = get_registry()
+            mdns_service = CastleMDNS(
+                hostname=registry.node.hostname,
+                gateway_port=registry.node.gateway_port,
+                api_port=settings.port,
+            )
+            mdns_service.start()
+            app.state.mdns = mdns_service
+        except Exception:
+            logger.exception("Failed to start mDNS")
+            mdns_service = None
+
     yield
 
     _shutting_down = True
     poll_task.cancel()
+
+    if mqtt_client:
+        await mqtt_client.stop()
+    if mdns_service:
+        mdns_service.stop()
+
     close_all_subscribers()
 
 
@@ -70,6 +117,7 @@ app.add_middleware(
 app.include_router(config_router)
 app.include_router(dashboard_router)
 app.include_router(logs_router)
+app.include_router(nodes_router)
 app.include_router(secrets_router)
 app.include_router(services_router)
 app.include_router(tools_router)

castle-api/src/castle_api/mdns.py

@@ -0,0 +1,130 @@
+"""mDNS discovery for castle mesh — zero-config LAN peer and broker discovery.
+
+Advertises this node as _castle._tcp and browses for:
+  - _castle._tcp  — peer castle nodes
+  - _mqtt._tcp    — MQTT broker address
+"""
+
+from __future__ import annotations
+
+import logging
+import socket
+
+from zeroconf import ServiceBrowser, ServiceInfo, ServiceStateChange, Zeroconf
+
+logger = logging.getLogger(__name__)
+
+CASTLE_SERVICE_TYPE = "_castle._tcp.local."
+MQTT_SERVICE_TYPE = "_mqtt._tcp.local."
+
+
+class CastleMDNS:
+    """Advertise this castle node and discover peers + MQTT broker via mDNS."""
+
+    def __init__(
+        self,
+        hostname: str,
+        gateway_port: int,
+        api_port: int,
+    ) -> None:
+        self._hostname = hostname
+        self._gateway_port = gateway_port
+        self._api_port = api_port
+        self._zeroconf: Zeroconf | None = None
+        self._browsers: list[ServiceBrowser] = []
+        self._service_info: ServiceInfo | None = None
+
+        # Discovered state
+        self.peers: dict[str, dict] = {}  # hostname -> {gateway_port, api_port, addresses}
+        self.mqtt_broker: dict | None = None  # {host, port} or None
+
+    def _on_service_state_change(
+        self,
+        zeroconf: Zeroconf,
+        service_type: str,
+        name: str,
+        state_change: ServiceStateChange,
+    ) -> None:
+        """Handle discovered/removed services."""
+        if state_change in (ServiceStateChange.Added, ServiceStateChange.Updated):
+            info = zeroconf.get_service_info(service_type, name)
+            if info is None:
+                return
+
+            if service_type == CASTLE_SERVICE_TYPE:
+                self._handle_castle_peer(info)
+            elif service_type == MQTT_SERVICE_TYPE:
+                self._handle_mqtt_broker(info)
+
+        elif state_change == ServiceStateChange.Removed:
+            if service_type == CASTLE_SERVICE_TYPE:
+                # Extract hostname from service name (format: "hostname._castle._tcp.local.")
+                peer_hostname = name.replace(f".{CASTLE_SERVICE_TYPE}", "")
+                if peer_hostname != self._hostname and peer_hostname in self.peers:
+                    del self.peers[peer_hostname]
+                    logger.info("mDNS: peer %s removed", peer_hostname)
+
+    def _handle_castle_peer(self, info: ServiceInfo) -> None:
+        """Process a discovered castle peer."""
+        props = {
+            k.decode() if isinstance(k, bytes) else k: v.decode() if isinstance(v, bytes) else v
+            for k, v in info.properties.items()
+        }
+        peer_hostname = props.get("hostname", "")
+        if not peer_hostname or peer_hostname == self._hostname:
+            return
+
+        addresses = [socket.inet_ntoa(addr) for addr in info.addresses if len(addr) == 4]
+
+        self.peers[peer_hostname] = {
+            "gateway_port": int(props.get("gateway_port", 9000)),
+            "api_port": int(props.get("api_port", 9020)),
+            "addresses": addresses,
+        }
+        logger.info("mDNS: discovered peer %s at %s", peer_hostname, addresses)
+
+    def _handle_mqtt_broker(self, info: ServiceInfo) -> None:
+        """Process a discovered MQTT broker."""
+        addresses = [socket.inet_ntoa(addr) for addr in info.addresses if len(addr) == 4]
+        if addresses:
+            self.mqtt_broker = {
+                "host": addresses[0],
+                "port": info.port,
+            }
+            logger.info("mDNS: discovered MQTT broker at %s:%d", addresses[0], info.port)
+
+    def start(self) -> None:
+        """Start advertising and browsing."""
+        self._zeroconf = Zeroconf()
+
+        # Advertise ourselves
+        self._service_info = ServiceInfo(
+            CASTLE_SERVICE_TYPE,
+            f"{self._hostname}.{CASTLE_SERVICE_TYPE}",
+            port=self._gateway_port,
+            properties={
+                "hostname": self._hostname,
+                "gateway_port": str(self._gateway_port),
+                "api_port": str(self._api_port),
+            },
+        )
+        self._zeroconf.register_service(self._service_info)
+        logger.info("mDNS: advertising %s on port %d", self._hostname, self._gateway_port)
+
+        # Browse for peers and MQTT broker
+        self._browsers.append(
+            ServiceBrowser(self._zeroconf, CASTLE_SERVICE_TYPE, handlers=[self._on_service_state_change])
+        )
+        self._browsers.append(
+            ServiceBrowser(self._zeroconf, MQTT_SERVICE_TYPE, handlers=[self._on_service_state_change])
+        )
+
+    def stop(self) -> None:
+        """Stop advertising and close."""
+        if self._zeroconf:
+            if self._service_info:
+                self._zeroconf.unregister_service(self._service_info)
+            self._zeroconf.close()
+            self._zeroconf = None
+        self._browsers.clear()
+        logger.info("mDNS: stopped")

castle-api/src/castle_api/mesh.py

@@ -0,0 +1,76 @@
+"""Mesh state manager — aggregates remote node registries."""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+
+from castle_core.registry import NodeRegistry
+
+logger = logging.getLogger(__name__)
+
+# Remote registries older than this are considered stale.
+STALE_TTL_SECONDS = 300  # 5 minutes
+
+
+@dataclass
+class RemoteNode:
+    """A remote node's registry and metadata."""
+
+    registry: NodeRegistry
+    last_seen: float = field(default_factory=time.time)
+    online: bool = True
+
+    @property
+    def is_stale(self) -> bool:
+        return (time.time() - self.last_seen) > STALE_TTL_SECONDS
+
+
+class MeshStateManager:
+    """Singleton holding remote node state discovered via MQTT.
+
+    Thread-safe for reads from the FastAPI request handlers.
+    Mutations happen only from the MQTT callback task.
+    """
+
+    def __init__(self) -> None:
+        self._nodes: dict[str, RemoteNode] = {}
+
+    def update_node(self, hostname: str, registry: NodeRegistry) -> None:
+        """Add or update a remote node's registry."""
+        self._nodes[hostname] = RemoteNode(registry=registry)
+        logger.info("Mesh: updated node %s (%d deployed)", hostname, len(registry.deployed))
+
+    def set_offline(self, hostname: str) -> None:
+        """Mark a node as offline (LWT received)."""
+        if hostname in self._nodes:
+            self._nodes[hostname].online = False
+            logger.info("Mesh: node %s went offline", hostname)
+
+    def remove_node(self, hostname: str) -> None:
+        """Remove a node entirely."""
+        if self._nodes.pop(hostname, None):
+            logger.info("Mesh: removed node %s", hostname)
+
+    def get_node(self, hostname: str) -> RemoteNode | None:
+        """Get a specific remote node."""
+        return self._nodes.get(hostname)
+
+    def all_nodes(self, *, include_stale: bool = False) -> dict[str, RemoteNode]:
+        """Return all remote nodes, optionally filtering out stale ones."""
+        if include_stale:
+            return dict(self._nodes)
+        return {h: n for h, n in self._nodes.items() if not n.is_stale}
+
+    def prune_stale(self) -> list[str]:
+        """Remove nodes that have gone stale. Returns list of pruned hostnames."""
+        pruned = [h for h, n in self._nodes.items() if n.is_stale]
+        for h in pruned:
+            del self._nodes[h]
+            logger.info("Mesh: pruned stale node %s", h)
+        return pruned
+
+
+# Module-level singleton — imported by MQTT client and API routes.
+mesh_state = MeshStateManager()

castle-api/src/castle_api/models.py

@@ -28,6 +28,7 @@ class ComponentSummary(BaseModel):
     system_dependencies: list[str] = []
     schedule: str | None = None
     installed: bool | None = None
+    node: str | None = None
 
 
 class ComponentDetail(ComponentSummary):
@@ -50,13 +51,55 @@ class StatusResponse(BaseModel):
     statuses: list[HealthStatus]
 
 
+class GatewayRoute(BaseModel):
+    """A single route in the gateway's reverse proxy table."""
+
+    path: str
+    target_port: int
+    component: str
+    node: str
+
+
 class GatewayInfo(BaseModel):
     """Gateway configuration summary."""
 
     port: int
+    hostname: str
     component_count: int
     service_count: int
     managed_count: int
+    routes: list[GatewayRoute] = []
+
+
+class NodeSummary(BaseModel):
+    """Summary of a discovered node in the mesh."""
+
+    hostname: str
+    gateway_port: int
+    deployed_count: int
+    service_count: int
+    is_local: bool = False
+    online: bool = True
+    is_stale: bool = False
+    last_seen: float | None = None
+
+
+class NodeDetail(NodeSummary):
+    """Full detail for a node, including its deployed components."""
+
+    deployed: list[ComponentSummary] = []
+
+
+class MeshStatus(BaseModel):
+    """Current state of the mesh coordination layer."""
+
+    enabled: bool = False
+    mqtt_connected: bool = False
+    mqtt_broker_host: str | None = None
+    mqtt_broker_port: int | None = None
+    mdns_enabled: bool = False
+    peer_count: int = 0
+    peers: list[str] = []
 
 
 class ServiceActionResponse(BaseModel):

castle-api/src/castle_api/mqtt_client.py

@@ -0,0 +1,255 @@
+"""MQTT client for inter-node mesh coordination.
+
+Topics:
+  castle/{hostname}/registry  — retained JSON NodeRegistry, published on connect
+  castle/{hostname}/status    — "online" (retained) / "offline" (LWT)
+
+On incoming messages from other hostnames, updates MeshStateManager.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+
+import paho.mqtt.client as mqtt
+
+from castle_core.registry import (
+    DeployedComponent,
+    NodeConfig,
+    NodeRegistry,
+)
+
+from castle_api.mesh import mesh_state
+from castle_api.stream import broadcast
+
+logger = logging.getLogger(__name__)
+
+
+def _registry_to_json(registry: NodeRegistry) -> str:
+    """Serialize a NodeRegistry to JSON for MQTT publishing.
+
+    Only includes fields needed for mesh routing — env vars, run_cmd,
+    and castle_root are excluded to avoid leaking secrets.
+    """
+    data: dict = {
+        "node": {
+            "hostname": registry.node.hostname,
+            "gateway_port": registry.node.gateway_port,
+        },
+        "deployed": {},
+    }
+
+    for name, comp in registry.deployed.items():
+        entry: dict = {
+            "runner": comp.runner,
+            "category": comp.category,
+        }
+        if comp.description:
+            entry["description"] = comp.description
+        if comp.port is not None:
+            entry["port"] = comp.port
+        if comp.health_path:
+            entry["health_path"] = comp.health_path
+        if comp.proxy_path:
+            entry["proxy_path"] = comp.proxy_path
+        if comp.schedule:
+            entry["schedule"] = comp.schedule
+        if comp.managed:
+            entry["managed"] = comp.managed
+        data["deployed"][name] = entry
+
+    return json.dumps(data)
+
+
+def _json_to_registry(payload: str) -> NodeRegistry:
+    """Deserialize a NodeRegistry from MQTT JSON payload."""
+    data = json.loads(payload)
+    node_data = data.get("node", {})
+    node = NodeConfig(
+        hostname=node_data.get("hostname", ""),
+        castle_root=node_data.get("castle_root"),
+        gateway_port=node_data.get("gateway_port", 9000),
+    )
+    deployed: dict[str, DeployedComponent] = {}
+    for name, comp_data in data.get("deployed", {}).items():
+        deployed[name] = DeployedComponent(
+            runner=comp_data.get("runner", "command"),
+            run_cmd=comp_data.get("run_cmd", []),
+            env=comp_data.get("env", {}),
+            description=comp_data.get("description"),
+            category=comp_data.get("category", "service"),
+            port=comp_data.get("port"),
+            health_path=comp_data.get("health_path"),
+            proxy_path=comp_data.get("proxy_path"),
+            schedule=comp_data.get("schedule"),
+            managed=comp_data.get("managed", False),
+        )
+    return NodeRegistry(node=node, deployed=deployed)
+
+
+class CastleMQTTClient:
+    """Async wrapper around paho-mqtt for castle mesh coordination."""
+
+    def __init__(
+        self,
+        local_hostname: str,
+        local_registry: NodeRegistry,
+        broker_host: str = "localhost",
+        broker_port: int = 1883,
+        loop: asyncio.AbstractEventLoop | None = None,
+    ) -> None:
+        self._local_hostname = local_hostname
+        self._local_registry = local_registry
+        self._broker_host = broker_host
+        self._broker_port = broker_port
+        self._loop = loop or asyncio.get_event_loop()
+
+        self._client = mqtt.Client(
+            callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
+            client_id=f"castle-{local_hostname}",
+            clean_session=True,
+        )
+
+        # LWT: if we disconnect unexpectedly, broker publishes "offline"
+        self._client.will_set(
+            f"castle/{local_hostname}/status",
+            payload="offline",
+            qos=1,
+            retain=True,
+        )
+
+        self._connected = False
+        self._client.on_connect = self._on_connect
+        self._client.on_disconnect = self._on_disconnect
+        self._client.on_message = self._on_message
+
+    @property
+    def connected(self) -> bool:
+        return self._connected
+
+    @property
+    def broker_host(self) -> str:
+        return self._broker_host
+
+    @property
+    def broker_port(self) -> int:
+        return self._broker_port
+
+    def _on_disconnect(
+        self,
+        client: mqtt.Client,
+        userdata: object,
+        flags: mqtt.DisconnectFlags,
+        rc: mqtt.ReasonCode,
+        properties: mqtt.Properties | None = None,
+    ) -> None:
+        self._connected = False
+        logger.info("Disconnected from MQTT broker (rc=%s)", rc)
+
+    def _on_connect(
+        self,
+        client: mqtt.Client,
+        userdata: object,
+        flags: mqtt.ConnectFlags,
+        rc: mqtt.ReasonCode,
+        properties: mqtt.Properties | None = None,
+    ) -> None:
+        """Called when connected to broker — publish our state and subscribe."""
+        if rc.is_failure:
+            logger.error("MQTT connect failed: %s", rc)
+            self._connected = False
+            return
+
+        self._connected = True
+        logger.info("Connected to MQTT broker at %s:%d", self._broker_host, self._broker_port)
+
+        # Publish our status as online (retained)
+        client.publish(
+            f"castle/{self._local_hostname}/status",
+            payload="online",
+            qos=1,
+            retain=True,
+        )
+
+        # Publish our registry (retained)
+        self.publish_registry(self._local_registry)
+
+        # Subscribe to all castle nodes
+        client.subscribe("castle/+/registry", qos=1)
+        client.subscribe("castle/+/status", qos=1)
+
+    def _on_message(
+        self,
+        client: mqtt.Client,
+        userdata: object,
+        msg: mqtt.MQTTMessage,
+    ) -> None:
+        """Process incoming MQTT messages from other nodes."""
+        try:
+            parts = msg.topic.split("/")
+            if len(parts) != 3 or parts[0] != "castle":
+                return
+
+            hostname = parts[1]
+            msg_type = parts[2]
+
+            # Ignore our own messages
+            if hostname == self._local_hostname:
+                return
+
+            payload = msg.payload.decode()
+
+            if msg_type == "registry":
+                registry = _json_to_registry(payload)
+                mesh_state.update_node(hostname, registry)
+                # Notify SSE clients about mesh change
+                asyncio.run_coroutine_threadsafe(
+                    broadcast("mesh", {"event": "node_updated", "hostname": hostname}),
+                    self._loop,
+                )
+
+            elif msg_type == "status":
+                if payload == "offline":
+                    mesh_state.set_offline(hostname)
+                    asyncio.run_coroutine_threadsafe(
+                        broadcast("mesh", {"event": "node_offline", "hostname": hostname}),
+                        self._loop,
+                    )
+
+        except Exception:
+            logger.exception("Error processing MQTT message on %s", msg.topic)
+
+    def publish_registry(self, registry: NodeRegistry) -> None:
+        """Publish (or re-publish) our local registry."""
+        self._local_registry = registry
+        self._client.publish(
+            f"castle/{self._local_hostname}/registry",
+            payload=_registry_to_json(registry),
+            qos=1,
+            retain=True,
+        )
+
+    async def start(self) -> None:
+        """Connect to the broker and start the network loop."""
+        self._client.connect_async(self._broker_host, self._broker_port)
+        self._client.loop_start()
+        logger.info(
+            "MQTT client starting (broker=%s:%d)",
+            self._broker_host,
+            self._broker_port,
+        )
+
+    async def stop(self) -> None:
+        """Disconnect and stop the network loop."""
+        # Publish offline status before disconnecting
+        self._client.publish(
+            f"castle/{self._local_hostname}/status",
+            payload="offline",
+            qos=1,
+            retain=True,
+        )
+        self._client.loop_stop()
+        self._client.disconnect()
+        logger.info("MQTT client stopped")

castle-api/src/castle_api/nodes.py

@@ -0,0 +1,115 @@
+"""Nodes router — discover and inspect mesh nodes."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request, status
+
+from castle_api.config import get_registry, settings
+from castle_api.mesh import mesh_state
+from castle_api.models import ComponentSummary, MeshStatus, NodeDetail, NodeSummary
+
+router = APIRouter(tags=["nodes"])
+
+
+def _local_node_summary(registry: object) -> NodeSummary:
+    """Build a NodeSummary for the local node from the registry."""
+    return NodeSummary(
+        hostname=registry.node.hostname,
+        gateway_port=registry.node.gateway_port,
+        deployed_count=len(registry.deployed),
+        service_count=sum(1 for d in registry.deployed.values() if d.port is not None),
+        is_local=True,
+        online=True,
+        is_stale=False,
+    )
+
+
+def _remote_node_summary(hostname: str, remote: object) -> NodeSummary:
+    """Build a NodeSummary from a RemoteNode."""
+    reg = remote.registry
+    return NodeSummary(
+        hostname=hostname,
+        gateway_port=reg.node.gateway_port,
+        deployed_count=len(reg.deployed),
+        service_count=sum(1 for d in reg.deployed.values() if d.port is not None),
+        is_local=False,
+        online=remote.online,
+        is_stale=remote.is_stale,
+        last_seen=remote.last_seen,
+    )
+
+
+def _deployed_to_summaries(registry: object, hostname: str) -> list[ComponentSummary]:
+    """Convert deployed components from a registry into ComponentSummary list."""
+    summaries = []
+    for name, d in registry.deployed.items():
+        summaries.append(
+            ComponentSummary(
+                id=name,
+                description=d.description,
+                category=d.category,
+                runner=d.runner,
+                port=d.port,
+                health_path=d.health_path,
+                proxy_path=d.proxy_path,
+                managed=d.managed,
+                schedule=d.schedule,
+                node=hostname,
+            )
+        )
+    return summaries
+
+
+@router.get("/mesh/status", response_model=MeshStatus)
+def get_mesh_status(request: Request) -> MeshStatus:
+    """Get the current state of the mesh coordination layer."""
+    mqtt_client = getattr(request.app.state, "mqtt_client", None)
+    mdns = getattr(request.app.state, "mdns", None)
+
+    peers = list(mesh_state.all_nodes(include_stale=True).keys())
+
+    return MeshStatus(
+        enabled=settings.mqtt_enabled,
+        mqtt_connected=mqtt_client.connected if mqtt_client else False,
+        mqtt_broker_host=mqtt_client.broker_host if mqtt_client else None,
+        mqtt_broker_port=mqtt_client.broker_port if mqtt_client else None,
+        mdns_enabled=settings.mdns_enabled,
+        peer_count=len(peers),
+        peers=peers,
+    )
+
+
+@router.get("/nodes", response_model=list[NodeSummary])
+def list_nodes() -> list[NodeSummary]:
+    """List all known nodes (local + discovered remote)."""
+    registry = get_registry()
+    nodes = [_local_node_summary(registry)]
+
+    for hostname, remote in mesh_state.all_nodes(include_stale=True).items():
+        nodes.append(_remote_node_summary(hostname, remote))
+
+    return nodes
+
+
+@router.get("/nodes/{hostname}", response_model=NodeDetail)
+def get_node(hostname: str) -> NodeDetail:
+    """Get detailed info for a specific node."""
+    registry = get_registry()
+
+    # Local node
+    if hostname == registry.node.hostname:
+        summary = _local_node_summary(registry)
+        deployed = _deployed_to_summaries(registry, hostname)
+        return NodeDetail(**summary.model_dump(), deployed=deployed)
+
+    # Remote node
+    remote = mesh_state.get_node(hostname)
+    if remote is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Node '{hostname}' not found",
+        )
+
+    summary = _remote_node_summary(hostname, remote)
+    deployed = _deployed_to_summaries(remote.registry, hostname)
+    return NodeDetail(**summary.model_dump(), deployed=deployed)

castle-api/src/castle_api/routes.py

@@ -2,20 +2,24 @@
 
 from __future__ import annotations
 
+import asyncio
 import shutil
 from pathlib import Path
 
 from fastapi import APIRouter, HTTPException, status
 
+from castle_core.config import GENERATED_DIR
 from castle_core.generators.caddyfile import generate_caddyfile_from_registry
 from castle_core.manifest import ComponentSpec, JobSpec, ServiceSpec
 
 from castle_api.config import get_castle_root, get_registry
+from castle_api.mesh import mesh_state
 from castle_api.health import check_all_health
 from castle_api.models import (
     ComponentDetail,
     ComponentSummary,
     GatewayInfo,
+    GatewayRoute,
     StatusResponse,
     SystemdInfo,
 )
@@ -173,15 +177,21 @@ def _summary_from_component(name: str, comp: ComponentSpec, root: Path) -> Compo
 
 
 @router.get("/components", response_model=list[ComponentSummary])
-def list_components() -> list[ComponentSummary]:
-    """List all components — deployed from registry, non-deployed from castle.yaml."""
+def list_components(include_remote: bool = False) -> list[ComponentSummary]:
+    """List all components — deployed from registry, non-deployed from castle.yaml.
+
+    Pass ?include_remote=true to include components from remote mesh nodes.
+    """
     registry = get_registry()
+    local_hostname = registry.node.hostname
     summaries: list[ComponentSummary] = []
     seen: set[str] = set()
 
     # Deployed components from registry
     for name, deployed in registry.deployed.items():
-        summaries.append(_summary_from_deployed(name, deployed))
+        s = _summary_from_deployed(name, deployed)
+        s.node = local_hostname
+        summaries.append(s)
         seen.add(name)
 
     # Non-deployed from castle.yaml (if repo available)
@@ -195,13 +205,17 @@ def list_components() -> list[ComponentSummary]:
             # Services not in registry
             for name, svc in config.services.items():
                 if name not in seen:
-                    summaries.append(_summary_from_service(name, svc, config))
+                    s = _summary_from_service(name, svc, config)
+                    s.node = local_hostname
+                    summaries.append(s)
                     seen.add(name)
 
             # Jobs not in registry
             for name, job in config.jobs.items():
                 if name not in seen:
-                    summaries.append(_summary_from_job(name, job, config))
+                    s = _summary_from_job(name, job, config)
+                    s.node = local_hostname
+                    summaries.append(s)
                     seen.add(name)
 
             # Backfill source from component refs for deployed items
@@ -223,6 +237,7 @@ def list_components() -> list[ComponentSummary]:
             # "what software exists", services/jobs are "how it runs".
             for name, comp in config.components.items():
                 summary = _summary_from_component(name, comp, root)
+                summary.node = local_hostname
                 # Skip if this exact category is already represented
                 # (e.g. a deployed tool already in the list)
                 if not any(s.id == name and s.category == summary.category for s in summaries):
@@ -230,6 +245,27 @@ def list_components() -> list[ComponentSummary]:
         except FileNotFoundError:
             pass
 
+    # Remote components from mesh (local wins on name conflicts)
+    if include_remote:
+        for hostname, remote in mesh_state.all_nodes().items():
+            for name, d in remote.registry.deployed.items():
+                if name not in seen:
+                    summaries.append(
+                        ComponentSummary(
+                            id=name,
+                            description=d.description,
+                            category=d.category,
+                            runner=d.runner,
+                            port=d.port,
+                            health_path=d.health_path,
+                            proxy_path=d.proxy_path,
+                            managed=d.managed,
+                            schedule=d.schedule,
+                            node=hostname,
+                        )
+                    )
+                    seen.add(name)
+
     return summaries
 
 
@@ -320,11 +356,42 @@ def get_gateway() -> GatewayInfo:
     deployed_count = len(registry.deployed)
     service_count = sum(1 for d in registry.deployed.values() if d.port is not None)
     managed_count = sum(1 for d in registry.deployed.values() if d.managed)
+
+    # Local routes
+    routes: list[GatewayRoute] = [
+        GatewayRoute(
+            path=d.proxy_path,
+            target_port=d.port,
+            component=name,
+            node=registry.node.hostname,
+        )
+        for name, d in registry.deployed.items()
+        if d.proxy_path and d.port
+    ]
+
+    # Remote routes from mesh (local paths take precedence)
+    local_paths = {r.path for r in routes}
+    for hostname, remote in mesh_state.all_nodes().items():
+        for name, d in remote.registry.deployed.items():
+            if d.proxy_path and d.port and d.proxy_path not in local_paths:
+                routes.append(
+                    GatewayRoute(
+                        path=d.proxy_path,
+                        target_port=d.port,
+                        component=name,
+                        node=hostname,
+                    )
+                )
+
+    routes.sort(key=lambda r: r.path)
+
     return GatewayInfo(
         port=registry.node.gateway_port,
+        hostname=registry.node.hostname,
         component_count=deployed_count,
         service_count=service_count,
         managed_count=managed_count,
+        routes=routes,
     )
 
 
@@ -333,3 +400,32 @@ def get_caddyfile() -> dict[str, str]:
     """Return the generated Caddyfile content."""
     registry = get_registry()
     return {"content": generate_caddyfile_from_registry(registry)}
+
+
+@router.post("/gateway/reload")
+async def reload_gateway() -> dict[str, str]:
+    """Regenerate Caddyfile and reload Caddy."""
+    registry = get_registry()
+    GENERATED_DIR.mkdir(parents=True, exist_ok=True)
+    caddyfile_path = GENERATED_DIR / "Caddyfile"
+
+    # Include remote registries for cross-node routing
+    remote_regs = {h: n.registry for h, n in mesh_state.all_nodes().items()}
+    caddyfile_path.write_text(
+        generate_caddyfile_from_registry(registry, remote_registries=remote_regs or None)
+    )
+
+    proc = await asyncio.create_subprocess_exec(
+        "systemctl", "--user", "reload", "castle-castle-gateway.service",
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    _, stderr = await proc.communicate()
+
+    if proc.returncode != 0:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Reload failed: {(stderr or b'').decode().strip()}",
+        )
+
+    return {"status": "ok"}

castle-api/tests/test_health.py

@@ -80,7 +80,27 @@ class TestGateway:
         assert response.status_code == 200
         data = response.json()
         assert data["port"] == 9000
+        assert data["hostname"] == "test-node"
         # Registry has 1 deployed component (test-svc)
         assert data["component_count"] == 1
         assert data["service_count"] == 1
         assert data["managed_count"] == 1
+
+    def test_gateway_routes(self, client: TestClient) -> None:
+        """Returns proxy routes from deployed components."""
+        response = client.get("/gateway")
+        data = response.json()
+        routes = data["routes"]
+        assert len(routes) == 1
+        route = routes[0]
+        assert route["path"] == "/test-svc"
+        assert route["target_port"] == 19000
+        assert route["component"] == "test-svc"
+        assert route["node"] == "test-node"
+
+    def test_gateway_routes_sorted(self, client: TestClient) -> None:
+        """Routes are sorted by path."""
+        response = client.get("/gateway")
+        data = response.json()
+        paths = [r["path"] for r in data["routes"]]
+        assert paths == sorted(paths)

castle-api/tests/test_mesh.py

@@ -0,0 +1,104 @@
+"""Tests for MeshStateManager."""
+
+import time
+
+from castle_core.registry import DeployedComponent, NodeConfig, NodeRegistry
+
+from castle_api.mesh import STALE_TTL_SECONDS, MeshStateManager, RemoteNode
+
+
+def _make_registry(hostname: str, deployed: dict | None = None) -> NodeRegistry:
+    return NodeRegistry(
+        node=NodeConfig(hostname=hostname, gateway_port=9000),
+        deployed=deployed or {},
+    )
+
+
+class TestRemoteNode:
+    """RemoteNode staleness tracking."""
+
+    def test_fresh_node_not_stale(self) -> None:
+        node = RemoteNode(registry=_make_registry("a"))
+        assert not node.is_stale
+
+    def test_old_node_is_stale(self) -> None:
+        node = RemoteNode(
+            registry=_make_registry("a"),
+            last_seen=time.time() - STALE_TTL_SECONDS - 1,
+        )
+        assert node.is_stale
+
+
+class TestMeshStateManager:
+    """MeshStateManager add/remove/stale operations."""
+
+    def test_update_and_get(self) -> None:
+        mgr = MeshStateManager()
+        reg = _make_registry("devbox")
+        mgr.update_node("devbox", reg)
+        node = mgr.get_node("devbox")
+        assert node is not None
+        assert node.registry.node.hostname == "devbox"
+        assert node.online is True
+
+    def test_set_offline(self) -> None:
+        mgr = MeshStateManager()
+        mgr.update_node("devbox", _make_registry("devbox"))
+        mgr.set_offline("devbox")
+        node = mgr.get_node("devbox")
+        assert node is not None
+        assert node.online is False
+
+    def test_remove_node(self) -> None:
+        mgr = MeshStateManager()
+        mgr.update_node("devbox", _make_registry("devbox"))
+        mgr.remove_node("devbox")
+        assert mgr.get_node("devbox") is None
+
+    def test_remove_nonexistent_is_safe(self) -> None:
+        mgr = MeshStateManager()
+        mgr.remove_node("nope")  # should not raise
+
+    def test_all_nodes_excludes_stale(self) -> None:
+        mgr = MeshStateManager()
+        mgr.update_node("fresh", _make_registry("fresh"))
+        mgr._nodes["stale"] = RemoteNode(
+            registry=_make_registry("stale"),
+            last_seen=time.time() - STALE_TTL_SECONDS - 1,
+        )
+        result = mgr.all_nodes()
+        assert "fresh" in result
+        assert "stale" not in result
+
+    def test_all_nodes_includes_stale_when_requested(self) -> None:
+        mgr = MeshStateManager()
+        mgr._nodes["stale"] = RemoteNode(
+            registry=_make_registry("stale"),
+            last_seen=time.time() - STALE_TTL_SECONDS - 1,
+        )
+        result = mgr.all_nodes(include_stale=True)
+        assert "stale" in result
+
+    def test_prune_stale(self) -> None:
+        mgr = MeshStateManager()
+        mgr.update_node("fresh", _make_registry("fresh"))
+        mgr._nodes["stale"] = RemoteNode(
+            registry=_make_registry("stale"),
+            last_seen=time.time() - STALE_TTL_SECONDS - 1,
+        )
+        pruned = mgr.prune_stale()
+        assert pruned == ["stale"]
+        assert mgr.get_node("stale") is None
+        assert mgr.get_node("fresh") is not None
+
+    def test_update_replaces_existing(self) -> None:
+        mgr = MeshStateManager()
+        mgr.update_node("devbox", _make_registry("devbox"))
+        new_reg = _make_registry(
+            "devbox",
+            {"svc": DeployedComponent(runner="python", run_cmd=["svc"])},
+        )
+        mgr.update_node("devbox", new_reg)
+        node = mgr.get_node("devbox")
+        assert node is not None
+        assert "svc" in node.registry.deployed

castle-api/tests/test_mqtt.py

@@ -0,0 +1,96 @@
+"""Tests for MQTT client serialization logic."""
+
+import json
+
+from castle_core.registry import DeployedComponent, NodeConfig, NodeRegistry
+
+from castle_api.mqtt_client import _json_to_registry, _registry_to_json
+
+
+def _make_registry() -> NodeRegistry:
+    return NodeRegistry(
+        node=NodeConfig(hostname="tower", castle_root="/data/repos/castle", gateway_port=9000),
+        deployed={
+            "my-svc": DeployedComponent(
+                runner="python",
+                run_cmd=["uv", "run", "my-svc"],
+                env={"PORT": "9001", "SECRET_KEY": "super-secret"},
+                description="My service",
+                category="service",
+                port=9001,
+                health_path="/health",
+                proxy_path="/my-svc",
+                managed=True,
+            ),
+            "my-job": DeployedComponent(
+                runner="command",
+                run_cmd=["my-job"],
+                category="job",
+                schedule="0 2 * * *",
+            ),
+        },
+    )
+
+
+class TestRegistrySerialization:
+    """Round-trip serialization of NodeRegistry to/from JSON."""
+
+    def test_round_trip(self) -> None:
+        original = _make_registry()
+        json_str = _registry_to_json(original)
+        restored = _json_to_registry(json_str)
+
+        assert restored.node.hostname == "tower"
+        assert restored.node.gateway_port == 9000
+
+    def test_deployed_components_preserved(self) -> None:
+        original = _make_registry()
+        restored = _json_to_registry(_registry_to_json(original))
+
+        assert "my-svc" in restored.deployed
+        svc = restored.deployed["my-svc"]
+        assert svc.runner == "python"
+        assert svc.port == 9001
+        assert svc.health_path == "/health"
+        assert svc.proxy_path == "/my-svc"
+        assert svc.managed is True
+
+    def test_job_fields_preserved(self) -> None:
+        original = _make_registry()
+        restored = _json_to_registry(_registry_to_json(original))
+
+        assert "my-job" in restored.deployed
+        job = restored.deployed["my-job"]
+        assert job.runner == "command"
+        assert job.schedule == "0 2 * * *"
+        assert job.category == "job"
+
+    def test_optional_fields_omitted(self) -> None:
+        """Fields like port, health_path are None when not set."""
+        reg = NodeRegistry(
+            node=NodeConfig(hostname="minimal"),
+            deployed={
+                "bare": DeployedComponent(runner="command", run_cmd=["bare"]),
+            },
+        )
+        restored = _json_to_registry(_registry_to_json(reg))
+        bare = restored.deployed["bare"]
+        assert bare.port is None
+        assert bare.health_path is None
+        assert bare.proxy_path is None
+        assert bare.schedule is None
+        assert bare.managed is False
+
+    def test_no_secrets_in_payload(self) -> None:
+        """env vars, run_cmd, and castle_root must not appear in MQTT payload."""
+        original = _make_registry()
+        json_str = _registry_to_json(original)
+        data = json.loads(json_str)
+
+        # No castle_root in node
+        assert "castle_root" not in data["node"]
+
+        # No env or run_cmd in any component
+        for name, comp in data["deployed"].items():
+            assert "env" not in comp, f"{name} has env in MQTT payload"
+            assert "run_cmd" not in comp, f"{name} has run_cmd in MQTT payload"

castle-api/tests/test_nodes.py

@@ -0,0 +1,91 @@
+"""Tests for nodes endpoints."""
+
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+from castle_core.registry import DeployedComponent, NodeConfig, NodeRegistry
+
+from castle_api.mesh import MeshStateManager
+
+
+class TestNodesList:
+    """GET /nodes endpoint tests."""
+
+    def test_returns_local_node(self, client: TestClient) -> None:
+        """Always returns the local node."""
+        response = client.get("/nodes")
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data) >= 1
+        local = data[0]
+        assert local["hostname"] == "test-node"
+        assert local["is_local"] is True
+        assert local["online"] is True
+
+    def test_local_node_counts(self, client: TestClient) -> None:
+        """Local node has correct deployment counts."""
+        response = client.get("/nodes")
+        data = response.json()
+        local = data[0]
+        assert local["deployed_count"] == 1  # test-svc
+        assert local["service_count"] == 1
+
+    def test_includes_remote_nodes(self, client: TestClient, registry_path: Path) -> None:
+        """Remote nodes from mesh state are included."""
+        import castle_api.mesh as mesh_mod
+
+        original = mesh_mod.mesh_state
+        try:
+            mgr = MeshStateManager()
+            remote_reg = NodeRegistry(
+                node=NodeConfig(hostname="devbox", gateway_port=9000),
+                deployed={
+                    "remote-svc": DeployedComponent(
+                        runner="python",
+                        run_cmd=["svc"],
+                        port=9050,
+                        category="service",
+                    ),
+                },
+            )
+            mgr.update_node("devbox", remote_reg)
+            mesh_mod.mesh_state = mgr
+
+            # Also patch the reference in the nodes module
+            import castle_api.nodes as nodes_mod
+
+            nodes_mod.mesh_state = mgr
+
+            response = client.get("/nodes")
+            data = response.json()
+            hostnames = [n["hostname"] for n in data]
+            assert "devbox" in hostnames
+            devbox = next(n for n in data if n["hostname"] == "devbox")
+            assert devbox["is_local"] is False
+            assert devbox["deployed_count"] == 1
+        finally:
+            mesh_mod.mesh_state = original
+            import castle_api.nodes as nodes_mod2
+
+            nodes_mod2.mesh_state = original
+
+
+class TestNodeDetail:
+    """GET /nodes/{hostname} endpoint tests."""
+
+    def test_local_node_detail(self, client: TestClient) -> None:
+        """Returns local node detail with deployed components."""
+        response = client.get("/nodes/test-node")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["hostname"] == "test-node"
+        assert data["is_local"] is True
+        assert len(data["deployed"]) == 1
+        assert data["deployed"][0]["id"] == "test-svc"
+        assert data["deployed"][0]["node"] == "test-node"
+
+    def test_unknown_node_returns_404(self, client: TestClient) -> None:
+        """Returns 404 for unknown hostname."""
+        response = client.get("/nodes/nonexistent")
+        assert response.status_code == 404