feat(mesh): fleet role + shared config/presence KV (Phase 2)

- add static 'role' (authority|follower) to NodeConfig/CastleConfig, wired from
  castle.yaml through the registry to the mesh wire; civil pinned authority
- castle-presence: TTL KV bucket each node renews (churn signal); delete-on-stop
- castle-config: authority-gated get/put + change-watch SSE (follower reconcile
  hook hangs off it)
- bound the NATS drain on stop() so shutdown can't hang
- tests: role config/registry round-trip, wire round-trip, role write-gating

Single-node verified; follower reconcile pending the second node.

castle-api/src/castle_api/mesh_wire.py

@@ -26,6 +26,8 @@ def registry_to_json(registry: NodeRegistry) -> str:
             # acme domain — lets peers build launch URLs (<subdomain>.<gateway_domain>)
             # for this node's exposed apps. Omitted when the node has no domain.
             "gateway_domain": registry.node.gateway_domain,
+            # fleet role — so peers know which node is the config/secret authority.
+            "role": registry.node.role,
         },
         "deployed": {},
     }
@@ -73,6 +75,7 @@ def json_to_registry(payload: str) -> NodeRegistry:
         castle_root=node_data.get("castle_root"),
         gateway_port=node_data.get("gateway_port", 9000),
         gateway_domain=node_data.get("gateway_domain"),
+        role=node_data.get("role", "follower"),
     )
     deployed: dict[str, Deployment] = {}
     for key, comp_data in data.get("deployed", {}).items():

castle-api/src/castle_api/nats_client.py

@@ -34,8 +34,11 @@ from castle_api.stream import broadcast
 logger = logging.getLogger(__name__)
 
 REGISTRY_BUCKET = "castle-registry"
+PRESENCE_BUCKET = "castle-presence"
+CONFIG_BUCKET = "castle-config"
 HEARTBEAT_SEC = 30.0
 PRUNE_SEC = 30.0
+PRESENCE_TTL = 90.0  # a node whose presence key expires within this is gone
 
 
 class CastleNATSClient:
@@ -52,6 +55,8 @@ class CastleNATSClient:
         self._servers = servers
         self._nc: nats.NATS | None = None
         self._kv = None
+        self._presence_kv = None
+        self._config_kv = None
         self._tasks: list[asyncio.Task] = []
         self._last_json: dict[str, str] = {}
         self._online: set[str] = set()
@@ -64,6 +69,11 @@ class CastleNATSClient:
     def servers(self) -> str | list[str]:
         return self._servers
 
+    @property
+    def role(self) -> str:
+        """This node's fleet role — 'authority' or 'follower'."""
+        return self._local_registry.node.role
+
     async def start(self) -> None:
         """Connect, publish our registry, seed state, and start watchers."""
         self._nc = await nats.connect(
@@ -72,22 +82,39 @@ class CastleNATSClient:
             max_reconnect_attempts=-1,  # reconnect forever — nodes come and go
         )
         js = self._nc.jetstream()
-        try:
-            self._kv = await js.key_value(REGISTRY_BUCKET)
-        except Exception:
-            self._kv = await js.create_key_value(
-                config=KeyValueConfig(bucket=REGISTRY_BUCKET, history=1)
-            )
+        self._kv = await self._ensure_bucket(js, REGISTRY_BUCKET, history=1)
+        # Presence: a short-TTL key each node renews; its expiry = the node is gone.
+        self._presence_kv = await self._ensure_bucket(
+            js, PRESENCE_BUCKET, history=1, ttl=PRESENCE_TTL
+        )
+        # Shared config: authority-written, followers watch + reconcile.
+        self._config_kv = await self._ensure_bucket(js, CONFIG_BUCKET, history=5)
 
         await self.publish_registry(self._local_registry)
+        await self._presence_kv.put(self._local_hostname, b"online")
         await self._seed_existing()
 
         self._tasks = [
             asyncio.create_task(self._watch_loop()),
+            asyncio.create_task(self._config_watch_loop()),
             asyncio.create_task(self._heartbeat_loop()),
             asyncio.create_task(self._prune_loop()),
         ]
-        logger.info("NATS mesh client started (servers=%s)", self._servers)
+        logger.info(
+            "NATS mesh client started (servers=%s, role=%s)",
+            self._servers,
+            self.role,
+        )
+
+    @staticmethod
+    async def _ensure_bucket(js, bucket: str, *, history: int = 1, ttl=None):
+        """Bind an existing KV bucket or create it."""
+        try:
+            return await js.key_value(bucket)
+        except Exception:
+            return await js.create_key_value(
+                config=KeyValueConfig(bucket=bucket, history=history, ttl=ttl)
+            )
 
     async def stop(self) -> None:
         """Delete our key (immediate offline to peers) and disconnect."""
@@ -100,9 +127,15 @@ class CastleNATSClient:
         if self._kv is not None:
             with contextlib.suppress(Exception):
                 await self._kv.delete(self._local_hostname)
-        if self._nc is not None:
+        if self._presence_kv is not None:
             with contextlib.suppress(Exception):
-                await self._nc.drain()
+                await self._presence_kv.delete(self._local_hostname)
+        if self._nc is not None:
+            # Bound the drain so a wedged connection can't hang systemd shutdown.
+            with contextlib.suppress(Exception):
+                await asyncio.wait_for(self._nc.drain(), timeout=5.0)
+            with contextlib.suppress(Exception):
+                await self._nc.close()
             self._nc = None
         logger.info("NATS mesh client stopped")
 
@@ -172,6 +205,46 @@ class CastleNATSClient:
             await asyncio.sleep(HEARTBEAT_SEC)
             with contextlib.suppress(Exception):
                 await self.publish_registry(self._local_registry)
+            if self._presence_kv is not None:
+                with contextlib.suppress(Exception):
+                    await self._presence_kv.put(self._local_hostname, b"online")
+
+    # --- Shared config (authority writes, followers reconcile) ---
+
+    async def get_shared_config(self, key: str) -> str | None:
+        """Read a shared-config value from the mesh (None if unset)."""
+        if self._config_kv is None:
+            return None
+        try:
+            entry = await self._config_kv.get(key)
+            return entry.value.decode() if entry.value else None
+        except Exception:
+            return None
+
+    async def put_shared_config(self, key: str, value: str) -> None:
+        """Write a shared-config value. Only the authority may write."""
+        if self.role != "authority":
+            raise PermissionError("only the authority node may write shared config")
+        if self._config_kv is None:
+            raise RuntimeError("config bucket not available")
+        await self._config_kv.put(key, value.encode())
+
+    async def _config_watch_loop(self) -> None:
+        """Watch shared config; announce changes so followers can reconcile.
+
+        The reconcile action (trigger `castle apply` on a follower) hangs off this
+        SSE event; it's inert on the authority and on a single node.
+        """
+        if self._config_kv is None:
+            return
+        watcher = await self._config_kv.watchall()
+        async for entry in watcher:
+            if entry is None:
+                continue
+            op = "delete" if entry.operation in ("DEL", "PURGE") else "put"
+            await broadcast(
+                "mesh", {"event": "config_changed", "key": entry.key, "op": op}
+            )
 
     async def _prune_loop(self) -> None:
         """Mark crashed peers (no refresh within the stale TTL) offline."""

castle-api/tests/test_mesh_wire.py

@@ -95,6 +95,18 @@ class TestRegistrySerialization:
         assert bare.schedule is None
         assert bare.managed is False
 
+    def test_node_role_preserved(self) -> None:
+        reg = NodeRegistry(
+            node=NodeConfig(hostname="civil", role="authority"), deployed={}
+        )
+        restored = json_to_registry(registry_to_json(reg))
+        assert restored.node.role == "authority"
+
+    def test_node_role_defaults_follower(self) -> None:
+        reg = NodeRegistry(node=NodeConfig(hostname="n"), deployed={})
+        restored = json_to_registry(registry_to_json(reg))
+        assert restored.node.role == "follower"
+
     def test_no_secrets_in_payload(self) -> None:
         """env vars, run_cmd, and castle_root must never appear on the wire."""
         original = _make_registry()

castle-api/tests/test_nats_client.py

@@ -0,0 +1,30 @@
+"""CastleNATSClient role gating — hermetic (no live NATS server needed).
+
+The write-gate is checked before touching the KV bucket, so a follower is denied
+without any connection.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from castle_core.registry import NodeConfig, NodeRegistry
+
+from castle_api.nats_client import CastleNATSClient
+
+
+def _client(role: str) -> CastleNATSClient:
+    reg = NodeRegistry(node=NodeConfig(hostname="n", role=role), deployed={})
+    return CastleNATSClient("n", reg, servers="nats://localhost:4222")
+
+
+def test_role_property() -> None:
+    assert _client("authority").role == "authority"
+    assert _client("follower").role == "follower"
+
+
+def test_follower_cannot_write_shared_config() -> None:
+    client = _client("follower")
+    with pytest.raises(PermissionError):
+        asyncio.run(client.put_shared_config("fleet/key", "value"))