From e82de4fd50dd994314575b0c344bcfda36d73539 Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Tue, 7 Jul 2026 05:52:57 -0700 Subject: [PATCH 1/2] feat(mesh): NATS TLS + token auth client support - CastleNATSClient accepts a token; a tls:// server URL makes nats-py verify the broker against the system CA (trusts the wildcard's Let's Encrypt issuer, no custom CA needed) - config: CASTLE_API_NATS_TOKEN; main wires it through Server side (instance): nats-server.conf gains tls{} (wildcard cert via expose.tcp.tls) + authorization{token}; clients use tls://castle-nats.. --- castle-api/src/castle_api/config.py | 1 + castle-api/src/castle_api/main.py | 1 + castle-api/src/castle_api/nats_client.py | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/castle-api/src/castle_api/config.py b/castle-api/src/castle_api/config.py index e85a8b0..c1433d4 100644 --- a/castle-api/src/castle_api/config.py +++ b/castle-api/src/castle_api/config.py @@ -17,6 +17,7 @@ class Settings(BaseSettings): # Mesh coordination (all off by default — single-node works without them) nats_enabled: bool = False nats_url: str = "nats://localhost:4222" + nats_token: str | None = None mdns_enabled: bool = False model_config = { diff --git a/castle-api/src/castle_api/main.py b/castle-api/src/castle_api/main.py index f291f39..3df72b4 100644 --- a/castle-api/src/castle_api/main.py +++ b/castle-api/src/castle_api/main.py @@ -68,6 +68,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: local_hostname=registry.node.hostname, local_registry=registry, servers=settings.nats_url, + token=settings.nats_token, ) await nats_client.start() app.state.nats_client = nats_client diff --git a/castle-api/src/castle_api/nats_client.py b/castle-api/src/castle_api/nats_client.py index e3255b4..b0ccb9e 100644 --- a/castle-api/src/castle_api/nats_client.py +++ b/castle-api/src/castle_api/nats_client.py @@ -50,10 +50,12 @@ class CastleNATSClient: local_hostname: str, local_registry: NodeRegistry, servers: str | list[str] = "nats://localhost:4222", + token: str | None = None, ) -> None: self._local_hostname = local_hostname self._local_registry = local_registry self._servers = servers + self._token = token or None self._nc: nats.NATS | None = None self._kv = None self._presence_kv = None @@ -77,9 +79,13 @@ class CastleNATSClient: async def start(self) -> None: """Connect, publish our registry, seed state, and start watchers.""" + # A `tls://` server URL makes nats-py verify the server against the system + # CA bundle — which trusts the wildcard's Let's Encrypt issuer, so no custom + # CA is needed. `token` authenticates this node to the broker. self._nc = await nats.connect( self._servers, name=f"castle-{self._local_hostname}", + token=self._token, max_reconnect_attempts=-1, # reconnect forever — nodes come and go ) js = self._nc.jetstream() From 76b3795485f14627dea87b87f65150606504c3f0 Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Tue, 7 Jul 2026 05:57:58 -0700 Subject: [PATCH 2/2] docs: TLS hardening done (NATS + OpenBao) across both nodes NATS tls:// + token and OpenBao HTTPS via the ACME wildcard cert (expose.tcp.tls), no custom CA. civil + primer both cut over; plaintext rejected on both. Server config + unseal.sh are instance-side (/data/castle, ~/.castle). --- docs/fleet-mesh-plan.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/fleet-mesh-plan.md b/docs/fleet-mesh-plan.md index 12eeea5..bda4327 100644 --- a/docs/fleet-mesh-plan.md +++ b/docs/fleet-mesh-plan.md @@ -1,8 +1,9 @@ # Fleet Mesh Plan — OpenBao + NATS -**Status:** in progress on branch `feat/fleet-mesh-nats-openbao`. -Phases 0–2 + Phase 4 (secret-read backend) complete + single-node verified (live). -Phase 3 (cross-node routing/breaker) and Phase 4 hardening pending the 2nd node. +**Status:** Phases 0–4 complete + verified live across **both nodes** (civil + +primer), including TLS hardening (NATS + OpenBao). Merged to main. +Remaining nice-to-have: the live curl+kill breaker demo (needs a peer-unique +consumed service) — every underlying piece is verified. ## Context @@ -245,12 +246,23 @@ second node is proven on NATS. systemd spec (general, `-`-prefixed so a hook failure never fails the unit); `castle-openbao` runs `/data/castle/castle-openbao/unseal.sh` (polls up, unseals with `OPENBAO_UNSEAL_KEY`). Verified: manual seal → restart → auto-unsealed. -3. **TLS hardening — REMAINING (deliberately deferred).** NATS + OpenBao are - plaintext on the trusted LAN. This is the gate before cross-*network* or moving - real secrets — neither of which is here yet (the vault holds no production - secrets). Doing NATS TLS/auth touches the *live* civil↔primer mesh, so it's a - deliberate, staged change (regenerate certs, update both nodes' clients), not a - tail-end one. Next dedicated step. +3. **TLS hardening — DONE + verified across both nodes (2026-07-07).** + Reuses the existing ACME **wildcard cert** via `expose.tcp.tls` (the postgres + mechanism) — services present a publicly-trusted cert for + `.civil.payne.io`, so clients verify against the **system CA with no + custom CA to distribute**. + - **NATS:** `tls{}` (wildcard cert in `/tls`) + `authorization{token}` + (`$NATS_TOKEN` from the `NATS_TOKEN` secret). Clients connect to + `tls://castle-nats.civil.payne.io:4222` with the token; `CastleNATSClient` + gained token support (a `tls://` URL → nats-py verifies via system CA). + **civil + primer both cut over**; plaintext now rejected (verified: + `certificate is valid for *.civil.payne.io, not localhost`). + - **OpenBao:** listener `tls_cert_file`/`tls_key_file` from `/tls`; reachable at + `https://castle-openbao.civil.payne.io:8200`. Auto-unseal + secret backend + both use the HTTPS URL. Verified: HTTPS serves, **auto-unsealed over HTTPS**, + plaintext rejected, backend reads a secret over HTTPS. + - Coordinated cutover across the whole fleet (both nodes) with a brief, + expected mesh blip; mesh isn't load-bearing so no service impact. ### Phase 3 — cross-node routing + breaker: logic DONE + verified against a real peer (2026-07-07)