From f94517887ec6f6c31ff01724183d3ad44bc7111e Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Tue, 7 Jul 2026 05:08:16 -0700 Subject: [PATCH] feat(secrets): pluggable secret backend with OpenBao read (Phase 4) - secret_backends.py: SecretBackend protocol, FileSecretBackend (historical), OpenBaoBackend (KV-v2 read + file fallback), build_backend() env-selected - _read_secret delegates to the active backend; default is file, so production is unchanged until CASTLE_SECRET_BACKEND=openbao - OpenBao token bootstraps from the file backend; missing/unreachable falls back - tests: file hit/miss, backend selection, unreachable + empty-token fallback Read path verified live against castle-openbao. Write path, auto-unseal-on-boot, and TLS hardening documented as remaining for full OpenBao production use. --- core/src/castle_core/config.py | 13 ++-- core/src/castle_core/secret_backends.py | 90 +++++++++++++++++++++++++ core/tests/test_secret_backends.py | 54 +++++++++++++++ docs/fleet-mesh-plan.md | 31 ++++++++- 4 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 core/src/castle_core/secret_backends.py create mode 100644 core/tests/test_secret_backends.py diff --git a/core/src/castle_core/config.py b/core/src/castle_core/config.py index 7e47969..f7fe9a6 100644 --- a/core/src/castle_core/config.py +++ b/core/src/castle_core/config.py @@ -319,10 +319,15 @@ def resolve_env_vars( def _read_secret(name: str) -> str: - """Read a secret from ~/.castle/secrets/. Returns placeholder if not found.""" - secret_path = SECRETS_DIR / name - if secret_path.exists(): - return secret_path.read_text().strip() + """Resolve a secret via the active backend (file by default; OpenBao opt-in). + + Returns a ```` placeholder if unresolved (never raises). + """ + from castle_core.secret_backends import build_backend + + value = build_backend(SECRETS_DIR).read(name) + if value is not None: + return value return f"" diff --git a/core/src/castle_core/secret_backends.py b/core/src/castle_core/secret_backends.py new file mode 100644 index 0000000..377aad0 --- /dev/null +++ b/core/src/castle_core/secret_backends.py @@ -0,0 +1,90 @@ +"""Pluggable secret backends for ``${secret:NAME}`` resolution. + +Default is the **file** backend (``~/.castle/secrets/``) — identical to the +historical behavior, so nothing changes unless a backend is explicitly selected +via ``CASTLE_SECRET_BACKEND``. + +The **openbao** backend reads from an OpenBao/Vault KV-v2 mount and falls back to +the file backend, which is also how it bootstraps: the OpenBao *token* itself is a +file secret (it can't live in the vault it unlocks). + +Selection (env, so it works in both the CLI and the systemd-run API): + CASTLE_SECRET_BACKEND file | openbao (default: file) + CASTLE_OPENBAO_ADDR http://localhost:8200 + CASTLE_OPENBAO_MOUNT castle (kv-v2 mount path) + CASTLE_OPENBAO_TOKEN_SECRET OPENBAO_TOKEN (file secret holding the token) +""" + +from __future__ import annotations + +import json +import os +import urllib.request +from pathlib import Path +from typing import Protocol + + +class SecretBackend(Protocol): + def read(self, name: str) -> str | None: ... + + +class FileSecretBackend: + """Reads ``/`` (the historical behavior).""" + + def __init__(self, secrets_dir: Path) -> None: + self._dir = secrets_dir + + def read(self, name: str) -> str | None: + path = self._dir / name + if path.exists(): + return path.read_text().strip() + return None + + +class OpenBaoBackend: + """Reads from an OpenBao/Vault KV-v2 mount; falls back to ``fallback``. + + A missing key, an auth failure, or an unreachable server all fall through to + the fallback — so a partly-migrated vault and the bootstrap token both resolve. + """ + + def __init__( + self, addr: str, token: str, mount: str, fallback: SecretBackend + ) -> None: + self._addr = addr.rstrip("/") + self._token = token + self._mount = mount + self._fallback = fallback + + def read(self, name: str) -> str | None: + value = self._read_bao(name) + if value is not None: + return value + return self._fallback.read(name) + + def _read_bao(self, name: str) -> str | None: + if not self._token: + return None + url = f"{self._addr}/v1/{self._mount}/data/{name}" + req = urllib.request.Request(url, headers={"X-Vault-Token": self._token}) + try: + with urllib.request.urlopen(req, timeout=5) as resp: # noqa: S310 + data = json.load(resp) + return data["data"]["data"].get("value") + except Exception: + return None + + +def build_backend(secrets_dir: Path) -> SecretBackend: + """Construct the active secret backend from the environment.""" + file_backend = FileSecretBackend(secrets_dir) + kind = os.environ.get("CASTLE_SECRET_BACKEND", "file").lower() + if kind == "openbao": + addr = os.environ.get("CASTLE_OPENBAO_ADDR", "http://localhost:8200") + mount = os.environ.get("CASTLE_OPENBAO_MOUNT", "castle") + token_secret = os.environ.get("CASTLE_OPENBAO_TOKEN_SECRET", "OPENBAO_TOKEN") + token = file_backend.read(token_secret) or os.environ.get( + "CASTLE_OPENBAO_TOKEN", "" + ) + return OpenBaoBackend(addr, token, mount, fallback=file_backend) + return file_backend diff --git a/core/tests/test_secret_backends.py b/core/tests/test_secret_backends.py new file mode 100644 index 0000000..855dcee --- /dev/null +++ b/core/tests/test_secret_backends.py @@ -0,0 +1,54 @@ +"""Tests for the pluggable secret backends (file default, OpenBao opt-in).""" + +from __future__ import annotations + +from pathlib import Path + +from castle_core.secret_backends import ( + FileSecretBackend, + OpenBaoBackend, + build_backend, +) + + +def test_file_backend_read_hit(tmp_path: Path) -> None: + (tmp_path / "MY_SECRET").write_text("value\n") + assert FileSecretBackend(tmp_path).read("MY_SECRET") == "value" + + +def test_file_backend_read_miss(tmp_path: Path) -> None: + assert FileSecretBackend(tmp_path).read("ABSENT") is None + + +def test_build_backend_defaults_to_file(tmp_path: Path, monkeypatch) -> None: + monkeypatch.delenv("CASTLE_SECRET_BACKEND", raising=False) + assert isinstance(build_backend(tmp_path), FileSecretBackend) + + +def test_build_backend_openbao_selected(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("CASTLE_SECRET_BACKEND", "openbao") + assert isinstance(build_backend(tmp_path), OpenBaoBackend) + + +def test_openbao_falls_back_to_file_when_unreachable(tmp_path: Path) -> None: + """An unreachable vault (or empty token) resolves via the file fallback.""" + (tmp_path / "ONLY_IN_FILE").write_text("from-file") + backend = OpenBaoBackend( + addr="http://127.0.0.1:1", # nothing listening + token="dummy", + mount="castle", + fallback=FileSecretBackend(tmp_path), + ) + assert backend.read("ONLY_IN_FILE") == "from-file" + assert backend.read("NOT_ANYWHERE") is None + + +def test_openbao_empty_token_uses_fallback(tmp_path: Path) -> None: + (tmp_path / "K").write_text("v") + backend = OpenBaoBackend( + addr="http://127.0.0.1:8200", + token="", # no token → never hits the network + mount="castle", + fallback=FileSecretBackend(tmp_path), + ) + assert backend.read("K") == "v" diff --git a/docs/fleet-mesh-plan.md b/docs/fleet-mesh-plan.md index ccd13f8..6a0c70e 100644 --- a/docs/fleet-mesh-plan.md +++ b/docs/fleet-mesh-plan.md @@ -1,7 +1,8 @@ # Fleet Mesh Plan — OpenBao + NATS **Status:** in progress on branch `feat/fleet-mesh-nats-openbao`. -Phases 0–2 complete + single-node verified (live). Phases 3–4 pending (need 2nd node). +Phases 0–2 + Phase 4 (secret-read backend) complete + single-node verified (live). +Phase 3 (cross-node routing/breaker) and Phase 4 hardening pending the 2nd node. ## Context @@ -218,6 +219,34 @@ second node is proven on NATS. **Pending second node:** the follower-side reconcile (watch `castle-config` → `castle apply`) is wired as an SSE hook but only meaningful with a peer. +### Phase 4 — secret-read backend DONE + verified + tested (2026-07-07) + +- New `core/castle_core/secret_backends.py`: `SecretBackend` protocol, + `FileSecretBackend` (the historical behavior), `OpenBaoBackend` (KV-v2 read with + file fallback), and `build_backend()` selecting via `CASTLE_SECRET_BACKEND` + (default **file** — production is byte-for-byte unchanged until opted in). +- `_read_secret` (the single chokepoint, `config.py`) now delegates to the active + backend. `${secret:NAME}` syntax untouched. +- OpenBao token bootstraps from the file backend (it can't live in the vault it + unlocks); a missing key / auth failure / unreachable server all fall through to + file, so a partly-migrated vault keeps working. +- Verified live against the running `castle-openbao`: a secret stored in the vault + resolves through `${secret:...}` in openbao mode; file-only secrets resolve via + fallback; missing → placeholder; **default file mode unchanged**. +- **Tests:** `core/tests/test_secret_backends.py` (file hit/miss, backend + selection, unreachable-vault + empty-token fallback). Suites: **206 core + 93 api**. + +**Remaining for full OpenBao production use (documented, not blocking — nothing +uses the backend until `CASTLE_SECRET_BACKEND=openbao`):** +1. **Write path** — `castle-api/secrets.py` CRUD still writes to files; in openbao + mode dashboard-set secrets land in file (resolve via fallback) rather than the + vault. Add `write`/`delete` to the backends + route the API writer through them. +2. **Auto-unseal on boot** (Phase 0 deferral) — OpenBao re-seals on container + restart. Wire an unseal step (manifest `exec_start_post` or a companion + oneshot reading `OPENBAO_UNSEAL_KEY`). Not urgent while the backend is unused. +3. **TLS hardening** — NATS + OpenBao are plaintext localhost. Required before + cross-network or moving real secrets: NATS mTLS + auth, OpenBao TLS listener. + ## Decisions (resolved) 1. **Discovery — both.** Keep mDNS for zero-config LAN peer discovery *and*