Add castle secret CLI and fail-loud unresolved-secret apply gate

Writing a secret by hand meant knowing the active backend and its layout;
getting it wrong (a file write on an OpenBao fleet) left the value where the
resolver never reads it, so ${secret:NAME} silently degraded to the literal
<MISSING_SECRET:NAME> placeholder that a service then used as its credential.

- `castle secret {list|set|get|rm}` reads/writes the ACTIVE backend, so there's
  no wrong store to pick. `set NAME` with no value reads a hidden prompt / stdin.
- `castle apply` (and --plan) now refuses to converge any deployment whose
  ${secret:...} refs don't resolve in the active backend: exits non-zero, writes
  nothing, and names each deployment + secret + the `castle secret set` fix.
- New helpers in core/config.py: active_secret_backend(), active_backend_name(),
  secret_refs(). Gate impl: deploy._unresolved_secrets() + ApplyResult.blocked.
- Tests: test_deploy_secret_gate.py, TestSecretRefs, test_secret.py. Docs: AGENTS.md.

core/src/castle_core/config.py

@@ -6,10 +6,14 @@ import os
 import re
 from dataclasses import InitVar, dataclass, field
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import yaml
 from pydantic import BaseModel, TypeAdapter
 
+if TYPE_CHECKING:
+    from castle_core.secret_backends import SecretBackend
+
 from castle_core.manifest import (
     AgentSpec,
     DeploymentSpec,
@@ -328,15 +332,46 @@ def _secrets_settings() -> dict:
         return {}
 
 
+def active_secret_backend() -> SecretBackend:
+    """The active secret backend (file or openbao), per ``secrets:`` + env.
+
+    The one place to get a handle for reading/writing/listing secrets, so callers
+    (the ``castle secret`` CLI, preflight checks) act on the *active* backend
+    rather than guessing the filesystem — the mistake that silently shadows an
+    OpenBao fleet's secret with a stray file.
+    """
+    from castle_core.secret_backends import build_backend
+
+    return build_backend(SECRETS_DIR, _secrets_settings())
+
+
+def active_backend_name() -> str:
+    """Human name of the active backend (``openbao`` | ``file``) for messages."""
+    return (os.environ.get("CASTLE_SECRET_BACKEND") or _secrets_settings().get("backend") or "file").lower()
+
+
 def read_secret(name: str) -> str | None:
     """Resolve a secret via the active backend, or None if unset.
 
     The public helper for code that reads secrets directly (DNS/supabase/etc.) so
     everything goes through the one backend rather than the filesystem.
     """
-    from castle_core.secret_backends import build_backend
+    return active_secret_backend().read(name)
 
-    return build_backend(SECRETS_DIR, _secrets_settings()).read(name)
+
+# ``${secret:NAME}`` references embedded in an env value (the grammar
+# :func:`resolve_env_split` resolves). Shared by the deploy-time preflight that
+# refuses to converge a deployment whose secrets the active backend can't resolve.
+_SECRET_REF_RE = re.compile(r"\$\{secret:([^}]+)\}")
+
+
+def secret_refs(value: str) -> list[str]:
+    """The secret names a value references via ``${secret:NAME}`` (order-preserving,
+    deduped)."""
+    seen: dict[str, None] = {}
+    for m in _SECRET_REF_RE.finditer(value):
+        seen.setdefault(m.group(1), None)
+    return list(seen)
 
 
 def _read_secret(name: str) -> str:

core/src/castle_core/deploy.py

@@ -97,6 +97,11 @@ class ApplyResult:
     # True for a `--plan` run: the diff was computed but nothing was written or
     # activated. Lets callers render "would activate…" vs "activated…".
     planned: bool = False
+    # Deployments refused because their ``${secret:NAME}`` references don't resolve
+    # in the active backend. A non-empty list means apply made NO changes and the
+    # caller should exit non-zero — better than starting a service with a bogus
+    # ``<MISSING_SECRET:…>`` value silently standing in for the real credential.
+    blocked: list[str] = field(default_factory=list)
 
     @property
     def changed(self) -> bool:
@@ -279,6 +284,7 @@ def apply(
     """
     import asyncio
 
+    from castle_core.config import active_backend_name
     from castle_core.lifecycle import activate, deactivate, is_active
 
     config = load_config(root)
@@ -291,6 +297,28 @@ def apply(
     ]
     names = [n for _k, n, _d in items]
 
+    # Fail loud on unresolved secrets BEFORE any render/write. Building a deployment
+    # writes its secret env file, so this gate runs first — a service is never
+    # rendered or (re)started with a bogus `<MISSING_SECRET:…>` value silently
+    # standing in for a real credential (the failure that shipped immich's DB
+    # password to the wrong backend). Aborts the whole run with a fix hint; nothing
+    # is written or reconciled.
+    blocked = _unresolved_secrets(items)
+    if blocked:
+        backend = active_backend_name()
+        result = ApplyResult(planned=plan)
+        result.blocked = [n for n, _ in blocked]
+        result.messages.append(
+            f"Error: unresolved secrets in the active '{backend}' backend — "
+            f"nothing was {'planned' if plan else 'applied'}."
+        )
+        for n, secrets in blocked:
+            for s in secrets:
+                result.messages.append(
+                    f"  {n}: ${{secret:{s}}} → not found. Fix: castle secret set {s}"
+                )
+        return result
+
     # Snapshot BEFORE rendering: liveness + current unit bytes (for restart-on-change).
     before_active = {(k, n): is_active(n, k, config) for k, n, _ in items}
     before_unit = {(k, n): _unit_bytes(n, k) for k, n, _ in items}
@@ -401,6 +429,47 @@ def _stack_preflight(
                 )
 
 
+def _unresolved_secrets(
+    items: Sequence[tuple[str, str, object]],
+) -> list[tuple[str, list[str]]]:
+    """Enabled deployments whose ``${secret:NAME}`` env references the active backend
+    can't resolve, as ``[(deployment, [missing_secret, ...]), ...]``.
+
+    This is the fail-loud gate for the footgun that shipped a service with a bogus
+    ``<MISSING_SECRET:…>`` string standing in for a real credential: a secret
+    written to the *wrong* backend (a file on an OpenBao fleet) never resolves, yet
+    the old path substituted the marker and started the service anyway. Checked
+    against the active backend so it's correct whichever backend is configured.
+    """
+    from castle_core.config import active_secret_backend, secret_refs
+
+    backend = active_secret_backend()
+    resolved: dict[str, bool] = {}  # cache: one backend read per distinct secret
+
+    def _missing(name: str) -> bool:
+        if name not in resolved:
+            try:
+                resolved[name] = backend.read(name) is not None
+            except Exception:
+                resolved[name] = False
+        return not resolved[name]
+
+    out: list[tuple[str, list[str]]] = []
+    for _k, n, spec in items:
+        if not getattr(spec, "enabled", True):
+            continue
+        defaults = getattr(spec, "defaults", None)
+        env = dict(defaults.env) if defaults and defaults.env else {}
+        missing: list[str] = []
+        for value in env.values():
+            for ref in secret_refs(str(value)):
+                if _missing(ref) and ref not in missing:
+                    missing.append(ref)
+        if missing:
+            out.append((n, missing))
+    return out
+
+
 def _record(result: ApplyResult, name: str, action: str) -> None:
     {
         "activate": result.activated,

core/tests/test_config.py

@@ -12,6 +12,7 @@ from castle_core.config import (
     resolve_env_split,
     resolve_env_vars,
     save_config,
+    secret_refs,
 )
 from castle_core.manifest import ProgramSpec, SystemdDeployment
 
@@ -182,6 +183,22 @@ class TestResolveEnvVars:
         assert resolved["API_KEY"] == "<MISSING_SECRET:NONEXISTENT>"
 
 
+class TestSecretRefs:
+    """Tests for extracting ${secret:NAME} references (the apply-gate input)."""
+
+    def test_none_when_no_refs(self) -> None:
+        assert secret_refs("plain") == []
+        assert secret_refs("${port}/${data_dir}") == []
+
+    def test_single_and_composite(self) -> None:
+        assert secret_refs("${secret:API_KEY}") == ["API_KEY"]
+        assert secret_refs("neo4j/${secret:NEO4J_PW}") == ["NEO4J_PW"]
+
+    def test_multiple_deduped_in_order(self) -> None:
+        value = "${secret:A}:${secret:B}:${secret:A}"
+        assert secret_refs(value) == ["A", "B"]
+
+
 class TestResolveEnvSplit:
     """Tests for the secret/plain partition used to keep secrets out of units."""
 

core/tests/test_deploy_secret_gate.py

@@ -0,0 +1,65 @@
+"""Tests for the apply-time unresolved-secret gate.
+
+`castle apply` must refuse to converge a deployment whose ``${secret:NAME}`` env
+references the active backend can't resolve — otherwise the value silently
+degrades to a ``<MISSING_SECRET:NAME>`` placeholder and the service starts with a
+bogus credential (the immich-DB-password-to-the-wrong-backend failure).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from castle_core.deploy import _unresolved_secrets
+
+
+def _spec(env: dict[str, str], enabled: bool = True) -> SimpleNamespace:
+    return SimpleNamespace(enabled=enabled, defaults=SimpleNamespace(env=env))
+
+
+@pytest.fixture
+def file_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    """Force the active backend to a temp file store with a known secret."""
+    secrets = tmp_path / "secrets"
+    secrets.mkdir()
+    (secrets / "PRESENT").write_text("value\n")
+    monkeypatch.setenv("CASTLE_SECRET_BACKEND", "file")
+    monkeypatch.setattr("castle_core.config.SECRETS_DIR", secrets)
+    return secrets
+
+
+def test_all_resolved_returns_empty(file_backend: Path) -> None:
+    items = [("service", "svc", _spec({"TOKEN": "${secret:PRESENT}"}))]
+    assert _unresolved_secrets(items) == []
+
+
+def test_missing_secret_is_reported(file_backend: Path) -> None:
+    items = [("service", "svc", _spec({"TOKEN": "${secret:ABSENT}"}))]
+    assert _unresolved_secrets(items) == [("svc", ["ABSENT"])]
+
+
+def test_disabled_deployment_skipped(file_backend: Path) -> None:
+    items = [("service", "svc", _spec({"TOKEN": "${secret:ABSENT}"}, enabled=False))]
+    assert _unresolved_secrets(items) == []
+
+
+def test_non_secret_placeholders_ignored(file_backend: Path) -> None:
+    items = [("service", "svc", _spec({"PORT": "${port}", "DIR": "${data_dir}"}))]
+    assert _unresolved_secrets(items) == []
+
+
+def test_composite_value_partial_missing(file_backend: Path) -> None:
+    # A URL mixing a present and an absent secret still flags the absent one.
+    env = {"URL": "postgres://u:${secret:PRESENT}@h/db?k=${secret:ABSENT}"}
+    assert _unresolved_secrets([("service", "svc", _spec(env))]) == [("svc", ["ABSENT"])]
+
+
+def test_multiple_deployments_only_broken_flagged(file_backend: Path) -> None:
+    items = [
+        ("service", "ok", _spec({"T": "${secret:PRESENT}"})),
+        ("service", "bad", _spec({"T": "${secret:ABSENT}"})),
+    ]
+    assert _unresolved_secrets(items) == [("bad", ["ABSENT"])]