feat: Enhance tool management and documentation in Castle

- Introduced category tools support in the `castle create` command.
- Added detailed guides for creating components in CLAUDE.md.
- Implemented new API endpoints for listing and retrieving tool details.
- Updated component and tool models to include additional metadata.
- Improved error handling and response structures in service actions.
- Enhanced documentation for component registry and web APIs.

dashboard-api/src/dashboard_api/main.py

@@ -19,16 +19,36 @@ from dashboard_api.logs import router as logs_router
 from dashboard_api.routes import router as dashboard_router
 from dashboard_api.secrets import router as secrets_router
 from dashboard_api.services import router as services_router
-from dashboard_api.stream import health_poll_loop, subscribe, unsubscribe
+from dashboard_api.stream import close_all_subscribers, health_poll_loop, subscribe, unsubscribe
+from dashboard_api.tools import router as tools_router
+
+# Set by _watch_shutdown when uvicorn begins its shutdown sequence.
+_shutting_down = False
+
+
+async def _watch_shutdown(server: uvicorn.Server) -> None:
+    """Poll uvicorn's should_exit flag and close SSE subscribers promptly."""
+    while not server.should_exit:
+        await asyncio.sleep(0.5)
+    global _shutting_down
+    _shutting_down = True
+    close_all_subscribers()
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     """Application lifespan handler."""
+    global _shutting_down
+    _shutting_down = False
+
     await bus.start()
     poll_task = asyncio.create_task(health_poll_loop())
+
     yield
+
+    _shutting_down = True
     poll_task.cancel()
+    close_all_subscribers()
     await bus.stop()
 
 
@@ -52,6 +72,7 @@ app.include_router(events_router)
 app.include_router(logs_router)
 app.include_router(secrets_router)
 app.include_router(services_router)
+app.include_router(tools_router)
 
 
 @app.get("/health")
@@ -70,6 +91,8 @@ async def sse_stream() -> StreamingResponse:
             yield "event: connected\ndata: {}\n\n"
             while True:
                 msg = await q.get()
+                if not msg:
+                    break
                 yield msg
         except asyncio.CancelledError:
             pass
@@ -85,12 +108,20 @@ async def sse_stream() -> StreamingResponse:
 
 def run() -> None:
     """Run the application with uvicorn."""
-    uvicorn.run(
+    config = uvicorn.Config(
         "dashboard_api.main:app",
         host=settings.host,
         port=settings.port,
         reload=False,
     )
+    server = uvicorn.Server(config)
+
+    async def serve_with_watcher() -> None:
+        watcher = asyncio.create_task(_watch_shutdown(server))
+        await server.serve()
+        watcher.cancel()
+
+    asyncio.run(serve_with_watcher())
 
 
 if __name__ == "__main__":

dashboard-api/src/dashboard_api/models.py

@@ -17,6 +17,10 @@ class ComponentSummary(BaseModel):
     category: str | None = None
     version: str | None = None
     tool_type: str | None = None
+    source: str | None = None
+    system_dependencies: list[str] = []
+    schedule: str | None = None
+    installed: bool | None = None
 
 
 class ComponentDetail(ComponentSummary):
@@ -54,3 +58,30 @@ class ServiceActionResponse(BaseModel):
     component: str
     action: str
     status: str
+
+
+class ToolSummary(BaseModel):
+    """Summary of a single tool."""
+
+    id: str
+    description: str | None = None
+    category: str | None = None
+    source: str | None = None
+    tool_type: str | None = None
+    version: str | None = None
+    runner: str | None = None
+    system_dependencies: list[str] = []
+    installed: bool = False
+
+
+class ToolCategory(BaseModel):
+    """Tools grouped by category."""
+
+    name: str
+    tools: list[ToolSummary]
+
+
+class ToolDetail(ToolSummary):
+    """Full detail for a single tool, including documentation."""
+
+    docs: str | None = None

dashboard-api/src/dashboard_api/routes.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import shutil
+
 from fastapi import APIRouter, HTTPException, status
 
 from castle_cli.config import load_config
@@ -33,6 +35,19 @@ def _summary_from_manifest(name: str, manifest: object) -> ComponentSummary:
         manifest.manage and manifest.manage.systemd and manifest.manage.systemd.enable
     )
 
+    # Extract cron schedule from first schedule trigger, if any
+    schedule = None
+    for t in manifest.triggers:
+        if t.type == "schedule":
+            schedule = t.cron
+            break
+
+    # Check if tool is actually installed on PATH
+    installed: bool | None = None
+    if manifest.install and manifest.install.path:
+        alias = manifest.install.path.alias or name
+        installed = shutil.which(alias) is not None
+
     return ComponentSummary(
         id=name,
         description=manifest.description,
@@ -45,6 +60,10 @@ def _summary_from_manifest(name: str, manifest: object) -> ComponentSummary:
         category=manifest.tool.category if manifest.tool else None,
         version=manifest.tool.version if manifest.tool else None,
         tool_type=manifest.tool.tool_type.value if manifest.tool else None,
+        source=manifest.tool.source if manifest.tool else None,
+        system_dependencies=manifest.tool.system_dependencies if manifest.tool else [],
+        schedule=schedule,
+        installed=installed,
     )
 
 

dashboard-api/src/dashboard_api/services.py

@@ -6,6 +6,7 @@ import asyncio
 import time
 
 from fastapi import APIRouter, HTTPException, status
+from starlette.responses import JSONResponse
 
 from castle_cli.config import load_config
 
@@ -17,6 +18,7 @@ from dashboard_api.stream import broadcast
 router = APIRouter(prefix="/services", tags=["services"])
 
 UNIT_PREFIX = "castle-"
+SELF_NAME = "dashboard-api"
 
 
 async def _systemctl(action: str, unit: str) -> tuple[bool, str]:
@@ -77,10 +79,25 @@ async def _broadcast_health_with_override(
     })
 
 
-async def _do_action(name: str, action: str) -> dict:
+async def _deferred_systemctl(action: str, unit: str, delay: float = 0.5) -> None:
+    """Run a systemctl action after a delay, allowing the HTTP response to flush."""
+    await asyncio.sleep(delay)
+    await _systemctl(action, unit)
+
+
+async def _do_action(name: str, action: str) -> JSONResponse:
     """Execute a systemctl action and broadcast updated health."""
     _validate_managed(name)
     unit = f"{UNIT_PREFIX}{name}.service"
+
+    # Self-restart: defer the systemctl call so the response can be sent first
+    if name == SELF_NAME and action in ("restart", "stop"):
+        asyncio.create_task(_deferred_systemctl(action, unit))
+        return JSONResponse(
+            status_code=202,
+            content={"component": name, "action": action, "status": "accepted"},
+        )
+
     ok, output = await _systemctl(action, unit)
     unit_status = await _get_unit_status(unit)
 
@@ -90,22 +107,24 @@ async def _do_action(name: str, action: str) -> dict:
     # Broadcast immediately with systemd status as the source of truth
     await _broadcast_health_with_override(name, unit_status)
 
-    return {"component": name, "action": action, "status": unit_status}
+    return JSONResponse(
+        content={"component": name, "action": action, "status": unit_status},
+    )
 
 
 @router.post("/{name}/start")
-async def start_service(name: str) -> dict:
+async def start_service(name: str) -> JSONResponse:
     """Start a systemd-managed service."""
     return await _do_action(name, "start")
 
 
 @router.post("/{name}/stop")
-async def stop_service(name: str) -> dict:
+async def stop_service(name: str) -> JSONResponse:
     """Stop a systemd-managed service."""
     return await _do_action(name, "stop")
 
 
 @router.post("/{name}/restart")
-async def restart_service(name: str) -> dict:
+async def restart_service(name: str) -> JSONResponse:
     """Restart a systemd-managed service."""
     return await _do_action(name, "restart")

dashboard-api/src/dashboard_api/stream.py

@@ -33,6 +33,16 @@ def unsubscribe(q: asyncio.Queue[str]) -> None:
         pass
 
 
+def close_all_subscribers() -> None:
+    """Unblock all SSE generators so they exit during shutdown."""
+    for q in list(_subscribers):
+        try:
+            q.put_nowait("")
+        except asyncio.QueueFull:
+            pass
+    _subscribers.clear()
+
+
 async def broadcast(event_type: str, data: dict) -> None:
     """Send an event to all connected SSE clients."""
     payload = f"event: {event_type}\ndata: {json.dumps(data)}\n\n"

dashboard-api/src/dashboard_api/tools.py

@@ -0,0 +1,177 @@
+"""Tools router — browse and inspect tool components."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, status
+
+from castle_cli.config import load_config
+from castle_cli.manifest import ComponentManifest
+
+from dashboard_api.config import settings
+from dashboard_api.models import ToolCategory, ToolDetail, ToolSummary
+
+router = APIRouter(tags=["tools"])
+
+
+def _tool_summary(name: str, manifest: ComponentManifest) -> ToolSummary:
+    """Build a ToolSummary from a manifest that has a tool spec."""
+    t = manifest.tool
+    assert t is not None
+    installed = bool(manifest.install and manifest.install.path and manifest.install.path.enable)
+    return ToolSummary(
+        id=name,
+        description=manifest.description,
+        category=t.category,
+        source=t.source,
+        tool_type=t.tool_type.value,
+        version=t.version,
+        runner=manifest.run.runner if manifest.run else None,
+        system_dependencies=t.system_dependencies,
+        installed=installed,
+    )
+
+
+def _find_md_for_tool(
+    root: Path,
+    source: str,
+    tool_name: str,
+    category: str | None = None,
+) -> Path | None:
+    """Find the .md documentation file for a tool source path."""
+    source_path = root / source
+    if source_path.is_file():
+        md = source_path.with_suffix(".md")
+        if md.exists():
+            return md
+    elif source_path.is_dir():
+        py_name = tool_name.replace("-", "_")
+        if category:
+            md = source_path / "src" / category / f"{py_name}.md"
+            if md.exists():
+                return md
+    return None
+
+
+def _strip_frontmatter(content: str) -> str:
+    """Strip YAML frontmatter from markdown content."""
+    if content.startswith("---\n"):
+        end = content.find("\n---\n", 4)
+        if end != -1:
+            content = content[end + 5:]
+    return content.strip()
+
+
+@router.get("/tools", response_model=list[ToolCategory])
+def list_tools() -> list[ToolCategory]:
+    """List tools grouped by category."""
+    config = load_config(settings.castle_root)
+    tools = {k: v for k, v in config.components.items() if v.tool}
+
+    by_category: dict[str, list[ToolSummary]] = {}
+    for name, manifest in tools.items():
+        cat = manifest.tool.category or "uncategorized"  # type: ignore[union-attr]
+        by_category.setdefault(cat, []).append(_tool_summary(name, manifest))
+
+    return [
+        ToolCategory(name=cat, tools=sorted(items, key=lambda t: t.id))
+        for cat, items in sorted(by_category.items())
+    ]
+
+
+@router.get("/tools/{name}", response_model=ToolDetail)
+def get_tool(name: str) -> ToolDetail:
+    """Get detailed info for a single tool."""
+    config = load_config(settings.castle_root)
+
+    if name not in config.components:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Component '{name}' not found",
+        )
+
+    manifest = config.components[name]
+    if not manifest.tool:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"'{name}' is not a tool",
+        )
+
+    summary = _tool_summary(name, manifest)
+    docs: str | None = None
+    t = manifest.tool
+    if t.source:
+        md_path = _find_md_for_tool(config.root, t.source, name, t.category)
+        if md_path and md_path.exists():
+            docs = _strip_frontmatter(md_path.read_text())
+            if not docs:
+                docs = None
+
+    return ToolDetail(**summary.model_dump(), docs=docs)
+
+
+@router.post("/tools/{name}/install")
+async def install_tool(name: str) -> dict:
+    """Install a tool to PATH via uv tool install."""
+    config = load_config(settings.castle_root)
+    if name not in config.components:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"'{name}' not found")
+
+    manifest = config.components[name]
+    if not manifest.tool or not manifest.tool.source:
+        raise HTTPException(status_code=400, detail=f"'{name}' has no tool source to install")
+
+    source_dir = config.root / manifest.tool.source
+    if not (source_dir / "pyproject.toml").exists():
+        raise HTTPException(status_code=400, detail=f"No pyproject.toml in {manifest.tool.source}")
+
+    proc = await asyncio.create_subprocess_exec(
+        "uv", "tool", "install", "--editable", str(source_dir), "--force",
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    output = (stdout or stderr or b"").decode().strip()
+
+    if proc.returncode != 0:
+        raise HTTPException(status_code=500, detail=output or "Install failed")
+
+    return {"component": name, "action": "install", "status": "ok"}
+
+
+@router.post("/tools/{name}/uninstall")
+async def uninstall_tool(name: str) -> dict:
+    """Uninstall a tool from PATH via uv tool uninstall."""
+    config = load_config(settings.castle_root)
+    if name not in config.components:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"'{name}' not found")
+
+    manifest = config.components[name]
+    if not manifest.tool or not manifest.tool.source:
+        raise HTTPException(status_code=400, detail=f"'{name}' has no tool source")
+
+    # uv tool uninstall uses the package name from pyproject.toml
+    source_dir = config.root / manifest.tool.source
+    # Try to read the package name; fall back to the source dir name
+    pkg_name = source_dir.name
+    pyproject = source_dir / "pyproject.toml"
+    if pyproject.exists():
+        import tomllib
+        with open(pyproject, "rb") as f:
+            data = tomllib.load(f)
+        pkg_name = data.get("project", {}).get("name", pkg_name)
+
+    proc = await asyncio.create_subprocess_exec(
+        "uv", "tool", "uninstall", pkg_name,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    output = (stdout or stderr or b"").decode().strip()
+
+    if proc.returncode != 0:
+        raise HTTPException(status_code=500, detail=output or "Uninstall failed")
+
+    return {"component": name, "action": "uninstall", "status": "ok"}

dashboard-api/tests/conftest.py

@@ -37,6 +37,19 @@ def castle_root(tmp_path: Path) -> Generator[Path, None, None]:
             "test-tool": {
                 "description": "Test tool",
                 "install": {"path": {"alias": "test-tool"}},
+                "tool": {
+                    "category": "document",
+                    "source": "tools/document",
+                    "system_dependencies": ["pandoc"],
+                },
+            },
+            "test-tool-2": {
+                "description": "Another test tool",
+                "tool": {
+                    "category": "utility",
+                    "version": "2.0.0",
+                    "tool_type": "script",
+                },
             },
         },
     }

dashboard-api/tests/test_health.py

@@ -72,6 +72,6 @@ class TestGateway:
         assert response.status_code == 200
         data = response.json()
         assert data["port"] == 9000
-        assert data["component_count"] == 2
+        assert data["component_count"] == 3
         assert data["service_count"] == 1
         assert data["managed_count"] == 1

dashboard-api/tests/test_tools.py

@@ -0,0 +1,105 @@
+"""Tests for tools endpoints."""
+
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+
+class TestToolsList:
+    """GET /tools endpoint tests."""
+
+    def test_returns_grouped_tools(self, client: TestClient) -> None:
+        """Returns tools grouped by category."""
+        response = client.get("/tools")
+        assert response.status_code == 200
+        data = response.json()
+        names = [cat["name"] for cat in data]
+        assert "document" in names
+        assert "utility" in names
+
+    def test_categories_sorted(self, client: TestClient) -> None:
+        """Categories are sorted alphabetically."""
+        response = client.get("/tools")
+        data = response.json()
+        names = [cat["name"] for cat in data]
+        assert names == sorted(names)
+
+    def test_tool_fields(self, client: TestClient) -> None:
+        """Tool summary has expected fields."""
+        response = client.get("/tools")
+        data = response.json()
+        doc_cat = next(c for c in data if c["name"] == "document")
+        tool = next(t for t in doc_cat["tools"] if t["id"] == "test-tool")
+        assert tool["description"] == "Test tool"
+        assert tool["category"] == "document"
+        assert tool["source"] == "tools/document"
+        assert tool["system_dependencies"] == ["pandoc"]
+
+    def test_installed_flag(self, client: TestClient) -> None:
+        """Tool with install.path is marked as installed."""
+        response = client.get("/tools")
+        data = response.json()
+        doc_cat = next(c for c in data if c["name"] == "document")
+        tool = next(t for t in doc_cat["tools"] if t["id"] == "test-tool")
+        assert tool["installed"] is True
+
+    def test_not_installed_flag(self, client: TestClient) -> None:
+        """Tool without install.path is not marked as installed."""
+        response = client.get("/tools")
+        data = response.json()
+        util_cat = next(c for c in data if c["name"] == "utility")
+        tool = next(t for t in util_cat["tools"] if t["id"] == "test-tool-2")
+        assert tool["installed"] is False
+
+    def test_service_excluded(self, client: TestClient) -> None:
+        """Services without tool spec are not listed."""
+        response = client.get("/tools")
+        data = response.json()
+        all_ids = [t["id"] for cat in data for t in cat["tools"]]
+        assert "test-svc" not in all_ids
+
+
+class TestToolDetail:
+    """GET /tools/{name} endpoint tests."""
+
+    def test_get_tool(self, client: TestClient) -> None:
+        """Returns detail for a known tool."""
+        response = client.get("/tools/test-tool")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["id"] == "test-tool"
+        assert data["category"] == "document"
+        assert data["system_dependencies"] == ["pandoc"]
+
+    def test_docs_from_file(self, client: TestClient, castle_root: Path) -> None:
+        """Reads documentation from .md file."""
+        # Create doc file matching the source lookup
+        doc_dir = castle_root / "tools" / "document" / "src" / "document"
+        doc_dir.mkdir(parents=True)
+        (doc_dir / "test_tool.md").write_text(
+            "---\ntitle: Test\n---\n\n# Test Tool\n\nUsage info here."
+        )
+        response = client.get("/tools/test-tool")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["docs"] is not None
+        assert "# Test Tool" in data["docs"]
+        # Frontmatter should be stripped
+        assert "---" not in data["docs"]
+
+    def test_no_docs(self, client: TestClient) -> None:
+        """Tool with no doc file returns null docs."""
+        response = client.get("/tools/test-tool")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["docs"] is None
+
+    def test_not_found(self, client: TestClient) -> None:
+        """Returns 404 for unknown component."""
+        response = client.get("/tools/nonexistent")
+        assert response.status_code == 404
+
+    def test_not_a_tool(self, client: TestClient) -> None:
+        """Returns 404 for component that is not a tool."""
+        response = client.get("/tools/test-svc")
+        assert response.status_code == 404