For real. Tools schema generation and ux tweaks.

castle-api/src/castle_api/config.py

@@ -20,6 +20,15 @@ class Settings(BaseSettings):
     nats_token: str | None = None
     mdns_enabled: bool = False
 
+    # LLM assist (off by default). Calls the litellm proxy's OpenAI-compatible
+    # /chat/completions to generate a tool-call schema for CLIs the deterministic
+    # parser can't structure (subcommand trees). The key is NOT stored here — it's
+    # read from the secret backend by name at call time.
+    llm_enabled: bool = False
+    llm_base_url: str = "https://litellm.civil.payne.io/v1"
+    llm_model: str = "qwen"
+    llm_api_key_secret: str = "LITELLM_MASTER_KEY"
+
     model_config = {
         "env_prefix": "CASTLE_API_",
         "env_file": ".env",

castle-api/src/castle_api/config_editor.py

@@ -6,7 +6,7 @@ import asyncio
 from pathlib import Path
 
 import yaml
-from fastapi import APIRouter, HTTPException, status
+from fastapi import APIRouter, Body, HTTPException, status
 from pydantic import BaseModel
 
 from castle_core.config import (
@@ -454,6 +454,80 @@ def delete_tool(name: str) -> dict:
     return _delete_deployment(name, kind="tool")
 
 
+@router.post("/tools/{name}/schema")
+async def generate_tool_schema(
+    name: str, deep: bool = False, assist: str | None = None
+) -> dict:
+    """Generate a *draft* tool-call schema (neutral core) from the tool's ``--help``.
+
+    Not saved — the client reviews/edits it and persists via ``PUT
+    /config/tools/{name}`` (the schema rides in the deployment config as
+    ``tool_schema``). Two modes:
+
+    * default (``assist`` unset) — deterministic: parse ``--help``. ``deep`` walks
+      subcommands. 422 if the tool isn't installed / emits no help.
+    * ``assist=llm`` — send the recursive ``--help`` to the litellm proxy for a
+      structured schema (the escape hatch for subcommand trees the parser can only
+      render as a ``command`` string). 503 if LLM assist is disabled; 502 on an
+      upstream/validation failure.
+    """
+    from castle_core.tool_schema import (
+        ToolSchemaError,
+        collect_tool_help,
+        derive_tool_schema,
+    )
+
+    config = get_config()
+    if config.deployment("tool", name) is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Tool '{name}' not found",
+        )
+
+    if assist == "llm":
+        from castle_api.config import settings
+        from castle_api.llm import LLMAssistError, generate_tool_schema_llm
+
+        if not settings.llm_enabled:
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="LLM assist is disabled (set CASTLE_API_LLM_ENABLED=true).",
+            )
+        try:
+            help_text = collect_tool_help(config, name)
+        except ToolSchemaError as e:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
+            )
+        try:
+            schema = await generate_tool_schema_llm(help_text, name)
+        except LLMAssistError as e:
+            raise HTTPException(
+                status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)
+            )
+        return {"ok": True, "schema": schema, "assist": "llm"}
+
+    try:
+        schema = derive_tool_schema(config, name, deep=deep)
+    except ToolSchemaError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+    return {"ok": True, "schema": schema}
+
+
+@router.post("/tools/schema/validate")
+def validate_tool_schema_endpoint(core: dict = Body(...)) -> dict:
+    """Deterministically validate a tool-call schema core (no LLM) — the shape and
+    that ``parameters`` is a valid JSON Schema. Lets the UI check a hand-edited
+    schema. Returns ``{valid, errors}``."""
+    from castle_core.tool_schema import validate_tool_schema_core
+
+    errors = validate_tool_schema_core(core)
+    return {"valid": not errors, "errors": errors}
+
+
 @router.put("/static/{name}")
 def save_static(name: str, request: ServiceConfigRequest) -> dict:
     """Create/update the *static* frontend named `name`."""

castle-api/src/castle_api/llm.py

@@ -0,0 +1,159 @@
+"""LLM assist — generate a tool-call schema from a CLI's --help via the litellm proxy.
+
+castle's first LLM-backed feature. The deterministic parser
+(``castle_core.tool_schema``) falls back to a single ``command`` string for
+subcommand trees (git, castle); this asks an LLM to read the recursive ``--help``
+and produce a *structured* neutral tool-call core instead.
+
+Goes through castle's litellm proxy over its OpenAI-compatible
+``/chat/completions`` (model-agnostic; the fleet standardizes on litellm), using
+forced tool-calling to constrain the output. Every response is validated
+deterministically (``validate_tool_schema_core``); on failure the errors are fed
+back to the model to repair, up to a cap — so a weak model's malformed draft is
+fixed rather than surfaced. The result is a reviewed draft — never auto-saved.
+"""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+
+from castle_core.config import read_secret
+from castle_core.tool_schema import validate_tool_schema_core
+
+from castle_api.config import settings
+
+_TIMEOUT = 60.0
+# Total attempts = 1 initial + repairs. Sonnet is valid first try; the extra
+# rounds salvage weaker models (repairing a bad enum, etc.).
+_MAX_ATTEMPTS = 3
+_SYSTEM = (
+    "You convert a command-line tool's --help output into a tool-call schema for "
+    "an AI agent. Call the `emit_tool_schema` function exactly once. Produce a "
+    "JSON Schema in `parameters` (type: object) whose properties let an agent "
+    "invoke the tool: one property per meaningful option or subcommand path, each "
+    "typed (boolean for flags, string for valued options, enum as a LIST of the "
+    "allowed values for fixed choices) with a concise description drawn from the "
+    "help. For a tool with subcommands, include a `subcommand` property (an enum "
+    "listing the available subcommands) plus the important shared options. `name` "
+    "must match ^[a-zA-Z0-9_-]{1,64}$. Do not invent options not in the help."
+)
+
+_EMIT_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "emit_tool_schema",
+        "description": "Emit the structured tool-call definition for this CLI.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string", "description": "Sanitized tool name."},
+                "description": {
+                    "type": "string",
+                    "description": "One-line description of what the tool does.",
+                },
+                "parameters": {
+                    "type": "object",
+                    "description": (
+                        "A JSON Schema object (type: object with a properties map) "
+                        "describing the tool's invocation arguments."
+                    ),
+                },
+            },
+            "required": ["name", "description", "parameters"],
+        },
+    },
+}
+
+
+class LLMAssistError(Exception):
+    """The LLM assist couldn't produce a valid schema (config, upstream, or output)."""
+
+
+def _extract_args(data: dict) -> dict | None:
+    """Pull the emit_tool_schema arguments out of a chat-completions response.
+
+    Returns the parsed arguments dict (even if not yet valid — the caller
+    validates), or None if the model didn't call the function / returned non-JSON.
+    """
+    try:
+        call = data["choices"][0]["message"]["tool_calls"][0]
+        args = call["function"]["arguments"]
+    except (KeyError, IndexError, TypeError):
+        return None
+    if isinstance(args, str):
+        try:
+            args = json.loads(args)
+        except json.JSONDecodeError:
+            return None
+    return args if isinstance(args, dict) else None
+
+
+async def _complete(messages: list[dict], key: str) -> dict:
+    """One forced-tool-call chat completion against the litellm proxy."""
+    payload = {
+        "model": settings.llm_model,
+        "messages": messages,
+        "tools": [_EMIT_TOOL],
+        "tool_choice": {"type": "function", "function": {"name": "emit_tool_schema"}},
+    }
+    url = f"{settings.llm_base_url.rstrip('/')}/chat/completions"
+    headers = {"Authorization": f"Bearer {key}"}
+    async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
+        try:
+            resp = await client.post(url, json=payload, headers=headers)
+        except httpx.HTTPError as e:
+            raise LLMAssistError(f"litellm request failed: {e}") from e
+    if resp.status_code >= 400:
+        raise LLMAssistError(f"litellm returned {resp.status_code}: {resp.text[:300]}")
+    return resp.json()
+
+
+def _repair_message(prior: dict, errors: list[str]) -> dict:
+    """A user turn that shows the invalid schema + its errors and asks for a fix."""
+    return {
+        "role": "user",
+        "content": (
+            "Your previous emit_tool_schema output failed validation:\n"
+            + "\n".join(f"- {e}" for e in errors)
+            + "\n\nHere is what you returned:\n"
+            + json.dumps(prior, indent=2)
+            + "\n\nCall emit_tool_schema again with every problem fixed. Remember: "
+            "`enum` must be a list of the allowed values, not a count."
+        ),
+    }
+
+
+async def generate_tool_schema_llm(help_text: str, name: str) -> dict:
+    """Ask the LLM for a structured neutral tool-call core, repairing invalid
+    output against ``validate_tool_schema_core`` until it passes (up to
+    ``_MAX_ATTEMPTS``). Raises ``LLMAssistError`` on config/upstream failure or if
+    it can't be made valid."""
+    key = read_secret(settings.llm_api_key_secret)
+    if not key:
+        raise LLMAssistError(
+            f"LLM assist enabled but secret '{settings.llm_api_key_secret}' is unset."
+        )
+    base = [
+        {"role": "system", "content": _SYSTEM},
+        {"role": "user", "content": f"Tool name: {name}\n\n--- help ---\n{help_text}"},
+    ]
+    messages = list(base)
+    errors = ["model did not call emit_tool_schema with JSON arguments"]
+    for _attempt in range(_MAX_ATTEMPTS):
+        data = await _complete(messages, key)
+        args = _extract_args(data)
+        if args is not None:
+            # Pin the name we were given so a model name-drift never costs a repair.
+            args["name"] = name
+            errors = validate_tool_schema_core(args)
+            if not errors:
+                return args
+        # Rebuild from base + a single repair turn (avoids provider-specific
+        # tool_call/tool_result threading; each repair is a fresh forced call).
+        messages = [*base, _repair_message(args or {}, errors)]
+    raise LLMAssistError(
+        f"could not produce a valid schema after {_MAX_ATTEMPTS} attempts: "
+        + "; ".join(errors)
+    )

castle-api/tests/test_tool_schema_api.py

@@ -0,0 +1,199 @@
+"""POST /config/tools/{name}/schema — derive an Anthropic tool schema from --help,
+and the tool_schema field round-tripping through the tool save path."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+import json
+
+from castle_core.config import load_config
+
+from castle_api.config import settings
+
+
+def _completion(args: dict) -> dict:
+    """A minimal chat-completions response carrying an emit_tool_schema call."""
+    return {
+        "choices": [
+            {"message": {"tool_calls": [
+                {"function": {"name": "emit_tool_schema", "arguments": json.dumps(args)}}
+            ]}}
+        ]
+    }
+
+
+_BAD_CORE = {
+    "name": "python3", "description": "d",
+    "parameters": {"type": "object", "properties": {"m": {"type": "string", "enum": 3}}},
+}
+_GOOD_CORE = {
+    "name": "python3", "description": "d",
+    "parameters": {"type": "object", "properties": {"m": {"type": "string"}}},
+}
+
+# A tool whose program falls back to the tool name for its executable; `python3`
+# is always on PATH, so derive succeeds deterministically.
+_PY_TOOL = {"program": "python3", "manager": "path"}
+
+
+class TestGenerateSchema:
+    def test_generate_returns_draft_not_saved(
+        self, client: TestClient, castle_root: Path
+    ) -> None:
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+
+        r = client.post("/config/tools/python3/schema")
+        assert r.status_code == 200, r.text
+        body = r.json()
+        schema = body["schema"]
+        # The stored draft is the neutral core (name/description/parameters),
+        # rendered to a provider envelope only on read.
+        assert schema["name"] == "python3"
+        assert "parameters" in schema
+        assert schema["parameters"]["properties"]
+
+        # It's a draft: nothing persisted until the client saves it.
+        cfg = load_config(castle_root)
+        assert cfg.tools["python3"].tool_schema is None
+
+    def test_unknown_tool_404(self, client: TestClient) -> None:
+        r = client.post("/config/tools/nope-not-a-tool/schema")
+        assert r.status_code == 404
+
+    def test_validate_endpoint_valid(self, client: TestClient) -> None:
+        r = client.post("/config/tools/schema/validate", json=_GOOD_CORE)
+        assert r.status_code == 200
+        assert r.json() == {"valid": True, "errors": []}
+
+    def test_validate_endpoint_invalid(self, client: TestClient) -> None:
+        r = client.post("/config/tools/schema/validate", json=_BAD_CORE)
+        assert r.status_code == 200
+        body = r.json()
+        assert body["valid"] is False and body["errors"]
+
+    def test_uninstalled_executable_422(
+        self, client: TestClient, castle_root: Path
+    ) -> None:
+        client.put(
+            "/config/tools/ghosttool",
+            json={"config": {"program": "ghosttool", "manager": "path"}},
+        )
+        r = client.post("/config/tools/ghosttool/schema")
+        assert r.status_code == 422
+        assert "PATH" in r.json()["detail"]
+
+    def test_assist_disabled_returns_503(
+        self, client: TestClient, castle_root: Path
+    ) -> None:
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+        # llm_enabled is False by default.
+        r = client.post("/config/tools/python3/schema?assist=llm")
+        assert r.status_code == 503
+        assert "disabled" in r.json()["detail"]
+
+    def test_assist_success_returns_draft(
+        self, client: TestClient, castle_root: Path, monkeypatch
+    ) -> None:
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+
+        async def _fake_llm(help_text: str, name: str) -> dict:
+            assert "python3" in help_text  # the recursive help was gathered
+            return {
+                "name": name,
+                "description": "the python interpreter",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"script": {"type": "string"}},
+                },
+            }
+
+        monkeypatch.setattr(settings, "llm_enabled", True)
+        monkeypatch.setattr("castle_api.llm.generate_tool_schema_llm", _fake_llm)
+
+        r = client.post("/config/tools/python3/schema?assist=llm")
+        assert r.status_code == 200, r.text
+        body = r.json()
+        assert body["assist"] == "llm"
+        assert body["schema"]["parameters"]["properties"] == {"script": {"type": "string"}}
+
+    def test_assist_upstream_error_returns_502(
+        self, client: TestClient, castle_root: Path, monkeypatch
+    ) -> None:
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+
+        async def _fail_llm(help_text: str, name: str) -> dict:
+            from castle_api.llm import LLMAssistError
+
+            raise LLMAssistError("litellm returned 500")
+
+        monkeypatch.setattr(settings, "llm_enabled", True)
+        monkeypatch.setattr("castle_api.llm.generate_tool_schema_llm", _fail_llm)
+
+        r = client.post("/config/tools/python3/schema?assist=llm")
+        assert r.status_code == 502
+        assert "litellm" in r.json()["detail"]
+
+    def test_assist_repairs_invalid_then_valid(
+        self, client: TestClient, castle_root: Path, monkeypatch
+    ) -> None:
+        """A malformed first response (bad enum) is fed back and repaired."""
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+        calls = {"n": 0}
+
+        async def _fake_complete(messages: list, key: str) -> dict:
+            calls["n"] += 1
+            return _completion(_BAD_CORE if calls["n"] == 1 else _GOOD_CORE)
+
+        monkeypatch.setattr(settings, "llm_enabled", True)
+        monkeypatch.setattr("castle_api.llm.read_secret", lambda name: "k")
+        monkeypatch.setattr("castle_api.llm._complete", _fake_complete)
+
+        r = client.post("/config/tools/python3/schema?assist=llm")
+        assert r.status_code == 200, r.text
+        assert calls["n"] == 2  # repaired on the second attempt
+        props = r.json()["schema"]["parameters"]["properties"]
+        assert props == {"m": {"type": "string"}}
+
+    def test_assist_unrepairable_returns_502(
+        self, client: TestClient, castle_root: Path, monkeypatch
+    ) -> None:
+        """Persistently invalid output exhausts the repair budget → 502."""
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+
+        async def _always_bad(messages: list, key: str) -> dict:
+            return _completion(_BAD_CORE)
+
+        monkeypatch.setattr(settings, "llm_enabled", True)
+        monkeypatch.setattr("castle_api.llm.read_secret", lambda name: "k")
+        monkeypatch.setattr("castle_api.llm._complete", _always_bad)
+
+        r = client.post("/config/tools/python3/schema?assist=llm")
+        assert r.status_code == 502
+        assert "attempts" in r.json()["detail"]
+
+    def test_schema_round_trips_through_save(
+        self, client: TestClient, castle_root: Path
+    ) -> None:
+        """Saving a tool with tool_schema persists it (PATCH merge), and clearing
+        it with null drops the field."""
+        client.put("/config/tools/python3", json={"config": _PY_TOOL})
+        schema = client.post("/config/tools/python3/schema").json()["schema"]
+
+        client.put(
+            "/config/tools/python3",
+            json={"config": {"program": "python3", "tool_schema": schema}},
+        )
+        cfg = load_config(castle_root)
+        assert cfg.tools["python3"].tool_schema is not None
+        assert cfg.tools["python3"].tool_schema["name"] == "python3"
+
+        # Clearing with null removes it (default None).
+        client.put(
+            "/config/tools/python3",
+            json={"config": {"program": "python3", "tool_schema": None}},
+        )
+        cfg = load_config(castle_root)
+        assert cfg.tools["python3"].tool_schema is None