diff --git a/README.md b/README.md index 257c5ca..e8c2309 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,10 @@ gateway: components: central-context: description: Content storage API + source: components/central-context run: - runner: python_uv_tool + runner: python tool: central-context - working_dir: components/central-context - env: - CENTRAL_CONTEXT_DATA_DIR: /data/castle/central-context - CENTRAL_CONTEXT_PORT: "9001" expose: http: internal: { port: 9001 } @@ -80,8 +77,30 @@ components: caddy: { path_prefix: /central-context } manage: systemd: {} + + notification-bridge: + description: Desktop notification forwarder + source: components/notification-bridge + run: + runner: python + tool: notification-bridge + defaults: + env: + CENTRAL_CONTEXT_URL: http://localhost:9001 + BUCKET_NAME: notifications + expose: + http: + internal: { port: 9002 } + health_path: /health + proxy: + caddy: { path_prefix: /notifications } + manage: + systemd: {} ``` +Convention-based env vars (`_DATA_DIR`, `_PORT`) are generated +automatically by `castle deploy`. Only non-convention values need `defaults.env`. + ## Architecture ``` diff --git a/TODO.md b/TODO.md index 2f554ce..eb3218b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,10 +1,7 @@ # TO DO - Remove devbox-connect. Instead, make it easy to copy an ssh tunnel command to expose all ports of all services to a remote box: `ssh -L 9000:localhost:9000 payne@dev.payne.io, etc. -- tool.uv.sources should be in cli, not in castle-api - Add a scripts dir? -- Maybe there's no real reason to have special handling for the `tools` dir (one md file per tool, put in categories, etc.). On the one hand, the categories help them share dependencies. On the other, there's not really a need to share dependencies because uv does just fine. Also, they prob don't need markdown files because their description can just be in the --help arg, and in the `castle.yaml` registration. Having them flat would allow us to just think of everything as a tool: a bash script tool, a python tool, a rust tool. They're all things just follow a std-in std-out pattern so they are unix-philosophy good. Daemons otoh, follow daemon patterns (env config, logging, long-running, port mapping, etc.) - - tools: use std-in/std-out unix philosophy - daemons or web-services: expose ports diff --git a/app/src/components/AddComponent.tsx b/app/src/components/AddComponent.tsx index c035a33..41d7897 100644 --- a/app/src/components/AddComponent.tsx +++ b/app/src/components/AddComponent.tsx @@ -4,7 +4,7 @@ import { Plus, X } from "lucide-react" const TEMPLATES: Record> = { service: { run: { - runner: "python_uv_tool", + runner: "python", tool: "", cwd: "", env: {}, diff --git a/app/src/lib/labels.ts b/app/src/lib/labels.ts index 3251f6b..12db52e 100644 --- a/app/src/lib/labels.ts +++ b/app/src/lib/labels.ts @@ -1,7 +1,6 @@ export const RUNNER_LABELS: Record = { - python_uv_tool: "Python (uv)", + python: "Python", command: "Command", - python_module: "Python module", container: "Container", node: "Node.js", remote: "Remote", diff --git a/castle-api/src/castle_api/routes.py b/castle-api/src/castle_api/routes.py index 8f10d0c..fc39b54 100644 --- a/castle-api/src/castle_api/routes.py +++ b/castle-api/src/castle_api/routes.py @@ -95,7 +95,7 @@ def _summary_from_manifest(name: str, manifest: object, root: Path) -> Component if runner is None and manifest.tool and manifest.tool.source: source_dir = root / manifest.tool.source if (source_dir / "pyproject.toml").exists(): - runner = "python_uv_tool" + runner = "python" elif source_dir.is_file(): runner = "command" diff --git a/castle-api/src/castle_api/tools.py b/castle-api/src/castle_api/tools.py index 2cbd688..0b27800 100644 --- a/castle-api/src/castle_api/tools.py +++ b/castle-api/src/castle_api/tools.py @@ -30,7 +30,7 @@ def _tool_summary( if runner is None and t.source and root: source_dir = root / t.source if (source_dir / "pyproject.toml").exists(): - runner = "python_uv_tool" + runner = "python" elif source_dir.is_file(): runner = "command" diff --git a/castle-api/tests/conftest.py b/castle-api/tests/conftest.py index 8c7a850..734e6c2 100644 --- a/castle-api/tests/conftest.py +++ b/castle-api/tests/conftest.py @@ -28,7 +28,7 @@ def castle_root(tmp_path: Path) -> Generator[Path, None, None]: "description": "Test service", "source": "test-svc", "run": { - "runner": "python_uv_tool", + "runner": "python", "tool": "test-svc", }, "expose": { @@ -73,7 +73,7 @@ def registry_path(tmp_path: Path, castle_root: Path) -> Generator[Path, None, No ), deployed={ "test-svc": DeployedComponent( - runner="python_uv_tool", + runner="python", run_cmd=["uv", "run", "test-svc"], env={ "TEST_SVC_PORT": "19000", diff --git a/castle-api/tests/test_health.py b/castle-api/tests/test_health.py index 392f4c9..6f090d6 100644 --- a/castle-api/tests/test_health.py +++ b/castle-api/tests/test_health.py @@ -55,7 +55,7 @@ class TestComponentDetail: data = response.json() assert data["id"] == "test-svc" assert "manifest" in data - assert data["manifest"]["runner"] == "python_uv_tool" + assert data["manifest"]["runner"] == "python" def test_not_found(self, client: TestClient) -> None: """Returns 404 for unknown component.""" diff --git a/castle.yaml b/castle.yaml index 6001ebd..8274a1a 100644 --- a/castle.yaml +++ b/castle.yaml @@ -26,7 +26,7 @@ components: on the LAN source: components/central-context run: - runner: python_uv_tool + runner: python tool: central-context manage: systemd: {} @@ -44,7 +44,7 @@ components: server. source: components/notification-bridge run: - runner: python_uv_tool + runner: python tool: notification-bridge defaults: env: @@ -65,7 +65,7 @@ components: description: Castle API source: castle-api run: - runner: python_uv_tool + runner: python tool: castle-api manage: systemd: {} @@ -98,8 +98,6 @@ components: install: path: alias: protonmail - tool: - source: components/protonmail/ backup-collect: description: Collect files from various sources into backup directory source: components/backup-collect @@ -117,7 +115,6 @@ components: manage: systemd: {} tool: - source: components/backup-collect/ system_dependencies: - rsync backup-data: @@ -136,7 +133,6 @@ components: timezone: America/Los_Angeles manage: systemd: {} - tool: {} castle-app: description: Castle web app source: app @@ -152,16 +148,12 @@ components: install: path: alias: devbox-connect - tool: - source: components/devbox-connect/ mbox2eml: description: MBOX to EML email converter source: components/mbox2eml install: path: alias: mbox2eml - tool: - source: components/mbox2eml/ android-backup: description: Backup Android device using ADB source: components/android-backup @@ -169,7 +161,6 @@ components: path: alias: android-backup tool: - source: components/android-backup/ system_dependencies: - adb browser: @@ -178,8 +169,6 @@ components: install: path: alias: browser - tool: - source: components/browser/ docx-extractor: description: Extract content and metadata from Word .docx files source: components/docx-extractor @@ -187,7 +176,6 @@ components: path: alias: docx-extractor tool: - source: components/docx-extractor/ system_dependencies: - pandoc docx2md: @@ -197,7 +185,6 @@ components: path: alias: docx2md tool: - source: components/docx2md/ system_dependencies: - pandoc gpt: @@ -206,16 +193,12 @@ components: install: path: alias: gpt - tool: - source: components/gpt/ html2text: description: Convert HTML content to plain text source: components/html2text install: path: alias: html2text - tool: - source: components/html2text/ md2pdf: description: Convert Markdown files to PDF source: components/md2pdf @@ -223,7 +206,6 @@ components: path: alias: md2pdf tool: - source: components/md2pdf/ system_dependencies: - pandoc - texlive-latex-base @@ -233,16 +215,12 @@ components: install: path: alias: mdscraper - tool: - source: components/mdscraper/ pdf-extractor: description: Extract content and metadata from PDF files source: components/pdf-extractor install: path: alias: pdf-extractor - tool: - source: components/pdf-extractor/ pdf2md: description: Convert PDF files to Markdown source: components/pdf2md @@ -250,7 +228,6 @@ components: path: alias: pdf2md tool: - source: components/pdf2md/ system_dependencies: - pandoc - poppler-utils @@ -260,21 +237,15 @@ components: install: path: alias: schedule - tool: - source: components/schedule/ search: description: Manage self-contained searchable collections of files source: components/search install: path: alias: search - tool: - source: components/search/ text-extractor: description: Extract content and metadata from text files source: components/text-extractor install: path: alias: text-extractor - tool: - source: components/text-extractor/ diff --git a/cli/src/castle_cli/commands/create.py b/cli/src/castle_cli/commands/create.py index 3fc3c2c..5c02e72 100644 --- a/cli/src/castle_cli/commands/create.py +++ b/cli/src/castle_cli/commands/create.py @@ -15,7 +15,7 @@ from castle_cli.manifest import ( ManageSpec, PathInstallSpec, ProxySpec, - RunPythonUvTool, + RunPython, SystemdSpec, ToolSpec, ) @@ -78,8 +78,8 @@ def run_create(args: argparse.Namespace) -> int: id=name, description=args.description or f"A castle {proj_type}", source=f"components/{name}", - run=RunPythonUvTool( - runner="python_uv_tool", + run=RunPython( + runner="python", tool=name, ), expose=ExposeSpec( diff --git a/cli/src/castle_cli/commands/deploy.py b/cli/src/castle_cli/commands/deploy.py index 87fe6ce..09e8727 100644 --- a/cli/src/castle_cli/commands/deploy.py +++ b/cli/src/castle_cli/commands/deploy.py @@ -174,7 +174,7 @@ def _build_deployed( def _build_run_cmd(run: object, env: dict[str, str]) -> list[str]: """Build a run command list from a RunSpec.""" match run.runner: - case "python_uv_tool": + case "python": resolved = shutil.which(run.tool) if not resolved: print( @@ -185,12 +185,6 @@ def _build_run_cmd(run: object, env: dict[str, str]) -> list[str]: if run.args: cmd.extend(run.args) return cmd - case "python_module": - python = run.python or shutil.which("python3") or "python3" - cmd = [python, "-m", run.module] - if run.args: - cmd.extend(run.args) - return cmd case "command": cmd = list(run.argv) resolved = shutil.which(cmd[0]) diff --git a/cli/src/castle_cli/commands/sync.py b/cli/src/castle_cli/commands/sync.py index 093e5fd..59c64df 100644 --- a/cli/src/castle_cli/commands/sync.py +++ b/cli/src/castle_cli/commands/sync.py @@ -22,7 +22,7 @@ def _sync_cmd(manifest: ComponentManifest) -> list[str] | None: return None match run.runner: - case "python_uv_tool" | "python_module": + case "python": return ["uv", "sync"] case "node": return [run.package_manager, "install"] @@ -80,13 +80,14 @@ def run_sync(args: argparse.Namespace) -> int: installed_dirs: set[Path] = set() for name, manifest in config.components.items(): - # Determine source directory — from tool.source or manifest.source - source = None - if manifest.tool and manifest.tool.source: - source = manifest.tool.source - elif manifest.run and manifest.run.runner == "python_uv_tool" and manifest.source_dir: - source = manifest.source_dir + # Install if: has install.path, or is a python runner service + if not ( + (manifest.install and manifest.install.path) + or (manifest.run and manifest.run.runner == "python") + ): + continue + source = manifest.source_dir if not source: continue diff --git a/cli/src/castle_cli/manifest.py b/cli/src/castle_cli/manifest.py index 17bfb51..fa0a4c1 100644 --- a/cli/src/castle_cli/manifest.py +++ b/cli/src/castle_cli/manifest.py @@ -23,8 +23,7 @@ from castle_core.manifest import ( # noqa: F401 — explicit re-exports for typ RunCommand, RunContainer, RunNode, - RunPythonModule, - RunPythonUvTool, + RunPython, RunRemote, RunSpec, SystemdSpec, diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index 2e2664b..681629e 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -20,7 +20,7 @@ def castle_root(tmp_path: Path) -> Generator[Path, None, None]: "description": "Test service", "source": "test-svc", "run": { - "runner": "python_uv_tool", + "runner": "python", "tool": "test-svc", }, "defaults": { diff --git a/core/src/castle_core/generators/systemd.py b/core/src/castle_core/generators/systemd.py index 3ed8ad3..dea671b 100644 --- a/core/src/castle_core/generators/systemd.py +++ b/core/src/castle_core/generators/systemd.py @@ -88,20 +88,13 @@ def manifest_to_exec_start(manifest: ComponentManifest, root: Path) -> str: raise ValueError(f"Component '{manifest.id}' has no run spec") match run.runner: - case "python_uv_tool": + case "python": uv_path = shutil.which("uv") or "uv" args_str = " ".join(run.args) if run.args else "" cmd = f"{uv_path} run {run.tool}" if args_str: cmd += f" {args_str}" return cmd - case "python_module": - python = run.python or shutil.which("python3") or "python3" - args_str = " ".join(run.args) if run.args else "" - cmd = f"{python} -m {run.module}" - if args_str: - cmd += f" {args_str}" - return cmd case "command": argv = list(run.argv) resolved = shutil.which(argv[0]) diff --git a/core/src/castle_core/manifest.py b/core/src/castle_core/manifest.py index 8aac433..28156e6 100644 --- a/core/src/castle_core/manifest.py +++ b/core/src/castle_core/manifest.py @@ -46,15 +46,8 @@ class RunCommand(RunBase): argv: list[str] = Field(min_length=1) -class RunPythonModule(RunBase): - runner: Literal["python_module"] - module: str - args: list[str] = Field(default_factory=list) - python: str | None = None - - -class RunPythonUvTool(RunBase): - runner: Literal["python_uv_tool"] +class RunPython(RunBase): + runner: Literal["python"] tool: str args: list[str] = Field(default_factory=list) @@ -84,9 +77,7 @@ class RunRemote(RunBase): RunSpec = Annotated[ - Union[ - RunCommand, RunPythonModule, RunPythonUvTool, RunContainer, RunNode, RunRemote - ], + Union[RunCommand, RunPython, RunContainer, RunNode, RunRemote], Field(discriminator="runner"), ] diff --git a/core/tests/conftest.py b/core/tests/conftest.py index 40ef2d1..58e6c13 100644 --- a/core/tests/conftest.py +++ b/core/tests/conftest.py @@ -20,7 +20,7 @@ def castle_root(tmp_path: Path) -> Generator[Path, None, None]: "description": "Test service", "source": "test-svc", "run": { - "runner": "python_uv_tool", + "runner": "python", "tool": "test-svc", }, "defaults": { diff --git a/core/tests/test_config.py b/core/tests/test_config.py index 59aff42..f7d00ff 100644 --- a/core/tests/test_config.py +++ b/core/tests/test_config.py @@ -60,7 +60,7 @@ class TestLoadConfig: """Service has correct RunSpec.""" config = load_config(castle_root) svc = config.components["test-svc"] - assert svc.run.runner == "python_uv_tool" + assert svc.run.runner == "python" assert svc.run.tool == "test-svc" assert svc.source == "test-svc" diff --git a/core/tests/test_manifest.py b/core/tests/test_manifest.py index 450ecce..27fa0ed 100644 --- a/core/tests/test_manifest.py +++ b/core/tests/test_manifest.py @@ -17,7 +17,7 @@ from castle_core.manifest import ( Role, RunCommand, RunContainer, - RunPythonUvTool, + RunPython, RunRemote, SystemdSpec, ToolSpec, @@ -32,7 +32,7 @@ class TestRoleDerivation: """Component with expose.http gets SERVICE role.""" m = ComponentManifest( id="svc", - run=RunPythonUvTool(runner="python_uv_tool", tool="svc"), + run=RunPython(runner="python", tool="svc"), expose=ExposeSpec(http=HttpExposeSpec(internal=HttpInternal(port=8000))), ) assert Role.SERVICE in m.roles @@ -114,7 +114,7 @@ class TestRoleDerivation: """Component can have multiple roles.""" m = ComponentManifest( id="multi", - run=RunPythonUvTool(runner="python_uv_tool", tool="multi"), + run=RunPython(runner="python", tool="multi"), expose=ExposeSpec(http=HttpExposeSpec(internal=HttpInternal(port=8000))), install=InstallSpec(path=PathInstallSpec(alias="multi")), ) @@ -125,7 +125,7 @@ class TestRoleDerivation: """Systemd + HTTP = SERVICE, not WORKER.""" m = ComponentManifest( id="svc", - run=RunPythonUvTool(runner="python_uv_tool", tool="svc"), + run=RunPython(runner="python", tool="svc"), expose=ExposeSpec(http=HttpExposeSpec(internal=HttpInternal(port=8000))), manage=ManageSpec(systemd=SystemdSpec()), ) @@ -170,7 +170,7 @@ class TestModelSerialization: m = ComponentManifest( id="svc", description="A service", - run=RunPythonUvTool(runner="python_uv_tool", tool="svc", cwd="svc"), + run=RunPython(runner="python", tool="svc", cwd="svc"), expose=ExposeSpec( http=HttpExposeSpec( internal=HttpInternal(port=9001), health_path="/health" @@ -180,6 +180,6 @@ class TestModelSerialization: manage=ManageSpec(systemd=SystemdSpec()), ) data = m.model_dump(exclude_none=True, exclude={"id", "roles"}) - assert data["run"]["runner"] == "python_uv_tool" + assert data["run"]["runner"] == "python" assert data["expose"]["http"]["internal"]["port"] == 9001 assert data["proxy"]["caddy"]["path_prefix"] == "/svc" diff --git a/core/tests/test_systemd.py b/core/tests/test_systemd.py index 4035ce3..a751184 100644 --- a/core/tests/test_systemd.py +++ b/core/tests/test_systemd.py @@ -55,7 +55,7 @@ class TestUnitGeneration: assert "Restart=on-failure" in unit def test_uses_uv_run(self, castle_root: Path) -> None: - """Unit file ExecStart uses uv run for python_uv_tool.""" + """Unit file ExecStart uses uv run for python runner.""" config = load_config(castle_root) manifest = config.components["test-svc"] unit = generate_unit(config, "test-svc", manifest) @@ -68,7 +68,7 @@ class TestUnitFromDeployed: def test_basic_service(self) -> None: """Generate a unit from a deployed component.""" deployed = DeployedComponent( - runner="python_uv_tool", + runner="python", run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"], env={"MY_SVC_PORT": "9001", "MY_SVC_DATA_DIR": "/data/castle/my-svc"}, description="My service", @@ -97,7 +97,7 @@ class TestUnitFromDeployed: def test_no_repo_paths(self) -> None: """Generated units must not reference repo paths.""" deployed = DeployedComponent( - runner="python_uv_tool", + runner="python", run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"], env={"DATA_DIR": "/data/castle/my-svc"}, description="Test", diff --git a/docs/component-registry.md b/docs/component-registry.md index bc8c149..f1632f7 100644 --- a/docs/component-registry.md +++ b/docs/component-registry.md @@ -15,12 +15,8 @@ components: my-service: description: Does something useful run: - runner: python_uv_tool + runner: python tool: my-service - cwd: my-service - env: - MY_SERVICE_DATA_DIR: /data/castle/my-service - MY_SERVICE_PORT: "9001" expose: http: internal: { port: 9001 } @@ -37,26 +33,22 @@ Each component declares **what it does** through these optional blocks: ### `run` — How to start it -Discriminated union on `runner`: +Discriminated union on `runner`. The runner encodes both the language/toolchain +(used by `castle sync`) and the deployment resolution (used by `castle deploy`): -| Runner | Use case | Key fields | -|--------|----------|------------| -| `python_uv_tool` | Python service/tool via uv | `tool`, `cwd`, `env` | -| `command` | Shell command | `argv`, `cwd`, `env` | -| `python_module` | Python -m invocation | `module`, `args`, `python` | -| `container` | Docker/Podman | `image`, `command`, `ports`, `volumes` | -| `node` | Node.js script | `script`, `package_manager` (npm/pnpm/yarn) | -| `remote` | External service | `base_url`, `health_url` | +| Runner | Sync | Deploy | Key fields | +|--------|------|--------|------------| +| `python` | `uv sync` | `which(tool)` → installed binary | `tool`, `args` | +| `command` | *(none)* | `which(argv[0])` → resolved path | `argv` | +| `container` | *(none)* | `podman run` | `image`, `command`, `ports`, `volumes` | +| `node` | `package_manager install` | `package_manager run script` | `script`, `package_manager` | +| `remote` | *(none)* | *(none — no local process)* | `base_url`, `health_url` | -**Services** use `python_uv_tool`: +**Services** use `python`: ```yaml run: - runner: python_uv_tool + runner: python tool: my-service # name in [project.scripts] - cwd: my-service # working directory relative to repo root - env: - MY_SERVICE_DATA_DIR: /data/castle/my-service - MY_SERVICE_PORT: "9001" ``` **Tools invoked by castle** (jobs, scheduled tasks) use `command`: @@ -298,7 +290,7 @@ and a `.timer` file. The Pydantic models live in `core/src/castle_core/manifest.py`. Key classes: - `ComponentManifest` — top-level model, has `roles` computed property -- `RunSpec` — discriminated union (RunPythonUvTool, RunCommand, etc.) +- `RunSpec` — discriminated union (RunPython, RunCommand, RunContainer, RunNode, RunRemote) - `TriggerSpec` — union (TriggerSchedule, TriggerManual, TriggerEvent, TriggerRequest) - `ExposeSpec`, `ProxySpec`, `ManageSpec`, `InstallSpec`, `ToolSpec`, `BuildSpec` - `CaddySpec`, `SystemdSpec`, `HttpExposeSpec`, `HttpInternal` diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..9b5db54 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,451 @@ +# Castle Design + +Castle is a personal software platform. It manages independent services, +tools, and frontends on a Linux machine using standard Unix primitives — +systemd for process supervision, Caddy for HTTP routing, the filesystem +for storage, and env vars for configuration. The `castle` CLI and API +provide a registry and coordination layer on top. + +The long-term goal: multiple Castle nodes (machines) that discover each +other and coordinate, forming a personal infrastructure mesh. Each node +is self-sufficient. The mesh is optional. + +## Principles + +1. **Unix-native.** Use the OS. systemd, journald, filesystem, signals, + env vars, DNS. Don't reimplement what Linux already provides. + +2. **Independence.** Components never depend on Castle. They accept + standard configuration (ports, data dirs, URLs) via env vars. A + Castle service is just a well-behaved Unix daemon that happens to + be registered in a manifest. + +3. **Declare capabilities, derive roles.** Components say what they + do (expose HTTP, run on a schedule, install to PATH). Castle infers + what they are (service, job, tool, frontend). No role labels. + +4. **Language-agnostic above the build line.** Below the build line, + every language is different (uv, pnpm, cargo, go). Above it, + everything is just processes, ports, files, and signals. Castle + operates above the line. + +5. **Separate source from runtime.** The repo is for development. The + runtime lives in standard Unix locations (`~/.castle/`, `/data/castle/`, + systemd units). Nothing running should point into the source tree. + +6. **AI-manageable.** The CLI and API exist so that AI assistants can + discover, create, and manage components programmatically. Humans + use the dashboard. Agents use the CLI and API. + +7. **Simple until proven otherwise.** Filesystem over databases. HTTP + over custom protocols. Shell commands over plugin systems. Add + complexity only when the simple thing actually fails. + +## Architecture Layers + +``` +┌─────────────────────────────────────────────┐ +│ Coordination │ +│ Node discovery, global registry, messaging │ +├─────────────────────────────────────────────┤ +│ Registry │ +│ Component spec, node config, CLI, API │ +├─────────────────────────────────────────────┤ +│ Runtime │ +│ systemd, Caddy, filesystem, journald │ +├─────────────────────────────────────────────┤ +│ Build │ +│ uv, pnpm, cargo, go build, etc. │ +└─────────────────────────────────────────────┘ +``` + +The critical boundary is between Build and Runtime. Below it, each +language has its own toolchain. Above it, everything is uniform — a +process that reads env vars, listens on a port, logs to stdout, and +responds to SIGTERM. + +### Build Layer + +Transforms source code into runnable artifacts. Castle does not abstract +over language toolchains — it just records the build commands and their +outputs. + +| Language | Toolchain | Artifact | +|----------|-----------|----------| +| Python | uv | Entry point in venv | +| Node/TS | pnpm | Static bundle (frontends) or node script | +| Rust | cargo | Binary | +| Go | go build | Binary | + +Castle's `build` spec is intentionally minimal: a list of shell commands +and a list of output paths. This works for any language without Castle +needing to understand the toolchain. + +For interpreted languages (Python, Node), Castle also needs to know the +runtime wrapper — how to invoke the artifact. This is what the `run` +spec's runner variants handle: + +- `python` — Python (sync via uv, deploy resolves installed binary) +- `node` — Node.js (sync via pnpm/npm) +- `command` — Direct execution (compiled binaries, shell scripts) +- `container` — Docker/Podman +- `remote` — External service (no local process) + +Compiled languages (Rust, Go) use `command` — once built, they're just +binaries. No Castle-specific runner needed. + +### Runtime Layer + +Manages running processes using standard Linux infrastructure. + +**systemd** handles process supervision: +- Start/stop/restart services +- Restart-on-failure policies (OTP's "let it crash") +- Dependency ordering via `After=` / `Wants=` +- Scheduled execution via `.timer` units +- Logging via journald (stdout/stderr capture) + +**Caddy** handles HTTP routing: +- Reverse proxy on port 9000 +- Path-based routing to services (`/api` → port 9020) +- Static file serving for frontends +- TLS termination + +**Filesystem** handles storage: +- Service data: `/data/castle//` +- Secrets: `~/.castle/secrets/` +- Generated config: `~/.castle/generated/` + +Castle generates systemd unit files and Caddyfile entries from the +registry. It doesn't run a daemon itself — it configures OS-level +infrastructure and gets out of the way. + +Critically, the runtime layer references only standard paths — never +the source tree. Systemd units point to installed binaries (on PATH +or in `~/.castle/bin/`), not to repo subdirectories. Caddy serves +from `~/.castle/static/`, not from build output directories in the repo. + +### Registry Layer + +The registry is the central concept in Castle. It tracks what components +exist, what they can do, and how they're configured. But it's not a +single thing — it's three distinct concepts: + +**1. Component spec** — what a component *is*. Description, capabilities, +build instructions, default configuration. This is source-level +information, version-controlled in the repo. It answers: "what components +could exist?" + +**2. Node config** — what's *deployed on this machine*, with what concrete +ports, data paths, and env vars. This is per-machine. Two Castle nodes +might run different subsets of components with different parameters. It +answers: "what's running here, and how?" + +**3. Runtime state** — what's *actually happening*. PIDs, health, uptime, +logs. This is ephemeral, owned by systemd and queried on demand. It +answers: "is it working?" + +#### Source vs. runtime split + +These map to two files: + +**`castle.yaml`** (in the repo, version-controlled) — Component specs: + +```yaml +components: + central-context: + description: Content storage API + source: components/central-context + run: + runner: python + tool: central-context + expose: + http: + internal: { port: 9001 } + health_path: /health + proxy: + caddy: + path_prefix: /central-context + manage: + systemd: {} +``` + +The spec says what the component *is* and what it *needs* — a port, a +data directory, HTTP exposure. Convention-based env vars (`_PORT`, +`_DATA_DIR`) are generated automatically during deploy. Only +non-convention values need `defaults.env`. + +**`~/.castle/registry.yaml`** (per-node, not in the repo) — Node config: + +```yaml +node: + hostname: tower + castle_root: /data/repos/castle + gateway_port: 9000 +deployed: + central-context: + runner: python + run_cmd: [/home/user/.local/bin/central-context] + env: + CENTRAL_CONTEXT_DATA_DIR: /data/castle/central-context + CENTRAL_CONTEXT_PORT: "9001" + roles: [service] + port: 9001 + health_path: /health + proxy_path: /central-context + managed: true +``` + +The node config says what's deployed *here* and with what concrete +values. `castle deploy` reads the spec from the repo, generates +convention-based env vars, resolves secrets, resolves binary paths, +and writes the registry. Systemd units and Caddyfile are then generated +from the registry — never from the spec directly. + +This separation means: +- The repo is just a repo. `git pull` doesn't affect running services. +- Multi-node works: sync the spec + deploy on each node, no repo needed. +- The spec is portable and version-controlled. The node config is local. +- AI agents read the node registry to know what's deployed and running. + +#### Interfaces + +Three interfaces expose the registry: + +- **CLI** (`castle`) — For AI agents and terminal users. Structured + output via `--json`. Commands for listing, inspecting, creating, + and managing components. +- **API** (`castle-api`) — For programmatic access over HTTP. Used by + the dashboard, other nodes, and remote agents. +- **Dashboard** (`castle-app`) — For human discoverability. Visual + overview of what's running, health status, logs. + +### Coordination Layer + +*Partially built. This section describes the target architecture.* + +Coordination handles discovery and communication — both between +components on a single node and across multiple Castle nodes. + +**Intra-node coordination** (current): +- Components find each other through the gateway (path-based routing) + or direct port access via env vars. +- The registry (CLI/API) provides discoverability. +- No service mesh or message broker required for basic operation. + +**Inter-node coordination** (future): +- Each Castle node runs the API, which exposes its component registry. +- Nodes discover each other via MQTT retained messages or mDNS/DNS-SD + (Avahi) for LAN environments. +- The gateway on each node can proxy to services on other nodes, + preserving path-based routing. Components don't know which node + they're talking to. +- MQTT provides pub/sub messaging for events, status, and coordination + across nodes. + +**Why MQTT over custom gossip:** +- Standard protocol, every language has a client library. +- Retained messages give new nodes an immediate view of the network. +- Topic-based routing maps naturally to `castle/{node}/{component}`. +- Works across networks (not just LAN like mDNS). +- Mosquitto is a single binary, simple to run as a Castle component. + +**Why mDNS/DNS-SD as a complement:** +- Zero-config LAN discovery via Avahi (already on most Linux systems). +- Each node advertises `_castle._tcp` — standard tooling works + (`avahi-browse`). +- Good for bootstrapping: find the MQTT broker without hardcoding + its address. + +## Component Contract + +Every Castle component, regardless of language, must satisfy a minimal +contract. This is what makes the system uniform above the build line. + +### Services (long-running daemons) + +| Requirement | Mechanism | +|-------------|-----------| +| Accept configuration | Env vars (prefixed by service name) | +| Declare its port | Env var, registered in `expose.http.internal.port` | +| Health endpoint | `GET /health` returns 200 | +| Data storage | Read `*_DATA_DIR` env var, write there | +| Logging | stdout for output, stderr for errors | +| Graceful shutdown | Handle SIGTERM, exit cleanly | +| Secrets | Read from env vars (Castle resolves `${secret:NAME}`) | +| No Castle dependency | Must run standalone with just env vars set | + +### Tools (CLI utilities) + +| Requirement | Mechanism | +|-------------|-----------| +| Input | File argument or stdin | +| Output | stdout (pipeable) | +| Errors/status | stderr | +| Exit codes | 0 success, non-zero failure | +| No interactive prompts | Scriptable by default | + +### Jobs (scheduled tasks) + +Same contract as tools, plus: + +| Requirement | Mechanism | +|-------------|-----------| +| Idempotent | Safe to re-run or run concurrently | +| Short-lived | Exit when done (oneshot systemd unit) | + +## Component Lifecycle + +The path from source to managed process: + +``` +source → [build] → artifact → [install] → available → [deploy] → managed +``` + +Each step is distinct: + +1. **Build** — Language-specific. Produces an artifact (binary, venv + entry point, static bundle). Castle records the commands but doesn't + execute them implicitly. + +2. **Install** — Makes the artifact available on the system. For tools: + `uv tool install` or compiled binary placed in `~/.castle/bin/`. For + services: same — the binary or entry point is on PATH or in a known + location. For frontends: built assets copied to `~/.castle/static/`. + +3. **Deploy** — Materializes the runtime configuration. Reads the + component spec, merges with node config, generates systemd units + and Caddyfile entries that reference *installed* artifacts — never + the source tree. Enables and starts services. + +For compiled languages (Rust, Go), build produces a standalone binary +and install is just placing it in `~/.castle/bin/`. For interpreted +languages (Python, Node), the runtime wrapper (uv, node) handles +finding the installed artifact. + +## Runtime Filesystem Layout + +What already exists and what the target looks like: + +``` +~/.castle/ ← Castle runtime home +├── registry.yaml ← Node config (what's deployed here) +├── generated/ ← Generated Caddyfile +│ └── Caddyfile +├── secrets/ ← Secret files (NAME → value) +│ └── PROTONMAIL_API_KEY +├── bin/ ← Compiled binaries, shims +│ └── my-go-tool +└── static/ ← Built frontend assets + └── castle-app/ + └── dist/ + +/data/castle/ ← Persistent service data +└── / + +~/.config/systemd/user/ ← Systemd units (standard location) +├── castle-central-context.service +├── castle-protonmail.service +├── castle-protonmail.timer +└── ... +``` + +Source (the repo) is referenced only during build and install. Everything +the runtime touches lives in `~/.castle/`, `/data/castle/`, or standard +systemd paths. + +## OTP as Design Guide + +Castle's architecture parallels Erlang/OTP, mapped onto Unix: + +| OTP Concept | Castle Equivalent | +|-------------|------------------| +| Application | Component (independent, self-contained) | +| Application resource file | Component spec in `castle.yaml` | +| Release config (sys.config) | Node config in `~/.castle/registry.yaml` | +| Release assembly | `castle deploy` (spec + node config → runtime) | +| Supervisor | systemd (restart policies, ordering) | +| Process | Running service/worker/job | +| Application env | Env vars | +| Node | A machine running Castle | +| epmd | mDNS / MQTT discovery | +| Distribution | Inter-node coordination via MQTT + gateway proxying | +| "Let it crash" | `restart: on-failure` in systemd | +| Global registry | Merged node registries via MQTT retained messages | + +The mapping is conceptual, not literal. Castle doesn't implement OTP +semantics — it uses OTP's *thinking* to guide which Unix primitives +to compose and how. + +Key OTP ideas that apply: +- **Isolation.** Components don't share state. Communication is + through explicit interfaces (HTTP, MQTT, filesystem paths). +- **Let it crash.** Services don't need elaborate error recovery. + systemd restarts them. Design for restartability, not immortality. +- **Supervision hierarchy.** systemd's dependency ordering provides + this. Services declare what they need to start after. +- **Location transparency.** Components talk to paths (`/api`, + `/central-context`), not to specific hosts or ports. The gateway + can remap these across nodes. +- **Spec vs. config.** In OTP, an application defines its structure + (the `.app` file) and a release provides the deployment config + (`sys.config`). Castle mirrors this: the component spec defines + structure, the node config provides deployment values. + +## Current State + +What exists today: + +- **CLI** — `castle` command, installed via `uv tool install --editable cli/` +- **Three packages** — `castle-core` (models, config, generators), + `castle-cli` (commands), `castle-api` (HTTP API) +- **Source/runtime split** — `castle.yaml` (spec) → `castle deploy` → + `~/.castle/registry.yaml` (node config). Systemd units and Caddyfile + generated from registry with fully resolved paths. No repo references + in runtime artifacts. +- **Convention-based env generation** — `castle deploy` auto-generates + `_DATA_DIR=/data/castle/` and `_PORT` from + the manifest. Only non-convention values need `defaults.env`. +- **Gateway** — Caddy on port 9000, Caddyfile generated from registry +- **API** — `castle-api` on port 9020, reads from registry (optional + castle.yaml fallback for non-deployed components) +- **Dashboard** — `castle-app` React/Vite frontend, static assets + served from `~/.castle/static/castle-app/` +- **Services** — central-context (content storage), notification-bridge + (desktop notification forwarder) +- **Jobs** — protonmail (email sync every 5 min), backup-collect (nightly), + backup-data (nightly restic backup) +- **Tools** — ~15 CLI utilities (pdf2md, docx2md, search, gpt, etc.) +- **Manifest** — `castle.yaml` with typed Pydantic models + +What doesn't exist yet: + +- **Multi-language support** — Rust and Go components (the abstractions + support them via `command` runner, but no examples exist yet) +- **Inter-node coordination** — MQTT broker, node discovery, cross-node + routing +- **Build automation** — Castle records build specs but doesn't + orchestrate builds (each project builds independently) + +## Technology Map + +| Concern | Technology | Status | +|---------|-----------|--------| +| Process supervision | systemd (user units) | Active | +| HTTP routing | Caddy (port 9000) | Active | +| Component specs | castle.yaml + Pydantic models | Active | +| Node config | `~/.castle/registry.yaml` | Active | +| CLI | castle (Python, uv) | Active | +| API | castle-api (FastAPI) | Active | +| Dashboard | castle-app (React, Vite, shadcn/ui) | Active | +| Python packaging | uv | Active | +| Node packaging | pnpm | Active | +| Linting | ruff (Python), ESLint (TS) | Active | +| Type checking | pyright (Python), tsc (TS) | Active | +| Testing | pytest (Python), Vitest (TS) | Active | +| Secrets | `~/.castle/secrets/` file-based | Active | +| Data storage | Filesystem (`/data/castle/`) | Active | +| Messaging | MQTT (Mosquitto) | Planned | +| Node discovery | mDNS/Avahi + MQTT | Planned | +| Rust packaging | cargo | Planned | +| Go packaging | go build | Planned | diff --git a/docs/web-apis.md b/docs/web-apis.md index 442137a..1ecd0b8 100644 --- a/docs/web-apis.md +++ b/docs/web-apis.md @@ -101,12 +101,10 @@ Castle passes config via env vars in castle.yaml: ```yaml components: my-service: + source: my-service run: - runner: python_uv_tool + runner: python tool: my-service - cwd: my-service - env: - MY_SERVICE_DATA_DIR: /data/castle/my-service expose: http: internal: { port: 9001 } @@ -117,6 +115,16 @@ components: systemd: {} ``` +Convention-based env vars (`MY_SERVICE_DATA_DIR`, `MY_SERVICE_PORT`) are +generated automatically by `castle deploy`. Only non-convention values +need `defaults.env`: + +```yaml + defaults: + env: + CENTRAL_CONTEXT_URL: http://localhost:9001 +``` + ## Application entry point ```python