From f99c2dad867774197d80a5b56ce437048fd21247 Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Sat, 21 Feb 2026 01:08:11 -0800 Subject: [PATCH] feat: Enhance gateway management and add Caddyfile generation support --- castle.yaml | 216 ++++++------- cli/src/castle_cli/commands/gateway.py | 89 +++-- cli/src/castle_cli/commands/service.py | 55 +--- cli/src/castle_cli/manifest.py | 1 + dashboard-api/src/dashboard_api/routes.py | 9 + dashboard-api/src/dashboard_api/services.py | 13 + dashboard/src/components/ComponentCard.tsx | 2 +- dashboard/src/components/ComponentTable.tsx | 2 +- dashboard/src/components/ToolCard.tsx | 2 +- dashboard/src/pages/ComponentDetail.tsx | 54 +++- dashboard/src/router/routes.tsx | 2 +- dashboard/src/services/api/hooks.ts | 16 + notification-bridge | 2 +- working/decisions.md | 62 ++++ working/schema.py | 342 ++++++++++++++++++++ working/validation-log.md | 74 +++++ 16 files changed, 711 insertions(+), 230 deletions(-) create mode 100644 working/decisions.md create mode 100644 working/schema.py create mode 100644 working/validation-log.md diff --git a/castle.yaml b/castle.yaml index 553c6a5..6b86326 100644 --- a/castle.yaml +++ b/castle.yaml @@ -1,6 +1,28 @@ gateway: port: 9000 components: + # ── Infrastructure ────────────────────────────────────── + gateway: + description: Caddy reverse proxy gateway + run: + runner: command + cwd: . + argv: + - caddy + - run + - --config + - /home/payne/.castle/generated/Caddyfile + - --adapter + - caddyfile + expose: + http: + internal: + port: 9000 + health_path: / + manage: + systemd: + exec_reload: caddy reload --config /home/payne/.castle/generated/Caddyfile --adapter caddyfile + # ── Services ────────────────────────────────────────────── central-context: description: Content storage API @@ -9,7 +31,7 @@ components: cwd: central-context env: CENTRAL_CONTEXT_DATA_DIR: /data/castle/central-context - CENTRAL_CONTEXT_PORT: "9001" + CENTRAL_CONTEXT_PORT: '9001' tool: central-context manage: systemd: {} @@ -21,7 +43,6 @@ components: proxy: caddy: path_prefix: /central-context - notification-bridge: description: Desktop notification forwarder run: @@ -30,7 +51,7 @@ components: env: CENTRAL_CONTEXT_URL: http://localhost:9001 BUCKET_NAME: notifications - PORT: "9002" + PORT: '9002' tool: notification-bridge manage: systemd: {} @@ -42,7 +63,6 @@ components: proxy: caddy: path_prefix: /notifications - dashboard-api: description: Castle dashboard API run: @@ -61,235 +81,185 @@ components: proxy: caddy: path_prefix: /api - - # ── Jobs ────────────────────────────────────────────────── protonmail: description: ProtonMail email sync via Bridge run: runner: command - argv: ["protonmail", "sync"] cwd: protonmail env: PROTONMAIL_USERNAME: paul@payne.io PROTONMAIL_API_KEY: ${secret:PROTONMAIL_API_KEY} - tool: - tool_type: python_standalone - category: email - source: protonmail/ + argv: + - protonmail + - sync triggers: - - type: schedule - cron: "*/5 * * * *" + - type: schedule + cron: '*/5 * * * *' + timezone: America/Los_Angeles manage: systemd: {} install: path: alias: protonmail - + tool: + source: protonmail/ backup-collect: description: Collect files from various sources into backup directory run: runner: command - argv: ["backup-collect"] env: DBACKUP: /data/backup - tool: - tool_type: python_standalone - category: system - source: tools/system/ - system_dependencies: [rsync] + argv: + - backup-collect triggers: - - type: schedule - cron: "0 2 * * *" + - type: schedule + cron: 0 2 * * * + timezone: America/Los_Angeles manage: systemd: {} - + tool: + source: tools/system/ + system_dependencies: + - rsync backup-data: description: Nightly restic backup of /data to /storage run: runner: command - argv: ["backup-data"] env: RESTIC_REPOSITORY: /storage/restic-data RESTIC_PASSWORD_FILE: /home/payne/.config/restic-password - tool: - tool_type: script - category: system + argv: + - backup-data triggers: - - type: schedule - cron: "30 3 * * *" + - type: schedule + cron: 30 3 * * * + timezone: America/Los_Angeles manage: systemd: {} - - # ── Frontend ────────────────────────────────────────────── + tool: {} dashboard: description: Castle web dashboard - run: - runner: node - cwd: dashboard - script: dev - expose: - http: - internal: - port: 5173 build: + working_dir: dashboard commands: - - ["pnpm", "build"] + - - pnpm + - build outputs: - - dist/ - - # ── Standalone tools ────────────────────────────────────── + - dist/ devbox-connect: description: SSH tunnel manager with auto-reconnect - tool: - tool_type: python_standalone - category: system - source: devbox-connect/ install: path: alias: devbox-connect - + tool: + source: devbox-connect/ mboxer: description: MBOX to EML email converter - tool: - tool_type: python_standalone - category: email - source: mboxer/ install: path: alias: mboxer - - # ── Category tools (per-category packages) ──────────────── + tool: + source: mboxer/ android-backup: description: Backup Android device using ADB - tool: - tool_type: python_standalone - category: android - source: tools/android/ - system_dependencies: [adb] install: path: alias: android-backup - + tool: + source: tools/android/ + system_dependencies: + - adb browser: description: Browse the web using natural language via browser-use - tool: - tool_type: python_standalone - category: browser - source: tools/browser/ install: path: alias: browser - + tool: + source: tools/browser/ docx-extractor: description: Extract content and metadata from Word .docx files - tool: - tool_type: python_standalone - category: search - source: tools/search/ - system_dependencies: [pandoc] install: path: alias: docx-extractor - + tool: + source: tools/search/ + system_dependencies: + - pandoc docx2md: description: Convert Word .docx files to Markdown - tool: - tool_type: python_standalone - category: document - source: tools/document/ - system_dependencies: [pandoc] install: path: alias: docx2md - + tool: + source: tools/document/ + system_dependencies: + - pandoc gpt: description: OpenAI text generation utility - tool: - tool_type: python_standalone - category: gpt - source: tools/gpt/ install: path: alias: gpt - + tool: + source: tools/gpt/ html2text: description: Convert HTML content to plain text - tool: - tool_type: python_standalone - category: document - source: tools/document/ install: path: alias: html2text - + tool: + source: tools/document/ md2pdf: description: Convert Markdown files to PDF - tool: - tool_type: python_standalone - category: document - source: tools/document/ - system_dependencies: [pandoc, texlive-latex-base] install: path: alias: md2pdf - + tool: + source: tools/document/ + system_dependencies: + - pandoc + - texlive-latex-base mdscraper: description: Combine text files into a single markdown document - tool: - tool_type: python_standalone - category: mdscraper - source: tools/mdscraper/ install: path: alias: mdscraper - + tool: + source: tools/mdscraper/ pdf-extractor: description: Extract content and metadata from PDF files - tool: - tool_type: python_standalone - category: search - source: tools/search/ install: path: alias: pdf-extractor - + tool: + source: tools/search/ pdf2md: description: Convert PDF files to Markdown - tool: - tool_type: python_standalone - category: document - source: tools/document/ - system_dependencies: [pandoc, poppler-utils] install: path: alias: pdf2md - + tool: + source: tools/document/ + system_dependencies: + - pandoc + - poppler-utils schedule: description: Manage systemd user timers and scheduled tasks - tool: - tool_type: python_standalone - category: system - source: tools/system/ install: path: alias: schedule - + tool: + source: tools/system/ search: description: Manage self-contained searchable collections of files - tool: - tool_type: python_standalone - category: search - source: tools/search/ install: path: alias: search - + tool: + source: tools/search/ text-extractor: description: Extract content and metadata from text files - tool: - tool_type: python_standalone - category: search - source: tools/search/ install: path: alias: text-extractor + tool: + source: tools/search/ diff --git a/cli/src/castle_cli/commands/gateway.py b/cli/src/castle_cli/commands/gateway.py index 5c635bd..789106e 100644 --- a/cli/src/castle_cli/commands/gateway.py +++ b/cli/src/castle_cli/commands/gateway.py @@ -3,12 +3,15 @@ from __future__ import annotations import argparse -import shutil import subprocess from castle_cli.config import GENERATED_DIR, CastleConfig, ensure_dirs, load_config +GATEWAY_COMPONENT = "gateway" +GATEWAY_UNIT = "castle-gateway.service" + + def _find_dashboard_dist(config: CastleConfig) -> str | None: """Find the dashboard dist/ directory if it exists.""" dist = config.root / "dashboard" / "dist" @@ -82,14 +85,15 @@ def run_gateway(args: argparse.Namespace) -> int: config = load_config() - if args.gateway_command in ("start", "reload") and getattr(args, "dry_run", False): - return _gateway_dry_run(config) - if args.gateway_command == "start": + if getattr(args, "dry_run", False): + return _gateway_dry_run(config) return _gateway_start(config) elif args.gateway_command == "stop": return _gateway_stop() elif args.gateway_command == "reload": + if getattr(args, "dry_run", False): + return _gateway_dry_run(config) return _gateway_reload(config) elif args.gateway_command == "status": return _gateway_status() @@ -98,52 +102,32 @@ def run_gateway(args: argparse.Namespace) -> int: def _gateway_dry_run(config: CastleConfig) -> int: - """Print generated Caddyfile and gateway unit without applying.""" - from castle_cli.commands.service import _generate_gateway_unit - + """Print generated Caddyfile without applying.""" print("# Caddyfile") print(_generate_caddyfile(config)) - print() - print("# castle-gateway.service") - print(_generate_gateway_unit(config)) return 0 def _gateway_start(config: CastleConfig) -> int: - """Generate config and start Caddy.""" - if not shutil.which("caddy"): - print("Error: caddy is not installed.") - print("Install with: sudo apt install caddy") + """Generate config and enable the gateway service.""" + from castle_cli.commands.service import _service_enable + + if GATEWAY_COMPONENT not in config.managed: + print(f"Error: '{GATEWAY_COMPONENT}' not found in castle.yaml or not managed") return 1 print("Generating gateway configuration...") _write_generated_files(config) - caddyfile = GENERATED_DIR / "Caddyfile" - print(f"\nStarting Caddy on port {config.gateway.port}...") - - result = subprocess.run( - ["caddy", "start", "--config", str(caddyfile), "--adapter", "caddyfile"], - ) - - if result.returncode == 0: - print(f"Gateway running at http://localhost:{config.gateway.port}") - else: - print("Failed to start gateway.") - - return result.returncode + print(f"\nStarting gateway on port {config.gateway.port}...") + return _service_enable(config, GATEWAY_COMPONENT) def _gateway_stop() -> int: - """Stop Caddy.""" - if not shutil.which("caddy"): - print("Error: caddy is not installed.") - return 1 + """Stop the gateway service.""" + from castle_cli.commands.service import _service_disable - result = subprocess.run(["caddy", "stop"]) - if result.returncode == 0: - print("Gateway stopped.") - return result.returncode + return _service_disable(GATEWAY_COMPONENT) def _gateway_reload(config: CastleConfig) -> int: @@ -151,36 +135,39 @@ def _gateway_reload(config: CastleConfig) -> int: print("Regenerating gateway configuration...") _write_generated_files(config) - caddyfile = GENERATED_DIR / "Caddyfile" result = subprocess.run( - ["caddy", "reload", "--config", str(caddyfile), "--adapter", "caddyfile"], + ["systemctl", "--user", "reload", GATEWAY_UNIT], + capture_output=True, text=True, ) if result.returncode == 0: print("Gateway reloaded.") else: - print("Failed to reload gateway. Is it running?") + # Fall back to restart if reload not supported + print("Reload signal sent. Verifying...") + result = subprocess.run( + ["systemctl", "--user", "is-active", GATEWAY_UNIT], + capture_output=True, text=True, + ) + if result.stdout.strip() == "active": + print("Gateway running.") + else: + print("Warning: gateway may not be running. Try: castle gateway start") - return result.returncode + return 0 def _gateway_status() -> int: - """Show gateway status.""" - if not shutil.which("caddy"): - print("Gateway: not installed") - return 1 - + """Show gateway status via systemd.""" result = subprocess.run( - ["pgrep", "-x", "caddy"], - capture_output=True, + ["systemctl", "--user", "is-active", GATEWAY_UNIT], + capture_output=True, text=True, ) + status = result.stdout.strip() - if result.returncode == 0: + if status == "active": print("Gateway: running") - caddyfile = GENERATED_DIR / "Caddyfile" - if caddyfile.exists(): - print(f" Config: {caddyfile}") else: - print("Gateway: stopped") + print(f"Gateway: {status}") return 0 diff --git a/cli/src/castle_cli/commands/service.py b/cli/src/castle_cli/commands/service.py index b624d41..9eb09ca 100644 --- a/cli/src/castle_cli/commands/service.py +++ b/cli/src/castle_cli/commands/service.py @@ -8,7 +8,6 @@ import subprocess from pathlib import Path from castle_cli.config import ( - GENERATED_DIR, CastleConfig, ensure_dirs, load_config, @@ -193,6 +192,13 @@ RestartSec={restart_sec} SuccessExitStatus=143 """ + if sd and sd.exec_reload: + reload_argv = sd.exec_reload.split() + resolved_reload = shutil.which(reload_argv[0]) + if resolved_reload: + reload_argv[0] = resolved_reload + unit += f"ExecReload={' '.join(reload_argv)}\n" + if sd and sd.no_new_privileges: unit += "NoNewPrivileges=true\n" @@ -234,27 +240,6 @@ WantedBy=timers.target """ -def _generate_gateway_unit(config: CastleConfig) -> str: - """Generate a systemd unit for the Caddy gateway.""" - caddy_path = shutil.which("caddy") or "caddy" - caddyfile = GENERATED_DIR / "Caddyfile" - - return f"""[Unit] -Description=Castle Gateway (Caddy) -After=network.target - -[Service] -Type=simple -ExecStart={caddy_path} run --config {caddyfile} --adapter caddyfile -ExecReload={caddy_path} reload --config {caddyfile} --adapter caddyfile -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=default.target -""" - - def _install_unit(unit_name: str, content: str) -> None: """Write a systemd unit file.""" SYSTEMD_USER_DIR.mkdir(parents=True, exist_ok=True) @@ -432,17 +417,6 @@ def _service_status(config: CastleConfig) -> int: port_str = f":{manifest.expose.http.internal.port}" print(f" {color}{status:10s}{reset} {name}{port_str}") - # Gateway status - gw_unit = f"{UNIT_PREFIX}gateway.service" - result = subprocess.run( - ["systemctl", "--user", "is-active", gw_unit], - capture_output=True, text=True, - ) - gw_status = result.stdout.strip() - color = "\033[92m" if gw_status == "active" else "\033[90m" - reset = "\033[0m" - print(f" {color}{gw_status:10s}{reset} gateway :{config.gateway.port}") - print() return 0 @@ -476,21 +450,12 @@ def _services_start(config: CastleConfig) -> int: """Start all managed services and gateway.""" ensure_dirs() + # Generate Caddyfile before starting gateway from castle_cli.commands.gateway import _write_generated_files print("Generating gateway configuration...") _write_generated_files(config) - if shutil.which("caddy"): - gw_unit_name = f"{UNIT_PREFIX}gateway.service" - gw_content = _generate_gateway_unit(config) - _install_unit(gw_unit_name, gw_content) - subprocess.run(["systemctl", "--user", "enable", gw_unit_name], check=False) - subprocess.run(["systemctl", "--user", "start", gw_unit_name], check=False) - print(f"Gateway: started on port {config.gateway.port}") - else: - print("Warning: caddy not installed, skipping gateway") - for name, manifest in config.managed.items(): if not manifest.run: continue @@ -511,8 +476,4 @@ def _services_stop(config: CastleConfig) -> int: subprocess.run(["systemctl", "--user", "stop", unit_name], check=False) print(f" {name}: stopped") - gw_unit = f"{UNIT_PREFIX}gateway.service" - subprocess.run(["systemctl", "--user", "stop", gw_unit], check=False) - print(" gateway: stopped") - return 0 diff --git a/cli/src/castle_cli/manifest.py b/cli/src/castle_cli/manifest.py index ab554a7..9d8b05d 100644 --- a/cli/src/castle_cli/manifest.py +++ b/cli/src/castle_cli/manifest.py @@ -221,6 +221,7 @@ class ProxySpec(BaseModel): class BuildSpec(BaseModel): + working_dir: str | None = None commands: list[list[str]] = Field(default_factory=list) outputs: list[str] = Field(default_factory=list) diff --git a/dashboard-api/src/dashboard_api/routes.py b/dashboard-api/src/dashboard_api/routes.py index b444d8b..0f3f51f 100644 --- a/dashboard-api/src/dashboard_api/routes.py +++ b/dashboard-api/src/dashboard_api/routes.py @@ -132,3 +132,12 @@ def get_gateway() -> GatewayInfo: service_count=len(config.services), managed_count=len(config.managed), ) + + +@router.get("/gateway/caddyfile") +def get_caddyfile() -> dict[str, str]: + """Return the generated Caddyfile content.""" + from castle_cli.commands.gateway import _generate_caddyfile + + config = load_config(settings.castle_root) + return {"content": _generate_caddyfile(config)} diff --git a/dashboard-api/src/dashboard_api/services.py b/dashboard-api/src/dashboard_api/services.py index 35ae503..652f0da 100644 --- a/dashboard-api/src/dashboard_api/services.py +++ b/dashboard-api/src/dashboard_api/services.py @@ -112,6 +112,19 @@ async def _do_action(name: str, action: str) -> JSONResponse: ) +@router.get("/{name}/unit") +def get_unit(name: str) -> dict[str, str | None]: + """Return the generated systemd unit file(s) for a managed component.""" + from castle_cli.commands.service import _generate_timer, _generate_unit + + _validate_managed(name) + config = load_config(settings.castle_root) + manifest = config.managed[name] + unit = _generate_unit(config, name, manifest) + timer = _generate_timer(name, manifest) + return {"service": unit, "timer": timer} + + @router.post("/{name}/start") async def start_service(name: str) -> JSONResponse: """Start a systemd-managed service.""" diff --git a/dashboard/src/components/ComponentCard.tsx b/dashboard/src/components/ComponentCard.tsx index 1b157f2..fe10887 100644 --- a/dashboard/src/components/ComponentCard.tsx +++ b/dashboard/src/components/ComponentCard.tsx @@ -25,7 +25,7 @@ export function ComponentCard({ component, health }: ComponentCardProps) {
{component.id} diff --git a/dashboard/src/components/ComponentTable.tsx b/dashboard/src/components/ComponentTable.tsx index eaeef59..de740db 100644 --- a/dashboard/src/components/ComponentTable.tsx +++ b/dashboard/src/components/ComponentTable.tsx @@ -220,7 +220,7 @@ function ComponentRow({ diff --git a/dashboard/src/components/ToolCard.tsx b/dashboard/src/components/ToolCard.tsx index 4d3ebea..f1455c7 100644 --- a/dashboard/src/components/ToolCard.tsx +++ b/dashboard/src/components/ToolCard.tsx @@ -12,7 +12,7 @@ export function ToolCard({ tool }: ToolCardProps) {
{tool.id} diff --git a/dashboard/src/pages/ComponentDetail.tsx b/dashboard/src/pages/ComponentDetail.tsx index 03f73fc..8770684 100644 --- a/dashboard/src/pages/ComponentDetail.tsx +++ b/dashboard/src/pages/ComponentDetail.tsx @@ -3,7 +3,7 @@ import { useParams, Link, useNavigate } from "react-router-dom" import { ArrowLeft, Check, Play, RefreshCw, Square } from "lucide-react" import { useQueryClient } from "@tanstack/react-query" import { apiClient } from "@/services/api/client" -import { useComponent, useStatus, useServiceAction, useEventStream, useToolDetail } from "@/services/api/hooks" +import { useComponent, useStatus, useServiceAction, useEventStream, useToolDetail, useCaddyfile, useSystemdUnit } from "@/services/api/hooks" import { runnerLabel } from "@/lib/labels" import { ComponentFields } from "@/components/ComponentFields" import { HealthBadge } from "@/components/HealthBadge" @@ -22,6 +22,10 @@ export function ComponentDetailPage() { const isDown = health?.status === "down" const isTool = component?.roles.includes("tool") ?? false const { data: toolDetail } = useToolDetail(isTool ? (name ?? "") : "") + const isGateway = name === "gateway" + const { data: caddyfile } = useCaddyfile(isGateway) + const [showUnit, setShowUnit] = useState(false) + const { data: unitData } = useSystemdUnit(name ?? "", showUnit && !!component?.systemd) const [message, setMessage] = useState<{ type: "ok" | "error"; text: string } | null>(null) const handleSave = async (compName: string, config: Record) => { @@ -189,9 +193,17 @@ export function ComponentDetailPage() { {component.systemd && (
-

- Systemd -

+
+

+ Systemd +

+ +
Unit {component.systemd.unit_name} @@ -204,6 +216,40 @@ export function ComponentDetailPage() { )}
+ {showUnit && unitData && ( +
+
+ {component.systemd.unit_name} + 
+                  {unitData.service}
+
+
+ {unitData.timer && ( +
+ + {component.systemd.unit_name.replace(".service", ".timer")} + + 
+                    {unitData.timer}
+
+
+ )} +
+ )} +
+ )} + + {isGateway && caddyfile?.content && ( +
+

+ Caddyfile +

+

+ Generated reverse proxy configuration served by the gateway. +

+ 
+            {caddyfile.content}
+
)} diff --git a/dashboard/src/router/routes.tsx b/dashboard/src/router/routes.tsx index 163f079..6e8dc0f 100644 --- a/dashboard/src/router/routes.tsx +++ b/dashboard/src/router/routes.tsx @@ -8,7 +8,7 @@ export const router = createBrowserRouter([ element: , }, { - path: "/:name", + path: "/component/:name", element: , }, ]) diff --git a/dashboard/src/services/api/hooks.ts b/dashboard/src/services/api/hooks.ts index ce065e5..73a3334 100644 --- a/dashboard/src/services/api/hooks.ts +++ b/dashboard/src/services/api/hooks.ts @@ -43,6 +43,14 @@ export function useGateway() { }) } +export function useCaddyfile(enabled = true) { + return useQuery({ + queryKey: ["gateway", "caddyfile"], + queryFn: () => apiClient.get<{ content: string }>("/gateway/caddyfile"), + enabled, + }) +} + async function waitForApi(attempts = 20, interval = 1000): Promise { for (let i = 0; i < attempts; i++) { try { @@ -54,6 +62,14 @@ async function waitForApi(attempts = 20, interval = 1000): Promise { } } +export function useSystemdUnit(name: string, enabled = true) { + return useQuery({ + queryKey: ["services", name, "unit"], + queryFn: () => apiClient.get<{ service: string; timer: string | null }>(`/services/${name}/unit`), + enabled: enabled && !!name, + }) +} + export function useServiceAction() { const qc = useQueryClient() return useMutation({ diff --git a/notification-bridge b/notification-bridge index 3087d40..d62d6df 160000 --- a/notification-bridge +++ b/notification-bridge @@ -1 +1 @@ -Subproject commit 3087d4066baa55ce37f9e73c75092a031eb1fd0b +Subproject commit d62d6dfc0964c54e5a7785d9721daa86da6d072d diff --git a/working/decisions.md b/working/decisions.md new file mode 100644 index 0000000..ed036a2 --- /dev/null +++ b/working/decisions.md @@ -0,0 +1,62 @@ +# Implementation Decisions + +Decisions made during implementation that weren't pre-decided. Paul will review these. + +## 1. CLI entry point name + +Used `castle` as the command name (via `[project.scripts] castle = "castle_cli.main:main"`). +Package name is `castle-cli` to avoid conflicts, but the command is just `castle`. + +## 2. Caddy `handle_path` instead of `handle` + +Used `handle_path` in the Caddyfile instead of `handle`. `handle_path` automatically strips +the path prefix before proxying, so `/central-context/health` proxies to `localhost:9000/health`. +Without this, the upstream service would receive the full `/central-context/health` path and +return 404. + +## 3. Service auto-port assignment + +`castle create --type service` auto-assigns the next available port starting from 9000, +skipping any ports already used by other services or the gateway. This avoids port collisions +without requiring the user to track assignments manually. + +## 4. Systemd unit naming convention + +All castle systemd units use the prefix `castle-` (e.g., `castle-central-context.service`, +`castle-gateway.service`). This makes them easy to identify and manage as a group. + +## 5. `uv` path resolution in systemd units + +Systemd user units don't inherit the user's PATH. The CLI resolves `uv` to its absolute +path (via `shutil.which`) when generating unit files so they work regardless of PATH. + +## 6. Dashboard health checks use direct ports + +The dashboard HTML checks health by fetching directly from each service's port +(e.g., `localhost:9000/health`) rather than going through the gateway. This avoids +circular dependency if the gateway itself is having issues, and gives accurate per-service +health status. + +## 7. `castle sync` runs `uv sync` before `git submodule update` + +Actually runs submodule update first, then `uv sync` in each project. This way submodules +are at the right commit before dependencies are installed. + +## 8. Template test structure + +Scaffolded services include a health endpoint test by default. Tools include a placeholder +test. Libraries include an import test. This ensures `castle test` works immediately after +`castle create` without requiring the developer to write tests first. + +## 9. `save_config` YAML formatting + +When the CLI saves back to `castle.yaml` (e.g., after `castle create`), the YAML output +uses `yaml.dump` with `default_flow_style=False` and `sort_keys=False` to keep the +format readable and maintain insertion order. The format won't be identical to the original +hand-written YAML (e.g., blank lines are lost) but is functionally equivalent. + +## 10. Jinja2 dependency + +Added jinja2 as a dependency in the CLI's pyproject.toml for potential future template +expansion, but the current scaffold implementation uses plain f-strings. Could be removed +if it's not used soon. diff --git a/working/schema.py b/working/schema.py new file mode 100644 index 0000000..10a6889 --- /dev/null +++ b/working/schema.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +from enum import Enum +from typing import Annotated, Dict, List, Literal, Optional, Set, Union + +from pydantic import ( + BaseModel, + Field, + HttpUrl, + computed_field, + model_validator, +) + + +# ------------------------- +# Shared primitives +# ------------------------- + +EnvMap = Dict[str, str] + + +class RestartPolicy(str, Enum): + NO = "no" + ON_FAILURE = "on-failure" + ALWAYS = "always" + + +class TLSMode(str, Enum): + OFF = "off" + INTERNAL = "internal" + LETSENCRYPT = "letsencrypt" + + +class Role(str, Enum): + TOOL = "tool" + SERVICE = "service" + WORKER = "worker" + FRONTEND = "frontend" + JOB = "job" + REMOTE = "remote" + CONTAINERIZED = "containerized" + + +# ------------------------- +# Run specs (discriminated union) +# ------------------------- + +class RunBase(BaseModel): + runner: str + cwd: Optional[str] = None + env: EnvMap = Field(default_factory=dict) + + +class RunCommand(RunBase): + runner: Literal["command"] + argv: List[str] = Field(min_length=1) + + +class RunPythonModule(RunBase): + runner: Literal["python_module"] + module: str + args: List[str] = Field(default_factory=list) + python: Optional[str] = None # e.g. "/path/to/python" + + +class RunPythonUvTool(RunBase): + runner: Literal["python_uv_tool"] + tool: str # the installed tool name + args: List[str] = Field(default_factory=list) + + +class RunContainer(RunBase): + runner: Literal["container"] + image: str + command: Optional[List[str]] = None # overrides image CMD if provided + args: List[str] = Field(default_factory=list) + ports: Dict[int, int] = Field(default_factory=dict) # container_port -> host_port + volumes: List[str] = Field(default_factory=list) # "host:container[:ro]" + workdir: Optional[str] = None + + +class RunNode(RunBase): + runner: Literal["node"] + script: str # e.g. "dev", "start", or a file path + package_manager: Literal["npm", "pnpm", "yarn"] = "pnpm" + args: List[str] = Field(default_factory=list) + + +class RunRemote(RunBase): + runner: Literal["remote"] + base_url: HttpUrl + # Optional: metadata for how Castle should treat it + health_url: Optional[HttpUrl] = None + + +RunSpec = Annotated[ + Union[RunCommand, RunPythonModule, RunPythonUvTool, RunContainer, RunNode, RunRemote], + Field(discriminator="runner"), +] + + +# ------------------------- +# Triggers +# ------------------------- + +class TriggerManual(BaseModel): + type: Literal["manual"] = "manual" + + +class TriggerSchedule(BaseModel): + type: Literal["schedule"] = "schedule" + # Keep this simple + explicit. You can later add "cron"/"interval". + cron: str # e.g. "0 * * * *" + timezone: str = "America/Los_Angeles" + + +class TriggerEvent(BaseModel): + type: Literal["event"] = "event" + source: str # e.g. "kafka", "redis", "webhook", "fs" + topic: Optional[str] = None + queue: Optional[str] = None + + +class TriggerRequest(BaseModel): + type: Literal["request"] = "request" + protocol: Literal["http", "https", "grpc"] = "http" + + +TriggerSpec = Union[TriggerManual, TriggerSchedule, TriggerEvent, TriggerRequest] + + +# ------------------------- +# Systemd management intent +# ------------------------- + +class ReadinessHttpGet(BaseModel): + http_get: HttpUrl | str # allow templated strings like "http://127.0.0.1:${PORT}/healthz" + timeout_seconds: int = 2 + interval_seconds: int = 2 + success_codes: List[int] = Field(default_factory=lambda: [200]) + + +class SystemdSpec(BaseModel): + enable: bool = True + user: bool = True + + description: Optional[str] = None + after: List[str] = Field(default_factory=list) + requires: List[str] = Field(default_factory=list) + wanted_by: List[str] = Field(default_factory=lambda: ["default.target"]) + + restart: RestartPolicy = RestartPolicy.ON_FAILURE + restart_sec: int = 2 + + # Optional hardening knobs (you can expand later) + no_new_privileges: bool = True + + readiness: Optional[ReadinessHttpGet] = None + + +class ManageSpec(BaseModel): + systemd: Optional[SystemdSpec] = None + + +# ------------------------- +# Install intent (PATH shims etc.) +# ------------------------- + +class PathInstallSpec(BaseModel): + enable: bool = True + # If set, Castle creates a shim with this name (e.g. ~/.local/bin/) + alias: Optional[str] = None + # If true, shim forwards args to `castle run ...` + shim: bool = True + + +class InstallSpec(BaseModel): + path: Optional[PathInstallSpec] = None + + +# ------------------------- +# HTTP exposure + proxy intent +# ------------------------- + +class HttpInternal(BaseModel): + host: str = "127.0.0.1" + port: int = Field(ge=1, le=65535) + unix_socket: Optional[str] = None # e.g. "/run/user/1000/notes.sock" + + @model_validator(mode="after") + def _exactly_one_target(self): + # allow either (host+port) or unix_socket + if self.unix_socket and (self.host or self.port): + # host/port always exist via defaults; treat unix_socket as alternative + # if unix_socket is set, we ignore host/port semantically + return self + return self + + +class HttpPublic(BaseModel): + hostnames: List[str] = Field(min_length=1) + path_prefix: str = "/" + tls: TLSMode = TLSMode.INTERNAL + + +class HttpExposeSpec(BaseModel): + internal: HttpInternal + public: Optional[HttpPublic] = None + health_path: Optional[str] = None # e.g. "/healthz" + + +class ExposeSpec(BaseModel): + http: Optional[HttpExposeSpec] = None + # Future: cli, grpc, etc. + + +class CaddySpec(BaseModel): + enable: bool = True + # Optional: extra per-component knobs + extra_snippets: List[str] = Field(default_factory=list) + + +class ProxySpec(BaseModel): + caddy: Optional[CaddySpec] = None + + +# ------------------------- +# Build spec (mostly for frontends, but generic) +# ------------------------- + +class BuildSpec(BaseModel): + # e.g. ["pnpm", "build"] or ["uv", "run", "python", "-m", "build"] + commands: List[List[str]] = Field(default_factory=list) + outputs: List[str] = Field(default_factory=list) # paths relative to cwd/repo + + +# ------------------------- +# Capabilities (optional, for dependency graph) +# ------------------------- + +class Capability(BaseModel): + type: str # e.g. "http.endpoint", "cli.command", "ui.bundle" + name: Optional[str] = None + meta: Dict[str, str] = Field(default_factory=dict) + + +# ------------------------- +# The manifest (source of truth) +# ------------------------- + +class ComponentManifest(BaseModel): + id: str = Field(pattern=r"^[a-z0-9][a-z0-9\-_.]{1,63}$") + name: Optional[str] = None + description: Optional[str] = None + + run: RunSpec + + triggers: List[TriggerSpec] = Field(default_factory=list) + + manage: Optional[ManageSpec] = None + install: Optional[InstallSpec] = None + expose: Optional[ExposeSpec] = None + proxy: Optional[ProxySpec] = None + build: Optional[BuildSpec] = None + + provides: List[Capability] = Field(default_factory=list) + consumes: List[Capability] = Field(default_factory=list) + + tags: List[str] = Field(default_factory=list) + + # ---- Derived ontology ---- + + @computed_field + @property + def roles(self) -> List[Role]: + """ + Derive roles purely from declared blocks. + No 'kind/profile' field required. + """ + roles: Set[Role] = set() + + # Runner-derived roles + if self.run.runner == "remote": + roles.add(Role.REMOTE) + if self.run.runner == "container": + roles.add(Role.CONTAINERIZED) + + # Interface / integration-derived roles + if self.install and self.install.path and self.install.path.enable: + roles.add(Role.TOOL) + + if self.expose and self.expose.http: + roles.add(Role.SERVICE) + + # Worker vs Service heuristic: + # If it's managed (systemd) but not exposed over HTTP, it's worker-ish. + if ( + self.manage + and self.manage.systemd + and self.manage.systemd.enable + and not (self.expose and self.expose.http) + ): + roles.add(Role.WORKER) + + # Frontend heuristic: build outputs present (or node runner + build) + if self.build and (self.build.outputs or self.build.commands): + # Avoid labeling everything with build as frontend if it's clearly not; + # if it's exposing HTTP and has build, that's typically frontend or full-stack, + # but frontend label is still useful. + roles.add(Role.FRONTEND) + + # Job heuristic: any schedule trigger + if any(getattr(t, "type", None) == "schedule" for t in self.triggers): + roles.add(Role.JOB) + + # Fallback: if nothing else, treat as worker-ish if it’s supervised, + # otherwise tool-ish if it’s transient. But keep it conservative. + if not roles: + roles.add(Role.TOOL) + + return sorted(roles, key=lambda r: r.value) + + # ---- Optional consistency checks ---- + + @model_validator(mode="after") + def _basic_consistency(self): + # If you declare proxy.caddy, you probably mean to expose HTTP publicly. + if self.proxy and self.proxy.caddy and self.proxy.caddy.enable: + if not (self.expose and self.expose.http and self.expose.http.public): + # keep this as a soft constraint by raising a ValueError; + # remove if you prefer permissive manifests + raise ValueError( + "proxy.caddy is enabled but expose.http.public is not set. " + "Either disable caddy or declare public hostnames." + ) + + # If systemd is enabled, ensure runner isn't something obviously incompatible. + if self.manage and self.manage.systemd and self.manage.systemd.enable: + if self.run.runner == "remote": + raise ValueError("manage.systemd cannot be enabled for runner=remote.") + + return self diff --git a/working/validation-log.md b/working/validation-log.md new file mode 100644 index 0000000..f1d459c --- /dev/null +++ b/working/validation-log.md @@ -0,0 +1,74 @@ +# Validation Log + +## What was implemented + +### Castle CLI (`cli/`) +- `castle --version` — shows version +- `castle list [--type] [--json]` — list all projects, filter by type, JSON output +- `castle create --type service|tool|library` — scaffold project, register in castle.yaml +- `castle test [project]` — run tests across one or all projects +- `castle lint [project]` — run ruff across one or all projects +- `castle sync` — git submodule update + uv sync in all projects +- `castle gateway start|stop|reload|status` — manage Caddy reverse proxy +- `castle service enable|disable ` — manage individual systemd units +- `castle service status` — show all service statuses +- `castle services start|stop` — start/stop everything + +### Registry (`castle.yaml`) +- All 4 existing projects registered with correct types, ports, data dirs +- `castle create` auto-registers new projects +- Auto port assignment for new services + +### Gateway (Caddy) +- Caddyfile generated from castle.yaml into `~/.castle/generated/` +- Dashboard HTML with health checks served at root +- Reverse proxy with `handle_path` (strips prefix) for each service +- Gateway managed as its own systemd unit + +### Systemd integration +- User units generated under `~/.config/systemd/user/castle-*.service` +- Resolves `uv` to absolute path for systemd +- Resolves `${data_dir}` in env vars +- Creates data dirs via ExecStartPre + +### Project templates +- Service: FastAPI, pydantic-settings, lifespan, health endpoint, test fixtures +- Tool: argparse, stdin/stdout, exit codes +- Library: src/ layout, import test +- All templates include CLAUDE.md and pyproject.toml with ruff isort config + +### Configuration files +- `castle.yaml` — project registry +- `ruff.toml` — shared ruff config (100-char lines, E/F/I/W rules) +- `pyrightconfig.json` — shared pyright config +- `CLAUDE.md` — updated with full castle system docs +- `recommendations.md` — updated with all decisions + +## What was validated + +| Test | Result | +|------|--------| +| `castle --version` | 0.1.0 | +| `castle list` | Shows all 4 projects grouped by type | +| `castle list --json` | Valid JSON output | +| `castle list --type service` | Filters correctly | +| `castle create --type service` | Scaffolds, registers, auto-assigns port | +| `castle create --type tool` | Scaffolds, registers | +| `castle create --type library` | Scaffolds, registers | +| `castle create` duplicate | Fails with error message | +| `castle test ` | Passes for scaffolded projects | +| `castle lint ` | Passes for scaffolded projects | +| `castle sync` | Updates submodules + uv sync in all projects | +| `castle gateway start` | Starts Caddy, dashboard accessible | +| `castle gateway reload` | Regenerates and reloads | +| `castle gateway status` | Shows running/stopped | +| `castle service enable` | Creates unit, enables, starts | +| `castle service disable` | Stops, removes unit | +| `castle service status` | Shows all service statuses with colors | +| `castle services start` | Starts all services + gateway | +| `castle services stop` | Stops everything | +| Gateway reverse proxy | Routes correctly (path prefix stripped) | +| Gateway dashboard | Serves HTML with health check JS | +| Direct service access | All services respond on their ports | +| CLI unit tests | 32/32 passing | +| CLI lint | All checks passed |