feat: Add new tools and configurations for Castle platform
- Introduced `uv.lock` for dependency management with various packages including `pytest`, `colorama`, and `pluggy`. - Added `pyrightconfig.json` for Python type checking configuration. - Expanded `recommendations.md` with detailed scaling recommendations and project management strategies. - Created shared `ruff.toml` for consistent linting across projects. - Developed `Castle Tools` with various utilities including Android backup, browser automation, document conversion, and search tools. - Implemented `backup-collect` and `schedule` tools for system administration tasks. - Enhanced `search` functionality with indexing and querying capabilities using Tantivy. - Added comprehensive documentation for each tool, including usage examples and installation instructions.
This commit is contained in:
424
docs/web-apis.md
Normal file
424
docs/web-apis.md
Normal file
@@ -0,0 +1,424 @@
|
||||
# Web APIs in Castle
|
||||
|
||||
How to build Python web APIs as castle service components. Based on the
|
||||
patterns used in [wild-cloud/api](https://github.com/civilsociety-dev/wild-cloud)
|
||||
and existing castle services (central-context, notification-bridge, event-bus).
|
||||
|
||||
## Stack
|
||||
|
||||
| Layer | Choice |
|
||||
|-------|--------|
|
||||
| **Framework** | FastAPI |
|
||||
| **Server** | uvicorn |
|
||||
| **Config** | pydantic-settings (env vars) |
|
||||
| **Validation** | Pydantic models |
|
||||
| **HTTP client** | httpx (async) |
|
||||
| **Testing** | pytest + FastAPI TestClient |
|
||||
| **Python** | 3.13+ for services |
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
my-service/
|
||||
├── src/my_service/
|
||||
│ ├── __init__.py # Package version
|
||||
│ ├── main.py # FastAPI app, lifespan, entry point
|
||||
│ ├── config.py # pydantic-settings
|
||||
│ ├── models.py # Request/response Pydantic models
|
||||
│ ├── routes.py # APIRouter with endpoints
|
||||
│ └── storage.py # Domain logic (no FastAPI imports)
|
||||
├── tests/
|
||||
│ ├── conftest.py # Fixtures (client, temp dirs)
|
||||
│ ├── test_api.py # Endpoint integration tests
|
||||
│ └── test_storage.py # Domain unit tests
|
||||
├── pyproject.toml
|
||||
└── CLAUDE.md
|
||||
```
|
||||
|
||||
Separation of concerns: routes handle HTTP, storage/core handles logic,
|
||||
models define schemas. Domain code never imports FastAPI.
|
||||
|
||||
## pyproject.toml
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "my-service"
|
||||
version = "0.1.0"
|
||||
description = "Does something useful"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0",
|
||||
"uvicorn>=0.34.0",
|
||||
"pydantic-settings>=2.0.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
my-service = "my_service.main:run"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/my_service"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
"httpx>=0.28.0",
|
||||
]
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Use pydantic-settings with an env prefix matching the service name:
|
||||
|
||||
```python
|
||||
# config.py
|
||||
from pathlib import Path
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
data_dir: Path = Path("./data")
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 9001
|
||||
|
||||
model_config = {
|
||||
"env_prefix": "MY_SERVICE_",
|
||||
"env_file": ".env",
|
||||
}
|
||||
|
||||
def ensure_data_dir(self) -> None:
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
```
|
||||
|
||||
Castle passes config via env vars in castle.yaml:
|
||||
|
||||
```yaml
|
||||
components:
|
||||
my-service:
|
||||
run:
|
||||
runner: python_uv_tool
|
||||
tool: my-service
|
||||
cwd: my-service
|
||||
env:
|
||||
MY_SERVICE_DATA_DIR: /data/castle/my-service
|
||||
expose:
|
||||
http:
|
||||
internal: { port: 9001 }
|
||||
health_path: /health
|
||||
proxy:
|
||||
caddy: { path_prefix: /my-service }
|
||||
manage:
|
||||
systemd: {}
|
||||
```
|
||||
|
||||
## Application entry point
|
||||
|
||||
```python
|
||||
# main.py
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
|
||||
from my_service.config import settings
|
||||
from my_service.routes import router
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
settings.ensure_data_dir()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="my-service",
|
||||
description="Does something useful",
|
||||
version="0.1.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def run() -> None:
|
||||
uvicorn.run(
|
||||
"my_service.main:app",
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
reload=False,
|
||||
)
|
||||
```
|
||||
|
||||
For services with async resources (HTTP clients, connections), initialize
|
||||
them in the lifespan and clean up after yield:
|
||||
|
||||
```python
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
settings.ensure_data_dir()
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
app.state.http_client = client
|
||||
yield
|
||||
```
|
||||
|
||||
## Routes
|
||||
|
||||
Use APIRouter with a prefix and tags. Map domain exceptions to HTTP status codes.
|
||||
|
||||
```python
|
||||
# routes.py
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
from my_service.models import ItemCreate, ItemResponse
|
||||
from my_service.storage import (
|
||||
ItemExistsError,
|
||||
ItemNotFoundError,
|
||||
create_item,
|
||||
get_item,
|
||||
list_items,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/items", tags=["items"])
|
||||
|
||||
|
||||
@router.post("", response_model=ItemResponse, status_code=status.HTTP_201_CREATED)
|
||||
def create(request: ItemCreate) -> ItemResponse:
|
||||
try:
|
||||
return create_item(request)
|
||||
except ItemExistsError as e:
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{item_id}", response_model=ItemResponse)
|
||||
def get(item_id: str) -> ItemResponse:
|
||||
try:
|
||||
return get_item(item_id)
|
||||
except ItemNotFoundError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found")
|
||||
|
||||
|
||||
@router.get("", response_model=list[ItemResponse])
|
||||
def list_all() -> list[ItemResponse]:
|
||||
return list_items()
|
||||
```
|
||||
|
||||
## Request/response models
|
||||
|
||||
Separate create models (what the client sends) from response models (what
|
||||
comes back). Use inheritance to avoid repetition.
|
||||
|
||||
```python
|
||||
# models.py
|
||||
from datetime import datetime
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ItemCreate(BaseModel):
|
||||
name: str = Field(..., description="Item name")
|
||||
content: str = Field(..., description="Item content")
|
||||
description: str | None = Field(default=None)
|
||||
|
||||
|
||||
class ItemResponse(BaseModel):
|
||||
name: str
|
||||
description: str | None
|
||||
created_at: datetime
|
||||
size_bytes: int
|
||||
checksum: str
|
||||
|
||||
|
||||
class ItemWithBody(ItemResponse):
|
||||
content: str
|
||||
```
|
||||
|
||||
## Error handling
|
||||
|
||||
Define domain exceptions in the storage/core layer. Map them to HTTP status
|
||||
codes in the route layer.
|
||||
|
||||
```python
|
||||
# storage.py
|
||||
class StorageError(Exception):
|
||||
pass
|
||||
|
||||
class ItemExistsError(StorageError):
|
||||
pass
|
||||
|
||||
class ItemNotFoundError(StorageError):
|
||||
pass
|
||||
|
||||
class InvalidNameError(StorageError):
|
||||
pass
|
||||
```
|
||||
|
||||
Mapping convention:
|
||||
|
||||
| Exception | HTTP Status |
|
||||
|-----------|-------------|
|
||||
| `NotFoundError` | 404 |
|
||||
| `ExistsError` / conflict | 409 |
|
||||
| `InvalidError` / bad input | 400 |
|
||||
| Unexpected | 500 (FastAPI default) |
|
||||
|
||||
## Storage
|
||||
|
||||
Castle services use filesystem storage with JSON metadata sidecars:
|
||||
|
||||
```
|
||||
/data/castle/my-service/
|
||||
└── bucket/
|
||||
├── item-name
|
||||
└── item-name.meta.json
|
||||
```
|
||||
|
||||
```python
|
||||
# storage.py
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from my_service.config import settings
|
||||
from my_service.models import ItemCreate, ItemResponse
|
||||
|
||||
|
||||
def create_item(request: ItemCreate) -> ItemResponse:
|
||||
checksum = hashlib.sha256(request.content.encode()).hexdigest()
|
||||
path = settings.data_dir / request.name
|
||||
meta_path = path.with_suffix(path.suffix + ".meta.json")
|
||||
|
||||
if path.exists():
|
||||
raise ItemExistsError(f"'{request.name}' already exists")
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
now = datetime.now()
|
||||
|
||||
metadata = ItemResponse(
|
||||
name=request.name,
|
||||
description=request.description,
|
||||
created_at=now,
|
||||
size_bytes=len(request.content.encode()),
|
||||
checksum=checksum,
|
||||
)
|
||||
|
||||
path.write_text(request.content, encoding="utf-8")
|
||||
meta_path.write_text(
|
||||
json.dumps(metadata.model_dump(), default=str, indent=2)
|
||||
)
|
||||
return metadata
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Fixtures
|
||||
|
||||
```python
|
||||
# tests/conftest.py
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from my_service import config
|
||||
from my_service.main import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_data_dir(tmp_path: Path) -> Generator[Path, None, None]:
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir()
|
||||
original = config.settings.data_dir
|
||||
config.settings.data_dir = data_dir
|
||||
yield data_dir
|
||||
config.settings.data_dir = original
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(temp_data_dir: Path) -> Generator[TestClient, None, None]:
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
```
|
||||
|
||||
### Endpoint tests
|
||||
|
||||
```python
|
||||
# tests/test_api.py
|
||||
from fastapi import status
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
class TestHealth:
|
||||
def test_health(self, client: TestClient) -> None:
|
||||
response = client.get("/health")
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
|
||||
class TestCreateItem:
|
||||
def test_create(self, client: TestClient) -> None:
|
||||
response = client.post(
|
||||
"/items",
|
||||
json={"name": "test", "content": "hello"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
data = response.json()
|
||||
assert data["name"] == "test"
|
||||
assert "checksum" in data
|
||||
|
||||
def test_duplicate_returns_409(self, client: TestClient) -> None:
|
||||
payload = {"name": "dup", "content": "hello"}
|
||||
client.post("/items", json=payload)
|
||||
response = client.post("/items", json=payload)
|
||||
assert response.status_code == status.HTTP_409_CONFLICT
|
||||
```
|
||||
|
||||
### Domain unit tests
|
||||
|
||||
```python
|
||||
# tests/test_storage.py
|
||||
from pathlib import Path
|
||||
|
||||
from my_service.models import ItemCreate
|
||||
from my_service.storage import create_item
|
||||
|
||||
|
||||
class TestCreateItem:
|
||||
def test_creates_files(self, temp_data_dir: Path) -> None:
|
||||
request = ItemCreate(name="test", content="hello")
|
||||
metadata = create_item(request)
|
||||
|
||||
assert metadata.name == "test"
|
||||
assert (temp_data_dir / "test").exists()
|
||||
assert (temp_data_dir / "test.meta.json").exists()
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
uv sync # Install deps
|
||||
uv run my-service # Run service
|
||||
uv run pytest tests/ -v # Run tests
|
||||
uv run ruff check . # Lint
|
||||
uv run ruff format . # Format
|
||||
```
|
||||
|
||||
## Scaffolding
|
||||
|
||||
`castle create` generates all of this automatically:
|
||||
|
||||
```bash
|
||||
castle create my-service --type service --description "Does something useful"
|
||||
```
|
||||
Reference in New Issue
Block a user