Rename Castle -> Wild PC across the repo
Repo-side rename only (Phases 1-3 of the migration plan); the live box (~/.castle, systemd units, /data/castle, domains) is a separate cutover. - Slug `castle` -> `wildpc`: CLI command, module names (wildpc_core/cli/api), dist names, entry point `wildpc = wildpc_cli.main:main`. - Identifiers: CastleConfig/NATSClient/DirError/MDNS -> Wildpc*. - Env/constants: CASTLE_* -> WILDPC_*; ~/.castle -> ~/.wildpc, castle.yaml -> wildpc.yaml, /data/castle -> /data/wildpc. - Systemd UNIT_PREFIX castle- -> wildpc-; own programs castle-api/gateway/etc. - Display prose "Castle" -> "Wild PC" in docs, agent-guide files, README, frontend. - Package dirs and bootstrap yaml renamed via git mv; lockfiles regenerated; redundant nested uv.lock files dropped (workspace root lock is authoritative). Tests: core 273, cli 47, wildpc-api 120 all pass. Frontend type-checks + builds. Fixed a stale test fixture (secret_env_path kind arg) broken pre-rename.
This commit is contained in:
3
core/src/wildpc_core/__init__.py
Normal file
3
core/src/wildpc_core/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""Wild PC core library - manifest models, configuration, and generators."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
157
core/src/wildpc_core/adopt.py
Normal file
157
core/src/wildpc_core/adopt.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Adopt an existing repo as a program — shared by the CLI (`wildpc program add`)
|
||||
and the API (`POST /programs/adopt`).
|
||||
|
||||
`create` scaffolds new code from a stack; `add` adopts code that already exists —
|
||||
a local path, or a git URL to clone later. Keeping the target parsing, stack /
|
||||
command sniffing, and ProgramSpec build here means both front-ends behave
|
||||
identically (the logic used to live only in the CLI).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import WildpcConfig
|
||||
from wildpc_core.manifest import BuildSpec, CommandsSpec, ProgramSpec
|
||||
|
||||
|
||||
class AdoptError(ValueError):
|
||||
"""A program can't be adopted (bad path, or the name already exists)."""
|
||||
|
||||
|
||||
def is_git_url(s: str) -> bool:
|
||||
return s.startswith(("http://", "https://", "git@", "ssh://")) or s.endswith(".git")
|
||||
|
||||
|
||||
def looks_like_program(src: Path) -> bool:
|
||||
"""Whether a directory holds something wildpc can adopt (a project manifest or
|
||||
a git repo). Used to flag candidates in the filesystem browser."""
|
||||
return (
|
||||
(src / ".git").exists()
|
||||
or (src / "pyproject.toml").exists()
|
||||
or (src / "Cargo.toml").exists()
|
||||
or (src / "package.json").exists()
|
||||
or (src / "Makefile").exists()
|
||||
or (src / "makefile").exists()
|
||||
)
|
||||
|
||||
|
||||
def detect_stack_commands(src: Path) -> tuple[str | None, dict[str, list[list[str]]]]:
|
||||
"""Detect (stack, commands) for a source dir.
|
||||
|
||||
Returns a stack name when the project fits a known one (so it inherits those
|
||||
defaults), otherwise an explicit commands map. Adoption takes source only; the
|
||||
deployment (and thus kind) is declared separately, so no kind is inferred here.
|
||||
"""
|
||||
commands: dict[str, list[list[str]]] = {}
|
||||
|
||||
pyproject = src / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
try:
|
||||
data = tomllib.loads(pyproject.read_text())
|
||||
except (OSError, tomllib.TOMLDecodeError):
|
||||
data = {}
|
||||
deps = " ".join(data.get("project", {}).get("dependencies", []))
|
||||
stack = "python-fastapi" if ("fastapi" in deps or "uvicorn" in deps) else "python-cli"
|
||||
return stack, commands
|
||||
|
||||
if (src / "Cargo.toml").exists():
|
||||
commands = {
|
||||
"build": [["cargo", "build", "--release"]],
|
||||
"test": [["cargo", "test"]],
|
||||
"lint": [["cargo", "clippy"]],
|
||||
"run": [["cargo", "run"]],
|
||||
}
|
||||
return None, commands
|
||||
|
||||
if (src / "package.json").exists():
|
||||
commands = {
|
||||
"build": [["pnpm", "build"]],
|
||||
"test": [["pnpm", "test"]],
|
||||
"lint": [["pnpm", "lint"]],
|
||||
}
|
||||
return None, commands
|
||||
|
||||
if (src / "Makefile").exists() or (src / "makefile").exists():
|
||||
commands = {"build": [["make"]], "test": [["make", "test"]]}
|
||||
return None, commands
|
||||
|
||||
return None, commands
|
||||
|
||||
|
||||
@dataclass
|
||||
class Adopted:
|
||||
"""The result of building a program spec from an adopt target — the caller
|
||||
persists it (``config.programs[name] = spec``; then save/write)."""
|
||||
|
||||
name: str
|
||||
spec: ProgramSpec
|
||||
source: str
|
||||
stack: str | None = None
|
||||
repo: str | None = None
|
||||
commands: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def build_adopted_program(
|
||||
config: WildpcConfig,
|
||||
target: str,
|
||||
name: str | None = None,
|
||||
description: str = "",
|
||||
) -> Adopted:
|
||||
"""Build (but do not save) a ProgramSpec adopting ``target``.
|
||||
|
||||
``target`` is a local path or a git URL. Raises :class:`AdoptError` if a local
|
||||
path doesn't exist or the resolved name already exists in the config. Does not
|
||||
mutate ``config`` — the caller assigns ``config.programs[name]`` and persists.
|
||||
"""
|
||||
repo_url: str | None = None
|
||||
|
||||
if is_git_url(target):
|
||||
repo_url = target
|
||||
name = name or Path(target.rstrip("/")).name.removesuffix(".git")
|
||||
# Default local clone location; cloned later via `wildpc clone`.
|
||||
source = str(config.repos_dir / name)
|
||||
src_path = Path(source)
|
||||
else:
|
||||
src_path = Path(target).expanduser().resolve()
|
||||
if not src_path.exists():
|
||||
raise AdoptError(f"path does not exist: {src_path}")
|
||||
source = str(src_path)
|
||||
name = name or src_path.name
|
||||
|
||||
if name in config.programs or config.deployments_named(name):
|
||||
raise AdoptError(f"'{name}' already exists in wildpc.yaml")
|
||||
|
||||
# Detect verbs from the working copy if we have one on disk. `kind` is derived
|
||||
# from a deployment, not stored on the program — so adoption takes source only;
|
||||
# declare a deployment separately (wildpc service/job create).
|
||||
stack: str | None = None
|
||||
detected: dict[str, list[list[str]]] = {}
|
||||
if src_path.exists():
|
||||
stack, detected = detect_stack_commands(src_path)
|
||||
|
||||
spec = ProgramSpec(
|
||||
id=name,
|
||||
description=description or f"Adopted from {target}",
|
||||
source=source,
|
||||
stack=stack,
|
||||
repo=repo_url,
|
||||
)
|
||||
# `build` is declared via BuildSpec; every other verb via CommandsSpec.
|
||||
if detected:
|
||||
build_cmds = detected.pop("build", None)
|
||||
if build_cmds:
|
||||
spec.build = BuildSpec(commands=build_cmds)
|
||||
if detected:
|
||||
spec.commands = CommandsSpec.model_validate(detected)
|
||||
|
||||
return Adopted(
|
||||
name=name,
|
||||
spec=spec,
|
||||
source=source,
|
||||
stack=stack,
|
||||
repo=repo_url,
|
||||
commands=sorted(detected),
|
||||
)
|
||||
117
core/src/wildpc_core/audit.py
Normal file
117
core/src/wildpc_core/audit.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""Consumption audit — *suggests* undeclared ``requires`` by matching a
|
||||
deployment's env endpoint values against known provider sockets.
|
||||
|
||||
This is the one place wildpc looks at env → dependency, and it does so **only to
|
||||
propose a declaration the user confirms** — never to write, and never to feed the
|
||||
graph or ``functional?``. The relationship graph stays strictly declaration-derived
|
||||
(see docs/relationships.md, "Env is derived *from* requires, never scraped *into*
|
||||
it"); this module is an explicit, opt-in lint that sits *on top* of it. A suggestion
|
||||
is accepted by writing a real ``requires`` (a declaration), at which point it stops
|
||||
being a suggestion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from wildpc_core.config import WildpcConfig
|
||||
from wildpc_core.relations import build_model
|
||||
|
||||
# Hosts that mean "a provider on this node" — a port match against them is a strong
|
||||
# signal (ports are unique per host). 172.17.0.1 is the docker bridge to the host.
|
||||
_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1", "172.17.0.1"}
|
||||
|
||||
# scheme://host:port OR host:port — anywhere inside an env value.
|
||||
_HOSTPORT = re.compile(r"(?:(?P<scheme>[a-z][\w+.-]*)://)?(?P<host>[\w.-]+):(?P<port>\d{2,5})")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Suggestion:
|
||||
consumer: str # the deployment whose env references the endpoint
|
||||
provider: str # the provider deployment the port resolves to
|
||||
env_var: str # the env var that revealed it
|
||||
endpoint: str # host:port as seen in the value
|
||||
protocol: str # the provider's socket protocol
|
||||
|
||||
|
||||
def _resolve(
|
||||
host: str,
|
||||
port: int,
|
||||
*,
|
||||
consumer: str,
|
||||
port_provider: dict[int, tuple[str, str]],
|
||||
declared: set[str],
|
||||
proposed: set[str],
|
||||
env_var: str,
|
||||
out: list[Suggestion],
|
||||
) -> None:
|
||||
"""Resolve a (host, port) to a provider and record a suggestion if it's a new,
|
||||
undeclared, local match. Only resolve local hosts (this node's providers) or a
|
||||
host that names the provider — avoids matching a coincidental external host that
|
||||
happens to share a port number."""
|
||||
prov = port_provider.get(port)
|
||||
if not prov:
|
||||
return
|
||||
pname, proto = prov
|
||||
if host not in _LOCAL_HOSTS and host != pname:
|
||||
return
|
||||
if pname == consumer or pname in declared or pname in proposed:
|
||||
return
|
||||
proposed.add(pname)
|
||||
out.append(Suggestion(consumer, pname, env_var, f"{host}:{port}", proto))
|
||||
|
||||
|
||||
def suggest_consumption(config: WildpcConfig) -> list[Suggestion]:
|
||||
"""Undeclared consumption suggestions, derived from env endpoint values.
|
||||
|
||||
Two shapes are recognized in a deployment's ``defaults.env``:
|
||||
(a) ``host:port`` inside a single value (a URL, a ``DATABASE_URL``); and
|
||||
(b) a split ``X_HOST`` + ``X_PORT`` pair (e.g. ``WILDPC_API_MQTT_HOST`` +
|
||||
``WILDPC_API_MQTT_PORT``). When the port resolves to a *local* provider's socket
|
||||
and the consumer doesn't already declare it, propose the edge — deduped per
|
||||
(consumer, provider). A bare ``*_PORT`` with no ``*_HOST`` is ignored: without an
|
||||
explicit host it can't be told apart from the deployment's own listen port."""
|
||||
model = build_model(config, check=False)
|
||||
# port -> (provider name, protocol); ports are unique per host, so a port match
|
||||
# against a local host is a confident resolution.
|
||||
port_provider: dict[int, tuple[str, str]] = {}
|
||||
for n in model.nodes:
|
||||
for ep in n.endpoints:
|
||||
port_provider.setdefault(ep.port, (n.name, ep.protocol))
|
||||
|
||||
out: list[Suggestion] = []
|
||||
for _kind, name, dep in config.all_deployments():
|
||||
env = dict(dep.defaults.env) if (dep.defaults and dep.defaults.env) else {}
|
||||
declared = {r.ref for r in getattr(dep, "requires", [])}
|
||||
proposed: set[str] = set()
|
||||
|
||||
def resolve(host: str, port: int, env_var: str) -> None:
|
||||
_resolve(
|
||||
host,
|
||||
port,
|
||||
consumer=name,
|
||||
port_provider=port_provider,
|
||||
declared=declared,
|
||||
proposed=proposed,
|
||||
env_var=env_var,
|
||||
out=out,
|
||||
)
|
||||
|
||||
# (a) host:port inside a single value.
|
||||
for var, val in env.items():
|
||||
for m in _HOSTPORT.finditer(str(val)):
|
||||
resolve(m.group("host"), int(m.group("port")), var)
|
||||
# (b) split X_HOST + X_PORT pair.
|
||||
for var, val in env.items():
|
||||
if not var.endswith("_HOST"):
|
||||
continue
|
||||
pvar = var[:-5] + "_PORT" # X_HOST -> X_PORT
|
||||
if pvar not in env:
|
||||
continue
|
||||
try:
|
||||
port = int(str(env[pvar]).strip())
|
||||
except ValueError:
|
||||
continue
|
||||
resolve(str(val).strip(), port, f"{var}+{pvar}")
|
||||
return out
|
||||
768
core/src/wildpc_core/config.py
Normal file
768
core/src/wildpc_core/config.py
Normal file
@@ -0,0 +1,768 @@
|
||||
"""Wild PC configuration and registry management."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import InitVar, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from wildpc_core.secret_backends import SecretBackend
|
||||
|
||||
from wildpc_core.manifest import (
|
||||
AgentSpec,
|
||||
DeploymentSpec,
|
||||
ProgramSpec,
|
||||
kind_for,
|
||||
)
|
||||
|
||||
# Validator for the manager-discriminated deployment union (it's an Annotated
|
||||
# Union, not a BaseModel, so it needs a TypeAdapter to parse a dict).
|
||||
_DEPLOYMENT_ADAPTER: TypeAdapter[DeploymentSpec] = TypeAdapter(DeploymentSpec)
|
||||
|
||||
|
||||
def _resolve_wildpc_home() -> Path:
|
||||
"""Resolve the wildpc home directory (config, code, artifacts, secrets).
|
||||
|
||||
Defaults to ~/.wildpc. Override with the WILDPC_HOME environment variable
|
||||
(supports ~ and relative paths, which are expanded and made absolute).
|
||||
"""
|
||||
override = os.environ.get("WILDPC_HOME")
|
||||
if override:
|
||||
return Path(override).expanduser().resolve()
|
||||
return Path.home() / ".wildpc"
|
||||
|
||||
|
||||
_DEFAULT_DATA_DIR = Path("/data/wildpc")
|
||||
_DEFAULT_REPOS_DIR = Path("/data/repos")
|
||||
|
||||
|
||||
class WildpcDirError(RuntimeError):
|
||||
"""A required wildpc directory can't be created (e.g. data_dir outside a writable
|
||||
parent). Carries an actionable message; surfaced to the CLI and the api instead of
|
||||
a bare PermissionError traceback."""
|
||||
|
||||
|
||||
def _resolve_root_path(
|
||||
env_var: str, yaml_value: object, anchor: Path, default: Path
|
||||
) -> Path:
|
||||
"""Resolve a configurable root with precedence: env var > wildpc.yaml > default.
|
||||
|
||||
`~` is expanded; a relative path is anchored to `anchor` (the dir containing
|
||||
wildpc.yaml) — never cwd, so the CLI (shell cwd) and the api service (unit cwd)
|
||||
resolve identically. The built-in default is returned as-is (so it compares equal
|
||||
for the "persist only when non-default" check in save_config)."""
|
||||
raw = os.environ.get(env_var) or yaml_value
|
||||
if not raw:
|
||||
return default
|
||||
p = Path(str(raw)).expanduser()
|
||||
if not p.is_absolute():
|
||||
p = anchor / p
|
||||
return p.resolve()
|
||||
|
||||
|
||||
WILDPC_HOME = _resolve_wildpc_home()
|
||||
CODE_DIR = WILDPC_HOME / "code"
|
||||
ARTIFACTS_DIR = WILDPC_HOME / "artifacts"
|
||||
SPECS_DIR = ARTIFACTS_DIR / "specs"
|
||||
CONTENT_DIR = ARTIFACTS_DIR / "content"
|
||||
SECRETS_DIR = WILDPC_HOME / "secrets"
|
||||
# data_dir and repos_dir are deliberately NOT module constants. Unlike the WILDPC_HOME
|
||||
# family above (env-or-default — the dir that *holds* wildpc.yaml can't be configured
|
||||
# inside it), these are per-instance settings read from wildpc.yaml. A module global
|
||||
# would be a second copy of that value, resolved once at import against one process's
|
||||
# environment — exactly what let the CLI and the api service drift. They live only on
|
||||
# the loaded WildpcConfig; read config.data_dir / config.repos_dir (see load_config).
|
||||
|
||||
# User tool directories — the single source of truth for "where our CLIs live".
|
||||
# Used both at build time (dev-verb subprocess PATH) and at run time (generated
|
||||
# systemd unit PATH) so a service sees the same tools wildpc used to build it.
|
||||
# Order matters: pnpm's modern standalone installer puts its shim in
|
||||
# $PNPM_HOME/bin, which must win over the bare dir (older installs leave a stale
|
||||
# version wrapper there). nvm/node is intentionally omitted — it's versioned and
|
||||
# brittle to hardcode. A program's node is resolved per-program from its declared
|
||||
# pin (.node-version/.nvmrc/engines) and prepended ahead of these dirs at both the
|
||||
# build subprocess (stacks._build_env) and the runtime unit PATH (Deployment
|
||||
# .path_prepend) — see wildpc_core.toolchains.
|
||||
USER_TOOL_PATH_DIRS = [
|
||||
Path.home() / ".local" / "bin",
|
||||
Path.home() / ".local" / "share" / "pnpm" / "bin",
|
||||
Path.home() / ".local" / "share" / "pnpm",
|
||||
Path.home() / ".deno" / "bin", # deno's installer target (supabase edge fns)
|
||||
Path("/usr/local/go/bin"),
|
||||
]
|
||||
|
||||
# Backwards-compat aliases (used by existing imports)
|
||||
GENERATED_DIR = SPECS_DIR
|
||||
STATIC_DIR = CONTENT_DIR
|
||||
|
||||
|
||||
def find_wildpc_root() -> Path:
|
||||
"""Find the wildpc config root (directory containing wildpc.yaml).
|
||||
|
||||
Search order:
|
||||
1. ~/.wildpc/wildpc.yaml (the canonical instance location)
|
||||
2. Walk up from cwd (for development/testing)
|
||||
"""
|
||||
# Canonical location first
|
||||
if (WILDPC_HOME / "wildpc.yaml").exists():
|
||||
return WILDPC_HOME
|
||||
# Fallback: walk up from cwd
|
||||
current = Path.cwd()
|
||||
while current != current.parent:
|
||||
if (current / "wildpc.yaml").exists():
|
||||
return current
|
||||
current = current.parent
|
||||
raise FileNotFoundError(
|
||||
f"Could not find wildpc.yaml.\nExpected at: {WILDPC_HOME / 'wildpc.yaml'}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GatewayConfig:
|
||||
"""Gateway configuration."""
|
||||
|
||||
port: int = 9000
|
||||
# None/"off" → HTTP-only gateway. "acme" → real Let's Encrypt wildcard cert
|
||||
# (*.domain) via a DNS-01 challenge; publicly trusted, no CA to install.
|
||||
tls: str | None = None
|
||||
# acme mode only: the zone for the wildcard cert and host-route subdomains
|
||||
# (e.g. "civil.payne.io" → host routes become <service>.civil.payne.io).
|
||||
domain: str | None = None
|
||||
acme_email: str | None = None
|
||||
acme_dns_provider: str = "cloudflare"
|
||||
# Public exposure via the Cloudflare tunnel (optional). `public_domain` is the
|
||||
# separate zone public services are published at (e.g. "pub.payne.io" →
|
||||
# <service>.pub.payne.io), kept distinct from `domain` so internal subdomain
|
||||
# names never appear in public DNS. `tunnel_id` is the cloudflared tunnel UUID.
|
||||
public_domain: str | None = None
|
||||
tunnel_id: str | None = None
|
||||
# acme mode only: emit the `events { on cert_obtained exec wildpc tls reconcile }`
|
||||
# hook so certs materialized onto raw-TCP services refresh on renewal. Requires
|
||||
# the events-exec plugin in the gateway's Caddy build — set true only once that
|
||||
# Caddy is installed (see docs/tcp-exposure.md §5). Default false keeps a
|
||||
# plugin-less gateway parseable.
|
||||
cert_hook: bool = False
|
||||
|
||||
|
||||
# Deployment kinds and the WildpcConfig store each lives in. Kind is STRUCTURAL —
|
||||
# a deployment's identity is (name, kind), so names are unique within a kind but may
|
||||
# collide across kinds (a `backup` tool + service + job coexist). `kind_for` (manifest)
|
||||
# stays only to validate that a spec's manager/schedule matches the store it's in.
|
||||
KINDS = ("service", "job", "tool", "static", "reference")
|
||||
_KIND_STORE = {
|
||||
"service": "services",
|
||||
"job": "jobs",
|
||||
"tool": "tools",
|
||||
"static": "statics",
|
||||
"reference": "references",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class WildpcConfig:
|
||||
"""Full wildpc configuration."""
|
||||
|
||||
root: Path
|
||||
gateway: GatewayConfig
|
||||
repo: Path | None
|
||||
programs: dict[str, ProgramSpec]
|
||||
# Per-kind deployment stores (the primary representation). Each is name-keyed and
|
||||
# unique within its kind; a name may appear in more than one store. There is no
|
||||
# single flat `deployments` dict — use `all_deployments()` / `deployments_named()`
|
||||
# / the kind store directly (`config.services[name]`).
|
||||
services: dict[str, DeploymentSpec] = field(default_factory=dict)
|
||||
jobs: dict[str, DeploymentSpec] = field(default_factory=dict)
|
||||
tools: dict[str, DeploymentSpec] = field(default_factory=dict)
|
||||
statics: dict[str, DeploymentSpec] = field(default_factory=dict)
|
||||
references: dict[str, DeploymentSpec] = field(default_factory=dict)
|
||||
# Launchable agent CLIs for the dashboard terminal UX (assistant-agnostic).
|
||||
# Optional; empty means the API falls back to a built-in default set.
|
||||
agents: dict[str, AgentSpec] = field(default_factory=dict)
|
||||
# Configurable roots — the single source of truth (no module-constant twin).
|
||||
# load_config sets them (env > wildpc.yaml > default); a bare constructor gets the
|
||||
# built-in defaults so tests/callers that don't care stay valid.
|
||||
data_dir: Path = field(default_factory=lambda: _DEFAULT_DATA_DIR)
|
||||
repos_dir: Path = field(default_factory=lambda: _DEFAULT_REPOS_DIR)
|
||||
# Fleet role: "authority" (may write shared config/secrets to the mesh) or
|
||||
# "follower" (reconciles from it). Static — pinned here, no election.
|
||||
role: str = "follower"
|
||||
# Construction convenience only (not stored): a flat name→spec dict is routed
|
||||
# into the per-kind stores by kind_for. Lets callers/tests hand us a flat map
|
||||
# without pre-splitting it; there is still no flat `deployments` attribute.
|
||||
deployments: InitVar[dict[str, DeploymentSpec] | None] = None
|
||||
|
||||
def __post_init__(self, deployments: dict[str, DeploymentSpec] | None) -> None:
|
||||
for name, spec in (deployments or {}).items():
|
||||
self.store_for(kind_for(spec))[name] = spec
|
||||
|
||||
def store_for(self, kind: str) -> dict[str, DeploymentSpec]:
|
||||
"""The name-keyed deployment store for a kind (service|job|tool|static|reference)."""
|
||||
return getattr(self, _KIND_STORE[kind])
|
||||
|
||||
def all_deployments(self) -> list[tuple[str, str, DeploymentSpec]]:
|
||||
"""Every deployment as `(kind, name, spec)`, kind-then-name ordered. The single
|
||||
shared iterate-all — the converge loop dispatches on `spec.manager`, so the
|
||||
machinery stays shared; only the namespace is split by kind."""
|
||||
out: list[tuple[str, str, DeploymentSpec]] = []
|
||||
for kind in KINDS:
|
||||
store = self.store_for(kind)
|
||||
out.extend((kind, name, store[name]) for name in sorted(store))
|
||||
return out
|
||||
|
||||
def deployment(self, kind: str, name: str) -> DeploymentSpec | None:
|
||||
"""A single deployment by its `(kind, name)` identity, or None."""
|
||||
return self.store_for(kind).get(name)
|
||||
|
||||
def deployments_named(self, name: str) -> list[tuple[str, DeploymentSpec]]:
|
||||
"""`(kind, spec)` for every kind that has a deployment with this bare name
|
||||
(≤5). Used where a caller has only a name and no kind (apply/restart/redirect)."""
|
||||
return [(kind, spec) for kind, n, spec in self.all_deployments() if n == name]
|
||||
|
||||
def deployments_of(self, program: str) -> list[tuple[str, str]]:
|
||||
"""A program's deployments as (deployment-name, kind) pairs, name-sorted.
|
||||
|
||||
A deployment belongs to a program when it names it (`program:`) or shares its
|
||||
name (the 1:1 tool/static case). Empty for a bare, undeployed program.
|
||||
"""
|
||||
return sorted(
|
||||
(name, kind)
|
||||
for kind, name, dep in self.all_deployments()
|
||||
if name == program or dep.program == program
|
||||
)
|
||||
|
||||
@property
|
||||
def frontends(self) -> dict[str, ProgramSpec]:
|
||||
"""Return programs that are frontends (have build outputs)."""
|
||||
return {
|
||||
k: v
|
||||
for k, v in self.programs.items()
|
||||
if v.build and (v.build.outputs or v.build.commands)
|
||||
}
|
||||
|
||||
|
||||
def resolve_env_split(
|
||||
env: dict[str, str], context: dict[str, str] | None = None
|
||||
) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""Resolve placeholders, splitting secret-bearing vars from plain ones.
|
||||
|
||||
Returns ``(plain, secret)``. A var is *secret-bearing* if its raw value
|
||||
contained a ``${secret:...}`` reference — including composite values like
|
||||
``neo4j/${secret:NEO4J_PASSWORD}``. Both dicts hold fully-resolved values;
|
||||
partitioning lets callers keep secrets out of unit files and process argv
|
||||
(routing them through a mode-0600 env file) while inlining the rest.
|
||||
|
||||
- ``${secret:NAME}`` resolves via the active secret backend (file or OpenBao).
|
||||
- ``${port}`` / ``${data_dir}`` / ``${name}`` / ``${public_url}`` (and
|
||||
anything else in ``context``) expand to wildpc's computed values, so a
|
||||
service maps them to whatever env var its program reads (e.g.
|
||||
``MY_PORT: ${port}``) without hardcoding or wildpc silently injecting a
|
||||
guessed var name. ``${public_url}`` is the service's gateway-facing base
|
||||
URL (``https://<name>.<domain>`` under acme) — the origin an app allowlists.
|
||||
"""
|
||||
context = context or {}
|
||||
plain: dict[str, str] = {}
|
||||
secret: dict[str, str] = {}
|
||||
for key, value in env.items():
|
||||
|
||||
def replace_var(match: re.Match[str]) -> str:
|
||||
ref = match.group(1)
|
||||
if ref.startswith("secret:"):
|
||||
return _read_secret(ref[7:])
|
||||
if ref in context:
|
||||
return context[ref]
|
||||
return match.group(0)
|
||||
|
||||
resolved = re.sub(r"\$\{([^}]+)\}", replace_var, value)
|
||||
if re.search(r"\$\{secret:[^}]+\}", value):
|
||||
secret[key] = resolved
|
||||
else:
|
||||
plain[key] = resolved
|
||||
return plain, secret
|
||||
|
||||
|
||||
def resolve_placeholders(value: str, context: dict[str, str] | None) -> str:
|
||||
"""Expand ``${key}`` refs in a single string from ``context``.
|
||||
|
||||
The one ``${...}`` grammar shared by env resolution (:func:`resolve_env_split`)
|
||||
and run-spec expansion (argv/volumes/env in a container launch), so a new
|
||||
placeholder only has to be added to the context dict, never to a second engine.
|
||||
Unknown refs — including ``${secret:...}`` — pass through untouched (secrets
|
||||
never belong in argv; they go via ``--env-file``). Write ``$${key}`` to emit a
|
||||
literal ``${key}`` (e.g. a container arg the container's own shell must expand).
|
||||
"""
|
||||
if not context:
|
||||
return value
|
||||
|
||||
def replace_var(match: re.Match[str]) -> str:
|
||||
ref = match.group(1)
|
||||
return context.get(ref, match.group(0))
|
||||
|
||||
# Split on the `$$` escape so an escaped `$${x}` never reaches the substitution
|
||||
# regex, then rejoin with a literal `$`.
|
||||
return "$".join(
|
||||
re.sub(r"\$\{([^}]+)\}", replace_var, part) for part in value.split("$$")
|
||||
)
|
||||
|
||||
|
||||
def resolve_env_vars(
|
||||
env: dict[str, str], context: dict[str, str] | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Resolve placeholders in env values (secrets included), preserving order.
|
||||
|
||||
Convenience wrapper over :func:`resolve_env_split` for callers that want a
|
||||
single flat dict. Prefer ``resolve_env_split`` when secrets must be kept out
|
||||
of generated artifacts.
|
||||
"""
|
||||
plain, secret = resolve_env_split(env, context)
|
||||
return {k: secret[k] if k in secret else plain[k] for k in env}
|
||||
|
||||
|
||||
def _secrets_settings() -> dict:
|
||||
"""The ``secrets:`` block of wildpc.yaml — selects the backend."""
|
||||
try:
|
||||
data = yaml.safe_load((WILDPC_HOME / "wildpc.yaml").read_text()) or {}
|
||||
return data.get("secrets") or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def active_secret_backend() -> SecretBackend:
|
||||
"""The active secret backend (file or openbao), per ``secrets:`` + env.
|
||||
|
||||
The one place to get a handle for reading/writing/listing secrets, so callers
|
||||
(the ``wildpc secret`` CLI, preflight checks) act on the *active* backend
|
||||
rather than guessing the filesystem — the mistake that silently shadows an
|
||||
OpenBao fleet's secret with a stray file.
|
||||
"""
|
||||
from wildpc_core.secret_backends import build_backend
|
||||
|
||||
return build_backend(SECRETS_DIR, _secrets_settings())
|
||||
|
||||
|
||||
def active_backend_name() -> str:
|
||||
"""Human name of the active backend (``openbao`` | ``file``) for messages."""
|
||||
return (os.environ.get("WILDPC_SECRET_BACKEND") or _secrets_settings().get("backend") or "file").lower()
|
||||
|
||||
|
||||
def read_secret(name: str) -> str | None:
|
||||
"""Resolve a secret via the active backend, or None if unset.
|
||||
|
||||
The public helper for code that reads secrets directly (DNS/supabase/etc.) so
|
||||
everything goes through the one backend rather than the filesystem.
|
||||
"""
|
||||
return active_secret_backend().read(name)
|
||||
|
||||
|
||||
# ``${secret:NAME}`` references embedded in an env value (the grammar
|
||||
# :func:`resolve_env_split` resolves). Shared by the deploy-time preflight that
|
||||
# refuses to converge a deployment whose secrets the active backend can't resolve.
|
||||
_SECRET_REF_RE = re.compile(r"\$\{secret:([^}]+)\}")
|
||||
|
||||
|
||||
def secret_refs(value: str) -> list[str]:
|
||||
"""The secret names a value references via ``${secret:NAME}`` (order-preserving,
|
||||
deduped)."""
|
||||
seen: dict[str, None] = {}
|
||||
for m in _SECRET_REF_RE.finditer(value):
|
||||
seen.setdefault(m.group(1), None)
|
||||
return list(seen)
|
||||
|
||||
|
||||
def _read_secret(name: str) -> str:
|
||||
"""Like :func:`read_secret` but returns a ``<MISSING_SECRET:...>`` placeholder
|
||||
for unresolved secrets (used by ``${secret:...}`` env substitution)."""
|
||||
value = read_secret(name)
|
||||
return value if value is not None else f"<MISSING_SECRET:{name}>"
|
||||
|
||||
|
||||
def _parse_program(name: str, data: dict) -> ProgramSpec:
|
||||
"""Parse a programs: entry into a ProgramSpec."""
|
||||
data_copy = dict(data)
|
||||
data_copy["id"] = name
|
||||
return ProgramSpec.model_validate(data_copy)
|
||||
|
||||
|
||||
def _parse_deployment(name: str, data: dict) -> DeploymentSpec:
|
||||
"""Parse a deployment entry (manager-discriminated shape) into a DeploymentSpec."""
|
||||
data_copy = dict(data)
|
||||
data_copy["id"] = name
|
||||
return _DEPLOYMENT_ADAPTER.validate_python(data_copy)
|
||||
|
||||
|
||||
def _load_resource_dir(directory: Path) -> dict[str, dict]:
|
||||
"""Load every *.yaml file in a resource directory.
|
||||
|
||||
The filename stem becomes the resource id. Returns a mapping of
|
||||
id → parsed YAML dict (empty mappings normalized to {}).
|
||||
"""
|
||||
result: dict[str, dict] = {}
|
||||
if not directory.is_dir():
|
||||
return result
|
||||
for path in sorted(directory.glob("*.yaml")):
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a YAML mapping")
|
||||
result[path.stem] = data
|
||||
return result
|
||||
|
||||
|
||||
def parse_gateway(gateway_data: dict) -> GatewayConfig:
|
||||
"""Build a GatewayConfig from a wildpc.yaml ``gateway:`` mapping.
|
||||
|
||||
The single parser shared by ``load_config`` and the API's whole-file editor,
|
||||
so a newly added gateway field can't be honored in one place and silently
|
||||
dropped in the other (which is exactly how ``cert_hook`` got wiped on a
|
||||
full-config save).
|
||||
"""
|
||||
return GatewayConfig(
|
||||
port=gateway_data.get("port", 9000),
|
||||
tls=gateway_data.get("tls"),
|
||||
domain=gateway_data.get("domain"),
|
||||
acme_email=gateway_data.get("acme_email"),
|
||||
acme_dns_provider=gateway_data.get("acme_dns_provider", "cloudflare"),
|
||||
public_domain=gateway_data.get("public_domain"),
|
||||
tunnel_id=gateway_data.get("tunnel_id"),
|
||||
cert_hook=gateway_data.get("cert_hook", False),
|
||||
)
|
||||
|
||||
|
||||
def load_config(root: Path | None = None) -> WildpcConfig:
|
||||
"""Load wildpc config: global wildpc.yaml + programs/ and deployments/ dirs."""
|
||||
if root is None:
|
||||
root = find_wildpc_root()
|
||||
|
||||
config_path = root / "wildpc.yaml"
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Wild PC config not found: {config_path}")
|
||||
|
||||
with open(config_path) as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
|
||||
gateway = parse_gateway(data.get("gateway", {}))
|
||||
|
||||
# repo: field points to the git repo for repo-relative sources
|
||||
repo_path: Path | None = None
|
||||
if data.get("repo"):
|
||||
repo_path = Path(data["repo"]).expanduser()
|
||||
|
||||
# Configurable roots: env > this file's data_dir/repos_dir > default, anchored to
|
||||
# `root` (the dir holding this wildpc.yaml) so a per-call load_config is correct
|
||||
# regardless of the import-time constants (which resolved against WILDPC_HOME).
|
||||
data_dir = _resolve_root_path(
|
||||
"WILDPC_DATA_DIR", data.get("data_dir"), root, _DEFAULT_DATA_DIR
|
||||
)
|
||||
repos_dir = _resolve_root_path(
|
||||
"WILDPC_REPOS_DIR", data.get("repos_dir"), root, _DEFAULT_REPOS_DIR
|
||||
)
|
||||
|
||||
programs: dict[str, ProgramSpec] = {}
|
||||
for name, comp_data in _load_resource_dir(root / "programs").items():
|
||||
prog = _parse_program(name, comp_data)
|
||||
# Resolve source paths to absolute
|
||||
if prog.source:
|
||||
if prog.source.startswith("repo:") and repo_path:
|
||||
# repo:wildpc-api → /data/repos/wildpc/wildpc-api
|
||||
prog.source = str(repo_path / prog.source[5:])
|
||||
elif not Path(prog.source).is_absolute():
|
||||
prog.source = str(root / prog.source)
|
||||
programs[name] = prog
|
||||
|
||||
stores = _load_deployments(root)
|
||||
_validate_subdomains(stores)
|
||||
|
||||
agents: dict[str, AgentSpec] = {
|
||||
name: AgentSpec.model_validate(spec or {})
|
||||
for name, spec in (data.get("agents") or {}).items()
|
||||
}
|
||||
|
||||
config = WildpcConfig(
|
||||
root=root,
|
||||
repo=repo_path,
|
||||
gateway=gateway,
|
||||
programs=programs,
|
||||
agents=agents,
|
||||
data_dir=data_dir,
|
||||
repos_dir=repos_dir,
|
||||
role=data.get("role", "follower"),
|
||||
**stores,
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _validate_subdomains(stores: dict[str, dict[str, DeploymentSpec]]) -> None:
|
||||
"""A gateway subdomain (``<name>.<domain>``) must be globally unique across kinds.
|
||||
Only HTTP-exposed kinds claim one: a proxied service, or a static site. A name may
|
||||
still be a tool/service/job trio (only the service is HTTP-exposed), but a service
|
||||
and a static can't share a name (they'd fight over the same subdomain)."""
|
||||
claimants: dict[str, list[str]] = {}
|
||||
for name, spec in stores["services"].items():
|
||||
if getattr(spec, "http_exposed", False):
|
||||
claimants.setdefault(name, []).append("service")
|
||||
for name in stores["statics"]:
|
||||
claimants.setdefault(name, []).append("static")
|
||||
for name, kinds in claimants.items():
|
||||
if len(kinds) > 1:
|
||||
raise ValueError(
|
||||
f"subdomain '{name}' is claimed by multiple HTTP-exposed deployments "
|
||||
f"({', '.join(kinds)}); a name can be HTTP-exposed by at most one kind"
|
||||
)
|
||||
|
||||
|
||||
def _load_deployments(root: Path) -> dict[str, dict[str, DeploymentSpec]]:
|
||||
"""Load the per-kind deployment stores for a config root.
|
||||
|
||||
Layout: ``deployments/<store>/<name>.yaml`` (store = services|jobs|tools|statics|
|
||||
references). Every file is routed to its store by ``kind_for(spec)`` — the dir is
|
||||
a namespace, the spec's manager/schedule is the source of truth.
|
||||
"""
|
||||
stores: dict[str, dict[str, DeploymentSpec]] = {s: {} for s in _KIND_STORE.values()}
|
||||
dep_dir = root / "deployments"
|
||||
for store in _KIND_STORE.values():
|
||||
for name, data in _load_resource_dir(dep_dir / store).items():
|
||||
spec = _parse_deployment(name, data)
|
||||
stores[_KIND_STORE[kind_for(spec)]][name] = spec
|
||||
return stores
|
||||
|
||||
|
||||
def _clean_for_yaml(data: object, preserve_keys: set[str] | None = None) -> object:
|
||||
"""Recursively remove empty lists and non-structural empty dicts."""
|
||||
if preserve_keys is None:
|
||||
preserve_keys = _STRUCTURAL_KEYS
|
||||
if isinstance(data, dict):
|
||||
cleaned = {}
|
||||
for k, v in data.items():
|
||||
v = _clean_for_yaml(v, preserve_keys)
|
||||
# Keep structural keys even if empty dict
|
||||
if k in preserve_keys and isinstance(v, dict):
|
||||
cleaned[k] = v
|
||||
continue
|
||||
# Skip empty collections
|
||||
if isinstance(v, (list, dict)) and not v:
|
||||
continue
|
||||
cleaned[k] = v
|
||||
return cleaned
|
||||
elif isinstance(data, list):
|
||||
return [_clean_for_yaml(item, preserve_keys) for item in data]
|
||||
return data
|
||||
|
||||
|
||||
# Keys whose presence is structurally significant even with all-default values.
|
||||
# We serialize these as empty dicts `{}` so they survive a roundtrip.
|
||||
_STRUCTURAL_KEYS = {
|
||||
"manage",
|
||||
"systemd",
|
||||
"expose",
|
||||
}
|
||||
|
||||
|
||||
def _spec_to_yaml_dict(spec: BaseModel) -> dict:
|
||||
"""Serialize a ProgramSpec or DeploymentSpec to a YAML-friendly dict."""
|
||||
exclude_fields = {"id"}
|
||||
full = spec.model_dump(mode="json", exclude_none=True, exclude=exclude_fields)
|
||||
minimal = spec.model_dump(
|
||||
mode="json", exclude_none=True, exclude=exclude_fields, exclude_defaults=True
|
||||
)
|
||||
|
||||
def merge(full_val: object, min_val: object | None, key: str = "") -> object:
|
||||
if isinstance(full_val, dict):
|
||||
result = {}
|
||||
for k, fv in full_val.items():
|
||||
mv = min_val.get(k) if isinstance(min_val, dict) else None
|
||||
if k in _STRUCTURAL_KEYS:
|
||||
merged = merge(fv, mv, k)
|
||||
if merged is not None:
|
||||
result[k] = merged
|
||||
elif mv is not None:
|
||||
result[k] = merge(fv, mv, k)
|
||||
elif isinstance(fv, dict):
|
||||
merged = merge(fv, None, k)
|
||||
if merged:
|
||||
result[k] = merged
|
||||
return result if result else ({} if key in _STRUCTURAL_KEYS else result)
|
||||
elif isinstance(full_val, list):
|
||||
if min_val is not None:
|
||||
return full_val
|
||||
return []
|
||||
else:
|
||||
if min_val is not None:
|
||||
return full_val
|
||||
return None
|
||||
|
||||
result = merge(full, minimal)
|
||||
cleaned = _clean_for_yaml(result)
|
||||
return cleaned if isinstance(cleaned, dict) else {}
|
||||
|
||||
|
||||
def _program_to_yaml_dict(spec: ProgramSpec, config: WildpcConfig) -> dict:
|
||||
"""Serialize a ProgramSpec, rewriting absolute source paths to relative."""
|
||||
d = _spec_to_yaml_dict(spec)
|
||||
if d.get("source") and Path(d["source"]).is_absolute():
|
||||
src = Path(d["source"])
|
||||
# If source is under repo, store as repo:relative
|
||||
if config.repo:
|
||||
try:
|
||||
d["source"] = "repo:" + str(src.relative_to(config.repo))
|
||||
return d
|
||||
except ValueError:
|
||||
pass
|
||||
# Otherwise store relative to config root
|
||||
try:
|
||||
d["source"] = str(src.relative_to(config.root))
|
||||
except ValueError:
|
||||
pass # not under root — keep absolute
|
||||
return d
|
||||
|
||||
|
||||
def _write_resource_dir(directory: Path, specs: dict[str, dict]) -> None:
|
||||
"""Write each spec to <directory>/<name>.yaml and prune orphaned files."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
for name, d in specs.items():
|
||||
with open(directory / f"{name}.yaml", "w") as f:
|
||||
yaml.dump(d, f, default_flow_style=False, sort_keys=False)
|
||||
# Prune files with no corresponding in-memory entry
|
||||
for path in directory.glob("*.yaml"):
|
||||
if path.stem not in specs:
|
||||
path.unlink()
|
||||
|
||||
|
||||
# Top-level wildpc.yaml keys that save_config owns and rewrites; anything else
|
||||
# (e.g. `secrets:`) is preserved verbatim so a rewrite can't drop it.
|
||||
_MANAGED_GLOBALS = {
|
||||
"gateway", "repo", "data_dir", "repos_dir", "agents", "role",
|
||||
"programs", "services", "jobs", "tools", "statics", "references", "deployments",
|
||||
}
|
||||
|
||||
|
||||
def write_deployment_file(config: WildpcConfig, kind: str, name: str) -> None:
|
||||
"""Write (or remove) a single deployment file — globals and other resources are
|
||||
left untouched. This is the PATCH primitive: a deployment edit persists only
|
||||
that deployment, never rewriting wildpc.yaml globals."""
|
||||
directory = config.root / "deployments" / _KIND_STORE[kind]
|
||||
path = directory / f"{name}.yaml"
|
||||
spec = config.store_for(kind).get(name)
|
||||
if spec is None:
|
||||
path.unlink(missing_ok=True)
|
||||
return
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
yaml.dump(_spec_to_yaml_dict(spec), f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
|
||||
def write_program_file(config: WildpcConfig, name: str) -> None:
|
||||
"""Write (or remove) a single program file — nothing else is touched."""
|
||||
directory = config.root / "programs"
|
||||
path = directory / f"{name}.yaml"
|
||||
spec = config.programs.get(name)
|
||||
if spec is None:
|
||||
path.unlink(missing_ok=True)
|
||||
return
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
yaml.dump(
|
||||
_program_to_yaml_dict(spec, config), f, default_flow_style=False, sort_keys=False
|
||||
)
|
||||
|
||||
|
||||
def save_config(config: WildpcConfig) -> None:
|
||||
"""Save wildpc config: global wildpc.yaml + programs/ and deployments/ dirs."""
|
||||
gateway_data: dict = {"port": config.gateway.port}
|
||||
if config.gateway.tls:
|
||||
gateway_data["tls"] = config.gateway.tls
|
||||
if config.gateway.domain:
|
||||
gateway_data["domain"] = config.gateway.domain
|
||||
if config.gateway.acme_email:
|
||||
gateway_data["acme_email"] = config.gateway.acme_email
|
||||
# Only persist the provider when non-default, to keep wildpc.yaml minimal.
|
||||
if (
|
||||
config.gateway.acme_dns_provider
|
||||
and config.gateway.acme_dns_provider != "cloudflare"
|
||||
):
|
||||
gateway_data["acme_dns_provider"] = config.gateway.acme_dns_provider
|
||||
if config.gateway.public_domain:
|
||||
gateway_data["public_domain"] = config.gateway.public_domain
|
||||
if config.gateway.tunnel_id:
|
||||
gateway_data["tunnel_id"] = config.gateway.tunnel_id
|
||||
if config.gateway.cert_hook:
|
||||
gateway_data["cert_hook"] = config.gateway.cert_hook
|
||||
data: dict = {"gateway": gateway_data}
|
||||
if config.repo:
|
||||
data["repo"] = str(config.repo)
|
||||
# Persist the configurable roots only when non-default, keeping wildpc.yaml minimal.
|
||||
# These MUST round-trip: save_config rewrites the file from scratch, so a root that
|
||||
# isn't re-emitted here would be silently dropped on the next apply.
|
||||
if config.data_dir != _DEFAULT_DATA_DIR:
|
||||
data["data_dir"] = str(config.data_dir)
|
||||
if config.repos_dir != _DEFAULT_REPOS_DIR:
|
||||
data["repos_dir"] = str(config.repos_dir)
|
||||
if config.agents:
|
||||
data["agents"] = {
|
||||
n: s.model_dump(exclude_none=True, exclude_defaults=True)
|
||||
for n, s in config.agents.items()
|
||||
}
|
||||
# MUST round-trip (save rewrites from scratch): the fleet role and the
|
||||
# `secrets:` backend block are not otherwise re-emitted, so they'd be silently
|
||||
# dropped on the next save — reverting the node to a follower on the file
|
||||
# backend. `role` lives on the config; `secrets` isn't modeled, so preserve it
|
||||
# from the existing file.
|
||||
if config.role and config.role != "follower":
|
||||
data["role"] = config.role
|
||||
# Preserve any top-level wildpc.yaml keys this writer doesn't model (e.g.
|
||||
# `secrets:`) — a full rewrite must never silently drop an unmanaged global.
|
||||
try:
|
||||
existing = yaml.safe_load((config.root / "wildpc.yaml").read_text()) or {}
|
||||
for k, v in existing.items():
|
||||
if k not in _MANAGED_GLOBALS and k not in data:
|
||||
data[k] = v
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
config_path = config.root / "wildpc.yaml"
|
||||
with open(config_path, "w") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
_write_resource_dir(
|
||||
config.root / "programs",
|
||||
{n: _program_to_yaml_dict(s, config) for n, s in config.programs.items()},
|
||||
)
|
||||
# Per-kind: deployments/<store>/<name>.yaml (each store pruned independently).
|
||||
dep_dir = config.root / "deployments"
|
||||
for kind, store in _KIND_STORE.items():
|
||||
_write_resource_dir(
|
||||
dep_dir / store,
|
||||
{n: _spec_to_yaml_dict(d) for n, d in config.store_for(kind).items()},
|
||||
)
|
||||
|
||||
|
||||
def ensure_dirs(config: WildpcConfig) -> None:
|
||||
"""Ensure wildpc directories exist. Takes the config so the data dir comes from the
|
||||
one source of truth (config.data_dir), not a process-resolved global."""
|
||||
WILDPC_HOME.mkdir(parents=True, exist_ok=True)
|
||||
CODE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
SPECS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONTENT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# The data dir can live outside $HOME (a dedicated volume), so its parent may be
|
||||
# unwritable or absent — fail loud with a fix, not a bare PermissionError.
|
||||
data_dir = config.data_dir
|
||||
try:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
raise WildpcDirError(
|
||||
f"Cannot create data dir {data_dir}: {e.strerror or e}. "
|
||||
f"Set data_dir: in {WILDPC_HOME / 'wildpc.yaml'} (or export WILDPC_DATA_DIR) "
|
||||
f"to a writable path, or create it: "
|
||||
f"sudo mkdir -p {data_dir} && sudo chown $(id -un) {data_dir}"
|
||||
) from e
|
||||
SECRETS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(SECRETS_DIR, 0o700)
|
||||
# Generated per-deployment secret env files (EnvironmentFile= / --env-file)
|
||||
# live here, kept out of unit files and process argv.
|
||||
secret_env_dir = SECRETS_DIR / "env"
|
||||
secret_env_dir.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(secret_env_dir, 0o700)
|
||||
1261
core/src/wildpc_core/deploy.py
Normal file
1261
core/src/wildpc_core/deploy.py
Normal file
File diff suppressed because it is too large
Load Diff
23
core/src/wildpc_core/generators/__init__.py
Normal file
23
core/src/wildpc_core/generators/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
"""Wild PC infrastructure generators."""
|
||||
|
||||
from wildpc_core.generators.caddyfile import (
|
||||
generate_caddyfile_from_registry,
|
||||
)
|
||||
from wildpc_core.generators.systemd import (
|
||||
cron_to_interval_sec,
|
||||
cron_to_oncalendar,
|
||||
generate_timer,
|
||||
generate_unit_from_deployed,
|
||||
timer_name,
|
||||
unit_name,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"cron_to_interval_sec",
|
||||
"cron_to_oncalendar",
|
||||
"generate_caddyfile_from_registry",
|
||||
"generate_timer",
|
||||
"generate_unit_from_deployed",
|
||||
"timer_name",
|
||||
"unit_name",
|
||||
]
|
||||
411
core/src/wildpc_core/generators/caddyfile.py
Normal file
411
core/src/wildpc_core/generators/caddyfile.py
Normal file
@@ -0,0 +1,411 @@
|
||||
"""Gateway routes + Caddyfile generation from the node registry.
|
||||
|
||||
A single source of truth: `compute_routes()` produces the structured list of
|
||||
gateway routes; `generate_caddyfile_from_registry()` renders that list to a
|
||||
Caddyfile, and the API serves the same list to the dashboard — so the route
|
||||
table always matches what Caddy actually does.
|
||||
|
||||
A route maps a public **address** (a path prefix `/foo`, or a host `foo.lan`) to
|
||||
a **target**, of one **kind**:
|
||||
- ``static`` — a built frontend's `dist/`; Caddy serves files (`file_server`).
|
||||
- ``proxy`` — a local service on a port; Caddy reverse-proxies.
|
||||
- ``remote`` — a service on another node; Caddy reverse-proxies cross-node.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import SPECS_DIR, WildpcConfig
|
||||
from wildpc_core.manifest import CaddyDeployment, SystemdDeployment
|
||||
from wildpc_core.registry import NodeRegistry
|
||||
|
||||
# DNS-01 provider → the env var the Caddyfile reads its API token from. The token
|
||||
# reaches Caddy via the gateway service's defaults.env (a mode-0600 EnvironmentFile).
|
||||
_DNS_TOKEN_ENV = {"cloudflare": "CLOUDFLARE_API_TOKEN"}
|
||||
|
||||
# Let's Encrypt staging directory — opt in via WILDPC_ACME_STAGING=1 to avoid the
|
||||
# production rate limits while testing, then cut over to prod (unset the env var).
|
||||
_ACME_STAGING_CA = "https://acme-staging-v02.api.letsencrypt.org/directory"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GatewayRoute:
|
||||
"""One gateway route: address → target."""
|
||||
|
||||
address: str # "/foo" (path prefix, served at /foo/*) or "foo.lan" (host)
|
||||
kind: str # "static" | "proxy" | "remote"
|
||||
target: str # static: serve dir; proxy: "localhost:PORT"/base_url; remote: "host:PORT"
|
||||
name: str | None = None # backing program/service
|
||||
node: str | None = None
|
||||
public: bool = False # also served under public_domain
|
||||
# Optional exact public-facing FQDN override (apex allowed). When set, the
|
||||
# public name is this instead of the derived <address>.<public_domain>.
|
||||
public_host: str | None = None
|
||||
# static routes only: SPA fallback (True) vs content-site serving (False).
|
||||
spa: bool = True
|
||||
|
||||
@property
|
||||
def is_host(self) -> bool:
|
||||
return not self.address.startswith("/")
|
||||
|
||||
|
||||
# (expose?, port, base_url) — expose=True → route <service-name>.<domain> here.
|
||||
ProxyTargets = tuple[bool, int | None, str | None]
|
||||
|
||||
|
||||
def service_proxy_targets(name: str, dep: SystemdDeployment) -> ProxyTargets:
|
||||
"""Derive a systemd deployment's gateway exposure from its spec.
|
||||
|
||||
The single source of truth shared by the registry build (``deploy``) and
|
||||
route computation (``compute_routes``), so they never disagree. A gateway
|
||||
route is HTTP-only: ``http_exposed`` requires ``reach != off`` *and* an HTTP
|
||||
port, so a raw-TCP service (``expose.tcp``) never yields a route here.
|
||||
"""
|
||||
port = None
|
||||
if dep.expose and dep.expose.http:
|
||||
port = dep.expose.http.internal.port
|
||||
return dep.http_exposed, port, None
|
||||
|
||||
|
||||
def _local_routes(
|
||||
config: WildpcConfig | None, registry: NodeRegistry
|
||||
) -> list[tuple[str, str, str, bool, str | None, bool]]:
|
||||
"""Each local deployment's route as ``(name, kind, target, public, public_host, spa)``.
|
||||
|
||||
``kind`` is ``static`` (a caddy deployment — file-serve a built dir) or
|
||||
``proxy`` (a proxied systemd process). Prefers ``wildpc.yaml``
|
||||
(``config.deployments``) so a regenerated Caddyfile reflects the current spec;
|
||||
falls back to the deployed registry snapshot when config isn't available.
|
||||
"""
|
||||
out: list[tuple[str, str, str, bool, str | None, bool]] = []
|
||||
if config is not None:
|
||||
# Only HTTP-exposed kinds route: static (file-serve) and service (proxy).
|
||||
# jobs/tools/references never do.
|
||||
for kind, name, dep in config.all_deployments():
|
||||
# A disabled deployment is defined but not running — no route (else it
|
||||
# would 502). `wildpc apply` converges it off.
|
||||
if not dep.enabled:
|
||||
continue
|
||||
if kind == "static" and isinstance(dep, CaddyDeployment):
|
||||
src = _program_source(config, dep.program)
|
||||
if src is not None:
|
||||
pub_host = dep.public_host if dep.public else None
|
||||
out.append((name, "static", str(src / dep.root), bool(dep.public), pub_host, dep.spa))
|
||||
elif kind == "service" and isinstance(dep, SystemdDeployment):
|
||||
expose, port, base_url = service_proxy_targets(name, dep)
|
||||
if expose and (port or base_url):
|
||||
pub_host = dep.public_host if dep.public else None
|
||||
out.append((name, "proxy", base_url or f"localhost:{port}", bool(dep.public), pub_host, True))
|
||||
return out
|
||||
# No config → route from the deployed registry snapshot.
|
||||
for _kind, name, d in registry.all():
|
||||
if not d.enabled:
|
||||
continue
|
||||
if d.static_root:
|
||||
out.append((name, "static", d.static_root, d.public, d.public_host, d.spa))
|
||||
elif d.subdomain and (d.port or d.base_url):
|
||||
out.append((name, "proxy", d.base_url or f"localhost:{d.port}", d.public, d.public_host, True))
|
||||
return out
|
||||
|
||||
|
||||
def _program_source(config: WildpcConfig | None, program: str | None):
|
||||
"""Absolute source dir of a referenced program (already resolved), or None."""
|
||||
if config is None or not program:
|
||||
return None
|
||||
prog = getattr(config, "programs", {}).get(program)
|
||||
if prog and prog.source:
|
||||
return Path(prog.source)
|
||||
return None
|
||||
|
||||
|
||||
def compute_routes(
|
||||
registry: NodeRegistry,
|
||||
config: WildpcConfig | None = None,
|
||||
remote_registries: dict[str, NodeRegistry] | None = None,
|
||||
) -> list[GatewayRoute]:
|
||||
"""Build the ordered list of gateway routes. Every route is a host route whose
|
||||
address is the service/frontend **name** (published at ``<name>.<domain>``);
|
||||
``proxy`` routes reverse-proxy a local port, ``static`` routes file-serve a
|
||||
frontend's dist. Path routes no longer exist. When ``remote_registries`` is
|
||||
given (online peers, keyed by hostname), ``remote`` routes are added for
|
||||
services this node **consumes** (a local ``requires`` ref satisfied by a peer)
|
||||
— so a consumed cross-node service is reachable at ``<ref>.<domain>``."""
|
||||
if config is None:
|
||||
try:
|
||||
from wildpc_core.config import load_config
|
||||
|
||||
config = load_config()
|
||||
except Exception:
|
||||
config = None
|
||||
|
||||
node = registry.node.hostname
|
||||
routes: list[GatewayRoute] = []
|
||||
|
||||
# Every route comes from a service. `static`-runner services file-serve their
|
||||
# built dir; everything else that's exposed reverse-proxies its port/base_url.
|
||||
# (Static frontends are `runner: static` services now — no separate program
|
||||
# branch, so routing derives from one place.)
|
||||
for name, kind, target, is_public, pub_host, spa in _local_routes(config, registry):
|
||||
routes.append(GatewayRoute(name, kind, target, name, node, is_public, pub_host, spa))
|
||||
|
||||
if remote_registries:
|
||||
routes.extend(_remote_routes(config, registry, remote_registries))
|
||||
|
||||
return routes
|
||||
|
||||
|
||||
def _remote_routes(
|
||||
config: WildpcConfig | None,
|
||||
registry: NodeRegistry,
|
||||
remote_registries: dict[str, NodeRegistry],
|
||||
) -> list[GatewayRoute]:
|
||||
"""Routes to services this node consumes from online peers.
|
||||
|
||||
A route is emitted for each local ``requires`` ref that (a) isn't satisfied
|
||||
locally and (b) is provided by an exposed service on some peer. The route is
|
||||
only present while the peer is (presence expiry removes the peer from
|
||||
``remote_registries``, which *is* the circuit-breaker: gone → no route)."""
|
||||
# Refs this node consumes.
|
||||
consumed: set[str] = set()
|
||||
local_names: set[str] = set()
|
||||
if config is not None:
|
||||
for _kind, name, dep in config.all_deployments():
|
||||
local_names.add(name)
|
||||
for req in getattr(dep, "requires", []) or []:
|
||||
ref = getattr(req, "ref", None)
|
||||
if ref and getattr(req, "kind", "deployment") == "deployment":
|
||||
consumed.add(ref)
|
||||
# Drop refs already satisfied locally.
|
||||
consumed -= local_names
|
||||
|
||||
out: list[GatewayRoute] = []
|
||||
for host, remote in sorted(remote_registries.items()):
|
||||
addr = remote.node.address or host
|
||||
for _kind, name, dep in remote.all():
|
||||
if name not in consumed:
|
||||
continue
|
||||
if dep.subdomain and dep.port:
|
||||
out.append(
|
||||
GatewayRoute(name, "remote", f"{addr}:{dep.port}", name, host)
|
||||
)
|
||||
consumed.discard(name) # first online provider wins
|
||||
return out
|
||||
|
||||
|
||||
def _host_matcher_block(label: str, host: str, target: str) -> list[str]:
|
||||
"""A `@host_X host <host> / handle @host_X { reverse_proxy <target> }` block.
|
||||
|
||||
Shared by the off-mode `:<port>` site and the acme wildcard site so the two
|
||||
can't drift. `label` names the matcher (service name, or the address)."""
|
||||
matcher = f"@host_{label.replace('-', '_').replace('.', '_')}"
|
||||
return [
|
||||
f" {matcher} host {host}",
|
||||
f" handle {matcher} {{",
|
||||
f" reverse_proxy {target}",
|
||||
" }",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def _host_remote_block(label: str, host: str, target: str) -> list[str]:
|
||||
"""A remote (cross-node) host route with a fail-fast breaker: a short dial
|
||||
timeout + passive health, so an unreachable peer 502s in ~2s instead of
|
||||
hanging. (Presence removal drops the route entirely — this guards the
|
||||
there-but-wedged case.)"""
|
||||
matcher = f"@host_{label.replace('-', '_').replace('.', '_')}"
|
||||
return [
|
||||
f" {matcher} host {host}",
|
||||
f" handle {matcher} {{",
|
||||
f" reverse_proxy {target} {{",
|
||||
" lb_try_duration 1s",
|
||||
" fail_duration 30s",
|
||||
" transport http {",
|
||||
" dial_timeout 2s",
|
||||
" }",
|
||||
" }",
|
||||
" }",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def _static_serve_lines(serve_dir: str, spa: bool, indent: str) -> list[str]:
|
||||
"""The `root`/`file_server` body for a static site. SPA sites get a fallback to
|
||||
the root `index.html` (client-side routing); content sites (Hugo) get plain
|
||||
`file_server`, which resolves directory indexes and 404s missing paths."""
|
||||
lines = [f"{indent}root * {serve_dir}"]
|
||||
if spa:
|
||||
lines.append(indent + "try_files {path} /index.html")
|
||||
lines.append(f"{indent}file_server")
|
||||
return lines
|
||||
|
||||
|
||||
def _host_static_block(label: str, host: str, serve_dir: str, spa: bool) -> list[str]:
|
||||
"""A host matcher that file-serves a frontend's dist."""
|
||||
matcher = f"@host_{label.replace('-', '_').replace('.', '_')}"
|
||||
return [
|
||||
f" {matcher} host {host}",
|
||||
f" handle {matcher} {{",
|
||||
*_static_serve_lines(serve_dir, spa, " "),
|
||||
" }",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def _public_site_block(host: str, kind: str, target: str, spa: bool = True) -> list[str]:
|
||||
"""A standalone Caddy site for a custom ``public_host`` (apex or another zone).
|
||||
|
||||
Unlike the ``*.<public_domain>`` wildcard site, an apex/foreign host isn't
|
||||
covered by a wildcard cert, so it gets its own site. Caddy issues that exact
|
||||
host's cert via the global ``acme_dns`` (DNS-01) — provided the gateway's
|
||||
Cloudflare token can edit the host's zone."""
|
||||
if kind == "static":
|
||||
body = _static_serve_lines(target, spa, " ")
|
||||
elif kind == "remote":
|
||||
body = [
|
||||
f" reverse_proxy {target} {{",
|
||||
" lb_try_duration 1s",
|
||||
" fail_duration 30s",
|
||||
" transport http {",
|
||||
" dial_timeout 2s",
|
||||
" }",
|
||||
" }",
|
||||
]
|
||||
else:
|
||||
body = [f" reverse_proxy {target}"]
|
||||
return [f"{host} {{", *body, "}", ""]
|
||||
|
||||
|
||||
# Wild PC's own control plane: the dashboard frontend and the API it calls. These
|
||||
# names are the subdomains they're published at in acme mode, and the pair served
|
||||
# on the :<port> site in off mode (no domain → no subdomains).
|
||||
_DASHBOARD = "wildpc"
|
||||
_API = "wildpc-api"
|
||||
|
||||
|
||||
def generate_caddyfile_from_registry(
|
||||
registry: NodeRegistry,
|
||||
remote_registries: dict[str, NodeRegistry] | None = None,
|
||||
) -> str:
|
||||
"""Render the routes to a Caddyfile. Every exposed service/frontend is a
|
||||
subdomain `<name>.<domain>`; there are no path routes.
|
||||
|
||||
Two modes, set by `gateway.tls`:
|
||||
|
||||
- **acme** — one `*.<domain>` site (a single DNS-01 wildcard cert) with a host
|
||||
matcher per route: `reverse_proxy` for services, `file_server` for frontends.
|
||||
The `:<port>` site just redirects to the dashboard subdomain (the "browse to
|
||||
the box by IP" entry).
|
||||
- **off / no domain** — no subdomains available, so the `:<port>` site serves
|
||||
wildpc's control plane only: the dashboard at `/` + `reverse_proxy /api/*` →
|
||||
wildpc-api (the one surviving path, for the dashboard's own backend). Other
|
||||
services are reachable at their `host:port` directly.
|
||||
"""
|
||||
routes = compute_routes(registry, None, remote_registries)
|
||||
node = registry.node
|
||||
gw_port = node.gateway_port
|
||||
mode = (node.gateway_tls or "").lower()
|
||||
domain = node.gateway_domain
|
||||
tls_acme = mode == "acme" and bool(domain)
|
||||
lines: list[str] = []
|
||||
|
||||
if tls_acme:
|
||||
# Global ACME options: LE account email + DNS-01 provider token (from env).
|
||||
provider = node.acme_dns_provider or "cloudflare"
|
||||
token_env = _DNS_TOKEN_ENV.get(provider, "CLOUDFLARE_API_TOKEN")
|
||||
lines.append("{")
|
||||
if node.acme_email:
|
||||
lines.append(f" email {node.acme_email}")
|
||||
lines.append(f" acme_dns {provider} {{env.{token_env}}}")
|
||||
if os.environ.get("WILDPC_ACME_STAGING") == "1":
|
||||
lines.append(f" acme_ca {_ACME_STAGING_CA}")
|
||||
# On issuance/renewal, refresh certs materialized onto raw-TCP services and
|
||||
# reload them (idempotent — a no-op when nothing rotated). Requires the
|
||||
# events-exec plugin in the gateway's Caddy build, so it's gated on the
|
||||
# durable `gateway.cert_hook` flag, set only once that Caddy is in place;
|
||||
# false → the block is omitted and a plugin-less gateway parses fine. See
|
||||
# docs/tcp-exposure.md §5.
|
||||
if getattr(node, "cert_hook", False):
|
||||
lines += [
|
||||
" events {",
|
||||
" on cert_obtained exec wildpc tls reconcile",
|
||||
" }",
|
||||
]
|
||||
lines += ["}", ""]
|
||||
# One wildcard site → a single cert covers every subdomain; a new service
|
||||
# needs no new cert or challenge.
|
||||
if routes:
|
||||
lines.append(f"*.{domain} {{")
|
||||
for r in routes:
|
||||
host = f"{r.address}.{domain}"
|
||||
if r.kind == "static":
|
||||
lines += _host_static_block(r.name or r.address, host, r.target, r.spa)
|
||||
elif r.kind == "remote":
|
||||
lines += _host_remote_block(r.name or r.address, host, r.target)
|
||||
else:
|
||||
lines += _host_matcher_block(r.name or r.address, host, r.target)
|
||||
lines.append("}")
|
||||
lines.append("")
|
||||
# Public exposure: public deployments are also reachable by their public
|
||||
# name so LAN clients can use it directly. Two shapes:
|
||||
# - default → <address>.<public_domain>, all served by one wildcard site
|
||||
# (*.<public_domain>, its own wildcard cert), when a node-wide
|
||||
# public_domain distinct from the internal zone is configured.
|
||||
# - override → an exact `public_host` (an apex or a name in another zone),
|
||||
# which a wildcard can't cover, so each gets a standalone site with its
|
||||
# own cert (DNS-01 via the global acme_dns).
|
||||
public_domain = node.public_domain
|
||||
public_routes = [r for r in routes if r.public]
|
||||
default_pub = [r for r in public_routes if not r.public_host]
|
||||
custom_pub = [r for r in public_routes if r.public_host]
|
||||
if public_domain and public_domain != domain and default_pub:
|
||||
lines.append(f"*.{public_domain} {{")
|
||||
for r in default_pub:
|
||||
host = f"{r.address}.{public_domain}"
|
||||
label = f"{r.name or r.address}_pub"
|
||||
if r.kind == "static":
|
||||
lines += _host_static_block(label, host, r.target, r.spa)
|
||||
elif r.kind == "remote":
|
||||
lines += _host_remote_block(label, host, r.target)
|
||||
else:
|
||||
lines += _host_matcher_block(label, host, r.target)
|
||||
lines.append("}")
|
||||
lines.append("")
|
||||
for r in custom_pub:
|
||||
if r.public_host: # always true (custom_pub filter); narrows the type
|
||||
lines += _public_site_block(r.public_host, r.kind, r.target, r.spa)
|
||||
# Redirect the bare gateway port to the dashboard subdomain.
|
||||
lines += [
|
||||
f":{gw_port} {{",
|
||||
f" redir https://{_DASHBOARD}.{domain}{{uri}}",
|
||||
"}",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
# off mode: HTTP-only control plane on :<port>.
|
||||
if mode == "acme" and not domain:
|
||||
lines.append("# gateway.tls=acme but gateway.domain is unset — serving the")
|
||||
lines.append("# control plane on the gateway port; services are port-only.")
|
||||
lines += ["{", " auto_https off", "}", "", f":{gw_port} {{"]
|
||||
|
||||
api = next((r for r in routes if r.name == _API and r.kind == "proxy"), None)
|
||||
if api is not None:
|
||||
lines += [
|
||||
" handle_path /api/* {",
|
||||
f" reverse_proxy {api.target}",
|
||||
" }",
|
||||
"",
|
||||
]
|
||||
app = next((r for r in routes if r.name == _DASHBOARD and r.kind == "static"), None)
|
||||
root = app.target if app is not None else str(SPECS_DIR / "app")
|
||||
lines += [
|
||||
" handle {",
|
||||
f" root * {root}",
|
||||
" try_files {path} /index.html",
|
||||
" file_server",
|
||||
" }",
|
||||
"}",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
157
core/src/wildpc_core/generators/dns.py
Normal file
157
core/src/wildpc_core/generators/dns.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Reconcile public DNS (Cloudflare CNAMEs) for tunnel-exposed services.
|
||||
|
||||
Wild PC owns the CNAMEs — across every zone the token can see — that point at its
|
||||
Cloudflare tunnel: on deploy it creates one per public host (each routed to the
|
||||
accessible zone whose name is its longest suffix, so apex and multi-zone hosts both
|
||||
work) and deletes any that point at this tunnel but no longer correspond to a
|
||||
public service. It **only ever touches records whose content is
|
||||
`<tunnel_id>.cfargotunnel.com`** — never other records in a zone — so a
|
||||
hand-managed A/CNAME in the same zone is safe.
|
||||
|
||||
Needs a Cloudflare API token with **DNS:Edit** on every target zone (Cloudflare's
|
||||
"Edit zone DNS" template — that single permission both lists the accessible zones
|
||||
and edits records; no separate Zone:Read is needed), stored at
|
||||
`~/.wildpc/secrets/CLOUDFLARE_PUBLIC_DNS_TOKEN`. Absent → this is a no-op and the
|
||||
caller falls back to surfacing the manual `cloudflared tunnel route dns` hints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
_API = "https://api.cloudflare.com/client/v4"
|
||||
|
||||
# Secret holding a Cloudflare token scoped to the PUBLIC zone (Zone:DNS:Edit).
|
||||
# Distinct from CLOUDFLARE_API_TOKEN (ACME, the internal civil zone) because the
|
||||
# public zone is typically a separate zone/account.
|
||||
PUBLIC_DNS_TOKEN = "CLOUDFLARE_PUBLIC_DNS_TOKEN"
|
||||
|
||||
|
||||
def public_dns_token() -> str | None:
|
||||
"""The public-zone DNS token from the active secret backend, or None."""
|
||||
from wildpc_core.config import read_secret
|
||||
|
||||
return read_secret(PUBLIC_DNS_TOKEN) or None
|
||||
|
||||
|
||||
def _api(token: str, method: str, path: str, body: dict | None = None) -> dict:
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(f"{_API}{path}", data=data, method=method)
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=15) as resp: # noqa: S310 (fixed API host)
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def _zone_for(host: str, zones: list[dict]) -> dict | None:
|
||||
"""The visible zone whose name is the longest suffix of ``host`` (or None).
|
||||
|
||||
Longest-suffix so an apex (``example.com`` in zone ``example.com``) and a
|
||||
subdomain in any accessible zone both resolve, even when zones nest.
|
||||
"""
|
||||
matches = [
|
||||
z
|
||||
for z in zones
|
||||
if host == z["name"] or host.endswith("." + z["name"])
|
||||
]
|
||||
return max(matches, key=lambda z: len(z["name"])) if matches else None
|
||||
|
||||
|
||||
def reconcile_public_dns(
|
||||
tunnel_id: str | None,
|
||||
desired_hosts: list[str],
|
||||
messages: list[str],
|
||||
token: str | None = None,
|
||||
) -> bool:
|
||||
"""Make the tunnel CNAMEs across every accessible zone exactly `desired_hosts`.
|
||||
|
||||
Each desired host is routed to the accessible zone whose name is its longest
|
||||
suffix (so apex hosts and hosts in different zones are handled), then per zone
|
||||
wildpc creates missing CNAMEs (proxied → the tunnel; Cloudflare flattens apex
|
||||
CNAMEs) and deletes wildpc-managed ones (content == `<tunnel_id>.cfargotunnel.com`)
|
||||
no longer desired. Never touches records pointing elsewhere. Scanning every
|
||||
visible zone also cleans up stale CNAMEs after a host moves zones or all public
|
||||
services are removed.
|
||||
|
||||
Returns True if reconciliation was attempted (a token was configured) — the
|
||||
caller then suppresses the manual route hints — or False if skipped (no token /
|
||||
no tunnel), so the caller can fall back to those hints.
|
||||
"""
|
||||
token = token or public_dns_token()
|
||||
if not (token and tunnel_id):
|
||||
return False
|
||||
target = f"{tunnel_id}.cfargotunnel.com"
|
||||
try:
|
||||
zones = (_api(token, "GET", "/zones?per_page=50").get("result")) or []
|
||||
if not zones:
|
||||
messages.append(
|
||||
"Warning: DNS token can't see any zone — public CNAMEs not "
|
||||
"reconciled. The token needs DNS:Edit (Cloudflare's 'Edit zone "
|
||||
"DNS' template) on the target zone(s)."
|
||||
)
|
||||
return True
|
||||
|
||||
# Route each desired host to its zone (longest-suffix match). Hosts with no
|
||||
# accessible zone can't be created — surface them rather than silently drop.
|
||||
desired_by_zone: dict[str, set[str]] = {z["id"]: set() for z in zones}
|
||||
for host in desired_hosts:
|
||||
z = _zone_for(host, zones)
|
||||
if z is None:
|
||||
messages.append(
|
||||
f"Warning: no accessible Cloudflare zone for public host "
|
||||
f"'{host}' — its CNAME was not created. The DNS token needs "
|
||||
f"DNS:Edit on that host's zone."
|
||||
)
|
||||
continue
|
||||
desired_by_zone[z["id"]].add(host)
|
||||
|
||||
created: list[str] = []
|
||||
removed: list[str] = []
|
||||
# Reconcile every visible zone (not just those with desired hosts) so a
|
||||
# CNAME orphaned by a host moving zones / going internal is cleaned up.
|
||||
for z in zones:
|
||||
zone_id = z["id"]
|
||||
recs = _api(
|
||||
token, "GET", f"/zones/{zone_id}/dns_records?type=CNAME&per_page=100"
|
||||
).get("result") or []
|
||||
managed = {r["name"]: r["id"] for r in recs if r.get("content") == target}
|
||||
desired = desired_by_zone[zone_id]
|
||||
for host in sorted(desired - set(managed)):
|
||||
_api(
|
||||
token,
|
||||
"POST",
|
||||
f"/zones/{zone_id}/dns_records",
|
||||
{"type": "CNAME", "name": host, "content": target, "proxied": True},
|
||||
)
|
||||
created.append(host)
|
||||
for host in sorted(set(managed) - desired):
|
||||
_api(token, "DELETE", f"/zones/{zone_id}/dns_records/{managed[host]}")
|
||||
removed.append(host)
|
||||
|
||||
if created or removed:
|
||||
parts = []
|
||||
if created:
|
||||
parts.append(f"+{len(created)} ({', '.join(sorted(created))})")
|
||||
if removed:
|
||||
parts.append(f"-{len(removed)} ({', '.join(sorted(removed))})")
|
||||
messages.append(f"Public DNS reconciled: {' '.join(parts)}")
|
||||
else:
|
||||
messages.append(f"Public DNS up to date ({len(desired_hosts)} CNAME(s)).")
|
||||
return True
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode(errors="replace")[:200]
|
||||
hint = ""
|
||||
if e.code == 403:
|
||||
# Reads can succeed while writes 403 — the token has DNS:Read but not
|
||||
# DNS:Edit. Point at the fix rather than the raw body.
|
||||
hint = (
|
||||
" → the token needs DNS:Edit (write), not just read — use "
|
||||
"Cloudflare's 'Edit zone DNS' template."
|
||||
)
|
||||
messages.append(f"Warning: public DNS reconcile failed (HTTP {e.code}): {body}{hint}")
|
||||
return True # token was present; don't also print stale manual hints
|
||||
except Exception as e: # noqa: BLE001 — DNS is best-effort; never fail a deploy
|
||||
messages.append(f"Warning: public DNS reconcile failed: {e}")
|
||||
return True
|
||||
234
core/src/wildpc_core/generators/systemd.py
Normal file
234
core/src/wildpc_core/generators/systemd.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""Systemd unit and timer generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import SECRETS_DIR, USER_TOOL_PATH_DIRS
|
||||
from wildpc_core.manifest import RestartPolicy, SystemdSpec
|
||||
from wildpc_core.registry import Deployment
|
||||
|
||||
SYSTEMD_USER_DIR = Path.home() / ".config" / "systemd" / "user"
|
||||
UNIT_PREFIX = "wildpc-"
|
||||
|
||||
# Generated mode-0600 env files holding a deployment's resolved secrets, kept out
|
||||
# of the unit file and the process argv (loaded via EnvironmentFile= / --env-file).
|
||||
SECRET_ENV_DIR = SECRETS_DIR / "env"
|
||||
|
||||
|
||||
def runtime_path(path_prepend: list[str] | tuple[str, ...] = ()) -> str:
|
||||
"""The PATH a wildpc service runs with: resolved toolchain dirs (e.g. a pinned
|
||||
node bin) + the user tool dirs that exist + system bins. This is the single
|
||||
definition of a service's runtime PATH — the unit generator writes it into
|
||||
``Environment=PATH`` and the dependency checker (``relations``) probes tools
|
||||
against it, so the two can never disagree about where a service finds its tools.
|
||||
"""
|
||||
dirs = list(path_prepend)
|
||||
dirs += [str(d) for d in USER_TOOL_PATH_DIRS if d.exists()]
|
||||
dirs += ["/usr/local/bin", "/usr/bin", "/bin"]
|
||||
return ":".join(dirs)
|
||||
|
||||
|
||||
def unit_basename(name: str, kind: str = "service") -> str:
|
||||
"""The systemd unit stem for a deployment. Jobs carry a ``-job`` marker so a
|
||||
service and a job can share a name (`wildpc-<name>.service` vs
|
||||
`wildpc-<name>-job.{service,timer}`); everything else is `wildpc-<name>`."""
|
||||
return f"{UNIT_PREFIX}{name}-job" if kind == "job" else f"{UNIT_PREFIX}{name}"
|
||||
|
||||
|
||||
def unit_name(service_name: str, kind: str = "service") -> str:
|
||||
"""Get the systemd `.service` unit name for a deployment of the given kind."""
|
||||
return f"{unit_basename(service_name, kind)}.service"
|
||||
|
||||
|
||||
def timer_name(service_name: str, kind: str = "job") -> str:
|
||||
"""Get the systemd `.timer` unit name (timers exist only for jobs)."""
|
||||
return f"{unit_basename(service_name, kind)}.timer"
|
||||
|
||||
|
||||
def secret_env_path(service_name: str, kind: str = "service") -> Path:
|
||||
"""Path to a deployment's generated secret env file (1:1 with its unit name)."""
|
||||
return SECRET_ENV_DIR / f"{unit_name(service_name, kind)}.env"
|
||||
|
||||
|
||||
def unit_env_file(deployed: Deployment, name: str) -> Path | None:
|
||||
"""The ``EnvironmentFile=`` path for a systemd-launched runner, or None.
|
||||
|
||||
Container runners load secrets via docker ``--env-file`` (baked into run_cmd),
|
||||
so systemd must not also read them — return None there. Only deployments that
|
||||
actually have secrets get a file.
|
||||
"""
|
||||
if deployed.launcher == "container" or not deployed.secret_env_keys:
|
||||
return None
|
||||
return secret_env_path(name, deployed.kind)
|
||||
|
||||
|
||||
def cron_to_oncalendar(cron: str) -> str:
|
||||
"""Best-effort conversion of cron expression to systemd OnCalendar.
|
||||
|
||||
Handles common patterns; falls back to using OnUnitActiveSec for the rest.
|
||||
"""
|
||||
parts = cron.strip().split()
|
||||
if len(parts) != 5:
|
||||
return ""
|
||||
|
||||
minute, hour, dom, month, dow = parts
|
||||
|
||||
# */N minutes → run every N minutes
|
||||
if (
|
||||
minute.startswith("*/")
|
||||
and hour == "*"
|
||||
and dom == "*"
|
||||
and month == "*"
|
||||
and dow == "*"
|
||||
):
|
||||
return "" # Use OnUnitActiveSec instead
|
||||
|
||||
# Specific time daily: "0 2 * * *" → "*-*-* 02:00:00"
|
||||
if dom == "*" and month == "*" and dow == "*":
|
||||
h = hour.zfill(2) if hour != "*" else "*"
|
||||
m = minute.zfill(2) if minute != "*" else "*"
|
||||
return f"*-*-* {h}:{m}:00"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def cron_to_interval_sec(cron: str) -> int | None:
|
||||
"""Extract interval seconds from */N cron patterns."""
|
||||
parts = cron.strip().split()
|
||||
if len(parts) != 5:
|
||||
return None
|
||||
minute, hour, dom, month, dow = parts
|
||||
if (
|
||||
minute.startswith("*/")
|
||||
and hour == "*"
|
||||
and dom == "*"
|
||||
and month == "*"
|
||||
and dow == "*"
|
||||
):
|
||||
try:
|
||||
return int(minute[2:]) * 60
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def generate_unit_from_deployed(
|
||||
name: str,
|
||||
deployed: Deployment,
|
||||
systemd_spec: SystemdSpec | None = None,
|
||||
env_file: Path | None = None,
|
||||
) -> str:
|
||||
"""Generate a systemd unit from a deployed component (registry-based).
|
||||
|
||||
No repo-relative paths — uses only resolved run_cmd and env from the registry.
|
||||
Secrets are never inlined as ``Environment=`` lines: ``env_file`` (when set)
|
||||
is loaded via ``EnvironmentFile=`` so the values stay out of the unit. The
|
||||
path is referenced fail-loud (no ``-`` prefix): a missing file blocks start.
|
||||
"""
|
||||
exec_start = " ".join(deployed.run_cmd)
|
||||
|
||||
env_lines = ""
|
||||
for key, value in deployed.env.items():
|
||||
env_lines += f"Environment={key}={value}\n"
|
||||
# Wild PC supplies a sensible default PATH (tool dirs + system bins). It is an
|
||||
# escape hatch, not a mandate: if the service pins its own PATH in defaults.env
|
||||
# (e.g. to add a versioned nvm node the tool dirs intentionally omit), respect
|
||||
# it rather than clobbering it with a trailing Environment=PATH line that would
|
||||
# win under systemd's last-assignment-wins rule. systemd does NOT expand
|
||||
# ${PATH} across Environment= lines, so a service that overrides PATH must
|
||||
# spell out the full value, tool dirs included.
|
||||
if "PATH" not in deployed.env:
|
||||
env_lines += f'Environment="PATH={runtime_path(deployed.path_prepend)}"\n'
|
||||
if env_file is not None:
|
||||
env_lines += f"EnvironmentFile={env_file}\n"
|
||||
|
||||
sd = systemd_spec
|
||||
description = deployed.description or name
|
||||
after = " ".join(sd.after) if sd and sd.after else "network.target"
|
||||
wanted_by = " ".join(sd.wanted_by) if sd else "default.target"
|
||||
|
||||
if deployed.schedule:
|
||||
unit = f"""[Unit]
|
||||
Description=Wild PC: {description}
|
||||
After={after}
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={exec_start}
|
||||
{env_lines}"""
|
||||
else:
|
||||
restart = (sd.restart if sd else RestartPolicy.ON_FAILURE).value
|
||||
restart_sec = sd.restart_sec if sd else 5
|
||||
unit = f"""[Unit]
|
||||
Description=Wild PC: {description}
|
||||
After={after}
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart={exec_start}
|
||||
{env_lines}Restart={restart}
|
||||
RestartSec={restart_sec}
|
||||
SuccessExitStatus=143
|
||||
"""
|
||||
# Explicit teardown (e.g. compose `down`) so the stack's networks/volumes
|
||||
# are reclaimed on stop rather than left dangling.
|
||||
if deployed.stop_cmd:
|
||||
unit += f"ExecStop={' '.join(deployed.stop_cmd)}\n"
|
||||
|
||||
if sd and sd.exec_reload:
|
||||
reload_argv = sd.exec_reload.split()
|
||||
resolved_reload = shutil.which(reload_argv[0])
|
||||
if resolved_reload:
|
||||
reload_argv[0] = resolved_reload
|
||||
unit += f"ExecReload={' '.join(reload_argv)}\n"
|
||||
|
||||
# Post-start hooks (e.g. OpenBao auto-unseal). `-` prefix → failure is ignored,
|
||||
# so a hiccup in the hook never fails the unit.
|
||||
for cmd in sd.exec_start_post if sd else []:
|
||||
argv = cmd.split()
|
||||
resolved = shutil.which(argv[0])
|
||||
if resolved:
|
||||
argv[0] = resolved
|
||||
unit += f"ExecStartPost=-{' '.join(argv)}\n"
|
||||
|
||||
if sd and sd.no_new_privileges:
|
||||
unit += "NoNewPrivileges=true\n"
|
||||
|
||||
unit += f"""
|
||||
[Install]
|
||||
WantedBy={wanted_by}
|
||||
"""
|
||||
return unit
|
||||
|
||||
|
||||
def generate_timer(
|
||||
name: str,
|
||||
schedule: str,
|
||||
description: str | None = None,
|
||||
) -> str:
|
||||
"""Generate a systemd timer unit from a cron schedule string."""
|
||||
description = description or name
|
||||
|
||||
# Try to convert cron to OnCalendar, fall back to OnUnitActiveSec
|
||||
on_calendar = cron_to_oncalendar(schedule)
|
||||
interval_sec = cron_to_interval_sec(schedule)
|
||||
|
||||
timer_lines = ""
|
||||
if on_calendar:
|
||||
timer_lines = f"OnCalendar={on_calendar}\n"
|
||||
elif interval_sec:
|
||||
timer_lines = f"OnBootSec=60\nOnUnitActiveSec={interval_sec}s\n"
|
||||
else:
|
||||
timer_lines = "OnBootSec=60\nOnUnitActiveSec=300\n"
|
||||
|
||||
return f"""[Unit]
|
||||
Description=Wild PC timer: {description}
|
||||
|
||||
[Timer]
|
||||
{timer_lines}Persistent=false
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
"""
|
||||
125
core/src/wildpc_core/generators/tunnel.py
Normal file
125
core/src/wildpc_core/generators/tunnel.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""Cloudflare tunnel (cloudflared) ingress config generation.
|
||||
|
||||
Projects ``public: true`` services onto the public internet at
|
||||
``<subdomain>.<public_domain>``, bridging each to the gateway's existing internal
|
||||
host site so the tunnel reuses all of Caddy's routing and TLS. The public zone and
|
||||
the internal zone are deliberately different (``pub.payne.io`` vs
|
||||
``civil.payne.io``) so internal subdomain names never appear in public DNS; the
|
||||
ingress rewrites the Host header and TLS SNI to the *internal* name so Caddy routes
|
||||
the request correctly and its wildcard cert validates.
|
||||
|
||||
The generated config is locally-managed (a ``config.yml`` with an explicit ingress
|
||||
list), not a remotely-managed token tunnel — so the public surface is exactly the
|
||||
set of ``public: true`` services, generated here, not something edited in a
|
||||
dashboard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from wildpc_core.config import SECRETS_DIR
|
||||
from wildpc_core.registry import Deployment, NodeRegistry
|
||||
|
||||
# Cloudflared tunnel credentials (from `cloudflared tunnel create`) live here as a
|
||||
# secret, one JSON per tunnel id. The generated config references this path.
|
||||
TUNNEL_CREDENTIALS_DIR = SECRETS_DIR / "cloudflared"
|
||||
|
||||
# In acme mode Caddy serves each host site on :443 with a publicly-valid cert, so
|
||||
# the tunnel origin is the gateway on 443 (not the :<gateway_port> redirect site).
|
||||
_GATEWAY_ORIGIN = "https://localhost:443"
|
||||
|
||||
|
||||
def tunnel_credentials_path(tunnel_id: str) -> Path:
|
||||
"""Path to a tunnel's credentials JSON (kept in the secret store)."""
|
||||
return TUNNEL_CREDENTIALS_DIR / f"{tunnel_id}.json"
|
||||
|
||||
|
||||
def public_fqdn(d: Deployment, node) -> str | None:
|
||||
"""The public-facing hostname for a deployment, or None if it has none.
|
||||
|
||||
A deployment may override its public name with an exact FQDN (``public_host`` —
|
||||
an apex like ``example.com`` or a name in another zone); otherwise it publishes
|
||||
at ``<subdomain>.<public_domain>`` using the node-wide default public domain.
|
||||
None when neither an override nor a default public domain is available.
|
||||
"""
|
||||
if d.public_host:
|
||||
return d.public_host
|
||||
if node.public_domain and d.subdomain:
|
||||
return f"{d.subdomain}.{node.public_domain}"
|
||||
return None
|
||||
|
||||
|
||||
def public_deployments(registry: NodeRegistry) -> list[tuple[str, Deployment]]:
|
||||
"""The deployed services flagged public (and actually routed), name-sorted."""
|
||||
return sorted(
|
||||
(
|
||||
(name, d)
|
||||
for _kind, name, d in registry.all()
|
||||
if d.public and d.subdomain
|
||||
),
|
||||
key=lambda nd: nd[0],
|
||||
)
|
||||
|
||||
|
||||
def public_hostnames(registry: NodeRegistry) -> list[str]:
|
||||
"""The public hostnames that need a DNS route.
|
||||
|
||||
Each is either a per-deployment ``public_host`` override or the default
|
||||
``<sub>.<public_domain>``; deployments with neither are skipped.
|
||||
"""
|
||||
node = registry.node
|
||||
hosts = [public_fqdn(d, node) for _, d in public_deployments(registry)]
|
||||
return [h for h in hosts if h]
|
||||
|
||||
|
||||
def generate_tunnel_config(registry: NodeRegistry) -> str | None:
|
||||
"""Render the cloudflared ``config.yml``, or None if there's nothing to serve.
|
||||
|
||||
Returns None when the tunnel isn't configured (no ``tunnel_id`` /
|
||||
``public_domain`` / ``gateway_domain``) or no service is public — deploy then
|
||||
removes any stale config and leaves the tunnel down.
|
||||
"""
|
||||
node = registry.node
|
||||
# A public deployment needs a tunnel + an internal host to bridge to; the
|
||||
# node-wide public_domain is only the *default* public name, so it isn't
|
||||
# required (a deployment may carry its own public_host override instead).
|
||||
if not (node.tunnel_id and node.gateway_domain):
|
||||
return None
|
||||
pubs = public_deployments(registry)
|
||||
if not pubs:
|
||||
return None
|
||||
|
||||
ingress: list[dict] = []
|
||||
for _name, d in pubs:
|
||||
public_host = public_fqdn(d, node)
|
||||
if not public_host:
|
||||
# public but no override and no default public domain — nothing to map.
|
||||
continue
|
||||
internal_host = f"{d.subdomain}.{node.gateway_domain}"
|
||||
ingress.append(
|
||||
{
|
||||
"hostname": public_host,
|
||||
"service": _GATEWAY_ORIGIN,
|
||||
# Bridge the public name to the internal host: Caddy routes by this
|
||||
# Host and its wildcard cert is issued for this SNI.
|
||||
"originRequest": {
|
||||
"originServerName": internal_host,
|
||||
"httpHostHeader": internal_host,
|
||||
},
|
||||
}
|
||||
)
|
||||
if not ingress:
|
||||
# Every public deployment was skipped (no override, no default domain).
|
||||
return None
|
||||
# Cloudflared requires a terminal catch-all; anything unmapped is refused.
|
||||
ingress.append({"service": "http_status:404"})
|
||||
|
||||
config = {
|
||||
"tunnel": node.tunnel_id,
|
||||
"credentials-file": str(tunnel_credentials_path(node.tunnel_id)),
|
||||
"ingress": ingress,
|
||||
}
|
||||
return yaml.dump(config, default_flow_style=False, sort_keys=False)
|
||||
159
core/src/wildpc_core/git.py
Normal file
159
core/src/wildpc_core/git.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""Git working-copy status and sync for programs whose source is a git repo.
|
||||
|
||||
Programs that declare a ``repo:`` URL are cloned once (``wildpc program clone``);
|
||||
this module lets a running wildpc *see how far behind* a working copy is and pull
|
||||
later updates. It is intentionally pull-only — it touches files on disk and never
|
||||
builds, applies, or restarts anything. Making the running artifact reflect the new
|
||||
code (rebuild a frontend, restart a service) stays an explicit, separate step via
|
||||
``wildpc apply`` / ``wildpc restart``.
|
||||
|
||||
Plain ``git`` via subprocess (matching ``wildpc program clone``); no GitPython.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
# Bound network calls (fetch/pull) so an unreachable remote can't hang a request.
|
||||
_FETCH_TIMEOUT = 20.0
|
||||
_PULL_TIMEOUT = 60.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitStatus:
|
||||
"""A program working copy's git state. ``ahead``/``behind`` are relative to the
|
||||
upstream tracking branch and reflect the *last fetch* (``git_status(fetch=True)``
|
||||
refreshes them). ``None`` counts mean "no upstream to compare against"."""
|
||||
|
||||
is_repo: bool
|
||||
branch: str | None = None
|
||||
upstream: str | None = None
|
||||
dirty: bool = False
|
||||
ahead: int | None = None
|
||||
behind: int | None = None
|
||||
detached: bool = False
|
||||
error: str | None = None
|
||||
|
||||
|
||||
def _git(
|
||||
source: Path, *args: str, timeout: float | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Run ``git -C <source> <args>`` capturing text output."""
|
||||
return subprocess.run(
|
||||
["git", "-C", str(source), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def is_git_repo(source: Path | None) -> bool:
|
||||
"""True when ``source`` is inside a git working tree."""
|
||||
if not source or not Path(source).is_dir():
|
||||
return False
|
||||
try:
|
||||
r = _git(
|
||||
Path(source), "rev-parse", "--is-inside-work-tree", timeout=_FETCH_TIMEOUT
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
return r.returncode == 0 and r.stdout.strip() == "true"
|
||||
|
||||
|
||||
def toplevel(source: Path | None) -> str | None:
|
||||
"""The absolute path of the git working copy ``source`` lives in, or None.
|
||||
|
||||
The natural identity of a *repo*: several programs whose sources share a
|
||||
toplevel are the same working copy (a monorepo). Adopted single-program repos
|
||||
are their own toplevel — the N=1 case."""
|
||||
if not source or not Path(source).is_dir():
|
||||
return None
|
||||
try:
|
||||
r = _git(Path(source), "rev-parse", "--show-toplevel", timeout=_FETCH_TIMEOUT)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return r.stdout.strip() or None if r.returncode == 0 else None
|
||||
|
||||
|
||||
def remote_url(source: Path | None) -> str | None:
|
||||
"""The ``origin`` remote URL of the working copy, or None (no remote)."""
|
||||
if not is_git_repo(source):
|
||||
return None
|
||||
r = _git(Path(source), "remote", "get-url", "origin") # type: ignore[arg-type]
|
||||
return r.stdout.strip() or None if r.returncode == 0 else None
|
||||
|
||||
|
||||
def git_status(source: Path | None, fetch: bool = True) -> GitStatus:
|
||||
"""The working copy's branch/dirty/ahead/behind state.
|
||||
|
||||
``fetch=True`` runs ``git fetch`` first (bounded, tolerant of an offline remote)
|
||||
so ``behind`` reflects the real remote; on fetch failure the counts fall back to
|
||||
the last-known values and ``error`` carries the reason. Never raises — a
|
||||
non-repo returns ``GitStatus(is_repo=False)`` so callers can just hide the UI.
|
||||
"""
|
||||
if not is_git_repo(source):
|
||||
return GitStatus(is_repo=False)
|
||||
src = Path(source) # type: ignore[arg-type]
|
||||
st = GitStatus(is_repo=True)
|
||||
|
||||
# Branch (or detached HEAD).
|
||||
branch = _git(src, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip()
|
||||
if branch == "HEAD":
|
||||
st.detached = True
|
||||
else:
|
||||
st.branch = branch
|
||||
|
||||
# Dirty working tree (staged, unstaged, or untracked).
|
||||
st.dirty = bool(_git(src, "status", "--porcelain").stdout.strip())
|
||||
|
||||
# Best-effort refresh from the remote; failure is non-fatal (offline, no remote).
|
||||
if fetch:
|
||||
try:
|
||||
fr = _git(src, "fetch", "--quiet", timeout=_FETCH_TIMEOUT)
|
||||
if fr.returncode != 0:
|
||||
st.error = (fr.stderr or fr.stdout).strip() or "git fetch failed"
|
||||
except subprocess.TimeoutExpired:
|
||||
st.error = "git fetch timed out"
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
st.error = str(e)
|
||||
|
||||
# Upstream tracking branch, then the ahead/behind split against it.
|
||||
up = _git(src, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}")
|
||||
if up.returncode == 0 and up.stdout.strip():
|
||||
st.upstream = up.stdout.strip()
|
||||
counts = _git(src, "rev-list", "--left-right", "--count", "@{u}...HEAD")
|
||||
if counts.returncode == 0:
|
||||
parts = counts.stdout.split()
|
||||
if len(parts) == 2:
|
||||
st.behind, st.ahead = int(parts[0]), int(parts[1])
|
||||
return st
|
||||
|
||||
|
||||
def head(source: Path | None) -> str | None:
|
||||
"""The working copy's current commit sha, or None if unavailable. Lets a caller
|
||||
tell whether a ``pull`` actually advanced the tree (before != after)."""
|
||||
if not is_git_repo(source):
|
||||
return None
|
||||
r = _git(Path(source), "rev-parse", "HEAD") # type: ignore[arg-type]
|
||||
return r.stdout.strip() or None if r.returncode == 0 else None
|
||||
|
||||
|
||||
def pull(source: Path | None) -> tuple[bool, str]:
|
||||
"""Fast-forward the working copy to its upstream (``git pull --ff-only``).
|
||||
|
||||
``--ff-only`` is deliberate: it refuses to merge, so a dirty or diverged tree
|
||||
fails cleanly with git's own message instead of creating a merge commit. Returns
|
||||
``(ok, combined_output)``.
|
||||
"""
|
||||
if not is_git_repo(source):
|
||||
return False, "not a git repository"
|
||||
try:
|
||||
r = _git(Path(source), "pull", "--ff-only", timeout=_PULL_TIMEOUT) # type: ignore[arg-type]
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "git pull timed out"
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
return False, str(e)
|
||||
out = (r.stdout + r.stderr).strip()
|
||||
return r.returncode == 0, out
|
||||
300
core/src/wildpc_core/lifecycle.py
Normal file
300
core/src/wildpc_core/lifecycle.py
Normal file
@@ -0,0 +1,300 @@
|
||||
"""Unified program lifecycle — a program is `active` when it's reachable in its mode.
|
||||
|
||||
`activate`/`deactivate` dispatch the right mechanism by behavior:
|
||||
- tool → on PATH (uv tool install / uninstall)
|
||||
- daemon / self-serving frontend / job → systemd enable+start / stop+disable
|
||||
- static frontend → served via the gateway (build + content present)
|
||||
|
||||
`is_active` reports the uniform state regardless of mechanism. The CLI/API verbs
|
||||
`install`/`uninstall` route through here (the words stay; the meaning is activate).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import WildpcConfig
|
||||
from wildpc_core.generators.systemd import (
|
||||
SYSTEMD_USER_DIR,
|
||||
generate_timer,
|
||||
generate_unit_from_deployed,
|
||||
secret_env_path,
|
||||
timer_name,
|
||||
unit_env_file,
|
||||
unit_name,
|
||||
)
|
||||
from wildpc_core.manifest import CaddyDeployment
|
||||
from wildpc_core.registry import REGISTRY_PATH, load_registry
|
||||
from wildpc_core.stacks import ActionResult, run_action
|
||||
|
||||
|
||||
def _systemctl_active(unit: str) -> bool:
|
||||
result = subprocess.run(
|
||||
["systemctl", "--user", "is-active", unit], capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip() in ("active", "waiting")
|
||||
|
||||
|
||||
_UV_TOOLS_CACHE: tuple[float, set[str]] | None = None
|
||||
|
||||
|
||||
def _uv_tool_packages() -> set[str]:
|
||||
"""Package names uv has installed as tools (`uv tool list`), briefly cached.
|
||||
|
||||
Authoritative for install detection: a program's *package* name can differ
|
||||
from the console script it exposes (e.g. `litellm-intent-router` installs the
|
||||
`intent-router` executable), so a `which(<program>)` check misses it.
|
||||
"""
|
||||
global _UV_TOOLS_CACHE
|
||||
now = time.monotonic()
|
||||
if _UV_TOOLS_CACHE is not None and now - _UV_TOOLS_CACHE[0] < 2.0:
|
||||
return _UV_TOOLS_CACHE[1]
|
||||
pkgs: set[str] = set()
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["uv", "tool", "list"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
for line in out.stdout.splitlines():
|
||||
# Package lines start at column 0 ("<name> vX.Y"); executables are
|
||||
# indented "- <exe>".
|
||||
if line and not line[0].isspace() and not line.startswith("-"):
|
||||
pkgs.add(line.split()[0])
|
||||
except Exception:
|
||||
pass
|
||||
_UV_TOOLS_CACHE = (now, pkgs)
|
||||
return pkgs
|
||||
|
||||
|
||||
def _own_venv_bins() -> set[str]:
|
||||
"""Bin dirs of the *running* interpreter's virtualenv.
|
||||
|
||||
Excluded from install detection: when the api server (or a dev `wildpc`) runs
|
||||
inside a project venv via `uv run`, a tool merely *colocated* there — e.g. an
|
||||
editable a dev accidentally `uv pip install`ed while that venv was active — must
|
||||
not read as "installed on PATH" for the user. We only strip *our own* venv, not
|
||||
every venv, so tools legitimately installed elsewhere on PATH still count.
|
||||
"""
|
||||
bins = {os.path.normpath(os.path.dirname(sys.executable))} # the interpreter's own bin
|
||||
bins.add(os.path.normpath(os.path.join(sys.prefix, "bin"))) # venv root → bin
|
||||
venv = os.environ.get("VIRTUAL_ENV")
|
||||
if venv:
|
||||
bins.add(os.path.normpath(os.path.join(venv, "bin")))
|
||||
return bins
|
||||
|
||||
|
||||
def _on_path(name: str) -> bool:
|
||||
"""Whether a tool is installed, script-name-independent and blind to our own venv.
|
||||
|
||||
Checks, in order: the console script on PATH (minus the running interpreter's
|
||||
own venv bin — see :func:`_own_venv_bins`), the script in ~/.local/bin (uv's
|
||||
install dir), and finally `uv tool list` by *package* name — the last catches
|
||||
tools whose executable is named differently from the program.
|
||||
"""
|
||||
exclude = _own_venv_bins()
|
||||
search = os.pathsep.join(
|
||||
p
|
||||
for p in (os.environ.get("PATH") or os.defpath).split(os.pathsep)
|
||||
if p and os.path.normpath(p) not in exclude
|
||||
)
|
||||
if shutil.which(name, path=search) is not None:
|
||||
return True
|
||||
if (Path.home() / ".local" / "bin" / name).exists():
|
||||
return True
|
||||
return name in _uv_tool_packages()
|
||||
|
||||
|
||||
def tool_installed(name: str) -> bool:
|
||||
"""Public: whether a tool (by program/package name) is installed on PATH."""
|
||||
return _on_path(name)
|
||||
|
||||
|
||||
def _svc_manager(name: str, kind: str, config: WildpcConfig) -> str | None:
|
||||
"""The manager for a deployment (name, kind), or None if not in config."""
|
||||
dep = config.deployment(kind, name)
|
||||
return dep.manager if dep is not None else None
|
||||
|
||||
|
||||
def _static_built(name: str, config: WildpcConfig) -> bool:
|
||||
"""Whether a static (caddy) deployment's served dir exists (assets are built)."""
|
||||
dep = config.statics.get(name)
|
||||
if not isinstance(dep, CaddyDeployment):
|
||||
return False
|
||||
comp = config.programs.get(dep.program or name)
|
||||
return bool(comp and comp.source and (Path(comp.source) / dep.root).is_dir())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def is_active(name: str, kind: str, config: WildpcConfig) -> bool:
|
||||
"""Whether a deployment (name, kind) is available in its mode, by manager."""
|
||||
manager = _svc_manager(name, kind, config)
|
||||
if manager == "systemd":
|
||||
unit = timer_name(name) if kind == "job" else unit_name(name, kind)
|
||||
return _systemctl_active(unit)
|
||||
if manager == "caddy":
|
||||
return _static_built(name, config) # served once its assets exist
|
||||
if manager == "path":
|
||||
return _on_path(name)
|
||||
if manager == "none":
|
||||
return True # remote: external, treated as available
|
||||
# No deployment — a bare program (e.g. a tool not yet given a path service).
|
||||
comp = config.programs.get(name)
|
||||
if comp is not None and comp.source:
|
||||
return _on_path(name)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Systemd enable/disable (extracted core; the CLI service command calls these)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def enable_service(name: str, kind: str, config: WildpcConfig) -> ActionResult:
|
||||
"""Generate+install the unit (and timer) from the registry, enable and start it."""
|
||||
if not REGISTRY_PATH.exists():
|
||||
return ActionResult(
|
||||
name, "activate", "error", "No registry. Run 'wildpc deploy' first."
|
||||
)
|
||||
registry = load_registry()
|
||||
deployed = registry.get(kind, name)
|
||||
if deployed is None:
|
||||
return ActionResult(
|
||||
name, "activate", "error", f"'{name}' not in registry; run 'wildpc deploy'."
|
||||
)
|
||||
if not deployed.managed:
|
||||
return ActionResult(
|
||||
name, "activate", "error", f"'{name}' is not a managed service."
|
||||
)
|
||||
|
||||
systemd_spec = None
|
||||
dep = config.deployment(kind, name)
|
||||
manage = getattr(dep, "manage", None)
|
||||
if manage:
|
||||
systemd_spec = manage.systemd
|
||||
|
||||
SYSTEMD_USER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
svc_unit = unit_name(name, kind)
|
||||
(SYSTEMD_USER_DIR / svc_unit).write_text(
|
||||
generate_unit_from_deployed(
|
||||
name, deployed, systemd_spec, env_file=unit_env_file(deployed, name)
|
||||
)
|
||||
)
|
||||
primary = svc_unit
|
||||
if deployed.schedule:
|
||||
tmr_unit = timer_name(name)
|
||||
(SYSTEMD_USER_DIR / tmr_unit).write_text(
|
||||
generate_timer(
|
||||
name, schedule=deployed.schedule, description=deployed.description
|
||||
)
|
||||
)
|
||||
primary = tmr_unit
|
||||
|
||||
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
||||
subprocess.run(["systemctl", "--user", "enable", primary], check=False)
|
||||
subprocess.run(["systemctl", "--user", "start", primary], check=False)
|
||||
status = "active" if _systemctl_active(primary) else "inactive"
|
||||
return ActionResult(
|
||||
name, "activate", "ok" if status == "active" else "error", f"{name}: {status}"
|
||||
)
|
||||
|
||||
|
||||
def disable_service(name: str, kind: str) -> ActionResult:
|
||||
"""Stop, disable, and remove the unit (and timer) for a service/job of a kind."""
|
||||
units = [unit_name(name, kind)]
|
||||
if kind == "job":
|
||||
units.append(timer_name(name))
|
||||
for unit in units:
|
||||
path = SYSTEMD_USER_DIR / unit
|
||||
if path.exists():
|
||||
subprocess.run(["systemctl", "--user", "stop", unit], check=False)
|
||||
subprocess.run(["systemctl", "--user", "disable", unit], check=False)
|
||||
path.unlink()
|
||||
# Drop the generated secret env file alongside the unit.
|
||||
secret_env_path(name, kind).unlink(missing_ok=True)
|
||||
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
||||
return ActionResult(name, "deactivate", "ok", f"{name}: deactivated")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Activate / deactivate (dispatch by behavior)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _program_for(name: str, kind: str, config: WildpcConfig):
|
||||
"""The program a deployment runs (its `program` ref, defaulting to the name)."""
|
||||
dep = config.deployment(kind, name)
|
||||
prog = (dep.program if dep else None) or name
|
||||
return prog, config.programs.get(prog)
|
||||
|
||||
|
||||
async def activate(name: str, kind: str, config: WildpcConfig, root: Path) -> ActionResult:
|
||||
"""Make a deployment (name, kind) available in its mode, dispatched by manager."""
|
||||
manager = _svc_manager(name, kind, config)
|
||||
|
||||
if manager == "systemd":
|
||||
# Ensure the program's binary is on PATH first (python), then enable the
|
||||
# unit. Skip the editable reinstall if it's already there.
|
||||
comp = config.programs.get(name)
|
||||
if comp is not None and (comp.stack or comp.commands) and not _on_path(name):
|
||||
res = await run_action("install", name, comp, root)
|
||||
if res.status != "ok":
|
||||
return res
|
||||
return enable_service(name, kind, config)
|
||||
|
||||
if manager == "caddy":
|
||||
# Served by the gateway — reload it so the route is live. Building the
|
||||
# assets is `wildpc program build` (the program's concern), not activation.
|
||||
subprocess.run(
|
||||
["systemctl", "--user", "reload", unit_name("wildpc-gateway")], check=False
|
||||
)
|
||||
return ActionResult(name, "activate", "ok", f"{name}: served via gateway")
|
||||
|
||||
if manager == "path":
|
||||
prog, comp = _program_for(name, kind, config)
|
||||
if comp is None:
|
||||
return ActionResult(name, "activate", "error", f"unknown program '{prog}'")
|
||||
if _on_path(prog): # already installed — skip the (slow) editable reinstall
|
||||
return ActionResult(name, "activate", "ok", f"{name}: on PATH")
|
||||
return await run_action("install", prog, comp, root)
|
||||
|
||||
if manager == "none":
|
||||
return ActionResult(name, "activate", "ok", f"{name}: external")
|
||||
|
||||
# No deployment — a bare tool program: install to PATH.
|
||||
comp = config.programs.get(name)
|
||||
if comp is not None:
|
||||
return await run_action("install", name, comp, root)
|
||||
return ActionResult(name, "activate", "error", f"'{name}' not found")
|
||||
|
||||
|
||||
async def deactivate(name: str, kind: str, config: WildpcConfig, root: Path) -> ActionResult:
|
||||
"""Take a deployment (name, kind) offline in its mode, dispatched by manager."""
|
||||
manager = _svc_manager(name, kind, config)
|
||||
|
||||
if manager == "systemd":
|
||||
return disable_service(name, kind)
|
||||
if manager == "caddy":
|
||||
return ActionResult(
|
||||
name, "deactivate", "ok",
|
||||
f"{name}: gateway-served — remove/disable the service to drop the route.",
|
||||
)
|
||||
if manager == "path":
|
||||
prog, comp = _program_for(name, kind, config)
|
||||
if comp is None:
|
||||
return ActionResult(name, "deactivate", "error", f"unknown program '{prog}'")
|
||||
return await run_action("uninstall", prog, comp, root)
|
||||
if manager == "none":
|
||||
return ActionResult(name, "deactivate", "ok", f"{name}: external")
|
||||
|
||||
comp = config.programs.get(name)
|
||||
if comp is not None:
|
||||
return await run_action("uninstall", name, comp, root)
|
||||
return ActionResult(name, "deactivate", "error", f"'{name}' not found")
|
||||
574
core/src/wildpc_core/manifest.py
Normal file
574
core/src/wildpc_core/manifest.py
Normal file
@@ -0,0 +1,574 @@
|
||||
"""Wild PC manifest models — program specs, service specs, job specs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Annotated, Literal, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
EnvMap = dict[str, str]
|
||||
|
||||
|
||||
class RestartPolicy(str, Enum):
|
||||
NO = "no"
|
||||
ON_FAILURE = "on-failure"
|
||||
ALWAYS = "always"
|
||||
|
||||
|
||||
class Reach(str, Enum):
|
||||
"""How far a deployment is exposed — a protocol-agnostic ladder.
|
||||
|
||||
``off`` → reachable only at its own host:port (no gateway route).
|
||||
``internal`` → reachable at ``<name>.<domain>`` (HTTP via the gateway, or TCP
|
||||
via bind + wildcard DNS).
|
||||
``public`` → *also* projected to the internet (HTTP via the tunnel origin;
|
||||
TCP via ``cloudflared access tcp``). Implies ``internal``.
|
||||
|
||||
``proxy``/``public`` survive as derived read-only accessors on the deployment.
|
||||
"""
|
||||
|
||||
OFF = "off"
|
||||
INTERNAL = "internal"
|
||||
PUBLIC = "public"
|
||||
|
||||
|
||||
def _validate_public_host(host: str | None, reach: Reach) -> None:
|
||||
"""Validate an optional ``public_host`` override on an exposable deployment.
|
||||
|
||||
A ``public_host`` only makes sense for a publicly-projected deployment, and
|
||||
must be a bare hostname (no scheme, path, port, or whitespace) — it becomes a
|
||||
tunnel ingress ``hostname`` and a Caddy site address verbatim.
|
||||
"""
|
||||
if host is None:
|
||||
return
|
||||
if reach != Reach.PUBLIC:
|
||||
raise ValueError(
|
||||
f"public_host is only valid with reach: public (got reach: {reach.value})"
|
||||
)
|
||||
bad = any(c.isspace() for c in host) or any(
|
||||
tok in host for tok in ("://", "/", ":")
|
||||
)
|
||||
if not host or host != host.strip(".") or bad:
|
||||
raise ValueError(
|
||||
f"public_host must be a bare hostname (e.g. example.com), got {host!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Launch specs — how systemd starts a process (discriminated union on `launcher`)
|
||||
# ---------------------
|
||||
#
|
||||
# A *launcher* is the process-launch mechanism for a systemd-managed deployment.
|
||||
# It is orthogonal to the deployment's `manager`: only systemd deployments have a
|
||||
# launcher, and it says only how the process starts — not how it's supervised.
|
||||
# The non-process managers (caddy/path/none) have no launcher; their fields live
|
||||
# on the deployment variant itself.
|
||||
|
||||
|
||||
class LaunchBase(BaseModel):
|
||||
launcher: str
|
||||
|
||||
|
||||
class RunCommand(LaunchBase):
|
||||
launcher: Literal["command"]
|
||||
argv: list[str] = Field(min_length=1)
|
||||
|
||||
|
||||
class RunPython(LaunchBase):
|
||||
launcher: Literal["python"]
|
||||
program: str
|
||||
args: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RunContainer(LaunchBase):
|
||||
launcher: Literal["container"]
|
||||
image: str
|
||||
command: list[str] | None = None
|
||||
args: list[str] = Field(default_factory=list)
|
||||
ports: dict[int, int] = Field(default_factory=dict)
|
||||
volumes: list[str] = Field(default_factory=list)
|
||||
env: EnvMap = Field(default_factory=dict)
|
||||
workdir: str | None = None
|
||||
# Run the container as this uid[:gid] (e.g. "${uid}:${gid}"). Running as the
|
||||
# invoking user makes bind-mounted data/secrets/certs readable with no chown —
|
||||
# see docs/tcp-exposure.md §4. None → the image's own default user.
|
||||
user: str | None = None
|
||||
# tmpfs mounts (e.g. ["/var/run/postgresql"]) for image runtime dirs that must
|
||||
# be writable when the container runs as a non-default uid.
|
||||
tmpfs: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RunNode(LaunchBase):
|
||||
launcher: Literal["node"]
|
||||
script: str
|
||||
package_manager: Literal["npm", "pnpm", "yarn"] = "pnpm"
|
||||
args: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RunCompose(LaunchBase):
|
||||
"""A multi-container stack supervised as one unit via ``docker compose``.
|
||||
|
||||
Unlike ``container`` (a single ``docker run``), compose owns the stack's own
|
||||
networking, startup ordering, and per-service health — Wild PC delegates rather
|
||||
than reinventing orchestration. The unit runs ``compose up`` attached
|
||||
(``Type=simple``) and tears the stack down via a generated ``ExecStop`` (down).
|
||||
Env/secrets flow through systemd ``Environment=``/``EnvironmentFile=`` (from
|
||||
``defaults.env``); compose interpolates them from the process environment.
|
||||
"""
|
||||
|
||||
launcher: Literal["compose"]
|
||||
file: str = "docker-compose.yml" # resolved relative to the program source
|
||||
project_name: str | None = None # ``-p``; defaults to ``wildpc-<name>``
|
||||
|
||||
|
||||
LaunchSpec = Annotated[
|
||||
Union[
|
||||
RunCommand,
|
||||
RunPython,
|
||||
RunContainer,
|
||||
RunNode,
|
||||
RunCompose,
|
||||
],
|
||||
Field(discriminator="launcher"),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Systemd management
|
||||
# ---------------------
|
||||
|
||||
|
||||
class ReadinessHttpGet(BaseModel):
|
||||
http_get: str
|
||||
timeout_seconds: int = 2
|
||||
interval_seconds: int = 2
|
||||
success_codes: list[int] = Field(default_factory=lambda: [200])
|
||||
|
||||
|
||||
class SystemdSpec(BaseModel):
|
||||
enable: bool = True
|
||||
user: bool = True
|
||||
description: str | None = None
|
||||
after: list[str] = Field(default_factory=list)
|
||||
requires: list[str] = Field(default_factory=list)
|
||||
wanted_by: list[str] = Field(default_factory=lambda: ["default.target"])
|
||||
restart: RestartPolicy = RestartPolicy.ON_FAILURE
|
||||
restart_sec: int = 2
|
||||
no_new_privileges: bool = True
|
||||
readiness: ReadinessHttpGet | None = None
|
||||
exec_reload: str | None = None
|
||||
# Commands run after the main process starts (systemd ``ExecStartPost=``), one
|
||||
# line each — e.g. an OpenBao auto-unseal step. Failures don't fail the unit.
|
||||
exec_start_post: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ManageSpec(BaseModel):
|
||||
systemd: SystemdSpec | None = None
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Exposure — HTTP (via the gateway) or raw TCP (bind + DNS)
|
||||
# ---------------------
|
||||
|
||||
|
||||
class HttpInternal(BaseModel):
|
||||
host: str = "127.0.0.1"
|
||||
port: int = Field(ge=1, le=65535)
|
||||
unix_socket: str | None = None
|
||||
|
||||
|
||||
class HttpExposeSpec(BaseModel):
|
||||
internal: HttpInternal
|
||||
health_path: str | None = None
|
||||
|
||||
|
||||
class TlsMaterial(str, Enum):
|
||||
"""What cert files wildpc materializes onto a service from the wildcard cert.
|
||||
|
||||
``off`` → the service does its own TLS (or none); wildpc stays out of it.
|
||||
``pair`` → cert.pem + key.pem (postgres, redis, most daemons).
|
||||
``combined`` → one file: key+cert concatenated (mongodb, haproxy, …).
|
||||
"""
|
||||
|
||||
OFF = "off"
|
||||
PAIR = "pair"
|
||||
COMBINED = "combined"
|
||||
|
||||
|
||||
class TlsSpec(BaseModel):
|
||||
"""Wild PC-managed TLS material for a raw-TCP service, cut from the gateway's
|
||||
ACME wildcard cert (valid for ``<name>.<domain>``) and refreshed on renewal.
|
||||
The service consumes the materialized files via the ``${tls_*}`` placeholders.
|
||||
"""
|
||||
|
||||
material: TlsMaterial = TlsMaterial.OFF
|
||||
# Optional zero-downtime reload argv (a single command) run after the cert is
|
||||
# re-materialized on renewal — e.g. ["systemctl", "--user", "reload", "wildpc-postgres"].
|
||||
# Default (None): wildpc restarts the deployment (fine at a ~60-day cadence).
|
||||
reload: list[str] | None = None
|
||||
|
||||
|
||||
class TcpExposeSpec(BaseModel):
|
||||
"""A raw-TCP service (postgres, redis, …). It doesn't ride the HTTP gateway:
|
||||
with ``reach: internal`` it's reachable at ``<name>.<domain>:<port>`` via the
|
||||
wildcard DNS record + the bound port (no Caddy route). Publishing the port on
|
||||
the LAN is the deployment's own job (a container's ``run.ports``, or a native
|
||||
service binding ``0.0.0.0``); wildpc doesn't rebind it, so there's no bind-host
|
||||
field here to imply otherwise. ``tls`` (optional) has wildpc drop the wildcard
|
||||
cert onto the service so it presents a trusted cert for ``<name>.<domain>``.
|
||||
"""
|
||||
|
||||
port: int = Field(ge=1, le=65535)
|
||||
tls: TlsSpec | None = None
|
||||
|
||||
|
||||
class ExposeSpec(BaseModel):
|
||||
http: HttpExposeSpec | None = None
|
||||
tcp: TcpExposeSpec | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _one_protocol(self) -> ExposeSpec:
|
||||
if self.http and self.tcp:
|
||||
raise ValueError("a deployment exposes http OR tcp, not both")
|
||||
return self
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Build spec
|
||||
# ---------------------
|
||||
|
||||
|
||||
class BuildSpec(BaseModel):
|
||||
commands: list[list[str]] = Field(default_factory=list)
|
||||
outputs: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Commands spec — per-program dev verb overrides
|
||||
# ---------------------
|
||||
|
||||
|
||||
class CommandsSpec(BaseModel):
|
||||
"""Per-program dev verb commands. Each verb is a list of argv lists run in
|
||||
sequence. A declared verb overrides the stack default; an absent verb falls
|
||||
back to the program's stack handler (if any), else the verb is unavailable.
|
||||
|
||||
This generalizes BuildSpec.commands to the rest of the verb contract, which
|
||||
is what lets a wired-in repo with no `stack` declare how it is linted/tested/run.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
lint: list[list[str]] | None = None
|
||||
test: list[list[str]] | None = None
|
||||
type_check: list[list[str]] | None = Field(default=None, alias="type-check")
|
||||
check: list[list[str]] | None = None
|
||||
run: list[list[str]] | None = None
|
||||
install: list[list[str]] | None = None
|
||||
uninstall: list[list[str]] | None = None
|
||||
|
||||
def for_verb(self, verb: str) -> list[list[str]] | None:
|
||||
"""Return the declared commands for a verb name (accepts 'type-check')."""
|
||||
return getattr(self, verb.replace("-", "_"), None)
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Capabilities
|
||||
# ---------------------
|
||||
|
||||
|
||||
class Requirement(BaseModel):
|
||||
"""A precondition — another **deployment** that must exist for this one to be
|
||||
*functional* (``ref`` = the target deployment's name). ``bind`` names the env
|
||||
var wildpc projects the target's URL into — env is derived *from* the
|
||||
requirement, never scraped back into it.
|
||||
|
||||
A deployment declares these in its ``requires`` list; ``kind`` defaults to
|
||||
``deployment`` (write just ``- ref: foo``). The ``system`` and ``tool`` kinds are
|
||||
not written here — a program's host-package preconditions live in
|
||||
``system_dependencies`` (synthesized as ``kind: system``), and its stack's
|
||||
toolchains (``uv``/``pnpm``/``hugo``/…) are synthesized as ``kind: tool`` — both by
|
||||
the relationship model for the ``functional?`` check. See docs/relationships.md.
|
||||
"""
|
||||
|
||||
kind: Literal["system", "deployment", "tool"] = "deployment"
|
||||
ref: str
|
||||
bind: str | None = None
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Defaults
|
||||
# ---------------------
|
||||
|
||||
|
||||
class DefaultsSpec(BaseModel):
|
||||
env: EnvMap = Field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Agent spec — a launchable agent CLI (assistant-agnostic)
|
||||
# ---------------------
|
||||
|
||||
|
||||
class SessionsSpec(BaseModel):
|
||||
"""Declarative session-history capability for an agent (optional).
|
||||
|
||||
Lets the dashboard show a unified picker of an agent's *own* past sessions
|
||||
without wildpc knowing anything agent-specific in code: it runs
|
||||
``list_command`` (which must print a JSON array of session objects), reads
|
||||
three named fields off each object, and launches ``command`` + ``resume``
|
||||
(with ``{id}`` substituted) to reopen one. Field names default to opencode's
|
||||
shape and are overridable per agent (dotted paths allowed, e.g.
|
||||
``config_summary.session_id``).
|
||||
"""
|
||||
|
||||
list_command: list[str] # argv → JSON array of session objects
|
||||
resume: list[str] # appended to the agent command; "{id}" is substituted
|
||||
id_field: str = "id"
|
||||
title_field: str = "title"
|
||||
time_field: str = "updated"
|
||||
|
||||
|
||||
class AgentSpec(BaseModel):
|
||||
"""A launchable agent CLI for the dashboard's terminal UX.
|
||||
|
||||
Wild PC just runs ``command args`` inside a pty at ``cwd``; it never parses
|
||||
the agent's output, so any interactive CLI works. This block only names the
|
||||
launch — live sessions (list/resume/kill) are a runtime concern, not config.
|
||||
"""
|
||||
|
||||
command: str
|
||||
args: list[str] = Field(default_factory=list)
|
||||
description: str | None = None
|
||||
cwd: str | None = None # defaults to the wildpc repo root when unset
|
||||
env: EnvMap = Field(default_factory=dict)
|
||||
# Extra args that open the agent's own session browser / continue its last
|
||||
# conversation (e.g. ["--resume"] or ["--continue"]). Optional and
|
||||
# agent-specific — wildpc just passes them through. Empty = no such affordance.
|
||||
resume_args: list[str] = Field(default_factory=list)
|
||||
# Optional: declares how to list + resume the agent's own sessions, so the
|
||||
# dashboard can render a unified session picker (see SessionsSpec).
|
||||
sessions: SessionsSpec | None = None
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Program spec — software identity
|
||||
# ---------------------
|
||||
|
||||
|
||||
class ProgramSpec(BaseModel):
|
||||
"""Software catalog entry — what exists."""
|
||||
|
||||
id: str = ""
|
||||
description: str | None = None
|
||||
# A program has NO kind of its own — kind is a *deployment* property. A program
|
||||
# is a catalog entry that has 0..N deployments, each with its own kind (see
|
||||
# kind_for and WildpcConfig.deployments_of).
|
||||
|
||||
source: str | None = None
|
||||
stack: str | None = None
|
||||
|
||||
# Wiring in existing repos: clone from `repo` (git URL) at optional `ref`;
|
||||
# `source` (when set) is the local working copy and takes precedence.
|
||||
repo: str | None = None
|
||||
ref: str | None = None
|
||||
|
||||
# Per-program dev verb overrides (declared verbs override the stack default).
|
||||
commands: CommandsSpec | None = None
|
||||
|
||||
# Host-package preconditions (apt packages / binaries) intrinsic to this
|
||||
# software. The relationship model checks these (`which`/`dpkg`) to derive the
|
||||
# `functional?` light. Deployment-to-deployment dependencies are NOT here — they
|
||||
# live on the deployment's `requires` (see DeploymentBase). See docs/relationships.md.
|
||||
system_dependencies: list[str] = Field(default_factory=list)
|
||||
install_extras: list[str] = Field(default_factory=list)
|
||||
version: str | None = None
|
||||
build: BuildSpec | None = None
|
||||
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
|
||||
@property
|
||||
def source_dir(self) -> str | None:
|
||||
"""Relative directory for this component's source, or None."""
|
||||
if self.source:
|
||||
return self.source.rstrip("/")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------
|
||||
# Deployment specs — a program materialized into the runtime
|
||||
# ---------------------
|
||||
#
|
||||
# A deployment is discriminated on its `manager` — who supervises/realizes it:
|
||||
# systemd → a process (or, with a `schedule`, a `.timer`)
|
||||
# caddy → a gateway route serving a static dir (file_server)
|
||||
# path → a CLI installed on PATH (uv tool install)
|
||||
# none → an external reference we only point at (remote)
|
||||
# The human "kind" (service/job/tool/static/reference) is *derived* from the
|
||||
# manager (+ schedule presence), never stored — see `kind_for`.
|
||||
|
||||
|
||||
class DeploymentBase(BaseModel):
|
||||
"""Fields common to every deployment, regardless of manager."""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
id: str = ""
|
||||
# The program this deployment materializes.
|
||||
program: str | None = None
|
||||
description: str | None = None
|
||||
defaults: DefaultsSpec | None = None
|
||||
# Deployment-to-deployment preconditions: other deployments this one needs
|
||||
# (e.g. a frontend that requires its API + the supabase substrate). Each entry
|
||||
# is `- ref: <deployment>` (+ optional `bind: ENV_VAR` to project the target's
|
||||
# URL into env). Drives the relationship graph's edges. See docs/relationships.md.
|
||||
requires: list[Requirement] = Field(default_factory=list)
|
||||
# Declared on/off state. `wildpc apply` converges reality to this: enabled
|
||||
# deployments are activated (service started, tool installed, route served),
|
||||
# disabled ones are deactivated but kept in the catalog. This is *desired
|
||||
# state*, not a runtime toggle — the only way to durably stop something.
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class SystemdDeployment(DeploymentBase):
|
||||
"""A process supervised by systemd — a *service*, or a *job* when scheduled."""
|
||||
|
||||
manager: Literal["systemd"]
|
||||
run: LaunchSpec
|
||||
# Present → a `.timer` (job); absent → a continuous `.service` (service).
|
||||
schedule: str | None = None
|
||||
timezone: str = "America/Los_Angeles"
|
||||
expose: ExposeSpec | None = None
|
||||
# How far this process is exposed (off | internal | public). See `Reach`.
|
||||
reach: Reach = Reach.OFF
|
||||
# Optional public hostname override (an exact FQDN, e.g. `api.example.com` or an
|
||||
# apex `example.com`). Only meaningful with `reach: public`; when set it is the
|
||||
# public-facing name instead of the derived `<name>.<gateway.public_domain>`.
|
||||
# Unset → the node-wide public domain is the default. See docs/tunnel-setup.md.
|
||||
public_host: str | None = None
|
||||
manage: ManageSpec | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_reach(self) -> SystemdDeployment:
|
||||
# An exposed reach needs a port to expose. Without an `expose` block the
|
||||
# reach silently no-ops — no route, no subdomain, no tunnel entry — so a
|
||||
# typo'd/omitted `expose` reads as success while the service is unreachable.
|
||||
# Reject it at load so the mistake surfaces (replaces the old
|
||||
# "public requires proxy" guard, now that reach is the canonical field).
|
||||
# Static frontends (manager: caddy) are inherently exposed and validated
|
||||
# elsewhere; this is a supervised process, which needs an explicit port.
|
||||
if self.reach != Reach.OFF and not self.expose:
|
||||
raise ValueError(
|
||||
f"reach: {self.reach.value} requires an `expose` block "
|
||||
"(expose.http or expose.tcp); a port-only process uses reach: off"
|
||||
)
|
||||
# Public raw-TCP (tunnel + Access) is a later step; guard it explicitly
|
||||
# rather than silently no-op'ing when a TCP service asks for it.
|
||||
if (
|
||||
self.reach == Reach.PUBLIC
|
||||
and self.expose
|
||||
and self.expose.tcp
|
||||
and not self.expose.http
|
||||
):
|
||||
raise ValueError(
|
||||
"reach: public for a raw-TCP service isn't supported yet "
|
||||
"(see docs/tcp-exposure.md step 5); use reach: internal"
|
||||
)
|
||||
_validate_public_host(self.public_host, self.reach)
|
||||
return self
|
||||
|
||||
# Derived, read-only back-compat accessors (not serialized) so existing
|
||||
# readers keep working while the stored/authored field is `reach`.
|
||||
@property
|
||||
def proxy(self) -> bool:
|
||||
return self.reach != Reach.OFF
|
||||
|
||||
@property
|
||||
def public(self) -> bool:
|
||||
return self.reach == Reach.PUBLIC
|
||||
|
||||
@property
|
||||
def http_exposed(self) -> bool:
|
||||
"""Exposed through the HTTP gateway at ``<name>.<domain>`` — the predicate
|
||||
for a Caddy route / subdomain. Requires ``reach != off`` *and* an HTTP
|
||||
port; a raw-TCP service (``expose.tcp``) is never HTTP-exposed."""
|
||||
return self.reach != Reach.OFF and bool(self.expose and self.expose.http)
|
||||
|
||||
@property
|
||||
def tcp_port(self) -> int | None:
|
||||
"""The raw-TCP port this service is exposed on, or None. Reachable at
|
||||
``<name>.<domain>:<port>`` when ``reach != off`` (bind + wildcard DNS)."""
|
||||
if self.reach != Reach.OFF and self.expose and self.expose.tcp:
|
||||
return self.expose.tcp.port
|
||||
return None
|
||||
|
||||
|
||||
class CaddyDeployment(DeploymentBase):
|
||||
"""A static site served by the gateway (Caddy ``file_server``) — no process.
|
||||
|
||||
The gateway *is* its runtime; it's inherently exposed at its subdomain. ``root``
|
||||
is the built dir to serve, relative to the program source (e.g. ``dist``/``public``).
|
||||
"""
|
||||
|
||||
manager: Literal["caddy"]
|
||||
root: str = "dist"
|
||||
# Serving strategy. `True` (default) → SPA fallback: any unmatched path serves
|
||||
# the root `index.html` so a client-side router (React/Vite) takes over. `False`
|
||||
# → content-site serving: the gateway resolves directory indexes (`/posts/` →
|
||||
# `/posts/index.html`) and 404s genuinely-missing paths — correct for Hugo and
|
||||
# other multi-page static sites, where the SPA fallback would swallow every
|
||||
# in-site link back to the homepage.
|
||||
spa: bool = True
|
||||
# A static site is inherently served at its subdomain, so `reach` is
|
||||
# `internal` or `public` (never `off`). `public` = also project via the tunnel.
|
||||
reach: Reach = Reach.INTERNAL
|
||||
# Optional public hostname override (exact FQDN, apex allowed). Only meaningful
|
||||
# with `reach: public`; see SystemdDeployment.public_host.
|
||||
public_host: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_reach(self) -> CaddyDeployment:
|
||||
if self.reach == Reach.OFF:
|
||||
raise ValueError("a static (caddy) deployment is always served; reach must be internal|public")
|
||||
_validate_public_host(self.public_host, self.reach)
|
||||
return self
|
||||
|
||||
@property
|
||||
def public(self) -> bool:
|
||||
return self.reach == Reach.PUBLIC
|
||||
|
||||
|
||||
class PathDeployment(DeploymentBase):
|
||||
"""A CLI installed on PATH via ``uv tool install`` — no process, no route.
|
||||
|
||||
Lifecycle is install/uninstall (what start/stop/enable/disable map to).
|
||||
"""
|
||||
|
||||
manager: Literal["path"]
|
||||
# An Anthropic Messages-API tool definition ({name, description, input_schema})
|
||||
# for handing this CLI to an agent. Derived from the tool's ``--help`` (see
|
||||
# ``wildpc_core.tool_schema``) but editable/overridable — a stored draft the
|
||||
# completion-context builder reads. None → not yet generated.
|
||||
tool_schema: dict | None = None
|
||||
|
||||
|
||||
class RemoteDeployment(DeploymentBase):
|
||||
"""An external service (another node) — we only reference/route it, never run it."""
|
||||
|
||||
manager: Literal["none"]
|
||||
base_url: str
|
||||
health_url: str | None = None
|
||||
|
||||
|
||||
DeploymentSpec = Annotated[
|
||||
Union[SystemdDeployment, CaddyDeployment, PathDeployment, RemoteDeployment],
|
||||
Field(discriminator="manager"),
|
||||
]
|
||||
|
||||
|
||||
def kind_for(spec: DeploymentSpec) -> str:
|
||||
"""The derived kind of a deployment: service|job|tool|static|reference."""
|
||||
if spec.manager == "systemd":
|
||||
return "job" if getattr(spec, "schedule", None) else "service"
|
||||
return {"caddy": "static", "path": "tool", "none": "reference"}[spec.manager]
|
||||
299
core/src/wildpc_core/registry.py
Normal file
299
core/src/wildpc_core/registry.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""Node registry — per-machine deployment state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from wildpc_core.config import CONTENT_DIR, SPECS_DIR
|
||||
|
||||
REGISTRY_PATH = SPECS_DIR / "registry.yaml"
|
||||
STATIC_DIR = CONTENT_DIR # backwards-compat alias
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeConfig:
|
||||
"""Per-node identity and settings."""
|
||||
|
||||
hostname: str = ""
|
||||
wildpc_root: str | None = None # repo path, for dev commands
|
||||
gateway_port: int = 9000
|
||||
# None/"off" → HTTP-only; "internal" → Caddy local-CA HTTPS; "acme" → Let's
|
||||
# Encrypt wildcard (*.gateway_domain) via DNS-01.
|
||||
gateway_tls: str | None = None
|
||||
gateway_domain: str | None = None # acme: zone for wildcard cert + host subdomains
|
||||
acme_email: str | None = None
|
||||
acme_dns_provider: str = "cloudflare"
|
||||
# Cloudflare tunnel: public services publish at <subdomain>.<public_domain>.
|
||||
public_domain: str | None = None
|
||||
tunnel_id: str | None = None
|
||||
# Emit the cert_obtained → `wildpc tls reconcile` hook (needs events-exec plugin).
|
||||
cert_hook: bool = False
|
||||
# Routable host peers use to reach this node's services (LAN IP/hostname).
|
||||
# Defaults to the hostname; set explicitly when the hostname isn't resolvable
|
||||
# cross-node. Used to build `remote` gateway routes to this node.
|
||||
address: str | None = None
|
||||
# Fleet role: "authority" may write shared config/secrets to the mesh;
|
||||
# "follower" reconciles from it. Static (no election) — the authority is
|
||||
# pinned in wildpc.yaml. When the authority is down, shared state is
|
||||
# read-only and every node keeps serving its own deployments from cache.
|
||||
role: str = "follower"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.hostname:
|
||||
self.hostname = socket.gethostname()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Deployment:
|
||||
"""A component deployed on this node with resolved runtime config."""
|
||||
|
||||
# Who supervises/realizes this deployment: systemd | caddy | path | none.
|
||||
manager: str
|
||||
run_cmd: list[str]
|
||||
# The systemd launch mechanism (python|command|container|compose|node), or
|
||||
# None for the non-process managers (caddy/path/none).
|
||||
launcher: str | None = None
|
||||
# Optional teardown command emitted as systemd ``ExecStop=`` (e.g. compose
|
||||
# ``down``). Empty for launchers whose stop is just SIGTERM to the ExecStart pid.
|
||||
stop_cmd: list[str] = field(default_factory=list)
|
||||
env: dict[str, str] = field(default_factory=dict)
|
||||
# Absolute dirs prepended to the unit's default PATH — a resolved toolchain the
|
||||
# default tool PATH omits (e.g. a program's pinned nvm node bin). Ignored when
|
||||
# the deployment sets its own PATH in env (an explicit PATH is a full override).
|
||||
path_prepend: list[str] = field(default_factory=list)
|
||||
# Names (never values) of secret-bearing env vars. Their resolved values live
|
||||
# only in the mode-0600 env file, never in env/run_cmd/registry — this is for
|
||||
# visibility (which secrets a deployment expects).
|
||||
secret_env_keys: list[str] = field(default_factory=list)
|
||||
description: str | None = None
|
||||
# Deployment identity is (name, kind) — a name may be shared across kinds.
|
||||
name: str = ""
|
||||
# Kind: service | job | tool | static | reference (structural — its store).
|
||||
kind: str = "service"
|
||||
stack: str | None = None
|
||||
port: int | None = None
|
||||
health_path: str | None = None
|
||||
# Exposed at <subdomain>.<gateway.domain> (the subdomain is the service name),
|
||||
# or None when the service is reachable only at its host:port.
|
||||
subdomain: str | None = None
|
||||
# Also projected to the public internet via the tunnel at
|
||||
# <subdomain>.<gateway.public_domain>. Requires subdomain.
|
||||
public: bool = False
|
||||
# Optional public hostname override (exact FQDN, apex allowed). When set it is
|
||||
# the public-facing name instead of the derived <subdomain>.<public_domain>.
|
||||
public_host: str | None = None
|
||||
# Raw-TCP exposure port (postgres, redis, …). Set → reachable at
|
||||
# <name>.<gateway.domain>:<tcp_port> via bind + wildcard DNS (no Caddy route).
|
||||
tcp_port: int | None = None
|
||||
# For `static` runner services: the absolute dir the gateway file_servers.
|
||||
# Set → the route is `static` (file_server) rather than `proxy` (reverse_proxy).
|
||||
static_root: str | None = None
|
||||
# For `static` routes: SPA fallback (True, default) vs content-site serving
|
||||
# (False, Hugo/multi-page). See CaddyDeployment.spa.
|
||||
spa: bool = True
|
||||
base_url: str | None = None
|
||||
schedule: str | None = None
|
||||
managed: bool = False
|
||||
# Declared desired state (from the deployment's `enabled:`). `wildpc apply`
|
||||
# activates enabled deployments and deactivates disabled ones. Default True.
|
||||
enabled: bool = True
|
||||
# Deployment `requires` (list of {kind, ref, bind}) — carried so the mesh can
|
||||
# draw cross-node consumption. A remote consumer's ref resolves against the
|
||||
# provider set across nodes.
|
||||
requires: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeRegistry:
|
||||
"""What's deployed on this node.
|
||||
|
||||
`deployed` is keyed by the composite identity ``"<kind>/<name>"`` so a service
|
||||
and a job (or tool) can share a bare name. Prefer the helpers over indexing
|
||||
`deployed` directly: `all()` (iterate), `get(kind, name)`, `named(name)`, `put()`.
|
||||
"""
|
||||
|
||||
node: NodeConfig
|
||||
deployed: dict[str, Deployment] = field(default_factory=dict)
|
||||
|
||||
@staticmethod
|
||||
def key(kind: str, name: str) -> str:
|
||||
return f"{kind}/{name}"
|
||||
|
||||
def all(self) -> list[tuple[str, str, Deployment]]:
|
||||
"""Every deployed unit as ``(kind, name, Deployment)``."""
|
||||
return [(d.kind, d.name, d) for d in self.deployed.values()]
|
||||
|
||||
def get(self, kind: str, name: str) -> Deployment | None:
|
||||
return self.deployed.get(self.key(kind, name))
|
||||
|
||||
def named(self, name: str) -> list[Deployment]:
|
||||
"""Every deployed unit sharing a bare name (across kinds)."""
|
||||
return [d for d in self.deployed.values() if d.name == name]
|
||||
|
||||
def put(self, deployed: Deployment) -> None:
|
||||
self.deployed[self.key(deployed.kind, deployed.name)] = deployed
|
||||
|
||||
|
||||
def load_registry(path: Path | None = None) -> NodeRegistry:
|
||||
"""Load the node registry from ~/.wildpc/registry.yaml."""
|
||||
if path is None:
|
||||
path = REGISTRY_PATH
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Registry not found: {path}\n"
|
||||
"Run 'wildpc deploy' to generate it from wildpc.yaml."
|
||||
)
|
||||
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not data:
|
||||
raise ValueError(f"Empty registry: {path}")
|
||||
|
||||
node_data = data.get("node", {})
|
||||
node = NodeConfig(
|
||||
hostname=node_data.get("hostname", ""),
|
||||
wildpc_root=node_data.get("wildpc_root"),
|
||||
gateway_port=node_data.get("gateway_port", 9000),
|
||||
gateway_tls=node_data.get("gateway_tls"),
|
||||
gateway_domain=node_data.get("gateway_domain"),
|
||||
acme_email=node_data.get("acme_email"),
|
||||
acme_dns_provider=node_data.get("acme_dns_provider", "cloudflare"),
|
||||
public_domain=node_data.get("public_domain"),
|
||||
tunnel_id=node_data.get("tunnel_id"),
|
||||
cert_hook=node_data.get("cert_hook", False),
|
||||
role=node_data.get("role", "follower"),
|
||||
address=node_data.get("address"),
|
||||
)
|
||||
|
||||
deployed: dict[str, Deployment] = {}
|
||||
for key, comp_data in data.get("deployed", {}).items():
|
||||
# Key is the composite "<kind>/<name>"; manager/kind are always present
|
||||
# (save_registry writes them). registry.yaml is regenerated every apply.
|
||||
kind, name = key.split("/", 1)
|
||||
deployed[NodeRegistry.key(kind, name)] = Deployment(
|
||||
manager=comp_data["manager"],
|
||||
launcher=comp_data.get("launcher"),
|
||||
run_cmd=comp_data.get("run_cmd", []),
|
||||
stop_cmd=comp_data.get("stop_cmd", []),
|
||||
env=comp_data.get("env", {}),
|
||||
path_prepend=comp_data.get("path_prepend", []),
|
||||
secret_env_keys=comp_data.get("secret_env_keys", []),
|
||||
description=comp_data.get("description"),
|
||||
name=name,
|
||||
kind=kind,
|
||||
stack=comp_data.get("stack"),
|
||||
port=comp_data.get("port"),
|
||||
health_path=comp_data.get("health_path"),
|
||||
subdomain=comp_data.get("subdomain"),
|
||||
public=comp_data.get("public", False),
|
||||
public_host=comp_data.get("public_host"),
|
||||
tcp_port=comp_data.get("tcp_port"),
|
||||
static_root=comp_data.get("static_root"),
|
||||
spa=comp_data.get("spa", True),
|
||||
base_url=comp_data.get("base_url"),
|
||||
schedule=comp_data.get("schedule"),
|
||||
managed=comp_data.get("managed", False),
|
||||
enabled=comp_data.get("enabled", True),
|
||||
requires=comp_data.get("requires", []),
|
||||
)
|
||||
|
||||
return NodeRegistry(node=node, deployed=deployed)
|
||||
|
||||
|
||||
def save_registry(registry: NodeRegistry, path: Path | None = None) -> None:
|
||||
"""Write the node registry to ~/.wildpc/registry.yaml."""
|
||||
if path is None:
|
||||
path = REGISTRY_PATH
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
data: dict = {
|
||||
"node": {
|
||||
"hostname": registry.node.hostname,
|
||||
"gateway_port": registry.node.gateway_port,
|
||||
},
|
||||
"deployed": {},
|
||||
}
|
||||
|
||||
if registry.node.wildpc_root:
|
||||
data["node"]["wildpc_root"] = registry.node.wildpc_root
|
||||
|
||||
if registry.node.gateway_tls:
|
||||
data["node"]["gateway_tls"] = registry.node.gateway_tls
|
||||
if registry.node.gateway_domain:
|
||||
data["node"]["gateway_domain"] = registry.node.gateway_domain
|
||||
if registry.node.acme_email:
|
||||
data["node"]["acme_email"] = registry.node.acme_email
|
||||
if registry.node.acme_dns_provider and registry.node.acme_dns_provider != "cloudflare":
|
||||
data["node"]["acme_dns_provider"] = registry.node.acme_dns_provider
|
||||
if registry.node.public_domain:
|
||||
data["node"]["public_domain"] = registry.node.public_domain
|
||||
if registry.node.tunnel_id:
|
||||
data["node"]["tunnel_id"] = registry.node.tunnel_id
|
||||
if registry.node.cert_hook:
|
||||
data["node"]["cert_hook"] = registry.node.cert_hook
|
||||
if registry.node.role and registry.node.role != "follower":
|
||||
data["node"]["role"] = registry.node.role
|
||||
if registry.node.address:
|
||||
data["node"]["address"] = registry.node.address
|
||||
|
||||
for key, comp in registry.deployed.items():
|
||||
entry: dict = {
|
||||
"manager": comp.manager,
|
||||
"run_cmd": comp.run_cmd,
|
||||
}
|
||||
if comp.launcher:
|
||||
entry["launcher"] = comp.launcher
|
||||
if comp.stop_cmd:
|
||||
entry["stop_cmd"] = comp.stop_cmd
|
||||
if comp.env:
|
||||
entry["env"] = comp.env
|
||||
if comp.path_prepend:
|
||||
entry["path_prepend"] = comp.path_prepend
|
||||
if comp.secret_env_keys:
|
||||
entry["secret_env_keys"] = comp.secret_env_keys
|
||||
if comp.description:
|
||||
entry["description"] = comp.description
|
||||
entry["kind"] = comp.kind
|
||||
if comp.stack:
|
||||
entry["stack"] = comp.stack
|
||||
if comp.port is not None:
|
||||
entry["port"] = comp.port
|
||||
if comp.health_path:
|
||||
entry["health_path"] = comp.health_path
|
||||
if comp.subdomain:
|
||||
entry["subdomain"] = comp.subdomain
|
||||
if comp.public:
|
||||
entry["public"] = comp.public
|
||||
if comp.public_host:
|
||||
entry["public_host"] = comp.public_host
|
||||
if comp.tcp_port is not None:
|
||||
entry["tcp_port"] = comp.tcp_port
|
||||
if comp.static_root:
|
||||
entry["static_root"] = comp.static_root
|
||||
# Only emit when non-default (content-site) — default-True omission keeps
|
||||
# existing registries byte-identical and matches the load-side default.
|
||||
if not comp.spa:
|
||||
entry["spa"] = comp.spa
|
||||
if comp.base_url:
|
||||
entry["base_url"] = comp.base_url
|
||||
if comp.schedule:
|
||||
entry["schedule"] = comp.schedule
|
||||
if comp.managed:
|
||||
entry["managed"] = comp.managed
|
||||
# Only emit when disabled — default-True omission keeps existing
|
||||
# registries byte-identical and matches the load-side default.
|
||||
if not comp.enabled:
|
||||
entry["enabled"] = comp.enabled
|
||||
if comp.requires:
|
||||
entry["requires"] = comp.requires
|
||||
data["deployed"][key] = entry
|
||||
|
||||
with open(path, "w") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, sort_keys=False)
|
||||
329
core/src/wildpc_core/relations.py
Normal file
329
core/src/wildpc_core/relations.py
Normal file
@@ -0,0 +1,329 @@
|
||||
"""The relationship model — derived, never stored. See docs/relationships.md.
|
||||
|
||||
Entities: **program**, **deployment**, **repo** (a repo is a git working copy;
|
||||
programs sharing a toplevel form a monorepo). Preconditions come from two encoded
|
||||
sources: a deployment's **`requires`** (other deployments it needs) and its
|
||||
program's **`system_dependencies`** (host packages). These are unified here into one
|
||||
requirement set (typed by ``kind``: ``deployment`` = must exist, ``system`` = must be
|
||||
installed). Everything else — repos, env wiring, fan-in, and the predicates
|
||||
``functional?`` / ``fresh?`` / ``deployed?`` — is computed here on demand.
|
||||
|
||||
Governing rule: *predicates are derived; we encode only the non-derivable.* So this
|
||||
module reads the encoded ``requires`` + ``system_dependencies`` and derives the rest.
|
||||
It does **not** scrape env for dependencies — env is generated *from* requirements,
|
||||
not the reverse.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core import git
|
||||
from wildpc_core.config import USER_TOOL_PATH_DIRS, WildpcConfig
|
||||
from wildpc_core.generators.systemd import runtime_path
|
||||
from wildpc_core.manifest import Requirement, SystemdDeployment
|
||||
from wildpc_core.stacks import ToolRequirement, tools_for
|
||||
|
||||
|
||||
@dataclass
|
||||
class Repo:
|
||||
key: str # url-safe slug (basename of the working copy)
|
||||
path: str # git toplevel
|
||||
url: str | None
|
||||
ref: str | None
|
||||
programs: list[str]
|
||||
deployments: list[str]
|
||||
behind: int | None = None # commits behind upstream (None = unknown/no upstream)
|
||||
dirty: bool = False
|
||||
fresh: bool | None = None # derived: at latest and clean (None = not evaluated)
|
||||
|
||||
@property
|
||||
def multi(self) -> bool:
|
||||
"""A monorepo — more than one program shares this working copy."""
|
||||
return len(self.programs) > 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class Edge:
|
||||
src: str # deployment name
|
||||
dst: str # target: a package (system) or another deployment
|
||||
kind: str # "system" | "deployment"
|
||||
bind: str | None = None # env var to project the target URL into (deployment)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Endpoint:
|
||||
"""A socket a deployment exposes. ``protocol`` is a display heuristic — the
|
||||
manifest has no protocol field (expose is http XOR tcp), so raw TCP is refined
|
||||
by well-known port (else ``tcp``). A real protocol field is future work."""
|
||||
|
||||
protocol: str # "http" | "tcp" | "pg" | "bolt" | "mqtt" | "redis"
|
||||
port: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
name: str # deployment name
|
||||
program: str | None
|
||||
kind: str # service|job|tool|static|reference
|
||||
repo: str | None
|
||||
depended_on_by: int # distinct deployments that require this one (fan-in)
|
||||
unmet: list[str] = field(default_factory=list) # unsatisfied requirements
|
||||
functional: bool = True # derived: all requirements satisfied
|
||||
fresh: bool | None = None # derived: its repo is at latest + clean
|
||||
deployed: bool | None = None # derived: active in the registry (None = unknown)
|
||||
reach: str | None = None # off|internal|public (systemd/caddy), else None
|
||||
endpoints: list[Endpoint] = field(default_factory=list) # sockets it exposes
|
||||
base_url: str | None = None # the target URL, for kind=="reference" (external)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
repos: list[Repo] = field(default_factory=list)
|
||||
nodes: list[Node] = field(default_factory=list)
|
||||
edges: list[Edge] = field(default_factory=list)
|
||||
|
||||
|
||||
def _program_of(name: str, dep: object) -> str:
|
||||
return getattr(dep, "program", None) or name
|
||||
|
||||
|
||||
def _slug(name: str, used: set[str]) -> str:
|
||||
base = name or "repo"
|
||||
key, n = base, 2
|
||||
while key in used:
|
||||
key, n = f"{base}-{n}", n + 1
|
||||
used.add(key)
|
||||
return key
|
||||
|
||||
|
||||
def derive_repos(config: WildpcConfig) -> dict[str, Repo]:
|
||||
"""Group programs by the git working copy their source lives in."""
|
||||
by_top: dict[str, list[str]] = {}
|
||||
for pname, prog in config.programs.items():
|
||||
top = git.toplevel(prog.source) if prog.source else None
|
||||
if top:
|
||||
by_top.setdefault(top, []).append(pname)
|
||||
|
||||
used: set[str] = set()
|
||||
repos: dict[str, Repo] = {}
|
||||
for top, progs in sorted(by_top.items()):
|
||||
progs = sorted(progs)
|
||||
url = next(
|
||||
(config.programs[p].repo for p in progs if config.programs[p].repo), None
|
||||
) or git.remote_url(Path(top))
|
||||
ref = (
|
||||
next(
|
||||
(config.programs[p].ref for p in progs if config.programs[p].ref), None
|
||||
)
|
||||
or git.git_status(Path(top), fetch=False).branch
|
||||
)
|
||||
deps = sorted(
|
||||
d for _k, d, dep in config.all_deployments() if _program_of(d, dep) in progs
|
||||
)
|
||||
repos[_slug(Path(top).name, used)] = Repo("", top, url, ref, progs, deps)
|
||||
for key, repo in repos.items():
|
||||
repo.key = key
|
||||
return repos
|
||||
|
||||
|
||||
def requirements_of(config: WildpcConfig, dep_name: str) -> list[Requirement]:
|
||||
"""The full requirement set for a deployment: its own ``requires`` (deployment
|
||||
dependencies), its program's ``system_dependencies`` synthesized as
|
||||
``kind: system`` requirements, and its stack's toolchains synthesized as
|
||||
``kind: tool`` requirements — de-duplicated by (kind, ref)."""
|
||||
reqs: list[Requirement] = []
|
||||
# A bare name may span kinds — union their requirements (plus each program's
|
||||
# host-package deps as `kind: system` and its stack's toolchains as `kind: tool`).
|
||||
for _kind, dep in config.deployments_named(dep_name):
|
||||
reqs += list(getattr(dep, "requires", []) or [])
|
||||
prog = config.programs.get(_program_of(dep_name, dep))
|
||||
if prog:
|
||||
reqs += [
|
||||
Requirement(kind="system", ref=pkg) for pkg in prog.system_dependencies
|
||||
]
|
||||
reqs += [
|
||||
Requirement(kind="tool", ref=t.command) for t in tools_for(prog.stack)
|
||||
]
|
||||
seen: set[tuple[str, str]] = set()
|
||||
out: list[Requirement] = []
|
||||
for r in reqs:
|
||||
if (r.kind, r.ref) not in seen:
|
||||
seen.add((r.kind, r.ref))
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def stack_tools_of(config: WildpcConfig, dep_name: str) -> dict[str, ToolRequirement]:
|
||||
"""command → its :class:`ToolRequirement`, for every stack toolchain the
|
||||
deployment(s) named ``dep_name`` need. The metadata (phase, install hint) behind
|
||||
the ``kind: tool`` requirements ``requirements_of`` synthesizes — used to check
|
||||
them (phase picks the PATH) and to hint a fix."""
|
||||
meta: dict[str, ToolRequirement] = {}
|
||||
for _kind, dep in config.deployments_named(dep_name):
|
||||
prog = config.programs.get(_program_of(dep_name, dep))
|
||||
if prog and prog.stack:
|
||||
for t in tools_for(prog.stack):
|
||||
meta.setdefault(t.command, t)
|
||||
return meta
|
||||
|
||||
|
||||
def _dpkg_installed(pkg: str) -> bool:
|
||||
"""Whether a dpkg package is installed (the authoritative 'installed' check on
|
||||
Debian/Ubuntu). Returns False where dpkg isn't available."""
|
||||
dpkg = shutil.which("dpkg")
|
||||
if not dpkg:
|
||||
return False
|
||||
r = subprocess.run([dpkg, "-s", pkg], capture_output=True, text=True)
|
||||
return r.returncode == 0 and "install ok installed" in r.stdout
|
||||
|
||||
|
||||
def _build_path() -> str:
|
||||
"""The PATH a *build/dev* verb runs with — the caller's own PATH plus the user
|
||||
tool dirs (mirrors ``stacks._build_env``, minus the per-program node pin resolved
|
||||
separately). Build-phase tools (pnpm, hugo, node) are checked against this."""
|
||||
dirs = [str(d) for d in USER_TOOL_PATH_DIRS if d.exists()]
|
||||
return ":".join([*dirs, os.environ.get("PATH", "")])
|
||||
|
||||
|
||||
def _tool_available(dep: object, tool: ToolRequirement) -> bool:
|
||||
"""Is a stack tool present *where the deployment needs it*? A ``run``/``both``
|
||||
tool of a systemd service must be on the **service's runtime PATH** (the curated
|
||||
unit PATH, which can differ from your shell — the drift this catches); every
|
||||
other case (build-only tools, or a non-service deployment like a static site) is
|
||||
checked against the build/dev PATH."""
|
||||
runs_service = isinstance(dep, SystemdDeployment)
|
||||
if tool.phase in ("run", "both") and runs_service:
|
||||
defaults = getattr(dep, "defaults", None)
|
||||
env_path = (defaults.env.get("PATH") if defaults else None) or None
|
||||
# `path_prepend` (resolved toolchain) lives on the registry deployment, not
|
||||
# the manifest one; absent here it's the base runtime PATH, which is what a
|
||||
# service without a pinned toolchain actually runs with.
|
||||
path = env_path or runtime_path(list(getattr(dep, "path_prepend", []) or ()))
|
||||
return shutil.which(tool.command, path=path) is not None
|
||||
return shutil.which(tool.command, path=_build_path()) is not None
|
||||
|
||||
|
||||
def _check(
|
||||
config: WildpcConfig,
|
||||
req: Requirement,
|
||||
dep: object | None = None,
|
||||
tool: ToolRequirement | None = None,
|
||||
) -> bool:
|
||||
"""Is a single requirement satisfied? The check is fixed by its ``kind``. For a
|
||||
``tool`` requirement, ``dep`` and ``tool`` supply the deployment context and the
|
||||
stack-tool metadata (phase decides which PATH to probe)."""
|
||||
if req.kind == "system":
|
||||
# `system_dependencies` holds PACKAGE names, not executables. A PATH lookup
|
||||
# only coincides with 'installed' when the package name equals its command
|
||||
# (pandoc, rsync) — for names like poppler-utils / texlive-latex-base /
|
||||
# docker-compose-plugin it never matches. Fast-path `which`, then ask the
|
||||
# package manager (the real meaning of 'installed').
|
||||
return shutil.which(req.ref) is not None or _dpkg_installed(req.ref)
|
||||
if req.kind == "deployment":
|
||||
return bool(config.deployments_named(req.ref))
|
||||
if req.kind == "tool":
|
||||
# No metadata (unknown stack) → don't raise a false alarm.
|
||||
return tool is None or _tool_available(dep, tool)
|
||||
return True
|
||||
|
||||
|
||||
def hint_for(req: Requirement, tool: ToolRequirement | None = None) -> str:
|
||||
"""A copyable next step for an *unmet* requirement — the piece that makes a
|
||||
diagnostic actionable. ``tool`` carries a stack tool's precise install command."""
|
||||
if req.kind == "tool":
|
||||
return tool.install_hint if tool else f"install {req.ref}"
|
||||
if req.kind == "system":
|
||||
return f"sudo apt install {req.ref}"
|
||||
if req.kind == "deployment":
|
||||
return f"create & apply the '{req.ref}' deployment"
|
||||
return ""
|
||||
|
||||
|
||||
# Well-known TCP ports → a friendlier protocol label (display heuristic only).
|
||||
_TCP_PROTOCOL = {5432: "pg", 7687: "bolt", 1883: "mqtt", 6379: "redis"}
|
||||
|
||||
|
||||
def _endpoints_of(dep: object) -> list[Endpoint]:
|
||||
"""The sockets a deployment exposes, derived from its manifest — reusing the
|
||||
``http_exposed`` / ``tcp_port`` accessors (SystemdDeployment only). Tools,
|
||||
statics, and references have no expose block and yield ``[]``."""
|
||||
eps: list[Endpoint] = []
|
||||
if not isinstance(dep, SystemdDeployment):
|
||||
return eps # only systemd services/jobs carry an expose block
|
||||
exp = dep.expose
|
||||
if dep.http_exposed and exp and exp.http:
|
||||
eps.append(Endpoint("http", exp.http.internal.port))
|
||||
if dep.tcp_port is not None:
|
||||
eps.append(Endpoint(_TCP_PROTOCOL.get(dep.tcp_port, "tcp"), dep.tcp_port))
|
||||
return eps
|
||||
|
||||
|
||||
def build_model(
|
||||
config: WildpcConfig,
|
||||
check: bool = True,
|
||||
active: set[str] | None = None,
|
||||
freshness: bool = False,
|
||||
) -> Model:
|
||||
"""Compute the relationship model.
|
||||
|
||||
- ``check`` (default): evaluate ``functional?`` (unmet requirements) via a live
|
||||
``which`` / registry probe. ``check=False`` → pure structural model.
|
||||
- ``active``: names of currently-active deployments → the ``deployed?``
|
||||
predicate (left ``None`` when the caller has no runtime view).
|
||||
- ``freshness``: also evaluate ``fresh?`` per repo (a ``git status``, no fetch —
|
||||
last-known — so it stays a local, network-free probe over many repos)."""
|
||||
|
||||
repos = derive_repos(config)
|
||||
if freshness:
|
||||
for repo in repos.values():
|
||||
st = git.git_status(Path(repo.path), fetch=False)
|
||||
repo.behind = st.behind
|
||||
repo.dirty = st.dirty
|
||||
repo.fresh = (st.behind == 0 or st.behind is None) and not st.dirty
|
||||
repo_of = {p: key for key, r in repos.items() for p in r.programs}
|
||||
fresh_of = {key: r.fresh for key, r in repos.items()}
|
||||
|
||||
edges: list[Edge] = []
|
||||
for _k, name, _d in config.all_deployments():
|
||||
for r in requirements_of(config, name):
|
||||
edges.append(Edge(name, r.ref, r.kind, r.bind))
|
||||
|
||||
fan_in = Counter(e.dst for e in edges if e.kind == "deployment")
|
||||
|
||||
nodes: list[Node] = []
|
||||
for _nk, name, dep in config.all_deployments():
|
||||
tmeta = stack_tools_of(config, name) if check else {}
|
||||
unmet = (
|
||||
[
|
||||
f"{r.kind}:{r.ref}"
|
||||
for r in requirements_of(config, name)
|
||||
if not _check(config, r, dep=dep, tool=tmeta.get(r.ref))
|
||||
]
|
||||
if check
|
||||
else []
|
||||
)
|
||||
prog_name = _program_of(name, dep)
|
||||
repo_key = repo_of.get(prog_name)
|
||||
nodes.append(
|
||||
Node(
|
||||
name=name,
|
||||
program=prog_name,
|
||||
kind=_nk,
|
||||
repo=repo_key,
|
||||
depended_on_by=fan_in.get(name, 0),
|
||||
unmet=unmet,
|
||||
functional=not unmet,
|
||||
fresh=fresh_of.get(repo_key) if (freshness and repo_key) else None,
|
||||
deployed=(name in active) if active is not None else None,
|
||||
reach=getattr(getattr(dep, "reach", None), "value", None),
|
||||
endpoints=_endpoints_of(dep),
|
||||
base_url=getattr(dep, "base_url", None),
|
||||
)
|
||||
)
|
||||
return Model(repos=list(repos.values()), nodes=nodes, edges=edges)
|
||||
175
core/src/wildpc_core/secret_backends.py
Normal file
175
core/src/wildpc_core/secret_backends.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""Pluggable secret backends for ``${secret:NAME}`` resolution.
|
||||
|
||||
Default is the **file** backend (``~/.wildpc/secrets/<name>``) — identical to the
|
||||
historical behavior, so nothing changes unless a backend is explicitly selected
|
||||
via ``WILDPC_SECRET_BACKEND``.
|
||||
|
||||
The **openbao** backend reads from an OpenBao/Vault KV-v2 mount and falls back to
|
||||
the file backend, which is also how it bootstraps: the OpenBao *token* itself is a
|
||||
file secret (it can't live in the vault it unlocks).
|
||||
|
||||
Selection (env, so it works in both the CLI and the systemd-run API):
|
||||
WILDPC_SECRET_BACKEND file | openbao (default: file)
|
||||
WILDPC_OPENBAO_ADDR http://localhost:8200
|
||||
WILDPC_OPENBAO_MOUNT wildpc (kv-v2 mount path)
|
||||
WILDPC_OPENBAO_TOKEN_SECRET OPENBAO_TOKEN (file secret holding the token)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class SecretBackend(Protocol):
|
||||
def read(self, name: str) -> str | None: ...
|
||||
def write(self, name: str, value: str) -> None: ...
|
||||
def delete(self, name: str) -> None: ...
|
||||
def list_names(self) -> list[str]: ...
|
||||
|
||||
|
||||
class FileSecretBackend:
|
||||
"""Reads/writes ``<secrets_dir>/<name>`` (the historical behavior)."""
|
||||
|
||||
def __init__(self, secrets_dir: Path) -> None:
|
||||
self._dir = secrets_dir
|
||||
|
||||
def read(self, name: str) -> str | None:
|
||||
path = self._dir / name
|
||||
if path.exists():
|
||||
return path.read_text().strip()
|
||||
return None
|
||||
|
||||
def write(self, name: str, value: str) -> None:
|
||||
self._dir.mkdir(parents=True, exist_ok=True)
|
||||
(self._dir / name).write_text(value.strip() + "\n")
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
path = self._dir / name
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
if not self._dir.exists():
|
||||
return []
|
||||
return sorted(f.name for f in self._dir.iterdir() if f.is_file())
|
||||
|
||||
|
||||
class OpenBaoBackend:
|
||||
"""Reads/writes an OpenBao/Vault KV-v2 mount. No file fallback — a missing key
|
||||
or unreachable server returns None (the bootstrap token is read separately by
|
||||
``build_backend`` via the file backend, not through here).
|
||||
|
||||
``node_prefix`` supports a shared vault with per-node overrides: a read tries
|
||||
``<node_prefix>/<name>`` first, then the shared ``<name>``. So a node-specific
|
||||
secret (e.g. that node's postgres password) lives at the prefixed path while
|
||||
shared secrets (a common token) live at the base — no name collision.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, addr: str, token: str, mount: str, node_prefix: str | None = None
|
||||
) -> None:
|
||||
self._addr = addr.rstrip("/")
|
||||
self._token = token
|
||||
self._mount = mount
|
||||
self._node_prefix = (node_prefix or "").strip("/")
|
||||
|
||||
def read(self, name: str) -> str | None:
|
||||
if self._node_prefix:
|
||||
override = self._read_bao(f"{self._node_prefix}/{name}")
|
||||
if override is not None:
|
||||
return override
|
||||
return self._read_bao(name)
|
||||
|
||||
def _read_bao(self, name: str) -> str | None:
|
||||
if not self._token:
|
||||
return None
|
||||
url = f"{self._addr}/v1/{self._mount}/data/{name}"
|
||||
try:
|
||||
data = self._request("GET", url)
|
||||
return data["data"]["data"].get("value")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def write(self, name: str, value: str) -> None:
|
||||
url = f"{self._addr}/v1/{self._mount}/data/{name}"
|
||||
self._request("POST", url, {"data": {"value": value.strip()}})
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
# Remove all versions (metadata delete), matching file-backend semantics.
|
||||
url = f"{self._addr}/v1/{self._mount}/metadata/{name}"
|
||||
self._request("DELETE", url)
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
# Drop folder entries (e.g. "nodes/") — those group per-node overrides.
|
||||
return sorted(k for k in self._list_path("") if not k.endswith("/"))
|
||||
|
||||
def _list_path(self, prefix: str) -> list[str]:
|
||||
sep = "/" if prefix else ""
|
||||
url = f"{self._addr}/v1/{self._mount}/metadata{sep}{prefix}?list=true"
|
||||
try:
|
||||
return self._request("GET", url).get("data", {}).get("keys", [])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def list_node_overrides(self) -> dict[str, list[str]]:
|
||||
"""Per-node secret overrides: ``{host: [names]}`` from ``<mount>/nodes/*``."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for entry in self._list_path("nodes"):
|
||||
if not entry.endswith("/"):
|
||||
continue
|
||||
host = entry.rstrip("/")
|
||||
names = [k for k in self._list_path(f"nodes/{host}") if not k.endswith("/")]
|
||||
if names:
|
||||
out[host] = sorted(names)
|
||||
return out
|
||||
|
||||
def _request(self, method: str, url: str, body: dict | None = None) -> dict:
|
||||
payload = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request( # noqa: S310
|
||||
url,
|
||||
data=payload,
|
||||
method=method,
|
||||
headers={
|
||||
"X-Vault-Token": self._token,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp: # noqa: S310
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def build_backend(secrets_dir: Path, settings: dict | None = None) -> SecretBackend:
|
||||
"""Construct the active secret backend.
|
||||
|
||||
Selection comes from ``settings`` (the ``secrets:`` block of wildpc.yaml), with
|
||||
environment variables overriding — so production is configured declaratively in
|
||||
wildpc.yaml while tests/CI can force a backend via env. Default: file.
|
||||
|
||||
The OpenBao **token** is still read from the file backend (the bootstrap root of
|
||||
trust — it can't live in the vault it unlocks); everything else comes from the
|
||||
vault with no file fallback.
|
||||
"""
|
||||
settings = settings or {}
|
||||
file_backend = FileSecretBackend(secrets_dir)
|
||||
kind = (os.environ.get("WILDPC_SECRET_BACKEND") or settings.get("backend") or "file").lower()
|
||||
if kind == "openbao":
|
||||
addr = os.environ.get("WILDPC_OPENBAO_ADDR") or settings.get(
|
||||
"addr"
|
||||
) or "http://localhost:8200"
|
||||
mount = os.environ.get("WILDPC_OPENBAO_MOUNT") or settings.get("mount") or "wildpc"
|
||||
token_secret = os.environ.get("WILDPC_OPENBAO_TOKEN_SECRET") or settings.get(
|
||||
"token_secret"
|
||||
) or "OPENBAO_TOKEN"
|
||||
token = file_backend.read(token_secret) or os.environ.get(
|
||||
"WILDPC_OPENBAO_TOKEN", ""
|
||||
)
|
||||
node_prefix = os.environ.get("WILDPC_OPENBAO_NODE_PREFIX") or settings.get(
|
||||
"node_prefix"
|
||||
)
|
||||
return OpenBaoBackend(addr, token, mount, node_prefix)
|
||||
return file_backend
|
||||
136
core/src/wildpc_core/stack_status.py
Normal file
136
core/src/wildpc_core/stack_status.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Stack dependency status — the derived, per-stack health the `wildpc stack`
|
||||
command, the ``GET /stacks`` API, and the dashboard Stacks page all render.
|
||||
|
||||
A *stack* declares the host toolchains it needs (``stacks.ToolRequirement``); this
|
||||
module answers, for each stack: which programs/deployments use it, whether its
|
||||
tools are present *where the using deployments need them* (run-phase tools against
|
||||
a service's runtime PATH — the drift the plain ``which`` a shell does misses), and
|
||||
the copyable fix when one is missing. It's the single source of truth so the CLI,
|
||||
API, and UI never disagree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from wildpc_core.config import WildpcConfig
|
||||
from wildpc_core.generators.systemd import runtime_path
|
||||
from wildpc_core.relations import _build_path, _tool_available
|
||||
from wildpc_core.stacks import ToolRequirement, available_stacks, get_handler, tools_for
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolStatus:
|
||||
command: str
|
||||
purpose: str
|
||||
phase: str # run | build | both
|
||||
present: bool # resolvable where the using deployments need it
|
||||
install_hint: str # copyable fix (shown when absent)
|
||||
version: str | None = None # best-effort `--version`, when present
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackStatus:
|
||||
name: str
|
||||
tools: list[ToolStatus] = field(default_factory=list)
|
||||
programs: list[str] = field(default_factory=list) # programs on this stack
|
||||
deployments: list[str] = field(default_factory=list) # their deployments
|
||||
verbs: list[str] = field(default_factory=list) # dev verbs the stack provides
|
||||
has_enabled_deployment: bool = False # ≥1 enabled deployment uses it
|
||||
|
||||
@property
|
||||
def in_use(self) -> bool:
|
||||
return bool(self.programs)
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
"""All needed tools present (vacuously true for a stack with no tools)."""
|
||||
return all(t.present for t in self.tools)
|
||||
|
||||
|
||||
def _version(command: str, path: str) -> str | None:
|
||||
"""Best-effort tool version — the first `<cmd> --version` line (falls back to
|
||||
`<cmd> version` for tools like hugo). Never raises; returns None on any trouble."""
|
||||
exe = shutil.which(command, path=path)
|
||||
if not exe:
|
||||
return None
|
||||
for argv in ([exe, "--version"], [exe, "version"]):
|
||||
try:
|
||||
r = subprocess.run(argv, capture_output=True, text=True, timeout=2)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
continue
|
||||
out = (r.stdout or r.stderr or "").strip()
|
||||
if r.returncode == 0 and out:
|
||||
return out.splitlines()[0].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _tool_status(
|
||||
tool: ToolRequirement,
|
||||
using_deps: list[object],
|
||||
*,
|
||||
with_version: bool,
|
||||
) -> ToolStatus:
|
||||
# Present where it's needed: a run/both tool must resolve for EVERY using
|
||||
# deployment (a service's runtime PATH); with no deployments, check generically.
|
||||
if using_deps:
|
||||
present = all(_tool_available(dep, tool) for dep in using_deps)
|
||||
elif tool.phase in ("run", "both"):
|
||||
present = shutil.which(tool.command, path=runtime_path()) is not None
|
||||
else:
|
||||
present = shutil.which(tool.command, path=_build_path()) is not None
|
||||
# A version is only meaningful when the tool is present; probe the path that
|
||||
# matched (runtime for run/both, build otherwise) so it reflects what's used.
|
||||
probe_path = runtime_path() if tool.phase in ("run", "both") else _build_path()
|
||||
version = _version(tool.command, probe_path) if (present and with_version) else None
|
||||
return ToolStatus(
|
||||
command=tool.command,
|
||||
purpose=tool.purpose,
|
||||
phase=tool.phase,
|
||||
present=present,
|
||||
install_hint=tool.install_hint,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
def stack_status(
|
||||
config: WildpcConfig, name: str, *, with_version: bool = True
|
||||
) -> StackStatus | None:
|
||||
"""The dependency status of one stack, or None if wildpc has no such handler."""
|
||||
handler = get_handler(name)
|
||||
if handler is None:
|
||||
return None
|
||||
programs = sorted(p for p, c in config.programs.items() if c.stack == name)
|
||||
deps: list[tuple[str, object]] = [] # (deployment-name, spec)
|
||||
enabled = False
|
||||
for _kind, dep_name, dep in config.all_deployments():
|
||||
prog = config.programs.get(dep.program or dep_name)
|
||||
if prog and prog.stack == name:
|
||||
deps.append((dep_name, dep))
|
||||
enabled = enabled or getattr(dep, "enabled", True)
|
||||
dep_specs = [d for _n, d in deps]
|
||||
return StackStatus(
|
||||
name=name,
|
||||
tools=[
|
||||
_tool_status(t, dep_specs, with_version=with_version)
|
||||
for t in tools_for(name)
|
||||
],
|
||||
programs=programs,
|
||||
deployments=sorted(n for n, _d in deps),
|
||||
verbs=sorted(handler.provides),
|
||||
has_enabled_deployment=enabled,
|
||||
)
|
||||
|
||||
|
||||
def all_stack_status(
|
||||
config: WildpcConfig, *, with_version: bool = True
|
||||
) -> list[StackStatus]:
|
||||
"""Dependency status for every stack wildpc knows — the Stacks catalog."""
|
||||
out = []
|
||||
for name in available_stacks():
|
||||
st = stack_status(config, name, with_version=with_version)
|
||||
if st is not None:
|
||||
out.append(st)
|
||||
return out
|
||||
938
core/src/wildpc_core/stacks.py
Normal file
938
core/src/wildpc_core/stacks.py
Normal file
@@ -0,0 +1,938 @@
|
||||
"""Stack protocol — lifecycle actions for each development stack."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import USER_TOOL_PATH_DIRS
|
||||
from wildpc_core.manifest import ProgramSpec
|
||||
from wildpc_core.toolchains import ToolchainError, resolve_node_bin
|
||||
|
||||
DEV_ACTIONS = ["build", "test", "lint", "format", "type-check", "check", "run"]
|
||||
INSTALL_ACTIONS = ["install", "uninstall"]
|
||||
ALL_ACTIONS = DEV_ACTIONS + INSTALL_ACTIONS
|
||||
|
||||
# Verbs a stack handler can provide (everything except `run`, which is declared-only).
|
||||
_STACK_VERBS = {
|
||||
"build",
|
||||
"test",
|
||||
"lint",
|
||||
"format",
|
||||
"type-check",
|
||||
"check",
|
||||
"install",
|
||||
"uninstall",
|
||||
}
|
||||
# Verbs whose handler method name differs from the verb spelling.
|
||||
_VERB_METHOD = {"type-check": "type_check"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionResult:
|
||||
"""Result of a program lifecycle action."""
|
||||
|
||||
program: str
|
||||
action: str
|
||||
status: str # "ok" | "error"
|
||||
output: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolRequirement:
|
||||
"""A host toolchain a stack needs — the declarative counterpart to the argv each
|
||||
handler hard-codes (``uv sync``, ``pnpm build``, …). Made explicit so wildpc can
|
||||
*check* whether the tool is present (on the box and, for run-phase tools, on the
|
||||
service's own PATH) and, when it isn't, show a copyable fix instead of failing
|
||||
mid-subprocess with a raw ``command not found``.
|
||||
|
||||
- ``phase`` — when the tool is needed. ``build`` tools run only at
|
||||
``wildpc apply``/build time (checked against the build env); ``run`` tools must
|
||||
also be on the *running service's* PATH (the curated systemd env, which can
|
||||
drift from your shell); ``both`` is checked in both places.
|
||||
- ``install_hint`` — the exact command a user can copy to install it.
|
||||
"""
|
||||
|
||||
command: (
|
||||
str # the executable, e.g. "uv" | "pnpm" | "node" | "hugo" | "deno" | "psql"
|
||||
)
|
||||
purpose: str # human phrase, e.g. "build & run Python programs"
|
||||
phase: str # "run" | "build" | "both"
|
||||
install_hint: str # copyable install command
|
||||
version_min: str | None = None
|
||||
|
||||
|
||||
def _build_env(node_source: Path | None = None) -> dict[str, str]:
|
||||
"""Build a subprocess env with user tool dirs on PATH.
|
||||
|
||||
``node_source`` is the program's source dir: if it pins a node version (see
|
||||
:mod:`wildpc_core.toolchains`), that node's bin dir goes on the front of PATH so
|
||||
the verb uses the program's node instead of whatever ambient node the caller
|
||||
happens to have (the CLI inherits your shell's; the wildpc-api build executor's
|
||||
default PATH has none). Raises :class:`ToolchainError` if the pin isn't installed.
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
dirs = [str(d) for d in USER_TOOL_PATH_DIRS if d.exists()]
|
||||
node_bin = resolve_node_bin(node_source)
|
||||
if node_bin is not None:
|
||||
dirs.insert(0, str(node_bin))
|
||||
if dirs:
|
||||
env["PATH"] = ":".join(dirs) + ":" + env.get("PATH", "")
|
||||
return env
|
||||
|
||||
|
||||
async def _run(
|
||||
cmd: list[str], cwd: Path, env: dict[str, str] | None = None
|
||||
) -> tuple[int, str]:
|
||||
"""Run a subprocess and return (returncode, combined output).
|
||||
|
||||
The verb runs in ``cwd`` (the program source), so that dir doubles as the node
|
||||
pin source — a pinned-but-missing node fails loud here rather than as a cryptic
|
||||
``node: not found`` mid-build."""
|
||||
try:
|
||||
run_env = _build_env(cwd)
|
||||
except ToolchainError as e:
|
||||
return 1, str(e)
|
||||
if env:
|
||||
run_env.update(env)
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
cwd=cwd,
|
||||
env=run_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
stdout, _ = await proc.communicate()
|
||||
return proc.returncode or 0, (stdout or b"").decode()
|
||||
|
||||
|
||||
def _vite_base(name: str) -> str:
|
||||
"""The base path a wildpc-served static frontend builds against.
|
||||
|
||||
Every frontend now serves at the **root of its own subdomain**
|
||||
(`<name>.<gateway.domain>`), so the base is always '/'. Exposed to the build
|
||||
as VITE_BASE (the vite.config reads `base: process.env.VITE_BASE ?? '/'`)."""
|
||||
return "/"
|
||||
|
||||
|
||||
def _source_dir(comp: ProgramSpec, root: Path) -> Path:
|
||||
"""Resolve source directory, raising ValueError if absent."""
|
||||
if not comp.source:
|
||||
raise ValueError("No source directory")
|
||||
return root / comp.source
|
||||
|
||||
|
||||
class StackHandler:
|
||||
"""Base class — subclasses implement each lifecycle action."""
|
||||
|
||||
# Whether this stack owns *persistent external state* (a database schema, a
|
||||
# bucket, …) that outlives a code delete. Drives whether `wildpc delete`
|
||||
# surfaces a data remnant / honors `--purge-data`. Overridden to True by the
|
||||
# stacks whose `teardown` actually destroys something.
|
||||
owns_data: bool = False
|
||||
|
||||
# The dev verbs this stack advertises — what `available_actions` offers and
|
||||
# `run_action` will dispatch. Defaults to every stack verb (the python/react/
|
||||
# supabase handlers implement them all); a narrow stack like hugo (build-only,
|
||||
# no native lint/test/type-check) overrides this so callers aren't offered verbs
|
||||
# that would only error. `check` composes the sub-verbs, so a handler that drops
|
||||
# lint/type-check/test also drops `check` implicitly.
|
||||
provides: set[str] = _STACK_VERBS
|
||||
|
||||
# The host toolchains this stack needs, declared so wildpc can check them and
|
||||
# hint a fix. Empty by default (a stackless / declared-command program depends on
|
||||
# nothing wildpc can name); each real handler overrides it. See `tools_for`.
|
||||
tools: tuple[ToolRequirement, ...] = ()
|
||||
|
||||
async def build(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def test(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def lint(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def format(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def type_check(
|
||||
self, name: str, comp: ProgramSpec, root: Path
|
||||
) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def check(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Composite: lint + type-check + test. Runs all, reports first failure."""
|
||||
for action_fn, action_name in [
|
||||
(self.lint, "lint"),
|
||||
(self.type_check, "type-check"),
|
||||
(self.test, "test"),
|
||||
]:
|
||||
result = await action_fn(name, comp, root)
|
||||
if result.status != "ok":
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="check",
|
||||
status="error",
|
||||
output=f"{action_name} failed:\n{result.output}",
|
||||
)
|
||||
return ActionResult(program=name, action="check", status="ok")
|
||||
|
||||
async def install(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def teardown(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Destroy the persistent external state this program's stack created
|
||||
(database schema, blobs, …) — the irreversible counterpart to a code
|
||||
delete. Distinct from `uninstall` (which only takes a program offline).
|
||||
|
||||
Default: nothing to tear down. Only stacks that set ``owns_data`` and own
|
||||
durable state override this; `wildpc delete --purge-data` invokes it.
|
||||
"""
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="teardown",
|
||||
status="ok",
|
||||
output=f"{name}: no persistent state to tear down.",
|
||||
)
|
||||
|
||||
|
||||
class PythonHandler(StackHandler):
|
||||
"""Handler for python-cli and python-fastapi stacks."""
|
||||
|
||||
tools = (
|
||||
ToolRequirement(
|
||||
command="uv",
|
||||
purpose="build & run Python programs",
|
||||
phase="both",
|
||||
install_hint="curl -LsSf https://astral.sh/uv/install.sh | sh",
|
||||
),
|
||||
)
|
||||
|
||||
async def build(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(["uv", "sync"], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="build",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def test(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
if not (src / "tests").exists():
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="test",
|
||||
status="ok",
|
||||
output="No tests directory found, skipping.",
|
||||
)
|
||||
rc, output = await _run(["uv", "run", "pytest", "tests/", "-v"], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="test",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def lint(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(["uv", "run", "ruff", "check", "."], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="lint",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def format(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(["uv", "run", "ruff", "format", "."], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="format",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def type_check(
|
||||
self, name: str, comp: ProgramSpec, root: Path
|
||||
) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(["uv", "run", "pyright"], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="type-check",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def install(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
pkg_spec = str(src)
|
||||
if comp.install_extras:
|
||||
pkg_spec += "[" + ",".join(comp.install_extras) + "]"
|
||||
rc, output = await _run(
|
||||
["uv", "tool", "install", "--editable", pkg_spec, "--force"], src
|
||||
)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
pkg_name = src.name
|
||||
pyproject = src / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
with open(pyproject, "rb") as f:
|
||||
data = tomllib.load(f)
|
||||
pkg_name = data.get("project", {}).get("name", pkg_name)
|
||||
rc, output = await _run(["uv", "tool", "uninstall", pkg_name], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="uninstall",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
# pnpm (10+) runs a deps-status check before `pnpm <script>` that wants to purge +
|
||||
# reinstall node_modules and aborts when there's no TTY
|
||||
# (ERR_PNPM_ABORTED_REMOVE_MODULES_DIR_NO_TTY). It's a false positive for a build —
|
||||
# the verb builds, it doesn't install — so skip the check and use the existing
|
||||
# modules. (The env-var form isn't honored; the CLI flag is.) CI=true guards any
|
||||
# other interactive prompt.
|
||||
_PNPM_ENV = {"CI": "true"}
|
||||
|
||||
|
||||
def _pnpm(*args: str) -> list[str]:
|
||||
"""A pnpm argv with the pre-run deps check disabled."""
|
||||
return ["pnpm", "--config.verify-deps-before-run=false", *args]
|
||||
|
||||
|
||||
class ReactViteHandler(StackHandler):
|
||||
"""Handler for react-vite stack."""
|
||||
|
||||
# Build-phase only: the output is a static `dist/` the gateway serves in place,
|
||||
# so no node/pnpm process runs at serve time. node is resolved per-program from
|
||||
# its pin (see toolchains); the hint names nvm, the ecosystem-standard installer.
|
||||
tools = (
|
||||
ToolRequirement(
|
||||
command="node",
|
||||
purpose="build the frontend (Vite/React toolchain)",
|
||||
phase="build",
|
||||
install_hint="nvm install --lts # or match the program's .node-version",
|
||||
),
|
||||
ToolRequirement(
|
||||
command="pnpm",
|
||||
purpose="install deps & run the Vite build",
|
||||
phase="build",
|
||||
install_hint="npm install -g pnpm # or: corepack enable pnpm",
|
||||
),
|
||||
)
|
||||
|
||||
async def build(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
# Build against the gateway serve root so absolute asset URLs resolve at /
|
||||
# (vite.config reads VITE_BASE=/). Removes the hand-tuned-base footgun.
|
||||
rc, output = await _run(
|
||||
_pnpm("build"), src, env={**_PNPM_ENV, "VITE_BASE": _vite_base(name)}
|
||||
)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="build",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def test(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(_pnpm("test"), src, env=_PNPM_ENV)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="test",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def lint(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(_pnpm("lint"), src, env=_PNPM_ENV)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="lint",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def format(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(_pnpm("format"), src, env=_PNPM_ENV)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="format",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def type_check(
|
||||
self, name: str, comp: ProgramSpec, root: Path
|
||||
) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(_pnpm("type-check"), src, env=_PNPM_ENV)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="type-check",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def install(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Build the static assets in place. The gateway serves them directly from
|
||||
<source>/<build.outputs[0]> — no copy into a central content dir."""
|
||||
result = await self.build(name, comp, root)
|
||||
if result.status != "ok":
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="error",
|
||||
output=f"Build failed:\n{result.output}",
|
||||
)
|
||||
outputs = comp.build.outputs if comp.build else []
|
||||
if not outputs:
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="error",
|
||||
output="No build outputs configured.",
|
||||
)
|
||||
dist = _source_dir(comp, root) / outputs[0]
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="ok",
|
||||
output=f"Built; served in place from {dist}",
|
||||
)
|
||||
|
||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Static frontends have no install footprint to remove (served in place).
|
||||
|
||||
Deactivating one means dropping its gateway route — handled by removing the
|
||||
program from the registry, not by deleting build output."""
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="uninstall",
|
||||
status="ok",
|
||||
output=f"{name}: served in place; nothing to uninstall.",
|
||||
)
|
||||
|
||||
|
||||
class HugoHandler(StackHandler):
|
||||
"""Handler for the hugo stack — a static site built by the Hugo generator.
|
||||
|
||||
Hugo has one real dev verb: **build** (`hugo --gc --minify` → `public/`), which
|
||||
a `caddy` deployment then serves in place at `<name>.<gateway.domain>`. There is
|
||||
no native lint/test/type-check, so `provides` is narrowed to the verbs that do
|
||||
something. A theme with an asset pipeline (e.g. Blowfish + Tailwind) declares its
|
||||
own two-step `build.commands` (`pnpm build` then `hugo …`), which override this
|
||||
default per the usual declared-command-wins resolution."""
|
||||
|
||||
provides = {"build", "install", "uninstall"}
|
||||
tools = (
|
||||
ToolRequirement(
|
||||
command="hugo",
|
||||
purpose="build the static site",
|
||||
phase="build",
|
||||
install_hint="sudo apt install hugo # or: snap install hugo",
|
||||
),
|
||||
)
|
||||
|
||||
async def build(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
src = _source_dir(comp, root)
|
||||
rc, output = await _run(["hugo", "--gc", "--minify"], src)
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="build",
|
||||
status="ok" if rc == 0 else "error",
|
||||
output=output,
|
||||
)
|
||||
|
||||
async def install(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Build the site in place. The gateway serves it directly from
|
||||
<source>/<build.outputs[0]> (default `public/`) — no copy step."""
|
||||
result = await self.build(name, comp, root)
|
||||
if result.status != "ok":
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="error",
|
||||
output=f"Build failed:\n{result.output}",
|
||||
)
|
||||
outputs = comp.build.outputs if comp.build else []
|
||||
dist = _source_dir(comp, root) / (outputs[0] if outputs else "public")
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="install",
|
||||
status="ok",
|
||||
output=f"Built; served in place from {dist}",
|
||||
)
|
||||
|
||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Static sites have no install footprint — served in place. Deactivating one
|
||||
means dropping its gateway route, not deleting build output."""
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="uninstall",
|
||||
status="ok",
|
||||
output=f"{name}: served in place; nothing to uninstall.",
|
||||
)
|
||||
|
||||
|
||||
def _migration_version(path: Path) -> str:
|
||||
"""The version key of a migration file — the leading token before '_'.
|
||||
|
||||
e.g. ``0001_init.sql`` → ``0001``. Recorded in ``schema_migrations`` so a
|
||||
redeploy applies only the unapplied files.
|
||||
"""
|
||||
return path.name.split("_", 1)[0]
|
||||
|
||||
|
||||
def plan_migrations(files: list[Path], applied: set[str]) -> list[Path]:
|
||||
"""Order migrations by filename and drop any whose version is already applied.
|
||||
|
||||
Forward-only and idempotent (mirrors Patch's runner): re-running applies only
|
||||
new files, never re-applies an existing one. Pure — no DB — so it's unit-tested
|
||||
without a substrate.
|
||||
"""
|
||||
return [
|
||||
p
|
||||
for p in sorted(files, key=lambda x: x.name)
|
||||
if _migration_version(p) not in applied
|
||||
]
|
||||
|
||||
|
||||
def _substrate_db_url() -> str | None:
|
||||
"""Best-effort Postgres URL for the shared substrate.
|
||||
|
||||
Prefers an explicit ``SUPABASE_DB_URL``; otherwise builds one from the
|
||||
generated ``SUPABASE_POSTGRES_PASSWORD`` secret against the substrate's direct
|
||||
Postgres port. The self-hosted substrate publishes Postgres on host **5433**
|
||||
(5432 is taken by another Postgres on this node — see the substrate compose),
|
||||
overridable via ``SUPABASE_DB_HOST_PORT``. Returns None if neither an explicit
|
||||
URL nor the secret is available (build then fails loud with guidance).
|
||||
"""
|
||||
explicit = os.environ.get("SUPABASE_DB_URL")
|
||||
if explicit:
|
||||
return explicit
|
||||
from wildpc_core.config import read_secret
|
||||
|
||||
pw = read_secret("SUPABASE_POSTGRES_PASSWORD")
|
||||
if pw:
|
||||
port = os.environ.get("SUPABASE_DB_HOST_PORT", "5433")
|
||||
return f"postgresql://postgres:{pw}@localhost:{port}/postgres"
|
||||
return None
|
||||
|
||||
|
||||
def app_schema(name: str) -> str:
|
||||
"""The dedicated Postgres schema a supabase app owns.
|
||||
|
||||
Each app is isolated in its own schema (named after the program, ``-``→``_``
|
||||
for a valid unquoted identifier) rather than sharing ``public``. That gives a
|
||||
clean teardown (``drop schema … cascade``) and a per-app ``schema_migrations``
|
||||
so migration version tokens never collide across apps.
|
||||
"""
|
||||
return name.replace("-", "_")
|
||||
|
||||
|
||||
# The privilege grant that makes an app schema reachable through PostgREST — the
|
||||
# canonical Supabase "expose a custom schema" snippet. Idempotent; run before
|
||||
# migrations so `alter default privileges` also covers the tables they create.
|
||||
# Exposure ALSO requires the schema in the substrate's PGRST_DB_SCHEMAS — wildpc
|
||||
# derives that from the registered supabase apps (`${supabase_app_schemas}`), so a
|
||||
# newly-added app needs a `wildpc deploy` + substrate restart to become routable.
|
||||
def _schema_setup_sql(schema: str) -> str:
|
||||
roles = "anon, authenticated, service_role"
|
||||
return (
|
||||
f'create schema if not exists "{schema}";\n'
|
||||
f'grant usage on schema "{schema}" to {roles};\n'
|
||||
f'grant all on all tables in schema "{schema}" to {roles};\n'
|
||||
f'grant all on all routines in schema "{schema}" to {roles};\n'
|
||||
f'grant all on all sequences in schema "{schema}" to {roles};\n'
|
||||
f'alter default privileges in schema "{schema}" '
|
||||
f"grant all on tables to {roles};\n"
|
||||
f'alter default privileges in schema "{schema}" '
|
||||
f"grant all on routines to {roles};\n"
|
||||
f'alter default privileges in schema "{schema}" '
|
||||
f"grant all on sequences to {roles};\n"
|
||||
)
|
||||
|
||||
|
||||
class SupabaseHandler(StackHandler):
|
||||
"""Stack handler for supabase apps (migrations + edge functions + static UI).
|
||||
|
||||
Each app is isolated in its **own Postgres schema** (``app_schema(name)``): the
|
||||
migration runner creates + grants that schema, tracks applied versions in
|
||||
``<schema>.schema_migrations``, and runs each migration with ``search_path``
|
||||
set to it — so migration SQL is schema-agnostic and version tokens never
|
||||
collide across apps. The static UI is served in place by the gateway (no
|
||||
process), so install/uninstall are no-ops like a frontend. `teardown` drops the
|
||||
schema (and thus every object the app created) in one shot.
|
||||
"""
|
||||
|
||||
owns_data = True
|
||||
tools = (
|
||||
ToolRequirement(
|
||||
command="psql",
|
||||
purpose="run schema migrations against the substrate DB",
|
||||
phase="both",
|
||||
install_hint="sudo apt install postgresql-client",
|
||||
),
|
||||
ToolRequirement(
|
||||
command="deno",
|
||||
purpose="serve/deploy edge functions",
|
||||
phase="both",
|
||||
install_hint="curl -fsSL https://deno.land/install.sh | sh",
|
||||
),
|
||||
)
|
||||
|
||||
async def build(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Apply unapplied migrations into the app's own schema (idempotent)."""
|
||||
src = _source_dir(comp, root)
|
||||
schema = app_schema(name)
|
||||
mig_dir = src / "migrations"
|
||||
files = sorted(mig_dir.glob("*.sql")) if mig_dir.is_dir() else []
|
||||
|
||||
psql = shutil.which("psql")
|
||||
url = _substrate_db_url()
|
||||
if not psql:
|
||||
return ActionResult(
|
||||
name,
|
||||
"build",
|
||||
"error",
|
||||
"psql not found — install postgresql-client to run migrations.",
|
||||
)
|
||||
if not url:
|
||||
return ActionResult(
|
||||
name,
|
||||
"build",
|
||||
"error",
|
||||
"No substrate DB URL. Set SUPABASE_DB_URL, or generate secrets "
|
||||
"(scripts/gen-keys.py) so SUPABASE_POSTGRES_PASSWORD exists.",
|
||||
)
|
||||
|
||||
# Ensure the app's schema exists, is PostgREST-exposable (grants), and has
|
||||
# its own tracking table — then read applied versions from THAT schema.
|
||||
rc, out = await _run(
|
||||
[
|
||||
psql,
|
||||
url,
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
_schema_setup_sql(schema),
|
||||
"-c",
|
||||
f'create table if not exists "{schema}".schema_migrations '
|
||||
"(version text primary key, applied_at timestamptz default now())",
|
||||
],
|
||||
src,
|
||||
)
|
||||
if rc != 0:
|
||||
return ActionResult(name, "build", "error", f"connect/init failed:\n{out}")
|
||||
|
||||
if not files:
|
||||
return ActionResult(
|
||||
name, "build", "ok", f"Schema '{schema}' ready. No migrations to apply."
|
||||
)
|
||||
|
||||
rc, out = await _run(
|
||||
[
|
||||
psql,
|
||||
url,
|
||||
"-tA",
|
||||
"-c",
|
||||
f'select version from "{schema}".schema_migrations',
|
||||
],
|
||||
src,
|
||||
)
|
||||
if rc != 0:
|
||||
return ActionResult(
|
||||
name, "build", "error", f"read migrations failed:\n{out}"
|
||||
)
|
||||
applied = {line.strip() for line in out.splitlines() if line.strip()}
|
||||
|
||||
pending = plan_migrations(files, applied)
|
||||
if not pending:
|
||||
return ActionResult(
|
||||
name,
|
||||
"build",
|
||||
"ok",
|
||||
f"All migrations already applied (schema {schema}).",
|
||||
)
|
||||
|
||||
log = []
|
||||
for path in pending:
|
||||
version = _migration_version(path)
|
||||
# search_path → the app schema, so migration SQL can write unqualified
|
||||
# names and they land in the app's schema (not public). File +
|
||||
# version-insert in ONE transaction: a failed migration records
|
||||
# nothing, so the next build safely retries it.
|
||||
rc, out = await _run(
|
||||
[
|
||||
psql,
|
||||
url,
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"--single-transaction",
|
||||
"-c",
|
||||
f'set search_path to "{schema}", public',
|
||||
"-f",
|
||||
str(path),
|
||||
"-c",
|
||||
f'insert into "{schema}".schema_migrations(version) '
|
||||
f"values('{version}')",
|
||||
],
|
||||
src,
|
||||
)
|
||||
if rc != 0:
|
||||
log.append(f"✗ {path.name}\n{out}")
|
||||
return ActionResult(name, "build", "error", "\n".join(log))
|
||||
log.append(f"✓ {path.name}")
|
||||
return ActionResult(name, "build", "ok", "\n".join(log))
|
||||
|
||||
async def teardown(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Drop the app's schema and everything in it (tables, its own
|
||||
schema_migrations, functions) in one statement — total and knowable
|
||||
because the app owns exactly one schema. The substrate's PGRST_DB_SCHEMAS
|
||||
drops the (now-absent) schema on the next `wildpc deploy` + restart.
|
||||
"""
|
||||
schema = app_schema(name)
|
||||
psql = shutil.which("psql")
|
||||
url = _substrate_db_url()
|
||||
if not psql or not url:
|
||||
return ActionResult(
|
||||
name,
|
||||
"teardown",
|
||||
"error",
|
||||
f"Cannot drop schema '{schema}': psql or substrate DB URL "
|
||||
'unavailable. Drop it manually: drop schema "%s" cascade;' % schema,
|
||||
)
|
||||
cwd = src if (src := (root / comp.source if comp.source else None)) else root
|
||||
rc, out = await _run(
|
||||
[
|
||||
psql,
|
||||
url,
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
f'drop schema if exists "{schema}" cascade',
|
||||
],
|
||||
cwd,
|
||||
)
|
||||
if rc != 0:
|
||||
return ActionResult(name, "teardown", "error", f"drop failed:\n{out}")
|
||||
return ActionResult(
|
||||
name,
|
||||
"teardown",
|
||||
"ok",
|
||||
f"Dropped schema '{schema}' (all tables + rows). Run `wildpc deploy` "
|
||||
"and restart the substrate to prune it from PGRST_DB_SCHEMAS.",
|
||||
)
|
||||
|
||||
async def _deno(
|
||||
self, name: str, action: str, comp: ProgramSpec, root: Path, args: list[str]
|
||||
) -> ActionResult:
|
||||
"""Run a `deno` subcommand over functions/, or skip cleanly if deno absent."""
|
||||
src = _source_dir(comp, root)
|
||||
fns = src / "functions"
|
||||
deno = shutil.which("deno")
|
||||
if not deno or not fns.is_dir():
|
||||
return ActionResult(
|
||||
name, action, "ok", f"{action}: skipped (no deno/functions)"
|
||||
)
|
||||
rc, out = await _run([deno, *args, "functions/"], src)
|
||||
return ActionResult(name, action, "ok" if rc == 0 else "error", out)
|
||||
|
||||
async def test(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
return await self._deno(name, "test", comp, root, ["test", "--allow-all"])
|
||||
|
||||
async def lint(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
return await self._deno(name, "lint", comp, root, ["lint"])
|
||||
|
||||
async def format(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
return await self._deno(name, "format", comp, root, ["fmt"])
|
||||
|
||||
async def type_check(
|
||||
self, name: str, comp: ProgramSpec, root: Path
|
||||
) -> ActionResult:
|
||||
return await self._deno(name, "type-check", comp, root, ["check"])
|
||||
|
||||
async def install(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
"""Static UI is served in place by the gateway; nothing to install."""
|
||||
return ActionResult(
|
||||
name, "install", "ok", f"{name}: served in place at /{name}/."
|
||||
)
|
||||
|
||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||
return ActionResult(
|
||||
name, "uninstall", "ok", f"{name}: served in place; nothing to remove."
|
||||
)
|
||||
|
||||
|
||||
HANDLERS: dict[str, StackHandler] = {
|
||||
"python-cli": PythonHandler(),
|
||||
"python-fastapi": PythonHandler(),
|
||||
"react-vite": ReactViteHandler(),
|
||||
"supabase": SupabaseHandler(),
|
||||
"hugo": HugoHandler(),
|
||||
}
|
||||
|
||||
|
||||
def get_handler(stack: str | None) -> StackHandler | None:
|
||||
"""Get the handler for a given stack, or None if unsupported."""
|
||||
if stack is None:
|
||||
return None
|
||||
return HANDLERS.get(stack)
|
||||
|
||||
|
||||
def available_stacks() -> list[str]:
|
||||
"""The stack names wildpc has handlers for — the single source of truth for the
|
||||
CLI ``--stack`` choices, the ``GET /stacks`` endpoint, and the dashboard select.
|
||||
"""
|
||||
return sorted(HANDLERS)
|
||||
|
||||
|
||||
def tools_for(stack: str | None) -> tuple[ToolRequirement, ...]:
|
||||
"""The host toolchains a stack declares it needs (empty for an unknown/absent
|
||||
stack). The single source of truth for the dependency checks in `relations`,
|
||||
`wildpc stack`, `wildpc doctor`, and the dashboard Stacks page."""
|
||||
handler = get_handler(stack)
|
||||
return handler.tools if handler is not None else ()
|
||||
|
||||
|
||||
def _declared_commands(comp: ProgramSpec, verb: str) -> list[list[str]] | None:
|
||||
"""Declared argv-lists for a verb, or None.
|
||||
|
||||
`build` is declared via BuildSpec.commands; every other verb via CommandsSpec.
|
||||
"""
|
||||
if verb == "build":
|
||||
if comp.build and comp.build.commands:
|
||||
return comp.build.commands
|
||||
return None
|
||||
if comp.commands is not None:
|
||||
return comp.commands.for_verb(verb)
|
||||
return None
|
||||
|
||||
|
||||
def _stack_provides(comp: ProgramSpec, verb: str) -> bool:
|
||||
"""Whether the program's stack handler advertises this verb.
|
||||
|
||||
Consults the handler's ``provides`` set (default: all of ``_STACK_VERBS``) so a
|
||||
build-only stack like hugo doesn't offer lint/test/type-check it can't run."""
|
||||
handler = get_handler(comp.stack)
|
||||
return bool(comp.source) and handler is not None and verb in handler.provides
|
||||
|
||||
|
||||
def is_available(comp: ProgramSpec, verb: str) -> bool:
|
||||
"""Whether a verb can be run for a program (declared command or stack default)."""
|
||||
if _declared_commands(comp, verb) is not None:
|
||||
return True
|
||||
if verb == "check":
|
||||
return any(is_available(comp, sub) for sub in ("lint", "type-check", "test"))
|
||||
return _stack_provides(comp, verb)
|
||||
|
||||
|
||||
def available_actions(comp: ProgramSpec) -> list[str]:
|
||||
"""Return the list of verbs available for a program (resolution-aware)."""
|
||||
if not comp.source:
|
||||
return []
|
||||
return [verb for verb in ALL_ACTIONS if is_available(comp, verb)]
|
||||
|
||||
|
||||
async def _run_declared(
|
||||
name: str, verb: str, cmds: list[list[str]], src: Path
|
||||
) -> ActionResult:
|
||||
"""Run declared argv-lists in sequence; stop at the first failure."""
|
||||
outputs: list[str] = []
|
||||
for argv in cmds:
|
||||
rc, output = await _run(argv, src)
|
||||
outputs.append(output)
|
||||
if rc != 0:
|
||||
return ActionResult(
|
||||
program=name, action=verb, status="error", output="".join(outputs)
|
||||
)
|
||||
return ActionResult(program=name, action=verb, status="ok", output="".join(outputs))
|
||||
|
||||
|
||||
async def run_action(
|
||||
verb: str, name: str, comp: ProgramSpec, root: Path
|
||||
) -> ActionResult:
|
||||
"""Resolve and run a verb: declared command → stack default → unavailable.
|
||||
|
||||
This is the single entry point callers should use; it replaces reaching for
|
||||
get_handler(...).<method>(...) directly so the override logic stays in one place.
|
||||
"""
|
||||
# `check` is a composite that must respect per-verb overrides — unless the
|
||||
# program declares its own `check`, run each available sub-verb via run_action.
|
||||
if verb == "check" and _declared_commands(comp, "check") is None:
|
||||
subs = [s for s in ("lint", "type-check", "test") if is_available(comp, s)]
|
||||
if not subs:
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="check",
|
||||
status="error",
|
||||
output="No checkable verbs available.",
|
||||
)
|
||||
sections: list[str] = []
|
||||
for sub in subs:
|
||||
result = await run_action(sub, name, comp, root)
|
||||
mark = "✓" if result.status == "ok" else "✗"
|
||||
body = result.output.strip()
|
||||
sections.append(f"{mark} {sub}" + (f"\n{body}" if body else ""))
|
||||
if result.status != "ok":
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action="check",
|
||||
status="error",
|
||||
output="\n\n".join(sections),
|
||||
)
|
||||
return ActionResult(
|
||||
program=name, action="check", status="ok", output="\n\n".join(sections)
|
||||
)
|
||||
|
||||
# 1. Declared command overrides the stack default.
|
||||
declared = _declared_commands(comp, verb)
|
||||
if declared is not None:
|
||||
try:
|
||||
src = _source_dir(comp, root)
|
||||
except ValueError:
|
||||
return ActionResult(
|
||||
program=name, action=verb, status="error", output="No source directory"
|
||||
)
|
||||
return await _run_declared(name, verb, declared, src)
|
||||
|
||||
# 2. Stack default.
|
||||
handler = get_handler(comp.stack)
|
||||
if handler is not None and verb in _STACK_VERBS:
|
||||
method = getattr(handler, _VERB_METHOD.get(verb, verb), None)
|
||||
if method is not None:
|
||||
return await method(name, comp, root)
|
||||
|
||||
# 3. Unavailable.
|
||||
return ActionResult(
|
||||
program=name,
|
||||
action=verb,
|
||||
status="error",
|
||||
output=f"Verb '{verb}' is not available for '{name}' "
|
||||
f"(no declared command and no stack handler provides it).",
|
||||
)
|
||||
240
core/src/wildpc_core/tls.py
Normal file
240
core/src/wildpc_core/tls.py
Normal file
@@ -0,0 +1,240 @@
|
||||
"""Wild PC-managed TLS material for raw-TCP services.
|
||||
|
||||
Cuts the gateway's ACME wildcard cert (valid for ``<name>.<domain>``) onto a
|
||||
service so it presents a *trusted* cert on its raw port, and refreshes it on
|
||||
renewal. Protocol-agnostic: wildpc only copies files (in the requested format)
|
||||
and signals the service — each deployment declares the format (``pair`` /
|
||||
``combined``) and how it consumes the files (via the ``${tls_*}`` placeholders).
|
||||
|
||||
Two entry points:
|
||||
- ``materialize_all`` — write/refresh the cert files, no reload (used by ``apply``,
|
||||
which (re)starts the service itself).
|
||||
- ``reconcile_tls`` — materialize *and* reload the services whose cert changed
|
||||
(used by ``wildpc tls reconcile`` and the Caddy ``cert_obtained`` hook).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from wildpc_core.config import WildpcConfig
|
||||
from wildpc_core.manifest import SystemdDeployment, TlsMaterial
|
||||
|
||||
_KEY_MODE_FILES = {"key.pem", "combined.pem"} # secret → 0600; certs → 0644
|
||||
|
||||
|
||||
def _caddy_data_dir() -> Path:
|
||||
xdg = os.environ.get("XDG_DATA_HOME")
|
||||
base = Path(xdg) if xdg else Path.home() / ".local" / "share"
|
||||
return base / "caddy"
|
||||
|
||||
|
||||
def wildcard_cert(domain: str) -> tuple[Path, Path] | None:
|
||||
"""``(crt, key)`` for ``*.<domain>`` from Caddy's store, or None.
|
||||
|
||||
Caddy stores it at ``certificates/<acme-dir>/wildcard_.<domain>/…``. Prefer a
|
||||
production cert over staging (``WILDPC_ACME_STAGING=1`` yields a staging dir).
|
||||
The ``.crt`` is the full chain (leaf + intermediates).
|
||||
"""
|
||||
store = _caddy_data_dir() / "certificates"
|
||||
if not store.is_dir():
|
||||
return None
|
||||
stem = f"wildcard_.{domain}"
|
||||
matches = sorted(
|
||||
store.glob(f"*/{stem}/{stem}.crt"),
|
||||
key=lambda p: 1 if "staging" in str(p) else 0, # prod (0) before staging (1)
|
||||
)
|
||||
for crt in matches:
|
||||
key = crt.with_suffix(".key")
|
||||
if key.exists():
|
||||
return crt, key
|
||||
return None
|
||||
|
||||
|
||||
def tls_dir_for(data_dir: Path, config_key: str) -> Path:
|
||||
"""Where a deployment's materialized cert files live (``${tls_dir}``). `data_dir`
|
||||
is the instance root (config.data_dir) — the single source of truth."""
|
||||
return data_dir / config_key / "tls"
|
||||
|
||||
|
||||
def _tls_of(dep: object) -> object | None:
|
||||
"""The active TlsSpec for a deployment, or None (not systemd / no tcp / off)."""
|
||||
if not isinstance(dep, SystemdDeployment):
|
||||
return None
|
||||
tcp = dep.expose.tcp if dep.expose else None
|
||||
tls = tcp.tls if tcp else None
|
||||
if not tls or tls.material == TlsMaterial.OFF:
|
||||
return None
|
||||
return tls
|
||||
|
||||
|
||||
_PEM_CERT = re.compile(
|
||||
rb"-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----\s*", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def _issuer_chain(crt: bytes) -> bytes:
|
||||
"""The issuer chain from a leaf+chain ``.crt``: every cert *after* the leaf
|
||||
(the intermediates), for ``${tls_ca}``. Empty when the ``.crt`` is a bare leaf
|
||||
(a genuine LE cert always ships an intermediate, so this is non-empty in
|
||||
practice)."""
|
||||
blocks = [m.group(0) for m in _PEM_CERT.finditer(crt)]
|
||||
return b"".join(blocks[1:])
|
||||
|
||||
|
||||
def _wanted_files(
|
||||
tls_dir: Path, material: TlsMaterial, crt: bytes, key: bytes
|
||||
) -> dict[Path, bytes]:
|
||||
"""The exact file set for a material choice. ``chain.pem`` (for ``${tls_ca}``)
|
||||
is the *issuer chain* — the intermediates only, leaf stripped — so it's a real
|
||||
CA bundle distinct from the leaf-bearing ``cert.pem``/``combined.pem``, always
|
||||
provided regardless of material."""
|
||||
files: dict[Path, bytes] = {tls_dir / "chain.pem": _issuer_chain(crt)}
|
||||
if material == TlsMaterial.PAIR:
|
||||
files[tls_dir / "cert.pem"] = crt
|
||||
files[tls_dir / "key.pem"] = key
|
||||
elif material == TlsMaterial.COMBINED:
|
||||
files[tls_dir / "combined.pem"] = key + crt
|
||||
return files
|
||||
|
||||
|
||||
def materialize_tls(config: WildpcConfig, name: str, dep: object) -> bool:
|
||||
"""Write ``dep``'s cert files from the wildcard, in its declared format.
|
||||
|
||||
Idempotent: returns ``False`` (no write) when the on-disk copy already matches
|
||||
the source, so it's safe to call on every ``apply`` and every renewal event.
|
||||
Returns ``True`` when files were (re)written. No reload here — see callers.
|
||||
"""
|
||||
tls = _tls_of(dep)
|
||||
if tls is None:
|
||||
return False
|
||||
domain = config.gateway.domain
|
||||
if not domain:
|
||||
return False
|
||||
src = wildcard_cert(domain)
|
||||
if src is None:
|
||||
return False
|
||||
crt_path, key_path = src
|
||||
crt, key = crt_path.read_bytes(), key_path.read_bytes()
|
||||
|
||||
config_key = dep.program or name # type: ignore[attr-defined]
|
||||
tls_dir = tls_dir_for(config.data_dir, config_key)
|
||||
wanted = _wanted_files(tls_dir, tls.material, crt, key) # type: ignore[attr-defined]
|
||||
|
||||
if all(p.exists() and p.read_bytes() == c for p, c in wanted.items()):
|
||||
return False # already current
|
||||
|
||||
tls_dir.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(tls_dir, 0o700)
|
||||
# Drop files left over from a previous material choice.
|
||||
for stale in ("cert.pem", "key.pem", "combined.pem", "chain.pem"):
|
||||
p = tls_dir / stale
|
||||
if p not in wanted and p.exists():
|
||||
p.unlink()
|
||||
for path, content in wanted.items():
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(content)
|
||||
os.chmod(path, 0o600 if path.name in _KEY_MODE_FILES else 0o644)
|
||||
return True
|
||||
|
||||
|
||||
def materialize_all(
|
||||
config: WildpcConfig,
|
||||
messages: list[str] | None = None,
|
||||
only: list[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Materialize certs for TLS-material deployments; no reload. For ``apply``,
|
||||
which starts/restarts the services itself.
|
||||
|
||||
``only`` scopes materialization to the given deployment names (what a scoped
|
||||
``wildpc apply <name>`` is converging). Left None → every deployment. Scoping
|
||||
keeps a scoped apply from rewriting an *unrelated* service's cert on disk
|
||||
without also reloading it (which would leave the file diverged from the running
|
||||
process until the next ``wildpc tls reconcile``)."""
|
||||
msgs = messages if messages is not None else []
|
||||
scope = set(only) if only is not None else None
|
||||
for _kind, name, dep in config.all_deployments():
|
||||
if scope is not None and name not in scope:
|
||||
continue
|
||||
if _tls_of(dep) is None:
|
||||
continue
|
||||
if materialize_tls(config, name, dep):
|
||||
msgs.append(f"tls: materialized cert for {name}")
|
||||
return msgs
|
||||
|
||||
|
||||
def wait_for_wildcard(
|
||||
config: WildpcConfig,
|
||||
names: list[str],
|
||||
messages: list[str] | None = None,
|
||||
timeout: float = 120.0,
|
||||
interval: float = 3.0,
|
||||
) -> list[str]:
|
||||
"""Block until the ACME wildcard cert exists, when an in-scope deployment needs
|
||||
wildpc-materialized TLS but the cert isn't issued yet.
|
||||
|
||||
On a fresh node the gateway reload during ``apply`` only *kicks off* DNS-01
|
||||
issuance of ``*.<domain>`` (seconds to a couple minutes); materializing right
|
||||
after would find no cert and start the TLS service pointed at missing files —
|
||||
and with ``gateway.cert_hook`` disabled (the default) nothing would later
|
||||
reconcile it. Waiting here lets ``apply`` bring the service up with its cert in
|
||||
place on first deploy. Bounded: on timeout it warns and returns so ``apply``
|
||||
still proceeds (rerun ``wildpc tls reconcile`` once the cert lands)."""
|
||||
msgs = messages if messages is not None else []
|
||||
needs = [
|
||||
n
|
||||
for n in names
|
||||
if any(_tls_of(spec) is not None for _k, spec in config.deployments_named(n))
|
||||
]
|
||||
if not needs:
|
||||
return msgs
|
||||
domain = config.gateway.domain
|
||||
if not domain or wildcard_cert(domain) is not None:
|
||||
return msgs # no acme domain, or the cert already exists — nothing to wait on
|
||||
msgs.append(f"tls: waiting for ACME wildcard *.{domain} to be issued…")
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(interval)
|
||||
if wildcard_cert(domain) is not None:
|
||||
msgs.append(f"tls: wildcard *.{domain} issued")
|
||||
return msgs
|
||||
msgs.append(
|
||||
f"tls: wildcard *.{domain} not ready after {int(timeout)}s — "
|
||||
f"{', '.join(needs)} may start without a cert; rerun `wildpc tls reconcile` "
|
||||
"once it is issued"
|
||||
)
|
||||
return msgs
|
||||
|
||||
|
||||
def _reload(name: str, tls: object, msgs: list[str]) -> None:
|
||||
reload_cmd = getattr(tls, "reload", None)
|
||||
if reload_cmd:
|
||||
subprocess.run(reload_cmd, check=False)
|
||||
msgs.append(f"tls: reloaded {name} (reload command)")
|
||||
else:
|
||||
subprocess.run(
|
||||
["systemctl", "--user", "restart", f"wildpc-{name}.service"], check=False
|
||||
)
|
||||
msgs.append(f"tls: restarted {name} to pick up rotated cert")
|
||||
|
||||
|
||||
def reconcile_tls(config: WildpcConfig, messages: list[str] | None = None) -> list[str]:
|
||||
"""Materialize certs and reload the services whose cert changed. Idempotent —
|
||||
a no-op when nothing rotated. Invoked by ``wildpc tls reconcile`` and the Caddy
|
||||
``cert_obtained`` hook. Logs its own outcome (the hook's ``exec`` swallows it)."""
|
||||
msgs = messages if messages is not None else []
|
||||
for _kind, name, dep in config.all_deployments():
|
||||
tls = _tls_of(dep)
|
||||
if tls is None:
|
||||
continue
|
||||
if materialize_tls(config, name, dep):
|
||||
msgs.append(f"tls: refreshed cert for {name}")
|
||||
_reload(name, tls, msgs)
|
||||
if not msgs:
|
||||
msgs.append("tls: all materialized certs current — nothing to do")
|
||||
return msgs
|
||||
393
core/src/wildpc_core/tool_schema.py
Normal file
393
core/src/wildpc_core/tool_schema.py
Normal file
@@ -0,0 +1,393 @@
|
||||
"""Derive LLM tool-call schemas from a CLI tool's ``--help``.
|
||||
|
||||
wildpc's in-process version of the standalone ``toolify`` tool: given a tool (a
|
||||
program with a ``path`` deployment), resolve its executable, run ``--help``, and
|
||||
build a tool-call definition.
|
||||
|
||||
Two parameter shapes, chosen automatically:
|
||||
|
||||
* **structured** — one typed property per option/positional, parsed from a
|
||||
standard argparse/click ``--help``, each carrying an ``x-cli`` executor hint.
|
||||
Used when the help is recognizable *and* the tool has no subcommands.
|
||||
* **command** — a single ``command`` string, full ``--help`` as description. The
|
||||
universal fallback for non-standard help (``jq``) and subcommand trees.
|
||||
|
||||
Schemas are built and stored in a **neutral** core (``{name, description,
|
||||
parameters}``); ``render_tool_schema`` wraps that in the Anthropic or OpenAI
|
||||
envelope on read. wildpc's feed defaults to OpenAI (litellm-native).
|
||||
|
||||
The extraction is intentionally duplicated from ``toolify`` rather than shared —
|
||||
``toolify`` is a standalone program that must never depend on wildpc. No LLM is
|
||||
used; the output is a deterministic function of the tool's ``--help``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from wildpc_core.config import WildpcConfig
|
||||
|
||||
__all__ = [
|
||||
"ToolSchemaError",
|
||||
"derive_tool_schema",
|
||||
"render_tool_schema",
|
||||
"tool_executable",
|
||||
]
|
||||
|
||||
_NAME_OK = re.compile(r"[^a-zA-Z0-9_-]")
|
||||
_HELP_TIMEOUT = 10
|
||||
_MAX_SUBCOMMANDS = 40
|
||||
_SKIP_OPTS = {"help", "version"}
|
||||
_CMD_HEADING = re.compile(
|
||||
r"^\s*(commands|subcommands|available commands)\s*:?\s*$", re.IGNORECASE
|
||||
)
|
||||
_OPT_HEADING = re.compile(r"^\s*(options|optional arguments)\s*:?\s*$", re.IGNORECASE)
|
||||
_POS_HEADING = re.compile(r"^\s*positional arguments\s*:?\s*$", re.IGNORECASE)
|
||||
|
||||
|
||||
class ToolSchemaError(Exception):
|
||||
"""The tool's executable couldn't be resolved or produced no help."""
|
||||
|
||||
|
||||
def _sanitize_name(raw: str) -> str:
|
||||
name = _NAME_OK.sub("_", raw).strip("_") or "tool"
|
||||
return name[:64]
|
||||
|
||||
|
||||
def _run_help(argv: list[str]) -> tuple[int | None, str]:
|
||||
"""Run ``argv + ['--help']`` → ``(returncode, help_text)``; never raises."""
|
||||
for flag in ("--help", "-h"):
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[*argv, flag],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
timeout=_HELP_TIMEOUT,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
continue
|
||||
text = proc.stdout.strip() or proc.stderr.strip()
|
||||
if text:
|
||||
return proc.returncode, text
|
||||
return None, ""
|
||||
|
||||
|
||||
def _section_entries(help_text: str, heading_re: re.Pattern[str]) -> list[list[str]]:
|
||||
"""Entries (each a list of lines) under headings matching ``heading_re``. A
|
||||
row at the section's minimum indent starts an entry; deeper rows continue it."""
|
||||
out: list[list[str]] = []
|
||||
lines = help_text.splitlines()
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
if not heading_re.match(lines[i]):
|
||||
i += 1
|
||||
continue
|
||||
i += 1
|
||||
body: list[str] = []
|
||||
while i < len(lines) and (not lines[i].strip() or lines[i][:1].isspace()):
|
||||
if lines[i].strip():
|
||||
body.append(lines[i])
|
||||
i += 1
|
||||
if not body:
|
||||
continue
|
||||
term = min(len(ln) - len(ln.lstrip()) for ln in body)
|
||||
cur: list[str] = []
|
||||
for ln in body:
|
||||
if (len(ln) - len(ln.lstrip())) == term:
|
||||
if cur:
|
||||
out.append(cur)
|
||||
cur = [ln]
|
||||
else:
|
||||
cur.append(ln)
|
||||
if cur:
|
||||
out.append(cur)
|
||||
return out
|
||||
|
||||
|
||||
def _extract_subcommands(help_text: str) -> list[str]:
|
||||
"""Subcommand names, or [] for a flat tool. Only click ``Commands:`` entries
|
||||
and argparse ``{a,b,c}`` choice rows count — a plain positional does not."""
|
||||
cmds: list[str] = []
|
||||
for entry in _section_entries(help_text, _CMD_HEADING):
|
||||
word = re.match(r"^([A-Za-z][\w-]*)\b", entry[0].strip())
|
||||
if word:
|
||||
cmds.append(word.group(1))
|
||||
for entry in _section_entries(help_text, _POS_HEADING):
|
||||
choice = re.match(r"^\{([^}]+)\}", entry[0].strip())
|
||||
if choice:
|
||||
cmds.extend(p.strip() for p in choice.group(1).split(","))
|
||||
seen: list[str] = []
|
||||
for c in cmds:
|
||||
if c and c not in seen:
|
||||
seen.append(c)
|
||||
return seen
|
||||
|
||||
|
||||
def _entry_head_and_desc(entry: list[str]) -> tuple[str, str]:
|
||||
m = re.match(r"^\s*(.*?)(?:\s{2,}(.*))?$", entry[0])
|
||||
head = (m.group(1) if m else entry[0]).strip()
|
||||
desc_parts = [m.group(2)] if m and m.group(2) else []
|
||||
desc_parts += [ln.strip() for ln in entry[1:]]
|
||||
return head, " ".join(p for p in desc_parts if p).strip()
|
||||
|
||||
|
||||
def _parse_option(entry: list[str]) -> tuple[str, dict] | None:
|
||||
head, desc = _entry_head_and_desc(entry)
|
||||
flags: list[str] = []
|
||||
metavar: str | None = None
|
||||
for tok in head.split(", "):
|
||||
fm = re.match(r"^(-{1,2}[\w-]+)(?:[ =](.+))?$", tok.strip())
|
||||
if not fm:
|
||||
continue
|
||||
flags.append(fm.group(1))
|
||||
if fm.group(2):
|
||||
metavar = fm.group(2).strip()
|
||||
longs = [f for f in flags if f.startswith("--")]
|
||||
canonical = longs[-1] if longs else (flags[0] if flags else None)
|
||||
if not canonical:
|
||||
return None
|
||||
key = canonical.lstrip("-").replace("-", "_")
|
||||
if key in _SKIP_OPTS:
|
||||
return None
|
||||
prop: dict = {}
|
||||
if metavar and metavar.startswith("{") and metavar.endswith("}"):
|
||||
prop["type"] = "string"
|
||||
prop["enum"] = [v.strip() for v in metavar[1:-1].split(",")]
|
||||
elif metavar:
|
||||
prop["type"] = "string"
|
||||
else:
|
||||
prop["type"] = "boolean"
|
||||
if desc:
|
||||
prop["description"] = desc
|
||||
prop["x-cli"] = {"flag": canonical, "value": bool(metavar)}
|
||||
return key, prop
|
||||
|
||||
|
||||
def _parse_positional(entry: list[str], order: int) -> tuple[str, dict] | None:
|
||||
head, desc = _entry_head_and_desc(entry)
|
||||
if head.startswith("{"):
|
||||
return None
|
||||
nm = re.match(r"^([A-Za-z][\w-]*)", head)
|
||||
if not nm:
|
||||
return None
|
||||
key = nm.group(1).replace("-", "_")
|
||||
prop: dict = {"type": "string"}
|
||||
if desc:
|
||||
prop["description"] = desc
|
||||
prop["x-cli"] = {"positional": True, "order": order}
|
||||
return key, prop
|
||||
|
||||
|
||||
def _summary(help_text: str, fallback: str) -> str:
|
||||
for para in re.split(r"\n\s*\n", help_text):
|
||||
p = para.strip()
|
||||
if not p or p.lower().startswith("usage:") or p.rstrip().endswith(":"):
|
||||
continue
|
||||
return " ".join(p.split())
|
||||
return fallback
|
||||
|
||||
|
||||
def _structured_core(name: str, help_text: str) -> dict | None:
|
||||
props: dict = {}
|
||||
required: list[str] = []
|
||||
for order, entry in enumerate(_section_entries(help_text, _POS_HEADING)):
|
||||
parsed = _parse_positional(entry, order)
|
||||
if parsed:
|
||||
key, prop = parsed
|
||||
props[key] = prop
|
||||
required.append(key)
|
||||
for entry in _section_entries(help_text, _OPT_HEADING):
|
||||
parsed = _parse_option(entry)
|
||||
if parsed:
|
||||
key, prop = parsed
|
||||
props[key] = prop
|
||||
if not props:
|
||||
return None
|
||||
parameters: dict = {"type": "object", "properties": props}
|
||||
if required:
|
||||
parameters["required"] = required
|
||||
return {
|
||||
"name": _sanitize_name(name),
|
||||
"description": _summary(help_text, f"Run the `{name}` command-line tool."),
|
||||
"parameters": parameters,
|
||||
}
|
||||
|
||||
|
||||
def _command_core(name: str, argv: list[str], help_text: str, deep: bool) -> dict:
|
||||
parts = [
|
||||
f"Run the `{name}` command-line tool. Provide the arguments in the "
|
||||
f"`command` parameter; the executable name is prepended automatically. "
|
||||
f"Below is the tool's help output.",
|
||||
f"\n$ {name} --help\n{help_text}",
|
||||
]
|
||||
if deep:
|
||||
for sub in _extract_subcommands(help_text)[:_MAX_SUBCOMMANDS]:
|
||||
rc, sub_help = _run_help([*argv, sub])
|
||||
if rc == 0 and sub_help:
|
||||
parts.append(f"\n$ {name} {sub} --help\n{sub_help}")
|
||||
return {
|
||||
"name": _sanitize_name(name),
|
||||
"description": "\n".join(parts),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"command": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
f"Arguments to pass to `{name}`. The full command line run "
|
||||
f"is `{name} ` followed by this string. Do not include the "
|
||||
f"leading `{name}`."
|
||||
),
|
||||
}
|
||||
},
|
||||
"required": ["command"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tool_executable(config: WildpcConfig, name: str) -> str:
|
||||
"""The console script to invoke for tool ``name`` — its first
|
||||
``[project.scripts]`` key (source of truth even when uninstalled), else the
|
||||
program name. Mirrors the CLI's tools lens."""
|
||||
comp = config.programs.get(name)
|
||||
src = getattr(comp, "source", None) if comp else None
|
||||
if src:
|
||||
pyproject = Path(src) / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
try:
|
||||
data = tomllib.loads(pyproject.read_text())
|
||||
scripts = data.get("project", {}).get("scripts", {})
|
||||
if scripts:
|
||||
return sorted(scripts.keys())[0]
|
||||
except (OSError, tomllib.TOMLDecodeError):
|
||||
pass
|
||||
return name
|
||||
|
||||
|
||||
def collect_tool_help(config: WildpcConfig, name: str) -> str:
|
||||
"""The full recursive ``--help`` text for tool ``name`` — top-level plus each
|
||||
subcommand's help, the raw material an LLM assist reads to structure a
|
||||
subcommand tree. Deterministic; raises ``ToolSchemaError`` if unresolved.
|
||||
"""
|
||||
exe_name = tool_executable(config, name)
|
||||
exe = shutil.which(exe_name)
|
||||
if exe is None:
|
||||
raise ToolSchemaError(
|
||||
f"`{exe_name}` is not on PATH — install the tool (enable it, then "
|
||||
f"`wildpc apply`) before generating its schema."
|
||||
)
|
||||
_, help_text = _run_help([exe])
|
||||
if not help_text:
|
||||
raise ToolSchemaError(f"`{exe_name} --help` produced no output.")
|
||||
parts = [f"$ {exe_name} --help\n{help_text}"]
|
||||
for sub in _extract_subcommands(help_text)[:_MAX_SUBCOMMANDS]:
|
||||
rc, sub_help = _run_help([exe, sub])
|
||||
if rc == 0 and sub_help:
|
||||
parts.append(f"\n$ {exe_name} {sub} --help\n{sub_help}")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def validate_tool_schema_core(core: object) -> list[str]:
|
||||
"""Deterministically validate a neutral tool-call core.
|
||||
|
||||
Returns a list of human-readable error strings (empty ⇒ valid). Checks the
|
||||
``{name, description, parameters}`` shape *and* that ``parameters`` is a valid
|
||||
JSON Schema (via ``jsonschema`` meta-validation, which catches malformed
|
||||
properties like an ``enum`` that isn't a list — the defect weak models hit).
|
||||
Shared by the LLM repair loop, the ``validate`` endpoint, and the accept gate.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
if not isinstance(core, dict):
|
||||
return ["schema must be a JSON object"]
|
||||
name = core.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
errors.append("`name` must be a non-empty string")
|
||||
elif not re.fullmatch(r"[a-zA-Z0-9_-]{1,64}", name):
|
||||
errors.append("`name` must match ^[a-zA-Z0-9_-]{1,64}$")
|
||||
if not isinstance(core.get("description"), str):
|
||||
errors.append("`description` must be a string")
|
||||
|
||||
params = core.get("parameters")
|
||||
if not isinstance(params, dict):
|
||||
errors.append("`parameters` must be an object")
|
||||
return errors
|
||||
if params.get("type") != "object":
|
||||
errors.append('`parameters.type` must be "object"')
|
||||
props = params.get("properties")
|
||||
if not isinstance(props, dict):
|
||||
errors.append("`parameters.properties` must be an object")
|
||||
elif not props:
|
||||
errors.append("`parameters.properties` is empty — no arguments captured")
|
||||
|
||||
try:
|
||||
import jsonschema
|
||||
from jsonschema.exceptions import SchemaError
|
||||
|
||||
try:
|
||||
jsonschema.Draft202012Validator.check_schema(params)
|
||||
except SchemaError as e:
|
||||
loc = "/".join(str(p) for p in e.absolute_path) or "parameters"
|
||||
errors.append(f"invalid JSON Schema at `{loc}`: {e.message}")
|
||||
except ImportError: # jsonschema not installed — structural checks stand alone
|
||||
pass
|
||||
return errors
|
||||
|
||||
|
||||
def is_tool_schema_core(obj: object) -> bool:
|
||||
"""True if ``obj`` is a valid neutral tool-call core (no validation errors)."""
|
||||
return not validate_tool_schema_core(obj)
|
||||
|
||||
|
||||
def derive_tool_schema(config: WildpcConfig, name: str, deep: bool = False) -> dict:
|
||||
"""Derive the neutral tool-call core for tool ``name`` from its ``--help``.
|
||||
|
||||
Structured params when the help is standard and flat; the command-string
|
||||
fallback for non-standard help / subcommand trees. Raises ``ToolSchemaError``
|
||||
if the executable isn't on PATH or emits no help.
|
||||
"""
|
||||
exe_name = tool_executable(config, name)
|
||||
exe = shutil.which(exe_name)
|
||||
if exe is None:
|
||||
raise ToolSchemaError(
|
||||
f"`{exe_name}` is not on PATH — install the tool (enable it, then "
|
||||
f"`wildpc apply`) before generating its schema."
|
||||
)
|
||||
_, help_text = _run_help([exe])
|
||||
if not help_text:
|
||||
raise ToolSchemaError(f"`{exe_name} --help` produced no output.")
|
||||
if not _extract_subcommands(help_text):
|
||||
structured = _structured_core(exe_name, help_text)
|
||||
if structured:
|
||||
return structured
|
||||
return _command_core(exe_name, [exe], help_text, deep)
|
||||
|
||||
|
||||
def render_tool_schema(core: dict, fmt: str = "openai") -> dict:
|
||||
"""Wrap a neutral core in a provider envelope.
|
||||
|
||||
``openai`` (default, litellm-native) → ``{type: function, function: {…}}``;
|
||||
``anthropic`` → ``{name, description, input_schema}``; ``neutral`` → as stored.
|
||||
"""
|
||||
if fmt == "neutral":
|
||||
return core
|
||||
if fmt == "anthropic":
|
||||
return {
|
||||
"name": core["name"],
|
||||
"description": core["description"],
|
||||
"input_schema": core["parameters"],
|
||||
}
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": core["name"],
|
||||
"description": core["description"],
|
||||
"parameters": core["parameters"],
|
||||
},
|
||||
}
|
||||
133
core/src/wildpc_core/toolchains.py
Normal file
133
core/src/wildpc_core/toolchains.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""Toolchain resolution — map a program's declared runtime version to a concrete
|
||||
binary directory on this box.
|
||||
|
||||
Today this covers **node**. Programs pin their node version the ecosystem-standard
|
||||
way — a ``.node-version`` / ``.nvmrc`` file, or ``package.json`` ``engines.node`` /
|
||||
``volta.node`` — so a program stays wildpc-independent (the prime directive: regular
|
||||
programs never depend on wildpc). Wild PC *reads* that pin and resolves it to an
|
||||
absolute ``.../bin`` directory it can put on PATH, at both execution sites that run a
|
||||
program's node:
|
||||
|
||||
- **build time** — the dev-verb subprocess (``stacks._build_env``), so ``wildpc
|
||||
program build`` uses the program's node regardless of who triggers it (your shell
|
||||
or the wildpc-api build executor);
|
||||
- **run time** — a ``launcher: node`` service's systemd unit PATH (via
|
||||
``deploy._build_deployed`` → the generated unit's ``Environment=PATH``).
|
||||
|
||||
Resolution scans nvm's versioned install layout (``WILDPC_NODE_VERSIONS_DIR``,
|
||||
default ``~/.nvm/versions/node``) — the versioned dir the default tool PATH
|
||||
intentionally omits. A pinned-but-not-installed version fails loud with an
|
||||
actionable ``nvm install`` hint rather than surfacing later as ``node: not found``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# A pin that is a bare, exact-ish version: "24", "24.14", "24.14.1" (optional "v").
|
||||
_EXACT_RE = re.compile(r"v?(\d+)(?:\.(\d+))?(?:\.(\d+))?$")
|
||||
# An installed nvm dir: v24.14.1
|
||||
_INSTALLED_RE = re.compile(r"v(\d+)\.(\d+)\.(\d+)$")
|
||||
# Wildcards/aliases that mean "newest installed".
|
||||
_NEWEST_ALIASES = {"", "*", "x", "node", "latest", "current", "lts", "lts/*"}
|
||||
|
||||
|
||||
class ToolchainError(Exception):
|
||||
"""A program pins a toolchain version that isn't installed on this box."""
|
||||
|
||||
|
||||
def node_versions_dir() -> Path:
|
||||
"""The directory nvm installs versioned node under (env-overridable)."""
|
||||
override = os.environ.get("WILDPC_NODE_VERSIONS_DIR")
|
||||
return Path(override) if override else Path.home() / ".nvm" / "versions" / "node"
|
||||
|
||||
|
||||
def read_node_pin(source: Path) -> str | None:
|
||||
"""The node version a program pins, read the ecosystem-standard way.
|
||||
|
||||
Precedence: ``.node-version`` → ``.nvmrc`` → ``package.json`` (``engines.node``,
|
||||
else ``volta.node``). Returns the raw spec string, or ``None`` if unpinned."""
|
||||
for fname in (".node-version", ".nvmrc"):
|
||||
f = source / fname
|
||||
if f.is_file():
|
||||
val = f.read_text().strip()
|
||||
if val:
|
||||
return val
|
||||
pkg = source / "package.json"
|
||||
if pkg.is_file():
|
||||
try:
|
||||
data = json.loads(pkg.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
for section in ("engines", "volta"):
|
||||
node = (data.get(section) or {}).get("node")
|
||||
if isinstance(node, str) and node.strip():
|
||||
return node.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _installed() -> list[tuple[tuple[int, int, int], Path]]:
|
||||
"""Installed (version, bin-dir) pairs, newest first."""
|
||||
root = node_versions_dir()
|
||||
out: list[tuple[tuple[int, int, int], Path]] = []
|
||||
if not root.is_dir():
|
||||
return out
|
||||
for child in root.iterdir():
|
||||
m = _INSTALLED_RE.match(child.name)
|
||||
if m and (child / "bin" / "node").exists():
|
||||
out.append(((int(m[1]), int(m[2]), int(m[3])), child / "bin"))
|
||||
out.sort(reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
def resolve_node_bin(source: Path | None) -> Path | None:
|
||||
"""Resolve a program's pinned node version to its ``.../bin`` dir on this box.
|
||||
|
||||
Returns ``None`` when the program pins nothing — wildpc injects no node, it does
|
||||
not guess. Raises :class:`ToolchainError` when a version *is* pinned but no
|
||||
matching version is installed.
|
||||
|
||||
Matching: an exact-ish pin (``24`` / ``24.14`` / ``24.14.1``) matches installed
|
||||
versions by that precision, newest wins. A range or alias (``>=24``, ``^24.1``,
|
||||
``lts/*``) is pinned by its leading major (or newest installed if it names none)
|
||||
— precise pins belong in ``.node-version``."""
|
||||
if source is None:
|
||||
return None
|
||||
pin = read_node_pin(source)
|
||||
if not pin:
|
||||
return None
|
||||
installed = _installed()
|
||||
|
||||
def newest_or_raise() -> Path:
|
||||
if installed:
|
||||
return installed[0][1]
|
||||
raise ToolchainError(
|
||||
f"node {pin!r} pinned (in {source}) but no node is installed under "
|
||||
f"{node_versions_dir()} — run `nvm install {pin}`"
|
||||
)
|
||||
|
||||
low = pin.strip().lower()
|
||||
if low in _NEWEST_ALIASES:
|
||||
return newest_or_raise()
|
||||
|
||||
exact = _EXACT_RE.match(low)
|
||||
if exact:
|
||||
parts = [int(g) for g in exact.groups() if g is not None]
|
||||
else:
|
||||
# A range/alias (>=24, ^24.1, ~24, 24.x): pin by leading major only.
|
||||
major = re.search(r"\d+", low)
|
||||
if not major:
|
||||
return newest_or_raise()
|
||||
parts = [int(major.group())]
|
||||
precision = len(parts)
|
||||
|
||||
for ver, bindir in installed: # newest first
|
||||
if list(ver[:precision]) == parts:
|
||||
return bindir
|
||||
raise ToolchainError(
|
||||
f"node {pin!r} pinned (in {source}) but not installed under "
|
||||
f"{node_versions_dir()} — run `nvm install {pin}`"
|
||||
)
|
||||
Reference in New Issue
Block a user