From add356dcf27f73e524c118294d332c7f2fcc193f Mon Sep 17 00:00:00 2001 From: Paul Payne Date: Sun, 5 Jul 2026 10:55:42 -0700 Subject: [PATCH] feat: relationship model (requires/repos/predicates) + git sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A derived, mostly-computed model of how programs, deployments, and repos relate, plus the git-sync surfaces that motivated it. See docs/relationships.md. Core: - `requires: [{kind, ref, version?, bind?}]` on programs + deployments — one precondition relation; `system_dependencies` is its `{kind: system}` alias. kind fixes meaning + check (system=installed, deployment=exists). - relations.py: derives repos (git toplevel / monorepo), fan-in, and the predicates functional?/fresh?/deployed? — nothing stored. - env is generated FROM a `{kind: deployment, bind}` requirement (target URL → consumer env), never scraped back into one; explicit defaults.env still wins. - git.py: working-copy status/pull, repo toplevel + remote url. Surfaces: - `castle graph` + GET /graph — the relationship diagnostic. - GET /repos, /repos/{key}/git|sync — repo-scoped sync (a repo is the sync unit; a monorepo backs several programs). GET /programs/{name}/git|sync + repo context. - Dashboard: Graph screen, program-page git status + repo-aware Sync, and a monorepo banner on Programs. Governing principle: predicates are derived; encode only the non-derivable, as a node or edge property. Pull-only sync — converge stays a separate step. --- app/src/components/Layout.tsx | 2 + app/src/components/MonorepoBanner.tsx | 68 ++++++++ app/src/components/detail/GitSyncRow.tsx | 78 +++++++++ app/src/pages/GraphPage.tsx | 148 ++++++++++++++++ app/src/pages/ProgramDetail.tsx | 40 ++++- app/src/pages/Programs.tsx | 3 + app/src/router/routes.tsx | 2 + app/src/services/api/hooks.ts | 71 ++++++++ app/src/types/index.ts | 79 +++++++++ castle-api/src/castle_api/graph.py | 30 ++++ castle-api/src/castle_api/main.py | 4 + castle-api/src/castle_api/programs.py | 86 ++++++++++ castle-api/src/castle_api/repos.py | 83 +++++++++ castle-api/tests/test_graph_repos.py | 63 +++++++ castle-api/tests/test_programs_git.py | 79 +++++++++ cli/src/castle_cli/commands/graph.py | 68 ++++++++ cli/src/castle_cli/main.py | 10 ++ core/src/castle_core/deploy.py | 49 +++++- core/src/castle_core/git.py | 159 ++++++++++++++++++ core/src/castle_core/manifest.py | 28 ++++ core/src/castle_core/relations.py | 204 +++++++++++++++++++++++ core/tests/test_git.py | 123 ++++++++++++++ core/tests/test_relations.py | 106 ++++++++++++ docs/relationships.md | 90 ++++++++++ 24 files changed, 1669 insertions(+), 4 deletions(-) create mode 100644 app/src/components/MonorepoBanner.tsx create mode 100644 app/src/components/detail/GitSyncRow.tsx create mode 100644 app/src/pages/GraphPage.tsx create mode 100644 castle-api/src/castle_api/graph.py create mode 100644 castle-api/src/castle_api/repos.py create mode 100644 castle-api/tests/test_graph_repos.py create mode 100644 castle-api/tests/test_programs_git.py create mode 100644 cli/src/castle_cli/commands/graph.py create mode 100644 core/src/castle_core/git.py create mode 100644 core/src/castle_core/relations.py create mode 100644 core/tests/test_git.py create mode 100644 core/tests/test_relations.py create mode 100644 docs/relationships.md diff --git a/app/src/components/Layout.tsx b/app/src/components/Layout.tsx index 9c03f6c..3f7a497 100644 --- a/app/src/components/Layout.tsx +++ b/app/src/components/Layout.tsx @@ -12,6 +12,7 @@ import { Package, Server, Share2, + Network, Wrench, X, type LucideIcon, @@ -38,6 +39,7 @@ const NAV: (NavLeaf | NavGroup)[] = [ ], }, { to: "/programs", label: "Programs", icon: Package }, + { to: "/graph", label: "Graph", icon: Network }, { to: "/mesh", label: "Mesh", icon: Share2 }, ] diff --git a/app/src/components/MonorepoBanner.tsx b/app/src/components/MonorepoBanner.tsx new file mode 100644 index 0000000..b7dc1fe --- /dev/null +++ b/app/src/components/MonorepoBanner.tsx @@ -0,0 +1,68 @@ +import { useState } from "react" +import { GitFork, Loader2, RefreshCw } from "lucide-react" +import { useRepos, useRepoSync } from "@/services/api/hooks" +import type { RepoSummary } from "@/types" + +// Repos that back more than one program (monorepos) get one row with a single +// repo-scoped Sync — the honest place for it, since a pull moves the whole working +// copy. Standalone programs (a repo of one) sync from their own program page. +export function MonorepoBanner() { + const { data: repos } = useRepos() + const monorepos = (repos ?? []).filter((r) => r.programs.length > 1) + if (monorepos.length === 0) return null + return ( +
+ {monorepos.map((r) => ( + + ))} +
+ ) +} + +function MonorepoRow({ repo }: { repo: RepoSummary }) { + const sync = useRepoSync() + const [msg, setMsg] = useState(null) + + const behind = repo.behind ?? 0 + const onSync = () => { + setMsg(null) + sync.mutate(repo.key, { + onSuccess: (d) => + setMsg(d.pulled ? `Pulled — ${d.deployments.join(", ")} may need restart/apply` : "Already up to date"), + onError: (e) => { + let t = String(e) + try { + t = JSON.parse((e as Error).message).detail ?? t + } catch { + t = (e as Error).message ?? t + } + setMsg(t) + }, + }) + } + + return ( +
+
+ + {repo.key} + monorepo · {repo.programs.join(", ")} + {behind > 0 ? ( + ● {behind} behind + ) : ( + ✓ up to date + )} + {repo.dirty && · dirty} + +
+ {msg &&
{msg}
} +
+ ) +} diff --git a/app/src/components/detail/GitSyncRow.tsx b/app/src/components/detail/GitSyncRow.tsx new file mode 100644 index 0000000..231f1bb --- /dev/null +++ b/app/src/components/detail/GitSyncRow.tsx @@ -0,0 +1,78 @@ +import { GitBranch, Loader2, RefreshCw } from "lucide-react" +import type { GitStatus } from "@/types" + +interface GitSyncRowProps { + status: GitStatus + program: string // the program whose page this is (excluded from "shared by") + loading: boolean // a fetch/refresh of the status is in flight + syncing: boolean // a git pull is in flight + onSync: () => void +} + +// The "Git" value cell in the Program Info card: branch · behind/ahead · dirty, +// plus a Sync (git pull) button. Pull-only — converge stays a separate step. For a +// monorepo, sync operates on the whole repo and lists the sibling programs. +export function GitSyncRow({ status, program, loading, syncing, onSync }: GitSyncRowProps) { + const behind = status.behind ?? 0 + const ahead = status.ahead ?? 0 + + return ( + + + + {status.detached ? "detached" : status.branch ?? "—"} + + + {status.behind === null ? ( + no upstream + ) : behind > 0 ? ( + + ● {behind} behind{ahead > 0 ? `, ${ahead} ahead` : ""} + + ) : ( + + ✓ up to date{ahead > 0 ? ` (${ahead} ahead)` : ""} + + )} + + + · {status.dirty ? "dirty" : "clean"} + + + {status.error && ( + + · fetch error + + )} + + {status.repo?.multi && ( + + · shared by {status.repo.programs.filter((p) => p !== program).join(", ")} + + )} + + + + ) +} diff --git a/app/src/pages/GraphPage.tsx b/app/src/pages/GraphPage.tsx new file mode 100644 index 0000000..3ea03c9 --- /dev/null +++ b/app/src/pages/GraphPage.tsx @@ -0,0 +1,148 @@ +import { AlertTriangle, CheckCircle2, GitFork, Layers, Link2, Package } from "lucide-react" +import { useGraph } from "@/services/api/hooks" +import type { GraphNode } from "@/types" + +// A read-only diagnostic of how programs/deployments relate — repos (provenance), +// `requires` edges (dependency), and the derived predicates functional?/fresh?. +// Everything is computed server-side from git + config; nothing is stored. +export function GraphPage() { + const { data, isLoading, error } = useGraph() + + if (isLoading) { + return
Loading…
+ } + if (error || !data) { + return
Failed to load graph.
+ } + + const monorepos = data.repos.filter((r) => r.programs.length > 1) + const staleRepos = data.repos.filter((r) => r.fresh === false) + const depEdges = data.edges.filter((e) => e.kind === "deployment") + const unhealthy = data.nodes.filter((n) => !n.functional) + const depended = data.nodes + .filter((n) => n.depended_on_by > 0) + .sort((a, b) => b.depended_on_by - a.depended_on_by) + + return ( +
+
+

+ Relationship Graph +

+

+ Derived, never stored — repos from git, requires edges, and the{" "} + functional? / fresh? predicates, computed on the fly. +

+
+ +
+ + + + +
+ + + {monorepos.length === 0 && ( +

No monorepos — every program has its own repo.

+ )} + {monorepos.map((r) => ( +
+ + {r.key} + + → {r.programs.join(", ")} +
+ ))} + {staleRepos.length > 0 && ( +

+ {staleRepos.length} repo(s) not fresh: {staleRepos.map((r) => r.key).join(", ")} +

+ )} + + + + {depEdges.length === 0 ? ( +

+ None declared yet — front-end/back-end deps have no encoded edge. +

+ ) : ( + depEdges.map((e, i) => ( +
+ {e.src} requires {e.dst} + {e.bind && → ${e.bind}} +
+ )) + )} + + + + {unhealthy.length === 0 ? ( +

+ All deployments functional. +

+ ) : ( + unhealthy.map((n) => ( +
+ ✗ {n.name}{" "} + unmet: {n.unmet.join(", ")} +
+ )) + )} + + + {depended.length > 0 && ( + + {depended.map((n: GraphNode) => ( +
+ {n.name} ← {n.depended_on_by} dependent(s) +
+ ))} + + )} +
+ ) +} + +function Stat({ label, value, sub, tone }: { label: string; value: number; sub?: string; tone?: "ok" | "warn" }) { + const color = tone === "warn" ? "text-amber-400" : tone === "ok" ? "text-green-400" : "" + return ( +
+
{value}
+
{label}
+ {sub &&
{sub}
} +
+ ) +} + +function Card({ + title, icon: Icon, note, children, +}: { + title: string + icon: typeof Layers + note?: string + children: React.ReactNode +}) { + return ( +
+

+ {title} +

+ {note &&

{note}

} +
{children}
+
+ ) +} + +function FreshBadge({ fresh, behind, dirty }: { fresh: boolean | null; behind: number | null; dirty: boolean }) { + if (fresh === null) return null + if (fresh) return fresh + const bits = [behind ? `${behind} behind` : null, dirty ? "dirty" : null].filter(Boolean) + return {bits.join(" · ") || "stale"} +} diff --git a/app/src/pages/ProgramDetail.tsx b/app/src/pages/ProgramDetail.tsx index db62cda..f70ffa0 100644 --- a/app/src/pages/ProgramDetail.tsx +++ b/app/src/pages/ProgramDetail.tsx @@ -1,17 +1,43 @@ import { useState } from "react" import { useParams } from "react-router-dom" -import { useProgram } from "@/services/api/hooks" +import { useProgram, useProgramGit, useProgramSync } from "@/services/api/hooks" import { subdomainUrl } from "@/lib/labels" import { DetailHeader } from "@/components/detail/DetailHeader" import { ConfigPanel } from "@/components/detail/ConfigPanel" import { DeploymentsSection } from "@/components/detail/DeploymentsSection" import { ProgramActions, ActionOutputPanel, type ActionOutput } from "@/components/ProgramActions" +import { GitSyncRow } from "@/components/detail/GitSyncRow" export function ProgramDetailPage() { const { name } = useParams<{ name: string }>() const { data: deployment, isLoading, error, refetch } = useProgram(name ?? "") + const git = useProgramGit(name ?? "", !!deployment?.repo) + const sync = useProgramSync() const [actionOutput, setActionOutput] = useState(null) + const handleSync = () => { + if (!deployment) return + setActionOutput(null) + sync.mutate(deployment.id, { + onSuccess: (data) => { + const lines = [data.output || (data.pulled ? "Pulled." : "Already up to date.")] + if (data.pulled && data.deployments.length) { + lines.push("", `May need a restart/apply: ${data.deployments.join(", ")}`) + } + setActionOutput({ action: "sync", text: lines.join("\n"), ok: true }) + }, + onError: (err) => { + let text = String(err) + try { + text = JSON.parse((err as Error).message).detail ?? text + } catch { + text = (err as Error).message ?? text + } + setActionOutput({ action: "sync", text, ok: false }) + }, + }) + } + if (isLoading) { return (
Loading...
@@ -85,6 +111,18 @@ export function ProgramDetailPage() { )} + {deployment.repo && git.data?.is_repo && ( + <> + Git + + + )} {deployment.version && ( <> Version diff --git a/app/src/pages/Programs.tsx b/app/src/pages/Programs.tsx index 2450311..f5c078a 100644 --- a/app/src/pages/Programs.tsx +++ b/app/src/pages/Programs.tsx @@ -1,5 +1,6 @@ import { usePrograms } from "@/services/api/hooks" import { ProgramList } from "@/components/ProgramList" +import { MonorepoBanner } from "@/components/MonorepoBanner" import { PageHeader } from "@/components/PageHeader" export function Programs() { @@ -9,6 +10,8 @@ export function Programs() {
+ + {isLoading ? (

Loading...

) : programs && programs.length > 0 ? ( diff --git a/app/src/router/routes.tsx b/app/src/router/routes.tsx index 29ebb9a..5d33bcc 100644 --- a/app/src/router/routes.tsx +++ b/app/src/router/routes.tsx @@ -7,6 +7,7 @@ import { Tools } from "@/pages/Tools" import { Programs } from "@/pages/Programs" import { GatewayPage } from "@/pages/GatewayPage" import { MeshPage } from "@/pages/MeshPage" +import { GraphPage } from "@/pages/GraphPage" import { ServiceDetailPage } from "@/pages/ServiceDetail" import { ScheduledDetailPage } from "@/pages/ScheduledDetail" import { ToolDetailPage } from "@/pages/ToolDetail" @@ -26,6 +27,7 @@ export const router = createBrowserRouter([ { path: "programs", element: }, { path: "gateway", element: }, { path: "mesh", element: }, + { path: "graph", element: }, { path: "services/:name", element: }, { path: "jobs/:name", element: }, { path: "tools/:name", element: }, diff --git a/app/src/services/api/hooks.ts b/app/src/services/api/hooks.ts index 7a18157..d063490 100644 --- a/app/src/services/api/hooks.ts +++ b/app/src/services/api/hooks.ts @@ -9,6 +9,10 @@ import type { JobDetail, ProgramSummary, ProgramDetail, + GitStatus, + GraphModel, + RepoSummary, + ProgramSyncResponse, StatusResponse, GatewayInfo, GatewayConfigRequest, @@ -226,6 +230,67 @@ export function useProgramAction() { }) } +// Git status of a program's working copy. The backend fetches from the remote, +// so this call is comparatively slow — hence its own query (not part of the +// program detail) with a short staleTime. `enabled` lets the caller skip it for +// programs with no repo. +export function useProgramGit(name: string, enabled = true) { + return useQuery({ + queryKey: ["programs", name, "git"], + queryFn: () => apiClient.get(`/programs/${name}/git`), + enabled: enabled && !!name, + staleTime: 30_000, + }) +} + +// Fast-forward a program's source (git pull). Pull-only — converge (restart/apply) +// stays a separate, explicit step. Refreshes the program and its git status. +export function useProgramSync() { + const qc = useQueryClient() + return useMutation({ + mutationFn: (name: string) => + apiClient.post(`/programs/${name}/sync`), + onSuccess: (_data, name) => { + qc.invalidateQueries({ queryKey: ["programs", name, "git"] }) + qc.invalidateQueries({ queryKey: ["programs"] }) + }, + }) +} + +export function useRepos() { + return useQuery({ + queryKey: ["repos"], + queryFn: () => apiClient.get("/repos"), + staleTime: 30_000, + }) +} + +// Fast-forward a whole repo (git pull the working copy). Pull-only — converge is +// separate. Refreshes repos, programs, and their git status. +export function useRepoSync() { + const qc = useQueryClient() + return useMutation({ + mutationFn: (key: string) => + apiClient.post(`/repos/${key}/sync`), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["repos"] }) + qc.invalidateQueries({ queryKey: ["programs"] }) + qc.invalidateQueries({ queryKey: ["graph"] }) + }, + }) +} + +// The derived relationship model (repos, requires edges, functional/fresh status). +// Fetches git status per repo server-side, so it's comparatively slow — its own +// query with a modest staleTime. +export function useGraph() { + return useQuery({ + queryKey: ["graph"], + queryFn: () => apiClient.get("/graph"), + staleTime: 30_000, + }) +} + export function useSaveGatewayConfig() { const qc = useQueryClient() return useMutation({ @@ -312,6 +377,12 @@ export function useEventStream() { qc.invalidateQueries({ queryKey: ["jobs"] }) }) + es.addEventListener("program-sync", () => { + // A program's source was pulled (possibly by another client) — refresh + // programs and their git status. + qc.invalidateQueries({ queryKey: ["programs"] }) + }) + es.addEventListener("mesh", () => { // A remote node updated or went offline — refresh mesh, nodes, and gateway qc.invalidateQueries({ queryKey: ["mesh"] }) diff --git a/app/src/types/index.ts b/app/src/types/index.ts index c6de932..26e8e5d 100644 --- a/app/src/types/index.ts +++ b/app/src/types/index.ts @@ -74,6 +74,85 @@ export interface ProgramDetail extends ProgramSummary { manifest: Record } +// Git state of a program's source working copy (GET /programs/{name}/git). +// ahead/behind are relative to the upstream tracking branch (null = no upstream); +// behind reflects the fetch the status call performed. +export interface GitStatus { + is_repo: boolean + branch: string | null + upstream: string | null + dirty: boolean + ahead: number | null + behind: number | null + detached: boolean + error: string | null + // The repo (git working copy) this program's source lives in. `multi` marks a + // monorepo shared by several programs — sync operates on the whole repo. + repo?: { + key: string + programs: string[] + multi: boolean + deployments: string[] + } | null +} + +// GET /repos — a repo (git working copy) with its members and last-known git state. +export interface RepoSummary { + key: string + path: string + url: string | null + ref: string | null + programs: string[] + deployments: string[] + branch: string | null + behind: number | null + dirty: boolean +} + +// GET /graph — the derived relationship model (docs/relationships.md). +export interface GraphRepo { + key: string + path: string + url: string | null + ref: string | null + programs: string[] + deployments: string[] + behind: number | null + dirty: boolean + fresh: boolean | null +} +export interface GraphNode { + name: string + program: string | null + kind: string + repo: string | null + depended_on_by: number + unmet: string[] + functional: boolean + fresh: boolean | null + deployed: boolean | null +} +export interface GraphEdge { + src: string + dst: string + kind: "system" | "deployment" + bind: string | null +} +export interface GraphModel { + repos: GraphRepo[] + nodes: GraphNode[] + edges: GraphEdge[] +} + +// POST /programs/{name}/sync — a fast-forward pull (no build/apply/restart). +export interface ProgramSyncResponse { + program: string + status: string + output: string + pulled: boolean + deployments: string[] // affected deployments that may need restart/apply +} + // Union for the shared ConfigPanel (ProgramFields / ServiceFields / JobFields) export type AnyDetail = ServiceDetail | JobDetail | ProgramDetail diff --git a/castle-api/src/castle_api/graph.py b/castle-api/src/castle_api/graph.py new file mode 100644 index 0000000..d74afd3 --- /dev/null +++ b/castle-api/src/castle_api/graph.py @@ -0,0 +1,30 @@ +"""The relationship model as JSON — repos, `requires` edges, derived status. + +Read-only diagnostic (see docs/relationships.md). Everything is computed on the +fly: repos from git, predicates (functional/fresh) from config + git. Nothing +stored. +""" + +from __future__ import annotations + +import dataclasses + +from fastapi import APIRouter + +from castle_core.relations import build_model + +from castle_api.config import get_config + +graph_router = APIRouter(tags=["graph"]) + + +@graph_router.get("/graph") +def get_graph() -> dict: + """The whole relationship model: repos (with freshness), deployment nodes (with + `functional?`), and `requires` edges.""" + model = build_model(get_config(), check=True, freshness=True) + return { + "repos": [dataclasses.asdict(r) for r in model.repos], + "nodes": [dataclasses.asdict(n) for n in model.nodes], + "edges": [dataclasses.asdict(e) for e in model.edges], + } diff --git a/castle-api/src/castle_api/main.py b/castle-api/src/castle_api/main.py index 9487192..7a71039 100644 --- a/castle-api/src/castle_api/main.py +++ b/castle-api/src/castle_api/main.py @@ -17,6 +17,8 @@ from castle_api.agents import router as agents_router from castle_api.config import get_registry, settings from castle_api.config_editor import router as config_router from castle_api.deploy_routes import router as deploy_router +from castle_api.graph import graph_router +from castle_api.repos import repos_router from castle_api.logs import router as logs_router from castle_api.routes import router as dashboard_router from castle_api.secrets import router as secrets_router @@ -127,6 +129,8 @@ app.include_router(services_router) app.include_router(programs_router) app.include_router(deploy_router) app.include_router(agents_router) +app.include_router(graph_router) +app.include_router(repos_router) @app.get("/health") diff --git a/castle-api/src/castle_api/programs.py b/castle-api/src/castle_api/programs.py index 77abadd..5c1ad3a 100644 --- a/castle-api/src/castle_api/programs.py +++ b/castle-api/src/castle_api/programs.py @@ -2,10 +2,14 @@ from __future__ import annotations +from dataclasses import asdict + from fastapi import APIRouter, HTTPException, status +from castle_core import git from castle_core.stacks import available_actions, available_stacks, run_action +from castle_api import stream from castle_api.config import get_config programs_router = APIRouter(tags=["programs"]) @@ -17,6 +21,88 @@ def list_stacks() -> list[str]: and keeps it in sync with the backend (no hardcoded frontend list).""" return available_stacks() + +# --------------------------------------------------------------------------- +# Git sync — pull a program's source working copy up to date (pull only; no +# build/apply/restart — converge stays an explicit, separate step). Declared +# BEFORE the generic /{action} route below so the literal `git`/`sync` segments +# win over the `{action}` path param (Starlette matches in declaration order). +# --------------------------------------------------------------------------- + + +def _program_source(name: str): + """The (program, config) for a named program, or raise the standard 404/400s.""" + config = get_config() + if name not in config.programs: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"'{name}' not found" + ) + return config.programs[name], config + + +@programs_router.get("/programs/{name}/git") +def program_git_status(name: str) -> dict: + """Git status of a program's working copy (branch, dirty, ahead/behind). + + Fetches from the remote first so ``behind`` is current. A program with no + source or a non-git source returns a benign ``{"is_repo": false}`` (not an + error) so the dashboard can simply hide the sync control.""" + comp, config = _program_source(name) + if not comp.source: + return {"is_repo": False} + out = asdict(git.git_status(comp.source, fetch=True)) + # Repo context: which repo this program's source lives in and who else shares it + # (a monorepo). Sync is a repo operation — the UI labels it and lists siblings. + from castle_core.relations import derive_repos + + for key, repo in derive_repos(config).items(): + if name in repo.programs: + out["repo"] = { + "key": key, + "programs": repo.programs, + "multi": repo.multi, + "deployments": repo.deployments, + } + break + return out + + +@programs_router.post("/programs/{name}/sync") +async def program_sync(name: str) -> dict: + """Fast-forward a program's working copy (``git pull --ff-only``). + + Pull-only: it updates the source on disk and reports which deployments may now + need a restart/apply, but does not build, apply, or restart anything itself.""" + comp, config = _program_source(name) + if not comp.source: + raise HTTPException(status_code=400, detail=f"'{name}' has no source directory") + if not git.is_git_repo(comp.source): + raise HTTPException( + status_code=400, detail=f"'{name}' source is not a git repository" + ) + + before = git.head(comp.source) + ok, output = git.pull(comp.source) + if not ok: + raise HTTPException(status_code=500, detail=output or "git pull failed") + + pulled = git.head(comp.source) != before + deployments = [dname for dname, _ in config.deployments_of(name)] + if pulled: + # Nudge other clients to refresh this program's git status. + await stream.broadcast( + "program-sync", {"program": name, "deployments": deployments} + ) + + return { + "program": name, + "status": "ok", + "output": output, + "pulled": pulled, + "deployments": deployments if pulled else [], + } + + # --------------------------------------------------------------------------- # Unified program action endpoint # --------------------------------------------------------------------------- diff --git a/castle-api/src/castle_api/repos.py b/castle-api/src/castle_api/repos.py new file mode 100644 index 0000000..b18a016 --- /dev/null +++ b/castle-api/src/castle_api/repos.py @@ -0,0 +1,83 @@ +"""Repo endpoints — a repo (git working copy) is the unit of sync. A monorepo backs +several programs; an adopted program is a repo of one. See docs/relationships.md. + +Sync is pull-only (fast-forward); converge (build/apply/restart) stays a separate, +explicit step. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +from fastapi import APIRouter, HTTPException, status + +from castle_core import git +from castle_core.relations import Repo, derive_repos + +from castle_api import stream +from castle_api.config import get_config + +repos_router = APIRouter(tags=["repos"]) + + +def _resolve(key: str) -> tuple[Repo, object]: + config = get_config() + repos = derive_repos(config) + if key not in repos: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"repo '{key}' not found" + ) + return repos[key], config + + +@repos_router.get("/repos") +def list_repos() -> list[dict]: + """Every repo with members and last-known git state (no fetch — fast).""" + out: list[dict] = [] + for repo in derive_repos(get_config()).values(): + st = git.git_status(Path(repo.path), fetch=False) + out.append( + { + **dataclasses.asdict(repo), + "branch": st.branch, + "behind": st.behind, + "dirty": st.dirty, + } + ) + return out + + +@repos_router.get("/repos/{key}/git") +def repo_git(key: str) -> dict: + """A repo's git status (fetches, so ``behind`` is current) plus its members.""" + repo, _ = _resolve(key) + return { + **dataclasses.asdict(git.git_status(Path(repo.path), fetch=True)), + "key": key, + "programs": repo.programs, + "deployments": repo.deployments, + } + + +@repos_router.post("/repos/{key}/sync") +async def repo_sync(key: str) -> dict: + """Fast-forward the repo's working copy. Pull-only — reports which deployments + may now need a restart/apply, but does not converge them.""" + repo, _ = _resolve(key) + before = git.head(Path(repo.path)) + ok, output = git.pull(Path(repo.path)) + if not ok: + raise HTTPException(status_code=500, detail=output or "git pull failed") + pulled = git.head(Path(repo.path)) != before + if pulled: + await stream.broadcast( + "repo-sync", {"repo": key, "deployments": repo.deployments} + ) + return { + "repo": key, + "status": "ok", + "output": output, + "pulled": pulled, + "deployments": repo.deployments if pulled else [], + } diff --git a/castle-api/tests/test_graph_repos.py b/castle-api/tests/test_graph_repos.py new file mode 100644 index 0000000..7dd5d29 --- /dev/null +++ b/castle-api/tests/test_graph_repos.py @@ -0,0 +1,63 @@ +"""Tests for the /graph diagnostic and /repos (repo-scoped sync) endpoints.""" + +import os +import subprocess +from pathlib import Path + +from fastapi.testclient import TestClient + +_ENV = { + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", +} + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(cwd), *args], check=True, + capture_output=True, text=True, env={**os.environ, **_ENV}) + + +def _commit(cwd: Path, fname: str) -> None: + (cwd / fname).write_text(fname) + _git(cwd, "add", fname) + _git(cwd, "commit", "-m", f"add {fname}") + + +def _setup(work: Path) -> None: + upstream = work.parent / f"{work.name}-upstream" + upstream.mkdir() + _git(upstream, "init", "-q", "-b", "main") + _commit(upstream, "a.txt") + _git(work.parent, "clone", "-q", str(upstream), str(work)) + + +class TestGraph: + def test_graph_shape(self, client: TestClient) -> None: + resp = client.get("/graph") + assert resp.status_code == 200 + body = resp.json() + assert {"repos", "nodes", "edges"} <= body.keys() + # every node carries the derived predicates + assert all("functional" in n for n in body["nodes"]) + + +class TestRepos: + def test_list_and_sync(self, client: TestClient, castle_root: Path) -> None: + _setup(castle_root / "wired-in") # program `wired-in` source → /wired-in + _commit(castle_root / "wired-in-upstream", "b.txt") # advance remote + + repos = {r["key"]: r for r in client.get("/repos").json()} + assert "wired-in" in repos + assert "wired-in" in repos["wired-in"]["programs"] + + gitinfo = client.get("/repos/wired-in/git").json() + assert gitinfo["is_repo"] is True and gitinfo["behind"] == 1 + + synced = client.post("/repos/wired-in/sync").json() + assert synced["pulled"] is True and "wired-in" in synced["deployments"] + assert (castle_root / "wired-in" / "b.txt").exists() + + def test_unknown_repo_404(self, client: TestClient) -> None: + assert client.get("/repos/nope/git").status_code == 404 + assert client.post("/repos/nope/sync").status_code == 404 diff --git a/castle-api/tests/test_programs_git.py b/castle-api/tests/test_programs_git.py new file mode 100644 index 0000000..427c2a4 --- /dev/null +++ b/castle-api/tests/test_programs_git.py @@ -0,0 +1,79 @@ +"""Tests for the program git-status / sync endpoints.""" + +import os +import subprocess +from pathlib import Path + +from fastapi.testclient import TestClient + +_ENV = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", +} + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run( + ["git", "-C", str(cwd), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_ENV}, + ) + + +def _commit(cwd: Path, fname: str) -> None: + (cwd / fname).write_text(fname) + _git(cwd, "add", fname) + _git(cwd, "commit", "-m", f"add {fname}") + + +def _setup_git_program(work: Path) -> None: + """Make `work` a git clone tracking an `-upstream` repo with one commit.""" + upstream = work.parent / f"{work.name}-upstream" + upstream.mkdir() + _git(upstream, "init", "-q", "-b", "main") + _commit(upstream, "a.txt") + _git(work.parent, "clone", "-q", str(upstream), str(work)) + + +class TestProgramGit: + def test_non_git_source_is_benign(self, client: TestClient) -> None: + """A program whose source isn't a git repo returns is_repo:false, not 500.""" + resp = client.get("/programs/test-tool/git") + assert resp.status_code == 200 + assert resp.json()["is_repo"] is False + + def test_status_behind_then_sync_pulls( + self, client: TestClient, castle_root: Path + ) -> None: + work = castle_root / "wired-in" # source: "wired-in" → /wired-in + _setup_git_program(work) + _commit(work.parent / "wired-in-upstream", "b.txt") # advance the remote + + status = client.get("/programs/wired-in/git") + assert status.status_code == 200 + body = status.json() + assert body["is_repo"] is True and body["branch"] == "main" + assert body["behind"] == 1 and body["ahead"] == 0 + + synced = client.post("/programs/wired-in/sync") + assert synced.status_code == 200 + s = synced.json() + assert s["status"] == "ok" and s["pulled"] is True + assert "wired-in" in s["deployments"] # affected deployment surfaced + assert (work / "b.txt").exists() + + # A second sync is a no-op: nothing pulled, no affected deployments. + again = client.post("/programs/wired-in/sync").json() + assert again["pulled"] is False and again["deployments"] == [] + + def test_sync_unknown_program_404(self, client: TestClient) -> None: + assert client.post("/programs/nope/sync").status_code == 404 + + def test_sync_non_git_source_400(self, client: TestClient) -> None: + assert client.post("/programs/test-tool/sync").status_code == 400 diff --git a/cli/src/castle_cli/commands/graph.py b/cli/src/castle_cli/commands/graph.py new file mode 100644 index 0000000..ba77b67 --- /dev/null +++ b/cli/src/castle_cli/commands/graph.py @@ -0,0 +1,68 @@ +"""castle graph — the relationship model: repos, `requires` edges, derived status. + +A read-only diagnostic (see docs/relationships.md). Nothing here is stored — repos +come from git, predicates (functional/fresh/deployed) are computed on the fly. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json + +from castle_cli.config import load_config + +BOLD, DIM, RESET = "\033[1m", "\033[2m", "\033[0m" +GREEN, RED, YELLOW, CYAN = "\033[32m", "\033[31m", "\033[33m", "\033[36m" + + +def run_graph(args: argparse.Namespace) -> int: + from castle_core.relations import build_model + + config = load_config() + model = build_model(config, check=True, freshness=True) + + if getattr(args, "json", False): + print( + json.dumps( + { + "repos": [dataclasses.asdict(r) for r in model.repos], + "nodes": [dataclasses.asdict(n) for n in model.nodes], + "edges": [dataclasses.asdict(e) for e in model.edges], + }, + indent=2, + ) + ) + return 0 + + monos = [r for r in model.repos if r.multi] + print(f"{BOLD}Repos{RESET} ({len(model.repos)}, {len(monos)} monorepo)") + for r in monos: + fresh = ( + "" + if r.fresh is None + else (f" {GREEN}fresh{RESET}" if r.fresh else f" {YELLOW}stale{RESET}") + ) + print(f" {CYAN}{r.key}{RESET}{fresh} {DIM}→ {', '.join(r.programs)}{RESET}") + + edges = [e for e in model.edges if e.kind == "deployment"] + print(f"\n{BOLD}requires{RESET} (deployment → deployment): {len(edges)}") + for e in edges: + bind = f" {DIM}→ ${e.bind}{RESET}" if e.bind else "" + print(f" {e.src} {DIM}requires{RESET} {e.dst}{bind}") + if not edges: + print(f" {DIM}(none declared — front-end/back-end deps have no encoded edge yet){RESET}") + + unhealthy = [n for n in model.nodes if not n.functional] + print(f"\n{BOLD}functional?{RESET} — {len(unhealthy)} with unmet requirements") + for n in unhealthy: + print(f" {RED}✗{RESET} {n.name} {DIM}unmet: {', '.join(n.unmet)}{RESET}") + if not unhealthy: + print(f" {GREEN}✓ all functional{RESET}") + + depended = sorted((n for n in model.nodes if n.depended_on_by), key=lambda n: -n.depended_on_by) + if depended: + print(f"\n{BOLD}widely depended-on{RESET}") + for n in depended: + print(f" {n.name} {DIM}← {n.depended_on_by} dependent(s){RESET}") + return 0 diff --git a/cli/src/castle_cli/main.py b/cli/src/castle_cli/main.py index c6ad78d..109772a 100644 --- a/cli/src/castle_cli/main.py +++ b/cli/src/castle_cli/main.py @@ -193,6 +193,12 @@ def build_parser() -> argparse.ArgumentParser: "doctor", help="Diagnose setup + runtime health, with next-step hints" ) + # Relationship model — repos, requires edges, and derived status. + p = subparsers.add_parser( + "graph", help="Show how programs/deployments relate (repos, requires, status)" + ) + p.add_argument("--json", action="store_true", help="Output as JSON") + # Cross-resource overview p = subparsers.add_parser("list", help="List programs, services, jobs, and tools") p.add_argument( @@ -335,6 +341,10 @@ def main() -> int: from castle_cli.commands.doctor import run_doctor return run_doctor(args) + if cmd == "graph": + from castle_cli.commands.graph import run_graph + + return run_graph(args) if cmd == "list": from castle_cli.commands.list_cmd import run_list diff --git a/core/src/castle_core/deploy.py b/core/src/castle_core/deploy.py index 537b17f..5f31233 100644 --- a/core/src/castle_core/deploy.py +++ b/core/src/castle_core/deploy.py @@ -244,7 +244,9 @@ def apply( return "restart" return "unchanged" - result = ApplyResult(registry=NodeRegistry(node=_node_config(config), deployed=desired)) + result = ApplyResult( + registry=NodeRegistry(node=_node_config(config), deployed=desired) + ) if plan: # No writes: for systemd, predict the new unit bytes by rendering to a string @@ -469,6 +471,36 @@ def _public_url( return None +def _target_url(config: CastleConfig, target_name: str) -> str | None: + """The base URL another deployment is reachable at — how a ``{kind: deployment, + bind: VAR}`` requirement projects its target into the consumer's env.""" + dep = config.deployments.get(target_name) + if dep is None: + return None + expose = getattr(dep, "expose", None) + http = getattr(expose, "http", None) if expose else None + tport = http.internal.port if http else None + return _public_url(config, target_name, getattr(dep, "http_exposed", False), tport) + + +def _requires_env(config: CastleConfig, name: str, config_key: str) -> dict[str, str]: + """Env generated FROM a deployment's ``requires`` — a ``{kind: deployment, + bind: VAR}`` requirement sets ``VAR`` to the target's URL. Env is derived from + the dependency, never scraped back into one (see docs/relationships.md).""" + dep = config.deployments[name] + prog = config.programs.get(config_key) + reqs = list(getattr(dep, "requires", []) or []) + if prog: + reqs += list(prog.requires) + out: dict[str, str] = {} + for r in reqs: + if r.kind == "deployment" and r.bind: + url = _target_url(config, r.ref) + if url: + out[r.bind] = url + return out + + def _supabase_app_schemas(config: CastleConfig) -> str: """The ``${supabase_app_schemas}`` placeholder: each registered supabase app's own schema, comma-prefixed and joined (or '' when there are none). @@ -621,8 +653,14 @@ def _build_deployed( # names to castle's computed values. Secret-bearing vars split out to a # mode-0600 file (never in the unit or argv). raw_env = dict(dep.defaults.env) if (dep.defaults and dep.defaults.env) else {} + # Env generated from `requires` ({kind: deployment, bind: VAR} → target URL). + # An explicit defaults.env value always wins — a hand-set var is never clobbered. + for var, url in _requires_env(config, name, config_key).items(): + raw_env.setdefault(var, url) public_url = _public_url(config, name, expose, port) - ctx = _env_context(name, config_key, port, public_url, _supabase_app_schemas(config)) + ctx = _env_context( + name, config_key, port, public_url, _supabase_app_schemas(config) + ) # ${tls_*}: paths to castle-materialized cert files for a TLS-material TCP # service. The deployment maps them into its own config (mount ${tls_dir} for a # container, or reference ${tls_cert}/${tls_key} directly for a native service). @@ -647,7 +685,12 @@ def _build_deployed( _ensure_python_tool(config, dep.program, messages) run_cmd = _build_run_cmd( - name, run, env, messages, source_dir, secret_env_file=secret_env_file, + name, + run, + env, + messages, + source_dir, + secret_env_file=secret_env_file, placeholders=ctx, ) stop_cmd = _build_stop_cmd(name, run, source_dir) diff --git a/core/src/castle_core/git.py b/core/src/castle_core/git.py new file mode 100644 index 0000000..c328234 --- /dev/null +++ b/core/src/castle_core/git.py @@ -0,0 +1,159 @@ +"""Git working-copy status and sync for programs whose source is a git repo. + +Programs that declare a ``repo:`` URL are cloned once (``castle program clone``); +this module lets a running castle *see how far behind* a working copy is and pull +later updates. It is intentionally pull-only — it touches files on disk and never +builds, applies, or restarts anything. Making the running artifact reflect the new +code (rebuild a frontend, restart a service) stays an explicit, separate step via +``castle apply`` / ``castle restart``. + +Plain ``git`` via subprocess (matching ``castle program clone``); no GitPython. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +# Bound network calls (fetch/pull) so an unreachable remote can't hang a request. +_FETCH_TIMEOUT = 20.0 +_PULL_TIMEOUT = 60.0 + + +@dataclass +class GitStatus: + """A program working copy's git state. ``ahead``/``behind`` are relative to the + upstream tracking branch and reflect the *last fetch* (``git_status(fetch=True)`` + refreshes them). ``None`` counts mean "no upstream to compare against".""" + + is_repo: bool + branch: str | None = None + upstream: str | None = None + dirty: bool = False + ahead: int | None = None + behind: int | None = None + detached: bool = False + error: str | None = None + + +def _git( + source: Path, *args: str, timeout: float | None = None +) -> subprocess.CompletedProcess[str]: + """Run ``git -C `` capturing text output.""" + return subprocess.run( + ["git", "-C", str(source), *args], + capture_output=True, + text=True, + timeout=timeout, + ) + + +def is_git_repo(source: Path | None) -> bool: + """True when ``source`` is inside a git working tree.""" + if not source or not Path(source).is_dir(): + return False + try: + r = _git( + Path(source), "rev-parse", "--is-inside-work-tree", timeout=_FETCH_TIMEOUT + ) + except (OSError, subprocess.SubprocessError): + return False + return r.returncode == 0 and r.stdout.strip() == "true" + + +def toplevel(source: Path | None) -> str | None: + """The absolute path of the git working copy ``source`` lives in, or None. + + The natural identity of a *repo*: several programs whose sources share a + toplevel are the same working copy (a monorepo). Adopted single-program repos + are their own toplevel — the N=1 case.""" + if not source or not Path(source).is_dir(): + return None + try: + r = _git(Path(source), "rev-parse", "--show-toplevel", timeout=_FETCH_TIMEOUT) + except (OSError, subprocess.SubprocessError): + return None + return r.stdout.strip() or None if r.returncode == 0 else None + + +def remote_url(source: Path | None) -> str | None: + """The ``origin`` remote URL of the working copy, or None (no remote).""" + if not is_git_repo(source): + return None + r = _git(Path(source), "remote", "get-url", "origin") # type: ignore[arg-type] + return r.stdout.strip() or None if r.returncode == 0 else None + + +def git_status(source: Path | None, fetch: bool = True) -> GitStatus: + """The working copy's branch/dirty/ahead/behind state. + + ``fetch=True`` runs ``git fetch`` first (bounded, tolerant of an offline remote) + so ``behind`` reflects the real remote; on fetch failure the counts fall back to + the last-known values and ``error`` carries the reason. Never raises — a + non-repo returns ``GitStatus(is_repo=False)`` so callers can just hide the UI. + """ + if not is_git_repo(source): + return GitStatus(is_repo=False) + src = Path(source) # type: ignore[arg-type] + st = GitStatus(is_repo=True) + + # Branch (or detached HEAD). + branch = _git(src, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + if branch == "HEAD": + st.detached = True + else: + st.branch = branch + + # Dirty working tree (staged, unstaged, or untracked). + st.dirty = bool(_git(src, "status", "--porcelain").stdout.strip()) + + # Best-effort refresh from the remote; failure is non-fatal (offline, no remote). + if fetch: + try: + fr = _git(src, "fetch", "--quiet", timeout=_FETCH_TIMEOUT) + if fr.returncode != 0: + st.error = (fr.stderr or fr.stdout).strip() or "git fetch failed" + except subprocess.TimeoutExpired: + st.error = "git fetch timed out" + except (OSError, subprocess.SubprocessError) as e: + st.error = str(e) + + # Upstream tracking branch, then the ahead/behind split against it. + up = _git(src, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") + if up.returncode == 0 and up.stdout.strip(): + st.upstream = up.stdout.strip() + counts = _git(src, "rev-list", "--left-right", "--count", "@{u}...HEAD") + if counts.returncode == 0: + parts = counts.stdout.split() + if len(parts) == 2: + st.behind, st.ahead = int(parts[0]), int(parts[1]) + return st + + +def head(source: Path | None) -> str | None: + """The working copy's current commit sha, or None if unavailable. Lets a caller + tell whether a ``pull`` actually advanced the tree (before != after).""" + if not is_git_repo(source): + return None + r = _git(Path(source), "rev-parse", "HEAD") # type: ignore[arg-type] + return r.stdout.strip() or None if r.returncode == 0 else None + + +def pull(source: Path | None) -> tuple[bool, str]: + """Fast-forward the working copy to its upstream (``git pull --ff-only``). + + ``--ff-only`` is deliberate: it refuses to merge, so a dirty or diverged tree + fails cleanly with git's own message instead of creating a merge commit. Returns + ``(ok, combined_output)``. + """ + if not is_git_repo(source): + return False, "not a git repository" + try: + r = _git(Path(source), "pull", "--ff-only", timeout=_PULL_TIMEOUT) # type: ignore[arg-type] + except subprocess.TimeoutExpired: + return False, "git pull timed out" + except (OSError, subprocess.SubprocessError) as e: + return False, str(e) + out = (r.stdout + r.stderr).strip() + return r.returncode == 0, out diff --git a/core/src/castle_core/manifest.py b/core/src/castle_core/manifest.py index a4f87a3..ad60ff3 100644 --- a/core/src/castle_core/manifest.py +++ b/core/src/castle_core/manifest.py @@ -283,6 +283,27 @@ class Capability(BaseModel): meta: dict[str, str] = Field(default_factory=dict) +class Requirement(BaseModel): + """A precondition — something that must be true for a program/deployment to be + *functional*. The ``kind`` fixes both the meaning and how it's checked (there is + no separate purpose tag): + + - ``system`` — a host package/binary must be installed (``ref`` = package). + - ``deployment`` — another deployment must exist/run (``ref`` = its name). + + ``version`` is reserved for a future constraint (unused now). ``bind`` (for a + ``deployment`` requirement) names the env var castle projects the target's URL + into — env is derived *from* the requirement, never scraped back into it. + + See docs/relationships.md. ``system_dependencies`` is the ``kind: system`` case. + """ + + kind: Literal["system", "deployment"] + ref: str + version: str | None = None + bind: str | None = None + + # --------------------- # Defaults # --------------------- @@ -363,6 +384,10 @@ class ProgramSpec(BaseModel): # Per-program dev verb overrides (declared verbs override the stack default). commands: CommandsSpec | None = None + # `requires` is the general precondition relation (see docs/relationships.md). + # `system_dependencies` is kept as the `{kind: system}` alias/back-compat; both + # are merged when evaluating what a program requires. + requires: list[Requirement] = Field(default_factory=list) system_dependencies: list[str] = Field(default_factory=list) install_extras: list[str] = Field(default_factory=list) version: str | None = None @@ -406,6 +431,9 @@ class DeploymentBase(BaseModel): ) description: str | None = None defaults: DefaultsSpec | None = None + # Runtime preconditions (e.g. another deployment that must exist). See + # docs/relationships.md; merged with the program's `requires` when evaluated. + requires: list[Requirement] = Field(default_factory=list) # Declared on/off state. `castle apply` converges reality to this: enabled # deployments are activated (service started, tool installed, route served), # disabled ones are deactivated but kept in the catalog. This is *desired diff --git a/core/src/castle_core/relations.py b/core/src/castle_core/relations.py new file mode 100644 index 0000000..e557515 --- /dev/null +++ b/core/src/castle_core/relations.py @@ -0,0 +1,204 @@ +"""The relationship model — derived, never stored. See docs/relationships.md. + +Entities: **program**, **deployment**, **repo** (a repo is a git working copy; +programs sharing a toplevel form a monorepo). One encoded relation, **`requires`** +(a precondition, typed by ``kind``: ``system`` = must be installed, ``deployment`` += must exist). Everything else — repos, env wiring, fan-in, and the predicates +``functional?`` / ``fresh?`` / ``deployed?`` — is computed here on demand. + +Governing rule: *predicates are derived; we encode only the non-derivable.* So this +module reads the encoded ``requires`` (plus ``system_dependencies`` as its +``kind: system`` alias) and derives the rest. It does **not** scrape env for +dependencies — env is generated *from* requirements, not the reverse. +""" + +from __future__ import annotations + +import shutil +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +from castle_core import git +from castle_core.config import CastleConfig +from castle_core.manifest import Requirement + + +@dataclass +class Repo: + key: str # url-safe slug (basename of the working copy) + path: str # git toplevel + url: str | None + ref: str | None + programs: list[str] + deployments: list[str] + behind: int | None = None # commits behind upstream (None = unknown/no upstream) + dirty: bool = False + fresh: bool | None = None # derived: at latest and clean (None = not evaluated) + + @property + def multi(self) -> bool: + """A monorepo — more than one program shares this working copy.""" + return len(self.programs) > 1 + + +@dataclass +class Edge: + src: str # deployment name + dst: str # target: a package (system) or another deployment + kind: str # "system" | "deployment" + bind: str | None = None # env var to project the target URL into (deployment) + + +@dataclass +class Node: + name: str # deployment name + program: str | None + kind: str # service|job|tool|static|reference + repo: str | None + depended_on_by: int # distinct deployments that require this one (fan-in) + unmet: list[str] = field(default_factory=list) # unsatisfied requirements + functional: bool = True # derived: all requirements satisfied + fresh: bool | None = None # derived: its repo is at latest + clean + deployed: bool | None = None # derived: active in the registry (None = unknown) + + +@dataclass +class Model: + repos: list[Repo] = field(default_factory=list) + nodes: list[Node] = field(default_factory=list) + edges: list[Edge] = field(default_factory=list) + + +def _program_of(name: str, dep: object) -> str: + return getattr(dep, "program", None) or name + + +def _slug(name: str, used: set[str]) -> str: + base = name or "repo" + key, n = base, 2 + while key in used: + key, n = f"{base}-{n}", n + 1 + used.add(key) + return key + + +def derive_repos(config: CastleConfig) -> dict[str, Repo]: + """Group programs by the git working copy their source lives in.""" + by_top: dict[str, list[str]] = {} + for pname, prog in config.programs.items(): + top = git.toplevel(prog.source) if prog.source else None + if top: + by_top.setdefault(top, []).append(pname) + + used: set[str] = set() + repos: dict[str, Repo] = {} + for top, progs in sorted(by_top.items()): + progs = sorted(progs) + url = next( + (config.programs[p].repo for p in progs if config.programs[p].repo), None + ) or git.remote_url(Path(top)) + ref = ( + next( + (config.programs[p].ref for p in progs if config.programs[p].ref), None + ) + or git.git_status(Path(top), fetch=False).branch + ) + deps = sorted( + d for d, dep in config.deployments.items() if _program_of(d, dep) in progs + ) + repos[_slug(Path(top).name, used)] = Repo("", top, url, ref, progs, deps) + for key, repo in repos.items(): + repo.key = key + return repos + + +def requirements_of(config: CastleConfig, dep_name: str) -> list[Requirement]: + """The full requirement set for a deployment: its own ``requires`` plus its + program's ``requires`` and ``system_dependencies`` (the ``kind: system`` alias), + de-duplicated by (kind, ref).""" + dep = config.deployments[dep_name] + prog = config.programs.get(_program_of(dep_name, dep)) + reqs: list[Requirement] = list(getattr(dep, "requires", []) or []) + if prog: + reqs += list(prog.requires) + reqs += [ + Requirement(kind="system", ref=pkg) for pkg in prog.system_dependencies + ] + seen: set[tuple[str, str]] = set() + out: list[Requirement] = [] + for r in reqs: + if (r.kind, r.ref) not in seen: + seen.add((r.kind, r.ref)) + out.append(r) + return out + + +def _check(config: CastleConfig, req: Requirement) -> bool: + """Is a single requirement satisfied? (The check is fixed by its kind.)""" + if req.kind == "system": + return shutil.which(req.ref) is not None + if req.kind == "deployment": + return req.ref in config.deployments + return True + + +def build_model( + config: CastleConfig, + check: bool = True, + active: set[str] | None = None, + freshness: bool = False, +) -> Model: + """Compute the relationship model. + + - ``check`` (default): evaluate ``functional?`` (unmet requirements) via a live + ``which`` / registry probe. ``check=False`` → pure structural model. + - ``active``: names of currently-active deployments → the ``deployed?`` + predicate (left ``None`` when the caller has no runtime view). + - ``freshness``: also evaluate ``fresh?`` per repo (a ``git status``, no fetch — + last-known — so it stays a local, network-free probe over many repos).""" + from castle_core.manifest import kind_for + + repos = derive_repos(config) + if freshness: + for repo in repos.values(): + st = git.git_status(Path(repo.path), fetch=False) + repo.behind = st.behind + repo.dirty = st.dirty + repo.fresh = (st.behind == 0 or st.behind is None) and not st.dirty + repo_of = {p: key for key, r in repos.items() for p in r.programs} + fresh_of = {key: r.fresh for key, r in repos.items()} + + edges: list[Edge] = [] + for name in config.deployments: + for r in requirements_of(config, name): + edges.append(Edge(name, r.ref, r.kind, r.bind)) + + fan_in = Counter(e.dst for e in edges if e.kind == "deployment") + + nodes: list[Node] = [] + for name, dep in config.deployments.items(): + unmet = ( + [ + f"{r.kind}:{r.ref}" + for r in requirements_of(config, name) + if not _check(config, r) + ] + if check + else [] + ) + repo_key = repo_of.get(_program_of(name, dep)) + nodes.append( + Node( + name=name, + program=_program_of(name, dep), + kind=kind_for(dep), + repo=repo_key, + depended_on_by=fan_in.get(name, 0), + unmet=unmet, + functional=not unmet, + fresh=fresh_of.get(repo_key) if (freshness and repo_key) else None, + deployed=(name in active) if active is not None else None, + ) + ) + return Model(repos=list(repos.values()), nodes=nodes, edges=edges) diff --git a/core/tests/test_git.py b/core/tests/test_git.py new file mode 100644 index 0000000..149c3e1 --- /dev/null +++ b/core/tests/test_git.py @@ -0,0 +1,123 @@ +"""Tests for git working-copy status/sync (core/src/castle_core/git.py).""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from castle_core import git as G + +# Identity so commits succeed without touching the user's global git config. +_ENV = { + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", +} + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run( + ["git", "-C", str(cwd), *args], + check=True, + capture_output=True, + text=True, + env={**_base_env(), **_ENV}, + ) + + +def _base_env() -> dict[str, str]: + import os + + return dict(os.environ) + + +def _commit(cwd: Path, fname: str, text: str) -> None: + (cwd / fname).write_text(text) + _git(cwd, "add", fname) + _git(cwd, "commit", "-m", f"add {fname}") + + +@pytest.fixture +def repos(tmp_path: Path): + """An `upstream` repo and a `work` clone of it (tracking upstream/main).""" + upstream = tmp_path / "upstream" + upstream.mkdir() + _git(upstream, "init", "-q", "-b", "main") + _commit(upstream, "a.txt", "one") + work = tmp_path / "work" + _git(tmp_path, "clone", "-q", str(upstream), str(work)) + return upstream, work + + +def test_non_repo_is_benign(tmp_path: Path) -> None: + assert G.is_git_repo(tmp_path / "nope") is False + st = G.git_status(tmp_path / "nope") + assert st.is_repo is False and st.branch is None + ok, out = G.pull(tmp_path / "nope") + assert ok is False and "not a git" in out + + +def test_status_clean_and_up_to_date(repos) -> None: + _, work = repos + st = G.git_status(work, fetch=True) + assert st.is_repo and st.branch == "main" + assert st.dirty is False + assert st.behind == 0 and st.ahead == 0 + assert st.upstream and st.upstream.endswith("main") + + +def test_behind_then_pull_fast_forwards(repos) -> None: + upstream, work = repos + _commit(upstream, "b.txt", "two") # advance the remote + st = G.git_status(work, fetch=True) + assert st.behind == 1 and st.ahead == 0 + + ok, out = G.pull(work) + assert ok is True, out + assert (work / "b.txt").exists() + + after = G.git_status(work, fetch=True) + assert after.behind == 0 and after.dirty is False + + +def test_conflicting_dirty_tree_blocks_pull(repos) -> None: + """A pull that would overwrite a locally-modified file is refused (ff-only never + merges), leaving the working copy untouched with git's own message.""" + upstream, work = repos + _commit(upstream, "a.txt", "upstream change") # remote touches a.txt... + (work / "a.txt").write_text("local uncommitted change") # ...so does the work tree + assert G.git_status(work, fetch=False).dirty is True + + ok, out = G.pull(work) + assert ok is False and out # "local changes would be overwritten" + assert (work / "a.txt").read_text() == "local uncommitted change" + + +def test_diverged_branch_blocks_ff_pull(repos) -> None: + """Local commits the remote doesn't have → --ff-only refuses (no merge commit).""" + upstream, work = repos + _commit(upstream, "b.txt", "remote two") + _commit(work, "c.txt", "local two") # work now has a commit upstream lacks + st = G.git_status(work, fetch=True) + assert st.behind == 1 and st.ahead == 1 + + ok, out = G.pull(work) + assert ok is False and out + + +def test_detached_head_reported(repos) -> None: + _, work = repos + head = subprocess.run( + ["git", "-C", str(work), "rev-parse", "HEAD"], + capture_output=True, + text=True, + env={**_base_env(), **_ENV}, + ).stdout.strip() + _git(work, "checkout", "-q", head) + st = G.git_status(work, fetch=False) + assert st.detached is True and st.branch is None diff --git a/core/tests/test_relations.py b/core/tests/test_relations.py new file mode 100644 index 0000000..45f1a11 --- /dev/null +++ b/core/tests/test_relations.py @@ -0,0 +1,106 @@ +"""Tests for the relationship model (core/src/castle_core/relations.py).""" + +from __future__ import annotations + +import pytest + +import castle_core.config as C +from castle_core import relations as R +from castle_core.manifest import ProgramSpec, Requirement, SystemdDeployment + + +def _dep(program: str) -> SystemdDeployment: + return SystemdDeployment.model_validate( + { + "manager": "systemd", + "program": program, + "run": {"launcher": "command", "argv": [program]}, + } + ) + + +def _cfg(programs: dict, deployments: dict) -> C.CastleConfig: + return C.CastleConfig( + root=None, + gateway=C.GatewayConfig(port=9000), + repo=None, + programs=programs, + deployments=deployments, + ) + + +def test_system_dependencies_is_the_system_requirement_alias() -> None: + """`system_dependencies` surfaces as a {kind: system} requirement.""" + cfg = _cfg( + {"t": ProgramSpec(id="t", system_dependencies=["pandoc"])}, {"t": _dep("t")} + ) + reqs = R.requirements_of(cfg, "t") + assert [(r.kind, r.ref) for r in reqs] == [("system", "pandoc")] + + +def test_requirements_merge_program_and_deployment_deduped() -> None: + prog = ProgramSpec( + id="web", + system_dependencies=["pandoc"], + requires=[Requirement(kind="deployment", ref="api", bind="API_URL")], + ) + dep = SystemdDeployment.model_validate( + { + "manager": "systemd", + "program": "web", + "run": {"launcher": "command", "argv": ["web"]}, + "requires": [{"kind": "system", "ref": "pandoc"}], + } # dup of program's + ) + cfg = _cfg( + {"web": prog, "api": ProgramSpec(id="api")}, {"web": dep, "api": _dep("api")} + ) + kinds = {(r.kind, r.ref) for r in R.requirements_of(cfg, "web")} + assert kinds == {("system", "pandoc"), ("deployment", "api")} # deduped + + +def test_deployment_edge_carries_bind_and_counts_fan_in() -> None: + """A {kind: deployment} requirement becomes an edge (with bind), and the target's + fan-in is the count of distinct dependents.""" + consumer = ProgramSpec( + id="web", requires=[Requirement(kind="deployment", ref="api", bind="API_URL")] + ) + consumer2 = ProgramSpec( + id="cli", requires=[Requirement(kind="deployment", ref="api")] + ) + cfg = _cfg( + {"web": consumer, "cli": consumer2, "api": ProgramSpec(id="api")}, + {"web": _dep("web"), "cli": _dep("cli"), "api": _dep("api")}, + ) + m = R.build_model(cfg, check=False) + edge = next(e for e in m.edges if e.src == "web" and e.dst == "api") + assert edge.kind == "deployment" and edge.bind == "API_URL" + api = next(n for n in m.nodes if n.name == "api") + assert api.depended_on_by == 2 # web + cli + + +def test_functional_predicate_reports_unmet(monkeypatch: pytest.MonkeyPatch) -> None: + """`functional?` is derived: a missing system package is unmet; a present + deployment requirement is satisfied.""" + prog = ProgramSpec( + id="web", + system_dependencies=["pandoc"], + requires=[Requirement(kind="deployment", ref="api")], + ) + cfg = _cfg( + {"web": prog, "api": ProgramSpec(id="api")}, + {"web": _dep("web"), "api": _dep("api")}, + ) + monkeypatch.setattr(R.shutil, "which", lambda _: None) # nothing installed + m = R.build_model(cfg, check=True) + web = next(n for n in m.nodes if n.name == "web") + assert web.unmet == ["system:pandoc"] # deployment:api exists → satisfied + assert web.functional is False + + +def test_missing_deployment_requirement_is_unmet() -> None: + prog = ProgramSpec(id="web", requires=[Requirement(kind="deployment", ref="ghost")]) + cfg = _cfg({"web": prog}, {"web": _dep("web")}) + m = R.build_model(cfg, check=True) + web = next(n for n in m.nodes if n.name == "web") + assert web.unmet == ["deployment:ghost"] and web.functional is False diff --git a/docs/relationships.md b/docs/relationships.md new file mode 100644 index 0000000..52e6833 --- /dev/null +++ b/docs/relationships.md @@ -0,0 +1,90 @@ +# Relationships: requires, repos, and derived predicates + +How castle models the relationships between **programs**, **deployments**, and +**repos** — and answers questions like *"is this functional?"*, *"is it fresh?"*, +*"is it deployed?"* — with the smallest possible amount of stored state. + +## The governing principle + +> **Predicates are always derived. Encode only what is not derivable.** + +A *predicate* is a question we ask about a program or deployment: `functional?`, +`fresh?`, `deployed?`. None of these are ever stored — each is a **function** over +data castle already has (git, config, the registry). When a predicate can't be +answered from derived data, find the one missing datum and ask: is it about the +**thing** (a node property) or about a **relationship** (an edge property)? Encode +*only* that datum. Everything else stays computed. + +This is the same instinct as `kind` (derived from `manager`) — we don't store what +we can compute, and a relationship that proves real and stable in the derived graph +is a candidate to *promote* into a first-class concept. Diagnostic → evidence → +abstraction, in that order. + +## Entities + +- **program** — the software catalog entry. +- **deployment** — a program realized on this node (`kind` derived from `manager`). +- **repo** — a git working copy. **Derived** from `git rev-parse --show-toplevel` + on each program's source; several programs sharing one toplevel is a *monorepo*. + Never stored. + +## The one encoded relation: `requires` + +Everything we were calling "substrate", "wiring", or "dependency" is one relation — +**`requires`** ("A must have B to be functional") — with a typed target. The +**kind fixes the meaning and the check**; there is no separate purpose/`for` tag: + +| kind | means | checked by | +|------|-------|-----------| +| `system` | the host package/binary must be **installed** | `which ` | +| `deployment` | another deployment must **exist / be running** | registry / config | + +```yaml +requires: + - { kind: system, ref: pandoc } # today's system_dependencies + - { kind: deployment, ref: astro-guru, bind: GURU_URL } + # - { kind: deployment, ref: litellm, version: ">=1" } # version: FUTURE, unused +``` + +`system_dependencies` is exactly the `{kind: system}` case and is kept as an alias. + +Only encode a `requires` edge that is **not derivable** and that **castle itself +must traverse** for an operation (status, bring-up order, group ops). Do **not** +duplicate what another layer already owns — systemd `Requires=`/`After=` for unit +ordering, uv/pnpm for build graphs. This is *castle's* slice, uncoupled from any +one package ecosystem. + +### Env is derived *from* `requires`, never scraped *into* it + +Reading dependencies out of env strings is unstable (formats vary; a static +frontend's API URL is baked into its bundle and invisible). The stable direction is +the reverse: from an encoded `{kind: deployment}` requirement castle **generates** +the wiring env — it knows the target's address (`.` / its port) and +projects it into the consumer's env, optionally under the var named by `bind`. Same +move as `${public_url}`, one step further. Dependency → env, never env → dependency. + +## Derived predicates + +Computed on demand from encoded `requires` + git + registry; nothing stored: + +- **`functional?`** — every `requires` is satisfied (system installed, deployment + exists). The unmet ones *are* the node's status (`doctor`/`status`). +- **`fresh?`** — the program's repo is at latest and clean (git status). +- **`deployed?`** — the deployment is active in the registry. + +New predicates are just new functions over the same preconditions — there is +nothing to "unify" in storage. + +## What's encoded vs derived (the whole surface) + +| datum | source | stored? | +|-------|--------|---------| +| repo / monorepo | git toplevel | derived | +| `fresh?` / `deployed?` / `functional?` | git / registry / requires | derived | +| env wiring for a dependency | the `requires` edge + target address | derived | +| fan-in ("widely depended-on") | count of requires | derived | +| a **non-derivable** requirement (frontend→backend, host package) | — | **encoded** (`requires`) | +| the env var to bind a dep's URL to, when non-conventional | — | **encoded** (`requires[].bind`) | + +Every irreducible found so far is an **edge** (a relationship); no new **node** +property has been needed yet — a sign the encoded surface stays tiny.