refactor: Restructure ~/.castle/ as the instance directory
Move all instance state into ~/.castle/ with clear separation: - code/ — user project source (was components/) - artifacts/ — generated specs, built content (was generated/, static/) - data/ — service runtime data (was /data/castle/) - secrets/ — credentials castle.yaml moves to ~/.castle/castle.yaml as the canonical location. Source paths use repo: prefix for git repo programs (castle-api, app) and relative paths for user projects (code/central-context). Add idempotent install.sh for bootstrapping infrastructure (Docker, Caddy, MQTT, Postgres, Neo4j) with interactive service detection. Remove components/ from repo (now in ~/.castle/code/), deinit submodules, remove .gitmodules.
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -1,2 +1,8 @@
|
|||||||
.venv
|
.venv
|
||||||
__pycache__
|
__pycache__
|
||||||
|
.ruff_cache
|
||||||
|
.pytest_cache
|
||||||
|
.claude
|
||||||
|
node_modules
|
||||||
|
castle.yaml
|
||||||
|
|
||||||
|
|||||||
6
.gitmodules
vendored
6
.gitmodules
vendored
@@ -1,6 +0,0 @@
|
|||||||
[submodule "central-context"]
|
|
||||||
path = components/central-context
|
|
||||||
url = https://github.com/payneio/central-context.git
|
|
||||||
[submodule "notification-bridge"]
|
|
||||||
path = components/notification-bridge
|
|
||||||
url = https://github.com/payneio/attention-firewall.git
|
|
||||||
@@ -274,11 +274,11 @@ async def apply_config() -> ApplyResponse:
|
|||||||
errors.append(f"Failed to restart {name}: {output}")
|
errors.append(f"Failed to restart {name}: {output}")
|
||||||
|
|
||||||
# Reload gateway
|
# Reload gateway
|
||||||
from castle_core.config import GENERATED_DIR, ensure_dirs
|
from castle_core.config import SPECS_DIR, ensure_dirs
|
||||||
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
||||||
|
|
||||||
ensure_dirs()
|
ensure_dirs()
|
||||||
caddyfile_path = GENERATED_DIR / "Caddyfile"
|
caddyfile_path = SPECS_DIR / "Caddyfile"
|
||||||
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
||||||
actions.append("Generated Caddyfile")
|
actions.append("Generated Caddyfile")
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, status
|
from fastapi import APIRouter, HTTPException, status
|
||||||
|
|
||||||
from castle_core.config import GENERATED_DIR
|
from castle_core.config import SPECS_DIR
|
||||||
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
||||||
from castle_core.manifest import ProgramSpec, JobSpec, ServiceSpec
|
from castle_core.manifest import ProgramSpec, JobSpec, ServiceSpec
|
||||||
from castle_core.stacks import available_actions
|
from castle_core.stacks import available_actions
|
||||||
@@ -834,8 +834,8 @@ def get_caddyfile() -> dict[str, str]:
|
|||||||
async def reload_gateway() -> dict[str, str]:
|
async def reload_gateway() -> dict[str, str]:
|
||||||
"""Regenerate Caddyfile and reload Caddy."""
|
"""Regenerate Caddyfile and reload Caddy."""
|
||||||
registry = get_registry()
|
registry = get_registry()
|
||||||
GENERATED_DIR.mkdir(parents=True, exist_ok=True)
|
SPECS_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
caddyfile_path = GENERATED_DIR / "Caddyfile"
|
caddyfile_path = SPECS_DIR / "Caddyfile"
|
||||||
|
|
||||||
# Include remote registries for cross-node routing
|
# Include remote registries for cross-node routing
|
||||||
remote_regs = {h: n.registry for h, n in mesh_state.all_nodes().items()}
|
remote_regs = {h: n.registry for h, n in mesh_state.all_nodes().items()}
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ def registry_path(tmp_path: Path, castle_root: Path) -> Generator[Path, None, No
|
|||||||
run_cmd=["uv", "run", "test-svc"],
|
run_cmd=["uv", "run", "test-svc"],
|
||||||
env={
|
env={
|
||||||
"TEST_SVC_PORT": "19000",
|
"TEST_SVC_PORT": "19000",
|
||||||
"TEST_SVC_DATA_DIR": "/data/castle/test-svc",
|
"TEST_SVC_DATA_DIR": "/home/user/.castle/data/test-svc",
|
||||||
},
|
},
|
||||||
description="Test service",
|
description="Test service",
|
||||||
behavior="daemon",
|
behavior="daemon",
|
||||||
|
|||||||
258
castle.yaml
258
castle.yaml
@@ -1,258 +0,0 @@
|
|||||||
gateway:
|
|
||||||
port: 9000
|
|
||||||
programs:
|
|
||||||
central-context:
|
|
||||||
description: Content storage API useful to get context into my data dir from anywhere
|
|
||||||
on the LAN
|
|
||||||
behavior: daemon
|
|
||||||
source: components/central-context
|
|
||||||
stack: python-fastapi
|
|
||||||
notification-bridge:
|
|
||||||
description: Desktop notification forwarder. This taps into your notifications
|
|
||||||
system on the desktop and will forward all notifications the the central context
|
|
||||||
server.
|
|
||||||
behavior: daemon
|
|
||||||
source: components/notification-bridge
|
|
||||||
stack: python-fastapi
|
|
||||||
install_extras:
|
|
||||||
- linux
|
|
||||||
castle-api:
|
|
||||||
description: Castle API
|
|
||||||
behavior: daemon
|
|
||||||
source: castle-api
|
|
||||||
stack: python-fastapi
|
|
||||||
protonmail:
|
|
||||||
description: ProtonMail email sync via Bridge
|
|
||||||
behavior: tool
|
|
||||||
source: components/protonmail
|
|
||||||
stack: python-cli
|
|
||||||
backup-collect:
|
|
||||||
description: Collect files from various sources into backup directory
|
|
||||||
behavior: tool
|
|
||||||
source: components/backup-collect
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- rsync
|
|
||||||
castle-app:
|
|
||||||
description: Castle web app
|
|
||||||
behavior: frontend
|
|
||||||
source: app
|
|
||||||
stack: react-vite
|
|
||||||
build:
|
|
||||||
commands:
|
|
||||||
- - pnpm
|
|
||||||
- build
|
|
||||||
outputs:
|
|
||||||
- dist/
|
|
||||||
devbox-connect:
|
|
||||||
description: SSH tunnel manager with auto-reconnect
|
|
||||||
behavior: tool
|
|
||||||
source: components/devbox-connect
|
|
||||||
stack: python-cli
|
|
||||||
mbox2eml:
|
|
||||||
description: MBOX to EML email converter
|
|
||||||
behavior: tool
|
|
||||||
source: components/mbox2eml
|
|
||||||
stack: python-cli
|
|
||||||
android-backup:
|
|
||||||
description: Backup Android device using ADB
|
|
||||||
behavior: tool
|
|
||||||
source: components/android-backup
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- adb
|
|
||||||
browser:
|
|
||||||
description: Browse the web using natural language via browser-use
|
|
||||||
behavior: tool
|
|
||||||
source: components/browser
|
|
||||||
stack: python-cli
|
|
||||||
docx-extractor:
|
|
||||||
description: Extract content and metadata from Word .docx files
|
|
||||||
behavior: tool
|
|
||||||
source: components/docx-extractor
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- pandoc
|
|
||||||
docx2md:
|
|
||||||
description: Convert Word .docx files to Markdown
|
|
||||||
behavior: tool
|
|
||||||
source: components/docx2md
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- pandoc
|
|
||||||
gpt:
|
|
||||||
description: OpenAI text generation utility
|
|
||||||
behavior: tool
|
|
||||||
source: components/gpt
|
|
||||||
stack: python-cli
|
|
||||||
html2text:
|
|
||||||
description: Convert HTML content to plain text
|
|
||||||
behavior: tool
|
|
||||||
source: components/html2text
|
|
||||||
stack: python-cli
|
|
||||||
md2pdf:
|
|
||||||
description: Convert Markdown files to PDF
|
|
||||||
behavior: tool
|
|
||||||
source: components/md2pdf
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- pandoc
|
|
||||||
- texlive-latex-base
|
|
||||||
mdscraper:
|
|
||||||
description: Combine text files into a single markdown document
|
|
||||||
behavior: tool
|
|
||||||
source: components/mdscraper
|
|
||||||
stack: python-cli
|
|
||||||
pdf-extractor:
|
|
||||||
description: Extract content and metadata from PDF files
|
|
||||||
behavior: tool
|
|
||||||
source: components/pdf-extractor
|
|
||||||
stack: python-cli
|
|
||||||
pdf2md:
|
|
||||||
description: Convert PDF files to Markdown
|
|
||||||
behavior: tool
|
|
||||||
source: components/pdf2md
|
|
||||||
stack: python-cli
|
|
||||||
system_dependencies:
|
|
||||||
- pandoc
|
|
||||||
- poppler-utils
|
|
||||||
schedule:
|
|
||||||
description: Manage systemd user timers and scheduled tasks
|
|
||||||
behavior: tool
|
|
||||||
source: components/schedule
|
|
||||||
stack: python-cli
|
|
||||||
search:
|
|
||||||
description: Manage self-contained searchable collections of files
|
|
||||||
behavior: tool
|
|
||||||
source: components/search
|
|
||||||
stack: python-cli
|
|
||||||
text-extractor:
|
|
||||||
description: Extract content and metadata from text files
|
|
||||||
behavior: tool
|
|
||||||
source: components/text-extractor
|
|
||||||
stack: python-cli
|
|
||||||
services:
|
|
||||||
castle-gateway:
|
|
||||||
description: Caddy reverse proxy gateway
|
|
||||||
run:
|
|
||||||
runner: command
|
|
||||||
argv:
|
|
||||||
- caddy
|
|
||||||
- run
|
|
||||||
- --config
|
|
||||||
- /home/payne/.castle/generated/Caddyfile
|
|
||||||
- --adapter
|
|
||||||
- caddyfile
|
|
||||||
expose:
|
|
||||||
http:
|
|
||||||
internal:
|
|
||||||
port: 9000
|
|
||||||
health_path: /
|
|
||||||
manage:
|
|
||||||
systemd:
|
|
||||||
exec_reload: caddy reload --config /home/payne/.castle/generated/Caddyfile
|
|
||||||
--adapter caddyfile
|
|
||||||
central-context:
|
|
||||||
component: central-context
|
|
||||||
run:
|
|
||||||
runner: python
|
|
||||||
program: central-context
|
|
||||||
expose:
|
|
||||||
http:
|
|
||||||
internal:
|
|
||||||
port: 9001
|
|
||||||
health_path: /health
|
|
||||||
proxy:
|
|
||||||
caddy:
|
|
||||||
path_prefix: /central-context
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
notification-bridge:
|
|
||||||
component: notification-bridge
|
|
||||||
run:
|
|
||||||
runner: python
|
|
||||||
program: notification-bridge
|
|
||||||
expose:
|
|
||||||
http:
|
|
||||||
internal:
|
|
||||||
port: 9002
|
|
||||||
health_path: /health
|
|
||||||
proxy:
|
|
||||||
caddy:
|
|
||||||
path_prefix: /notifications
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
defaults:
|
|
||||||
env:
|
|
||||||
CENTRAL_CONTEXT_URL: http://localhost:9001
|
|
||||||
BUCKET_NAME: notifications
|
|
||||||
PORT: '9002'
|
|
||||||
castle-api:
|
|
||||||
component: castle-api
|
|
||||||
run:
|
|
||||||
runner: python
|
|
||||||
program: castle-api
|
|
||||||
expose:
|
|
||||||
http:
|
|
||||||
internal:
|
|
||||||
port: 9020
|
|
||||||
health_path: /health
|
|
||||||
proxy:
|
|
||||||
caddy:
|
|
||||||
path_prefix: /api
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
castle-mqtt:
|
|
||||||
description: MQTT broker for inter-node mesh coordination
|
|
||||||
run:
|
|
||||||
runner: container
|
|
||||||
image: eclipse-mosquitto:2
|
|
||||||
ports:
|
|
||||||
'1883': 1883
|
|
||||||
volumes:
|
|
||||||
- /data/castle/castle-mqtt/config:/mosquitto/config
|
|
||||||
- /data/castle/castle-mqtt/data:/mosquitto/data
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
jobs:
|
|
||||||
protonmail-sync:
|
|
||||||
component: protonmail
|
|
||||||
description: Sync ProtonMail email
|
|
||||||
run:
|
|
||||||
runner: command
|
|
||||||
argv:
|
|
||||||
- protonmail
|
|
||||||
- sync
|
|
||||||
schedule: '*/5 * * * *'
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
defaults:
|
|
||||||
env:
|
|
||||||
PROTONMAIL_USERNAME: paul@payne.io
|
|
||||||
PROTONMAIL_API_KEY: ${secret:PROTONMAIL_API_KEY}
|
|
||||||
backup-collect:
|
|
||||||
component: backup-collect
|
|
||||||
description: Collect files from various sources into backup directory
|
|
||||||
run:
|
|
||||||
runner: command
|
|
||||||
argv:
|
|
||||||
- backup-collect
|
|
||||||
schedule: 0 2 * * *
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
defaults:
|
|
||||||
env:
|
|
||||||
DBACKUP: /data/backup
|
|
||||||
backup-data:
|
|
||||||
description: Nightly restic backup of /data to /storage
|
|
||||||
run:
|
|
||||||
runner: command
|
|
||||||
argv:
|
|
||||||
- backup-data
|
|
||||||
schedule: 30 3 * * *
|
|
||||||
manage:
|
|
||||||
systemd: {}
|
|
||||||
defaults:
|
|
||||||
env:
|
|
||||||
RESTIC_REPOSITORY: /storage/restic-data
|
|
||||||
RESTIC_PASSWORD_FILE: /home/payne/.config/restic-password
|
|
||||||
@@ -8,8 +8,9 @@ import subprocess
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from castle_core.config import (
|
from castle_core.config import (
|
||||||
GENERATED_DIR,
|
CONTENT_DIR,
|
||||||
STATIC_DIR,
|
DATA_DIR,
|
||||||
|
SPECS_DIR,
|
||||||
CastleConfig,
|
CastleConfig,
|
||||||
ensure_dirs,
|
ensure_dirs,
|
||||||
load_config,
|
load_config,
|
||||||
@@ -31,8 +32,6 @@ from castle_core.registry import (
|
|||||||
load_registry,
|
load_registry,
|
||||||
save_registry,
|
save_registry,
|
||||||
)
|
)
|
||||||
|
|
||||||
DATA_ROOT = Path("/data/castle")
|
|
||||||
SYSTEMD_USER_DIR = Path.home() / ".config" / "systemd" / "user"
|
SYSTEMD_USER_DIR = Path.home() / ".config" / "systemd" / "user"
|
||||||
|
|
||||||
|
|
||||||
@@ -88,7 +87,7 @@ def run_deploy(args: argparse.Namespace) -> int:
|
|||||||
_generate_systemd_units(config, registry)
|
_generate_systemd_units(config, registry)
|
||||||
|
|
||||||
# Generate Caddyfile from registry
|
# Generate Caddyfile from registry
|
||||||
caddyfile_path = GENERATED_DIR / "Caddyfile"
|
caddyfile_path = SPECS_DIR / "Caddyfile"
|
||||||
caddyfile_content = generate_caddyfile_from_registry(registry)
|
caddyfile_content = generate_caddyfile_from_registry(registry)
|
||||||
caddyfile_path.write_text(caddyfile_content)
|
caddyfile_path.write_text(caddyfile_content)
|
||||||
print(f"Caddyfile written: {caddyfile_path}")
|
print(f"Caddyfile written: {caddyfile_path}")
|
||||||
@@ -130,7 +129,7 @@ def _build_deployed_service(
|
|||||||
if svc.manage and svc.manage.systemd and not svc.manage.systemd.enable:
|
if svc.manage and svc.manage.systemd and not svc.manage.systemd.enable:
|
||||||
managed = False
|
managed = False
|
||||||
if managed:
|
if managed:
|
||||||
env[f"{prefix}_DATA_DIR"] = str(DATA_ROOT / name)
|
env[f"{prefix}_DATA_DIR"] = str(DATA_DIR / name)
|
||||||
|
|
||||||
# Port convention (if exposed)
|
# Port convention (if exposed)
|
||||||
port = None
|
port = None
|
||||||
@@ -183,7 +182,7 @@ def _build_deployed_job(
|
|||||||
env: dict[str, str] = {}
|
env: dict[str, str] = {}
|
||||||
|
|
||||||
# Data dir convention
|
# Data dir convention
|
||||||
env[f"{prefix}_DATA_DIR"] = str(DATA_ROOT / name)
|
env[f"{prefix}_DATA_DIR"] = str(DATA_DIR / name)
|
||||||
|
|
||||||
# Merge defaults.env (overrides conventions)
|
# Merge defaults.env (overrides conventions)
|
||||||
if job.defaults and job.defaults.env:
|
if job.defaults and job.defaults.env:
|
||||||
@@ -274,23 +273,22 @@ def _print_deployed(name: str, deployed: DeployedComponent) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _copy_app_static(config: CastleConfig) -> None:
|
def _copy_app_static(config: CastleConfig) -> None:
|
||||||
"""Copy castle-app build output to ~/.castle/static/castle-app/."""
|
"""Copy frontend build outputs to ~/.castle/static/<name>/."""
|
||||||
if "castle-app" not in config.programs:
|
for name, comp in config.programs.items():
|
||||||
return
|
if comp.behavior != "frontend":
|
||||||
|
continue
|
||||||
|
if not (comp.build and comp.build.outputs):
|
||||||
|
continue
|
||||||
|
|
||||||
comp = config.programs["castle-app"]
|
source_dir = comp.source_dir or name
|
||||||
if not (comp.build and comp.build.outputs):
|
for output in comp.build.outputs:
|
||||||
return
|
src = config.root / source_dir / output
|
||||||
|
if src.exists():
|
||||||
source_dir = comp.source_dir or "app"
|
dest = CONTENT_DIR / name
|
||||||
for output in comp.build.outputs:
|
if dest.exists():
|
||||||
src = config.root / source_dir / output
|
shutil.rmtree(dest)
|
||||||
if src.exists():
|
shutil.copytree(src, dest)
|
||||||
dest = STATIC_DIR / "castle-app"
|
print(f" Static: {src} → {dest}")
|
||||||
if dest.exists():
|
|
||||||
shutil.rmtree(dest)
|
|
||||||
shutil.copytree(src, dest)
|
|
||||||
print(f" Static: {src} → {dest}")
|
|
||||||
|
|
||||||
|
|
||||||
def _generate_systemd_units(config: CastleConfig, registry: NodeRegistry) -> None:
|
def _generate_systemd_units(config: CastleConfig, registry: NodeRegistry) -> None:
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from castle_core.config import GENERATED_DIR
|
from castle_core.config import SPECS_DIR
|
||||||
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
||||||
from castle_core.registry import REGISTRY_PATH, load_registry
|
from castle_core.registry import REGISTRY_PATH, load_registry
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ def _write_generated_files() -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
registry = load_registry()
|
registry = load_registry()
|
||||||
caddyfile_path = GENERATED_DIR / "Caddyfile"
|
caddyfile_path = SPECS_DIR / "Caddyfile"
|
||||||
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
||||||
print(f" Generated {caddyfile_path}")
|
print(f" Generated {caddyfile_path}")
|
||||||
|
|
||||||
|
|||||||
@@ -271,11 +271,11 @@ def _services_start(config: CastleConfig) -> int:
|
|||||||
|
|
||||||
ensure_dirs()
|
ensure_dirs()
|
||||||
|
|
||||||
from castle_core.config import GENERATED_DIR
|
from castle_core.config import SPECS_DIR
|
||||||
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
from castle_core.generators.caddyfile import generate_caddyfile_from_registry
|
||||||
|
|
||||||
registry = load_registry()
|
registry = load_registry()
|
||||||
caddyfile_path = GENERATED_DIR / "Caddyfile"
|
caddyfile_path = SPECS_DIR / "Caddyfile"
|
||||||
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
caddyfile_path.write_text(generate_caddyfile_from_registry(registry))
|
||||||
print(f"Generated {caddyfile_path}")
|
print(f"Generated {caddyfile_path}")
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,13 @@
|
|||||||
from castle_core.config import * # noqa: F401, F403
|
from castle_core.config import * # noqa: F401, F403
|
||||||
from castle_core.config import ( # noqa: F401 — explicit re-exports for type checkers
|
from castle_core.config import ( # noqa: F401 — explicit re-exports for type checkers
|
||||||
CASTLE_HOME,
|
CASTLE_HOME,
|
||||||
GENERATED_DIR,
|
CODE_DIR,
|
||||||
|
ARTIFACTS_DIR,
|
||||||
|
SPECS_DIR,
|
||||||
|
CONTENT_DIR,
|
||||||
|
DATA_DIR,
|
||||||
SECRETS_DIR,
|
SECRETS_DIR,
|
||||||
|
GENERATED_DIR,
|
||||||
STATIC_DIR,
|
STATIC_DIR,
|
||||||
CastleConfig,
|
CastleConfig,
|
||||||
GatewayConfig,
|
GatewayConfig,
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "android-backup"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Backup Android device using ADB"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
android-backup = "android_backup.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/android_backup"]
|
|
||||||
File diff suppressed because it is too large
Load Diff
8
components/android-backup/uv.lock
generated
8
components/android-backup/uv.lock
generated
@@ -1,8 +0,0 @@
|
|||||||
version = 1
|
|
||||||
revision = 3
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "android-backup"
|
|
||||||
version = "0.1.0"
|
|
||||||
source = { editable = "." }
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "backup-collect"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Collect files from various sources into backup directory"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
backup-collect = "backup_collect.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/backup_collect"]
|
|
||||||
@@ -1,289 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
backup-collect: Collect files from various sources into backup directory
|
|
||||||
|
|
||||||
Incrementally backs up specified directories to the central backup location
|
|
||||||
using rsync. Creates hardlinks to unchanged files from previous backups to
|
|
||||||
save disk space.
|
|
||||||
|
|
||||||
Usage: backup-collect [options]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
backup-collect # Backup using default configuration
|
|
||||||
backup-collect -c custom.json # Use custom configuration file
|
|
||||||
backup-collect --full # Perform a full backup instead of incremental
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import subprocess
|
|
||||||
import datetime
|
|
||||||
import logging
|
|
||||||
|
|
||||||
# Constants
|
|
||||||
DEFAULT_CONFIG_DIR = os.path.expanduser("~/.config/toolkit")
|
|
||||||
DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_CONFIG_DIR, "backup-config.json")
|
|
||||||
DEFAULT_BACKUP_DIR = os.environ.get("DBACKUP", "/data/backup")
|
|
||||||
|
|
||||||
# Setup logging
|
|
||||||
logging.basicConfig(
|
|
||||||
level=logging.INFO,
|
|
||||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
|
||||||
)
|
|
||||||
logger = logging.getLogger("backup-collect")
|
|
||||||
|
|
||||||
|
|
||||||
def create_default_config(config_path):
|
|
||||||
"""Create a default configuration file if none exists."""
|
|
||||||
default_config = {
|
|
||||||
"sources": [
|
|
||||||
{
|
|
||||||
"name": "config",
|
|
||||||
"paths": [
|
|
||||||
os.path.expanduser("~/.config"),
|
|
||||||
],
|
|
||||||
"exclusions": [
|
|
||||||
os.path.expanduser("~/.config/Code/Cache/*"),
|
|
||||||
"*/Cache/*",
|
|
||||||
"*/CacheStorage/*",
|
|
||||||
],
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"retention": {"local": 7},
|
|
||||||
"schedule": {"incremental": "daily", "full": "weekly"},
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create config directory if it doesn't exist
|
|
||||||
os.makedirs(os.path.dirname(config_path), exist_ok=True)
|
|
||||||
|
|
||||||
with open(config_path, "w") as f:
|
|
||||||
json.dump(default_config, f, indent=2)
|
|
||||||
|
|
||||||
logger.info(f"Created default configuration at {config_path}")
|
|
||||||
return default_config
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(config_path):
|
|
||||||
"""Load the configuration file or create default if it doesn't exist."""
|
|
||||||
if not os.path.exists(config_path):
|
|
||||||
return create_default_config(config_path)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(config_path, "r") as f:
|
|
||||||
config = json.load(f)
|
|
||||||
logger.info(f"Loaded configuration from {config_path}")
|
|
||||||
return config
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error loading configuration: {e}")
|
|
||||||
logger.info("Using default configuration")
|
|
||||||
return create_default_config(config_path + ".default")
|
|
||||||
|
|
||||||
|
|
||||||
def run_backup(source, backup_dir, date_str, incremental=True):
|
|
||||||
"""Run rsync backup for a given source configuration."""
|
|
||||||
source_name = source["name"]
|
|
||||||
source_paths = source["paths"]
|
|
||||||
exclusions = source.get("exclusions", [])
|
|
||||||
|
|
||||||
# Create dated and latest directory names
|
|
||||||
dated_dir = os.path.join(backup_dir, f"{source_name}-{date_str}")
|
|
||||||
latest_dir = os.path.join(backup_dir, f"{source_name}-latest")
|
|
||||||
previous_dir = os.path.join(backup_dir, f"{source_name}-previous")
|
|
||||||
|
|
||||||
logger.info(f"Backing up {source_name} to {dated_dir}")
|
|
||||||
|
|
||||||
# Create backup directory if it doesn't exist
|
|
||||||
os.makedirs(dated_dir, exist_ok=True)
|
|
||||||
|
|
||||||
for source_path in source_paths:
|
|
||||||
# Expand user paths like ~/.config
|
|
||||||
expanded_path = os.path.expanduser(source_path)
|
|
||||||
|
|
||||||
if not os.path.exists(expanded_path):
|
|
||||||
logger.warning(f"Source path does not exist: {expanded_path}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Prepare rsync command
|
|
||||||
rsync_cmd = ["rsync", "-av", "--delete", "--block-size=131072"]
|
|
||||||
|
|
||||||
# Add exclusions
|
|
||||||
for exclusion in exclusions:
|
|
||||||
rsync_cmd.extend(["--exclude", exclusion])
|
|
||||||
|
|
||||||
# For incremental backups, use hardlinks to previous backup
|
|
||||||
if incremental and os.path.exists(latest_dir):
|
|
||||||
rsync_cmd.extend(["--link-dest", latest_dir])
|
|
||||||
|
|
||||||
# Add source and destination with path preservation
|
|
||||||
if os.path.isdir(expanded_path):
|
|
||||||
# For directories, preserve the trailing slash behavior
|
|
||||||
source_with_slash = (
|
|
||||||
expanded_path if expanded_path.endswith("/") else expanded_path + "/"
|
|
||||||
)
|
|
||||||
rsync_cmd.extend(["-R", source_with_slash]) # -R preserves relative path
|
|
||||||
rsync_cmd.append(dated_dir + "/")
|
|
||||||
else:
|
|
||||||
# For files, preserve the full path structure
|
|
||||||
rsync_cmd.extend(["-R", expanded_path]) # -R preserves relative path
|
|
||||||
rsync_cmd.append(dated_dir + "/")
|
|
||||||
|
|
||||||
# Execute rsync
|
|
||||||
logger.info(f"Running: {' '.join(rsync_cmd)}")
|
|
||||||
try:
|
|
||||||
# Use check=False to continue even if there are errors
|
|
||||||
result = subprocess.run(
|
|
||||||
rsync_cmd, check=False, capture_output=True, text=True
|
|
||||||
)
|
|
||||||
if result.returncode == 0:
|
|
||||||
logger.info(f"Backup of {expanded_path} completed successfully")
|
|
||||||
else:
|
|
||||||
logger.error(f"Backup failed for {expanded_path}: {result.stderr}")
|
|
||||||
return False
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Backup failed for {expanded_path}: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Update symlinks
|
|
||||||
try:
|
|
||||||
# Move previous latest to previous if it exists
|
|
||||||
if os.path.exists(latest_dir) and os.path.islink(latest_dir):
|
|
||||||
# Remove existing previous symlink first
|
|
||||||
if os.path.exists(previous_dir) or os.path.islink(previous_dir):
|
|
||||||
os.unlink(previous_dir)
|
|
||||||
os.symlink(os.readlink(latest_dir), previous_dir)
|
|
||||||
|
|
||||||
# Update latest symlink
|
|
||||||
if os.path.exists(latest_dir) or os.path.islink(latest_dir):
|
|
||||||
os.unlink(latest_dir)
|
|
||||||
os.symlink(os.path.basename(dated_dir), latest_dir)
|
|
||||||
logger.info(f"Updated symlinks for {source_name}")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to update symlinks: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_old_backups(source, backup_dir, retention):
|
|
||||||
"""Remove old backups beyond the retention period."""
|
|
||||||
source_name = source["name"]
|
|
||||||
logger.info(f"Cleaning up old backups for {source_name}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Find all dated backups for this source
|
|
||||||
backups = sorted(
|
|
||||||
[
|
|
||||||
d
|
|
||||||
for d in os.listdir(backup_dir)
|
|
||||||
if os.path.isdir(os.path.join(backup_dir, d))
|
|
||||||
and d.startswith(f"{source_name}-")
|
|
||||||
and not os.path.islink(os.path.join(backup_dir, d))
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Keep only the most recent 'retention' number of backups
|
|
||||||
if len(backups) > retention:
|
|
||||||
to_delete = backups[:-retention]
|
|
||||||
for backup in to_delete:
|
|
||||||
backup_path = os.path.join(backup_dir, backup)
|
|
||||||
logger.info(f"Removing old backup: {backup_path}")
|
|
||||||
# Use subprocess to remove directories that might have read-only files
|
|
||||||
subprocess.run(["rm", "-rf", backup_path], check=True)
|
|
||||||
|
|
||||||
logger.info(f"Cleanup completed, kept {min(len(backups), retention)} backups")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error during cleanup: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Collect files from various sources into backup directory",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1]
|
|
||||||
if __doc__
|
|
||||||
else "", # Use the docstring as extended help
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-c", "--config", help=f"Path to config file (default: {DEFAULT_CONFIG_FILE})"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-b",
|
|
||||||
"--backup-dir",
|
|
||||||
help=f"Backup directory (default: $DBACKUP or {DEFAULT_BACKUP_DIR})",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-f",
|
|
||||||
"--full",
|
|
||||||
action="store_true",
|
|
||||||
help="Perform a full backup instead of incremental",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-v", "--verbose", action="store_true", help="Enable verbose output"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-d", "--debug", action="store_true", help="Enable debug output"
|
|
||||||
)
|
|
||||||
parser.add_argument("--version", action="version", version="backup-collect 1.0.0")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Set verbosity
|
|
||||||
if args.verbose or args.debug:
|
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
# Extra debug info
|
|
||||||
if args.debug:
|
|
||||||
logging.getLogger().setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
# Load configuration
|
|
||||||
config_path = args.config or DEFAULT_CONFIG_FILE
|
|
||||||
config = load_config(config_path)
|
|
||||||
|
|
||||||
# Determine backup directory
|
|
||||||
backup_dir = args.backup_dir or DEFAULT_BACKUP_DIR
|
|
||||||
os.makedirs(backup_dir, exist_ok=True)
|
|
||||||
logger.info(f"Using backup directory: {backup_dir}")
|
|
||||||
|
|
||||||
# Current date for backup naming
|
|
||||||
date_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
|
|
||||||
# Determine if using incremental backup
|
|
||||||
incremental = not args.full
|
|
||||||
logger.info(f"Performing {'incremental' if incremental else 'full'} backup")
|
|
||||||
|
|
||||||
success = True
|
|
||||||
|
|
||||||
# Debug output of all config
|
|
||||||
if args.debug:
|
|
||||||
logger.debug(f"Using config: {json.dumps(config, indent=2)}")
|
|
||||||
logger.debug(f"Backup dir: {backup_dir}")
|
|
||||||
logger.debug(f"Sources to backup: {len(config.get('sources', []))}")
|
|
||||||
for source in config.get("sources", []):
|
|
||||||
logger.debug(f"Source: {source['name']}, Paths: {source['paths']}")
|
|
||||||
|
|
||||||
# Process each source
|
|
||||||
for source in config.get("sources", []):
|
|
||||||
logger.info(f"Processing source: {source['name']}")
|
|
||||||
if not run_backup(source, backup_dir, date_str, incremental):
|
|
||||||
logger.error(f"Backup failed for {source['name']}")
|
|
||||||
success = False
|
|
||||||
|
|
||||||
# Cleanup old backups
|
|
||||||
retention = config.get("retention", {}).get("local", 7)
|
|
||||||
cleanup_old_backups(source, backup_dir, retention)
|
|
||||||
|
|
||||||
if success:
|
|
||||||
logger.info("✅ All backups completed successfully")
|
|
||||||
return 0
|
|
||||||
else:
|
|
||||||
logger.error("One or more backups failed")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "browser"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Browser automation tool powered by browser-use"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = [
|
|
||||||
"python-dotenv>=1.0.0",
|
|
||||||
"browser-use>=0.1.0",
|
|
||||||
"langchain-openai>=0.1.0",
|
|
||||||
"langchain-anthropic>=0.1.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
browser = "browser.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/browser"]
|
|
||||||
@@ -1,199 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Browser automation tool powered by browser-use.
|
|
||||||
|
|
||||||
This tool allows executing browser automation from command line instructions.
|
|
||||||
It takes instruction text and uses browser-use to automate browser tasks.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
from typing import Dict, Any
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
|
|
||||||
# Load environment variables for API keys
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
# Import browser-use and related dependencies
|
|
||||||
try:
|
|
||||||
from browser_use import Agent, Browser, BrowserConfig, BrowserContextConfig
|
|
||||||
|
|
||||||
# Optional imports for different LLM providers
|
|
||||||
openai_available = False
|
|
||||||
anthropic_available = False
|
|
||||||
|
|
||||||
try:
|
|
||||||
from langchain_openai import ChatOpenAI
|
|
||||||
|
|
||||||
openai_available = True
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
from langchain_anthropic import ChatAnthropic
|
|
||||||
|
|
||||||
anthropic_available = True
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
except ImportError:
|
|
||||||
sys.stderr.write("Error: Required dependencies not found. Install with:\n")
|
|
||||||
sys.stderr.write("pip install browser-use python-dotenv\n")
|
|
||||||
sys.stderr.write("pip install langchain-openai # For OpenAI models\n")
|
|
||||||
sys.stderr.write("pip install langchain-anthropic # For Anthropic models\n")
|
|
||||||
sys.stderr.write("patchright install chromium\n")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
async def run_browser_task(
|
|
||||||
task: str,
|
|
||||||
provider: str = "openai",
|
|
||||||
model: str = "gpt-4o",
|
|
||||||
max_steps: int = 25,
|
|
||||||
max_actions_per_step: int = 4,
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Run a browser task based on the given instruction.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
task: The instruction text describing what to do in the browser
|
|
||||||
provider: The model provider to use
|
|
||||||
model: The specific model name to use
|
|
||||||
max_steps: Maximum number of steps the agent can take
|
|
||||||
max_actions_per_step: Maximum number of actions per step
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
The result from the browser-use agent
|
|
||||||
"""
|
|
||||||
# Configure the browser (headless mode is required for environments without X11 display)
|
|
||||||
browser = Browser(
|
|
||||||
config=BrowserConfig(
|
|
||||||
headless=True,
|
|
||||||
new_context_config=BrowserContextConfig(
|
|
||||||
viewport_expansion=0,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Select and configure the appropriate LLM based on provider
|
|
||||||
if provider.lower() == "anthropic":
|
|
||||||
if not anthropic_available:
|
|
||||||
raise ImportError(
|
|
||||||
"Anthropic integration not available. Install with: pip install langchain-anthropic"
|
|
||||||
)
|
|
||||||
if not os.environ.get("ANTHROPIC_API_KEY"):
|
|
||||||
raise ValueError("ANTHROPIC_API_KEY not found in environment variables")
|
|
||||||
llm = ChatAnthropic(model=model) # type: ignore
|
|
||||||
elif provider.lower() == "openai":
|
|
||||||
if not openai_available:
|
|
||||||
raise ImportError(
|
|
||||||
"OpenAI integration not available. Install with: pip install langchain-openai"
|
|
||||||
)
|
|
||||||
if not os.environ.get("OPENAI_API_KEY"):
|
|
||||||
raise ValueError("OPENAI_API_KEY not found in environment variables")
|
|
||||||
llm = ChatOpenAI(model=model) # type: ignore
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unsupported provider: {provider}")
|
|
||||||
|
|
||||||
# Create and run the agent
|
|
||||||
agent = Agent(
|
|
||||||
task=task,
|
|
||||||
llm=llm,
|
|
||||||
browser=browser,
|
|
||||||
max_actions_per_step=max_actions_per_step,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Execute the task with a maximum number of steps
|
|
||||||
result = await agent.run(max_steps=max_steps)
|
|
||||||
return result # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Execute browser automation from command line instructions"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"instruction", nargs="*", help="Instruction text for browser automation"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--provider",
|
|
||||||
"-p",
|
|
||||||
choices=["openai", "anthropic"],
|
|
||||||
default="openai",
|
|
||||||
help="The model provider to use (default: openai)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--model", "-m", help="The model to use (default depends on provider)"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--file", "-f", help="Read instruction from file instead of command line"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-steps",
|
|
||||||
type=int,
|
|
||||||
default=25,
|
|
||||||
help="Maximum number of steps the agent can take (default: 25)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-actions",
|
|
||||||
type=int,
|
|
||||||
default=4,
|
|
||||||
help="Maximum number of actions per step (default: 4)",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Set default model based on provider if not specified
|
|
||||||
if not args.model:
|
|
||||||
if args.provider == "anthropic":
|
|
||||||
args.model = "claude-3-opus-20240229"
|
|
||||||
else: # openai
|
|
||||||
args.model = "gpt-4o"
|
|
||||||
|
|
||||||
# Get instruction from file, command line args, or stdin
|
|
||||||
if args.file:
|
|
||||||
try:
|
|
||||||
with open(args.file, "r") as f:
|
|
||||||
instruction = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error reading file: {e}\n")
|
|
||||||
sys.exit(1)
|
|
||||||
elif args.instruction:
|
|
||||||
instruction = " ".join(args.instruction)
|
|
||||||
else:
|
|
||||||
# Check if there's input from stdin
|
|
||||||
if not sys.stdin.isatty():
|
|
||||||
instruction = sys.stdin.read().strip()
|
|
||||||
else:
|
|
||||||
parser.print_help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if not instruction:
|
|
||||||
sys.stderr.write("Error: No instruction provided\n")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
print(f"Running browser task: {instruction}")
|
|
||||||
print(f"Using model: {args.provider}/{args.model}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = asyncio.run(
|
|
||||||
run_browser_task(
|
|
||||||
task=instruction,
|
|
||||||
provider=args.provider,
|
|
||||||
model=args.model,
|
|
||||||
max_steps=args.max_steps,
|
|
||||||
max_actions_per_step=args.max_actions,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
print("\nTask completed!")
|
|
||||||
if "output" in result:
|
|
||||||
print(result["output"])
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error executing browser task: {e}\n")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
5793
components/browser/uv.lock
generated
5793
components/browser/uv.lock
generated
File diff suppressed because it is too large
Load Diff
Submodule components/central-context deleted from 495e6d7c1f
@@ -1,219 +0,0 @@
|
|||||||
# devbox-connect
|
|
||||||
|
|
||||||
SSH tunnel manager for connecting local ports to your devbox.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- **Simple YAML configuration** - Define tunnels in a readable config file
|
|
||||||
- **Auto-reconnect** - Automatically reconnects with exponential backoff when connections drop
|
|
||||||
- **Multiple tunnels** - Manage many port forwards to one or more hosts
|
|
||||||
- **Windows service support** - Run as a background service with auto-start
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install with uv
|
|
||||||
uv tool install git+https://github.com/YOUR_USERNAME/devbox-connect
|
|
||||||
|
|
||||||
# Or install from local directory
|
|
||||||
cd devbox-connect
|
|
||||||
uv tool install .
|
|
||||||
```
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
1. Create a configuration file `tunnels.yaml`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
user: your-username
|
|
||||||
key_file: ~/.ssh/id_rsa
|
|
||||||
|
|
||||||
tunnels:
|
|
||||||
- name: web-dev
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8080
|
|
||||||
|
|
||||||
- name: database
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 5432
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Start the tunnels:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
devbox-connect -c tunnels.yaml start
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Access your devbox services locally:
|
|
||||||
- `localhost:8080` → devbox:8080
|
|
||||||
- `localhost:5432` → devbox:5432
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```
|
|
||||||
devbox-connect [-c CONFIG] COMMAND
|
|
||||||
|
|
||||||
Commands:
|
|
||||||
start Start tunnels and keep running (default)
|
|
||||||
status Show configured tunnels
|
|
||||||
validate Validate configuration file
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-c, --config PATH Path to config file (default: tunnels.yaml)
|
|
||||||
-v, --verbose Enable verbose output
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
See `tunnels.example.yaml` for a complete example.
|
|
||||||
|
|
||||||
### Simple Format (single host)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
user: username
|
|
||||||
key_file: ~/.ssh/id_rsa # Optional
|
|
||||||
|
|
||||||
tunnels:
|
|
||||||
- name: web
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8080
|
|
||||||
local_port: 8080 # Optional, defaults to remote_port
|
|
||||||
|
|
||||||
- name: jupyter
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8888
|
|
||||||
local_port: 9999 # Use different local port
|
|
||||||
```
|
|
||||||
|
|
||||||
### Grouped Format (multiple hosts)
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
hosts:
|
|
||||||
- host: devbox1.example.com
|
|
||||||
user: username
|
|
||||||
tunnels:
|
|
||||||
- name: web
|
|
||||||
remote_port: 8080
|
|
||||||
|
|
||||||
- host: devbox2.example.com
|
|
||||||
user: username
|
|
||||||
tunnels:
|
|
||||||
- name: api
|
|
||||||
remote_port: 3000
|
|
||||||
```
|
|
||||||
|
|
||||||
### Configuration Options
|
|
||||||
|
|
||||||
| Option | Description | Default |
|
|
||||||
|--------|-------------|---------|
|
|
||||||
| `user` | SSH username | (required) |
|
|
||||||
| `host` | Remote hostname | (required) |
|
|
||||||
| `key_file` | Path to SSH private key | SSH agent/default |
|
|
||||||
| `remote_port` | Port on remote host | (required) |
|
|
||||||
| `local_port` | Local port to listen on | Same as remote_port |
|
|
||||||
| `remote_host` | Host on remote side | `localhost` |
|
|
||||||
| `reconnect_delay` | Initial reconnect delay (seconds) | `5` |
|
|
||||||
| `max_reconnect_delay` | Max reconnect delay | `60` |
|
|
||||||
|
|
||||||
### Forwarding Through Devbox
|
|
||||||
|
|
||||||
You can access services on other hosts through your devbox:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
tunnels:
|
|
||||||
- name: internal-db
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 5432
|
|
||||||
remote_host: internal-db.corp # Accessed via devbox
|
|
||||||
```
|
|
||||||
|
|
||||||
## Windows Service
|
|
||||||
|
|
||||||
To run devbox-connect as a Windows service that starts automatically:
|
|
||||||
|
|
||||||
### Prerequisites
|
|
||||||
|
|
||||||
1. Install NSSM (Non-Sucking Service Manager):
|
|
||||||
```powershell
|
|
||||||
winget install nssm
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Install devbox-connect:
|
|
||||||
```powershell
|
|
||||||
uv tool install .
|
|
||||||
```
|
|
||||||
|
|
||||||
### Install Service
|
|
||||||
|
|
||||||
Run PowerShell as Administrator:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\service\install-service.ps1 -ConfigPath C:\path\to\tunnels.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manage Service
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
# Check status
|
|
||||||
Get-Service DevboxConnect
|
|
||||||
|
|
||||||
# Start/Stop
|
|
||||||
Start-Service DevboxConnect
|
|
||||||
Stop-Service DevboxConnect
|
|
||||||
|
|
||||||
# View logs
|
|
||||||
Get-Content $env:LOCALAPPDATA\devbox-connect\service.log -Tail 50
|
|
||||||
|
|
||||||
# Uninstall
|
|
||||||
.\service\install-service.ps1 -Uninstall
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manual Run (without service)
|
|
||||||
|
|
||||||
```batch
|
|
||||||
service\run-manual.bat C:\path\to\tunnels.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
## SSH Key Setup
|
|
||||||
|
|
||||||
devbox-connect uses SSH key authentication. Ensure your key is set up:
|
|
||||||
|
|
||||||
1. Generate a key (if needed):
|
|
||||||
```bash
|
|
||||||
ssh-keygen -t ed25519 -f ~/.ssh/devbox_key
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Copy to devbox:
|
|
||||||
```bash
|
|
||||||
ssh-copy-id -i ~/.ssh/devbox_key user@devbox.example.com
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Reference in config:
|
|
||||||
```yaml
|
|
||||||
key_file: ~/.ssh/devbox_key
|
|
||||||
```
|
|
||||||
|
|
||||||
Or use SSH agent (key_file not needed if agent has your key loaded).
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Connection refused
|
|
||||||
- Check the remote service is running on the specified port
|
|
||||||
- Verify you can SSH to the host manually: `ssh user@devbox`
|
|
||||||
|
|
||||||
### Permission denied
|
|
||||||
- Check your SSH key is correct and has proper permissions
|
|
||||||
- On Windows, ensure key file isn't world-readable
|
|
||||||
|
|
||||||
### Port already in use
|
|
||||||
- Change `local_port` to an unused port
|
|
||||||
- Check what's using the port: `netstat -an | findstr :8080`
|
|
||||||
|
|
||||||
### Tunnels disconnect frequently
|
|
||||||
- Check network stability
|
|
||||||
- Increase `reconnect_delay` and `max_reconnect_delay`
|
|
||||||
- Some networks/firewalls drop idle connections; the remote service may need keepalive
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
MIT
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "devbox-connect"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "SSH tunnel manager for connecting to devbox ports"
|
|
||||||
readme = "README.md"
|
|
||||||
requires-python = ">=3.10"
|
|
||||||
dependencies = [
|
|
||||||
"paramiko>=3.4.0",
|
|
||||||
"pyyaml>=6.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
devbox-connect = "devbox_connect.cli:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/devbox_connect"]
|
|
||||||
|
|
||||||
[tool.ruff]
|
|
||||||
line-length = 100
|
|
||||||
target-version = "py310"
|
|
||||||
|
|
||||||
[tool.ruff.lint]
|
|
||||||
select = ["E", "F", "I", "W"]
|
|
||||||
|
|
||||||
[tool.pyright]
|
|
||||||
pythonVersion = "3.10"
|
|
||||||
typeCheckingMode = "standard"
|
|
||||||
@@ -1,202 +0,0 @@
|
|||||||
# install-service.ps1 - Install devbox-connect as a Windows service using NSSM
|
|
||||||
#
|
|
||||||
# Prerequisites:
|
|
||||||
# 1. Install NSSM: winget install nssm
|
|
||||||
# 2. Install devbox-connect: uv tool install .
|
|
||||||
#
|
|
||||||
# Usage (run as Administrator):
|
|
||||||
# .\install-service.ps1 -ConfigPath C:\path\to\tunnels.yaml
|
|
||||||
# .\install-service.ps1 -Uninstall
|
|
||||||
# .\install-service.ps1 -Status
|
|
||||||
|
|
||||||
param(
|
|
||||||
[string]$ConfigPath,
|
|
||||||
[switch]$Uninstall,
|
|
||||||
[switch]$Status,
|
|
||||||
[string]$ServiceName = "DevboxConnect"
|
|
||||||
)
|
|
||||||
|
|
||||||
$ErrorActionPreference = "Stop"
|
|
||||||
|
|
||||||
function Test-Administrator {
|
|
||||||
$identity = [Security.Principal.WindowsIdentity]::GetCurrent()
|
|
||||||
$principal = New-Object Security.Principal.WindowsPrincipal($identity)
|
|
||||||
return $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
|
|
||||||
}
|
|
||||||
|
|
||||||
function Get-NssmPath {
|
|
||||||
$nssm = Get-Command nssm -ErrorAction SilentlyContinue
|
|
||||||
if ($nssm) {
|
|
||||||
return $nssm.Source
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check common locations
|
|
||||||
$paths = @(
|
|
||||||
"C:\Program Files\nssm\nssm.exe",
|
|
||||||
"C:\Program Files (x86)\nssm\nssm.exe",
|
|
||||||
"$env:LOCALAPPDATA\Microsoft\WinGet\Packages\*nssm*\nssm.exe"
|
|
||||||
)
|
|
||||||
|
|
||||||
foreach ($path in $paths) {
|
|
||||||
$found = Get-Item $path -ErrorAction SilentlyContinue | Select-Object -First 1
|
|
||||||
if ($found) {
|
|
||||||
return $found.FullName
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $null
|
|
||||||
}
|
|
||||||
|
|
||||||
function Get-DevboxConnectPath {
|
|
||||||
# Find the uv-installed devbox-connect
|
|
||||||
$uvToolPath = "$env:LOCALAPPDATA\uv\tools\devbox-connect"
|
|
||||||
if (Test-Path $uvToolPath) {
|
|
||||||
# Find the Python executable in the venv
|
|
||||||
$pythonPath = Join-Path $uvToolPath "Scripts\python.exe"
|
|
||||||
if (Test-Path $pythonPath) {
|
|
||||||
return @{
|
|
||||||
Python = $pythonPath
|
|
||||||
Module = "devbox_connect.cli"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try to find via uv tool list
|
|
||||||
$uvPath = Get-Command uv -ErrorAction SilentlyContinue
|
|
||||||
if ($uvPath) {
|
|
||||||
$toolDir = & uv tool dir 2>$null
|
|
||||||
if ($toolDir -and (Test-Path "$toolDir\devbox-connect")) {
|
|
||||||
$pythonPath = Join-Path $toolDir "devbox-connect\Scripts\python.exe"
|
|
||||||
if (Test-Path $pythonPath) {
|
|
||||||
return @{
|
|
||||||
Python = $pythonPath
|
|
||||||
Module = "devbox_connect.cli"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $null
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check for admin rights
|
|
||||||
if (-not (Test-Administrator)) {
|
|
||||||
Write-Error "This script must be run as Administrator"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Handle status check
|
|
||||||
if ($Status) {
|
|
||||||
$service = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
|
||||||
if ($service) {
|
|
||||||
Write-Host "Service '$ServiceName' status: $($service.Status)"
|
|
||||||
Write-Host "Startup type: $($service.StartType)"
|
|
||||||
} else {
|
|
||||||
Write-Host "Service '$ServiceName' is not installed"
|
|
||||||
}
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Handle uninstall
|
|
||||||
if ($Uninstall) {
|
|
||||||
$nssm = Get-NssmPath
|
|
||||||
if (-not $nssm) {
|
|
||||||
Write-Error "NSSM not found. Install with: winget install nssm"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "Stopping service..."
|
|
||||||
& $nssm stop $ServiceName 2>$null
|
|
||||||
|
|
||||||
Write-Host "Removing service..."
|
|
||||||
& $nssm remove $ServiceName confirm
|
|
||||||
|
|
||||||
Write-Host "Service '$ServiceName' removed"
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Install service
|
|
||||||
if (-not $ConfigPath) {
|
|
||||||
Write-Error "ConfigPath is required. Usage: .\install-service.ps1 -ConfigPath C:\path\to\tunnels.yaml"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if (-not (Test-Path $ConfigPath)) {
|
|
||||||
Write-Error "Config file not found: $ConfigPath"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
$ConfigPath = (Resolve-Path $ConfigPath).Path
|
|
||||||
|
|
||||||
# Find NSSM
|
|
||||||
$nssm = Get-NssmPath
|
|
||||||
if (-not $nssm) {
|
|
||||||
Write-Error @"
|
|
||||||
NSSM (Non-Sucking Service Manager) not found.
|
|
||||||
|
|
||||||
Install with:
|
|
||||||
winget install nssm
|
|
||||||
|
|
||||||
Or download from: https://nssm.cc/download
|
|
||||||
"@
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Find devbox-connect
|
|
||||||
$devboxConnect = Get-DevboxConnectPath
|
|
||||||
if (-not $devboxConnect) {
|
|
||||||
Write-Error @"
|
|
||||||
devbox-connect not found. Install with:
|
|
||||||
uv tool install .
|
|
||||||
|
|
||||||
Or from a directory containing the project:
|
|
||||||
uv tool install /path/to/devbox-connect
|
|
||||||
"@
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "Found devbox-connect at: $($devboxConnect.Python)"
|
|
||||||
Write-Host "Config file: $ConfigPath"
|
|
||||||
Write-Host ""
|
|
||||||
|
|
||||||
# Remove existing service if present
|
|
||||||
$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
|
||||||
if ($existing) {
|
|
||||||
Write-Host "Removing existing service..."
|
|
||||||
& $nssm stop $ServiceName 2>$null
|
|
||||||
& $nssm remove $ServiceName confirm
|
|
||||||
}
|
|
||||||
|
|
||||||
# Install the service
|
|
||||||
Write-Host "Installing service '$ServiceName'..."
|
|
||||||
|
|
||||||
& $nssm install $ServiceName $devboxConnect.Python "-m" $devboxConnect.Module "-c" $ConfigPath "start"
|
|
||||||
|
|
||||||
# Configure service
|
|
||||||
& $nssm set $ServiceName DisplayName "Devbox Connect SSH Tunnels"
|
|
||||||
& $nssm set $ServiceName Description "Maintains SSH tunnels to devbox for port forwarding"
|
|
||||||
& $nssm set $ServiceName Start SERVICE_AUTO_START
|
|
||||||
& $nssm set $ServiceName AppStdout "$env:LOCALAPPDATA\devbox-connect\service.log"
|
|
||||||
& $nssm set $ServiceName AppStderr "$env:LOCALAPPDATA\devbox-connect\service.log"
|
|
||||||
& $nssm set $ServiceName AppRotateFiles 1
|
|
||||||
& $nssm set $ServiceName AppRotateBytes 1048576
|
|
||||||
|
|
||||||
# Create log directory
|
|
||||||
$logDir = "$env:LOCALAPPDATA\devbox-connect"
|
|
||||||
if (-not (Test-Path $logDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $logDir | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "Service '$ServiceName' installed successfully!"
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "Commands:"
|
|
||||||
Write-Host " Start: Start-Service $ServiceName"
|
|
||||||
Write-Host " Stop: Stop-Service $ServiceName"
|
|
||||||
Write-Host " Status: Get-Service $ServiceName"
|
|
||||||
Write-Host " Logs: Get-Content $logDir\service.log -Tail 50"
|
|
||||||
Write-Host ""
|
|
||||||
Write-Host "Starting service..."
|
|
||||||
|
|
||||||
Start-Service $ServiceName
|
|
||||||
Get-Service $ServiceName
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
@echo off
|
|
||||||
REM run-manual.bat - Run devbox-connect manually (not as a service)
|
|
||||||
REM
|
|
||||||
REM Usage:
|
|
||||||
REM run-manual.bat - Uses tunnels.yaml in current directory
|
|
||||||
REM run-manual.bat C:\path\to\config - Uses specified config file
|
|
||||||
|
|
||||||
setlocal
|
|
||||||
|
|
||||||
set CONFIG=%1
|
|
||||||
if "%CONFIG%"=="" set CONFIG=tunnels.yaml
|
|
||||||
|
|
||||||
echo Starting devbox-connect with config: %CONFIG%
|
|
||||||
echo Press Ctrl+C to stop
|
|
||||||
echo.
|
|
||||||
|
|
||||||
devbox-connect -c "%CONFIG%" start
|
|
||||||
|
|
||||||
pause
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
"""SSH tunnel manager for connecting to devbox ports."""
|
|
||||||
|
|
||||||
__version__ = "0.1.0"
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
"""Command-line interface for devbox-connect."""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import signal
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from .config import ConfigError, load_config
|
|
||||||
from .tunnel import TunnelManager, TunnelState
|
|
||||||
|
|
||||||
# ANSI color codes
|
|
||||||
GREEN = "\033[92m"
|
|
||||||
RED = "\033[91m"
|
|
||||||
YELLOW = "\033[93m"
|
|
||||||
CYAN = "\033[96m"
|
|
||||||
RESET = "\033[0m"
|
|
||||||
BOLD = "\033[1m"
|
|
||||||
|
|
||||||
# State to color mapping
|
|
||||||
STATE_COLORS = {
|
|
||||||
TunnelState.CONNECTED: GREEN,
|
|
||||||
TunnelState.CONNECTING: YELLOW,
|
|
||||||
TunnelState.DISCONNECTED: RED,
|
|
||||||
TunnelState.ERROR: RED,
|
|
||||||
}
|
|
||||||
|
|
||||||
STATE_SYMBOLS = {
|
|
||||||
TunnelState.CONNECTED: "[OK]",
|
|
||||||
TunnelState.CONNECTING: "[..]",
|
|
||||||
TunnelState.DISCONNECTED: "[--]",
|
|
||||||
TunnelState.ERROR: "[!!]",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logging(verbose: bool) -> None:
|
|
||||||
"""Configure logging."""
|
|
||||||
level = logging.DEBUG if verbose else logging.INFO
|
|
||||||
format_str = "%(asctime)s - %(levelname)s - %(message)s" if verbose else "%(message)s"
|
|
||||||
logging.basicConfig(level=level, format=format_str)
|
|
||||||
|
|
||||||
|
|
||||||
def print_banner() -> None:
|
|
||||||
"""Print application banner."""
|
|
||||||
print(f"{BOLD}{CYAN}devbox-connect{RESET} - SSH Tunnel Manager")
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
def print_status(manager: TunnelManager) -> None:
|
|
||||||
"""Print current tunnel status."""
|
|
||||||
status = manager.get_status()
|
|
||||||
|
|
||||||
for host_key, tunnels in status.items():
|
|
||||||
print(f"{BOLD}{host_key}{RESET}")
|
|
||||||
for name, tunnel_status in tunnels.items():
|
|
||||||
state = tunnel_status.state
|
|
||||||
color = STATE_COLORS[state]
|
|
||||||
symbol = STATE_SYMBOLS[state]
|
|
||||||
local_port = tunnel_status.config.local_port or tunnel_status.config.remote_port
|
|
||||||
remote = f"{tunnel_status.config.remote_host}:{tunnel_status.config.remote_port}"
|
|
||||||
|
|
||||||
line = f" {color}{symbol}{RESET} {name}: localhost:{local_port} -> {remote}"
|
|
||||||
|
|
||||||
if tunnel_status.connections > 0:
|
|
||||||
line += f" ({tunnel_status.connections} active)"
|
|
||||||
|
|
||||||
if tunnel_status.error:
|
|
||||||
line += f" {RED}({tunnel_status.error}){RESET}"
|
|
||||||
|
|
||||||
print(line)
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
def on_status_change(host: str, tunnel: str, state: TunnelState) -> None:
|
|
||||||
"""Callback for tunnel status changes."""
|
|
||||||
color = STATE_COLORS[state]
|
|
||||||
symbol = STATE_SYMBOLS[state]
|
|
||||||
logging.info(f"{color}{symbol}{RESET} {host}/{tunnel}: {state.value}")
|
|
||||||
|
|
||||||
|
|
||||||
def run_manager(config_path: Path, verbose: bool) -> int:
|
|
||||||
"""Run the tunnel manager."""
|
|
||||||
setup_logging(verbose)
|
|
||||||
print_banner()
|
|
||||||
|
|
||||||
# Load configuration
|
|
||||||
try:
|
|
||||||
config = load_config(config_path)
|
|
||||||
except ConfigError as e:
|
|
||||||
logging.error(f"Configuration error: {e}")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
if not config.hosts:
|
|
||||||
logging.error("No tunnels configured")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Count tunnels
|
|
||||||
total_tunnels = sum(len(h.tunnels) for h in config.hosts)
|
|
||||||
logging.info(f"Loaded {total_tunnels} tunnel(s) for {len(config.hosts)} host(s)")
|
|
||||||
print()
|
|
||||||
|
|
||||||
# Create and start manager
|
|
||||||
manager = TunnelManager(config, on_status_change=on_status_change)
|
|
||||||
|
|
||||||
# Handle shutdown signals
|
|
||||||
shutdown_requested = False
|
|
||||||
|
|
||||||
def signal_handler(signum: int, frame: object) -> None:
|
|
||||||
nonlocal shutdown_requested
|
|
||||||
if shutdown_requested:
|
|
||||||
# Force exit on second signal
|
|
||||||
sys.exit(1)
|
|
||||||
shutdown_requested = True
|
|
||||||
print(f"\n{YELLOW}Shutting down...{RESET}")
|
|
||||||
manager.stop()
|
|
||||||
|
|
||||||
signal.signal(signal.SIGINT, signal_handler)
|
|
||||||
signal.signal(signal.SIGTERM, signal_handler)
|
|
||||||
|
|
||||||
# Start tunnels
|
|
||||||
manager.start()
|
|
||||||
|
|
||||||
# Wait a moment for connections
|
|
||||||
time.sleep(2)
|
|
||||||
|
|
||||||
# Print initial status
|
|
||||||
print_status(manager)
|
|
||||||
print(f"{CYAN}Press Ctrl+C to stop{RESET}")
|
|
||||||
print()
|
|
||||||
|
|
||||||
# Keep running until shutdown
|
|
||||||
try:
|
|
||||||
while not shutdown_requested:
|
|
||||||
time.sleep(1)
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
pass
|
|
||||||
|
|
||||||
manager.stop()
|
|
||||||
print(f"{GREEN}Stopped{RESET}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def show_status(config_path: Path) -> int:
|
|
||||||
"""Show status of configured tunnels (quick check)."""
|
|
||||||
setup_logging(verbose=False)
|
|
||||||
|
|
||||||
try:
|
|
||||||
config = load_config(config_path)
|
|
||||||
except ConfigError as e:
|
|
||||||
logging.error(f"Configuration error: {e}")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print_banner()
|
|
||||||
print(f"Config: {config_path}")
|
|
||||||
print()
|
|
||||||
|
|
||||||
for host in config.hosts:
|
|
||||||
print(f"{BOLD}{host.user}@{host.host}:{host.port}{RESET}")
|
|
||||||
for tunnel in host.tunnels:
|
|
||||||
local_port = tunnel.local_port or tunnel.remote_port
|
|
||||||
remote = f"{tunnel.remote_host}:{tunnel.remote_port}"
|
|
||||||
print(f" - {tunnel.name}: localhost:{local_port} -> {remote}")
|
|
||||||
print()
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def validate_config(config_path: Path) -> int:
|
|
||||||
"""Validate configuration file."""
|
|
||||||
setup_logging(verbose=False)
|
|
||||||
|
|
||||||
try:
|
|
||||||
config = load_config(config_path)
|
|
||||||
total_tunnels = sum(len(h.tunnels) for h in config.hosts)
|
|
||||||
print(f"{GREEN}Configuration valid{RESET}")
|
|
||||||
print(f" Hosts: {len(config.hosts)}")
|
|
||||||
print(f" Tunnels: {total_tunnels}")
|
|
||||||
return 0
|
|
||||||
except ConfigError as e:
|
|
||||||
print(f"{RED}Configuration invalid: {e}{RESET}")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
"""Main entry point."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog="devbox-connect",
|
|
||||||
description="SSH tunnel manager for connecting to devbox ports",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-c",
|
|
||||||
"--config",
|
|
||||||
type=Path,
|
|
||||||
default=Path("tunnels.yaml"),
|
|
||||||
help="Path to configuration file (default: tunnels.yaml)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-v",
|
|
||||||
"--verbose",
|
|
||||||
action="store_true",
|
|
||||||
help="Enable verbose output",
|
|
||||||
)
|
|
||||||
|
|
||||||
subparsers = parser.add_subparsers(dest="command", help="Commands")
|
|
||||||
|
|
||||||
# start command (default)
|
|
||||||
start_parser = subparsers.add_parser("start", help="Start tunnels (default)")
|
|
||||||
start_parser.add_argument("-v", "--verbose", action="store_true")
|
|
||||||
|
|
||||||
# status command
|
|
||||||
subparsers.add_parser("status", help="Show configured tunnels")
|
|
||||||
|
|
||||||
# validate command
|
|
||||||
subparsers.add_parser("validate", help="Validate configuration file")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Handle default command
|
|
||||||
command = args.command or "start"
|
|
||||||
|
|
||||||
if command == "start":
|
|
||||||
verbose = getattr(args, "verbose", False)
|
|
||||||
return run_manager(args.config, verbose)
|
|
||||||
elif command == "status":
|
|
||||||
return show_status(args.config)
|
|
||||||
elif command == "validate":
|
|
||||||
return validate_config(args.config)
|
|
||||||
else:
|
|
||||||
parser.print_help()
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,162 +0,0 @@
|
|||||||
"""Configuration loading and validation."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class TunnelConfig:
|
|
||||||
"""Configuration for a single SSH tunnel."""
|
|
||||||
|
|
||||||
name: str
|
|
||||||
host: str
|
|
||||||
remote_port: int
|
|
||||||
local_port: int | None = None # Defaults to remote_port if not specified
|
|
||||||
user: str | None = None # Defaults to global user
|
|
||||||
key_file: str | None = None # Defaults to global key_file
|
|
||||||
remote_host: str = "localhost" # The host on the remote side to connect to
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
if self.local_port is None:
|
|
||||||
self.local_port = self.remote_port
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class HostConfig:
|
|
||||||
"""Configuration for a host with multiple tunnels."""
|
|
||||||
|
|
||||||
host: str
|
|
||||||
user: str
|
|
||||||
key_file: str | None = None
|
|
||||||
port: int = 22
|
|
||||||
tunnels: list[TunnelConfig] = field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Config:
|
|
||||||
"""Root configuration."""
|
|
||||||
|
|
||||||
hosts: list[HostConfig] = field(default_factory=list)
|
|
||||||
reconnect_delay: int = 5 # Seconds between reconnection attempts
|
|
||||||
max_reconnect_delay: int = 60 # Max delay with exponential backoff
|
|
||||||
|
|
||||||
|
|
||||||
class ConfigError(Exception):
|
|
||||||
"""Configuration error."""
|
|
||||||
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(path: Path) -> Config:
|
|
||||||
"""Load configuration from a YAML file."""
|
|
||||||
if not path.exists():
|
|
||||||
raise ConfigError(f"Config file not found: {path}")
|
|
||||||
|
|
||||||
with open(path) as f:
|
|
||||||
data = yaml.safe_load(f)
|
|
||||||
|
|
||||||
if not data:
|
|
||||||
raise ConfigError("Empty configuration file")
|
|
||||||
|
|
||||||
return _parse_config(data)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_config(data: dict) -> Config:
|
|
||||||
"""Parse configuration dictionary into Config object."""
|
|
||||||
config = Config(
|
|
||||||
reconnect_delay=data.get("reconnect_delay", 5),
|
|
||||||
max_reconnect_delay=data.get("max_reconnect_delay", 60),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle simple format: list of tunnels with host info per tunnel
|
|
||||||
if "tunnels" in data:
|
|
||||||
config.hosts = _parse_simple_format(data["tunnels"], data)
|
|
||||||
# Handle grouped format: hosts with nested tunnels
|
|
||||||
elif "hosts" in data:
|
|
||||||
config.hosts = _parse_grouped_format(data["hosts"])
|
|
||||||
else:
|
|
||||||
raise ConfigError("Config must contain either 'tunnels' or 'hosts' key")
|
|
||||||
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_simple_format(tunnels: list[dict], global_config: dict) -> list[HostConfig]:
|
|
||||||
"""Parse simple format where each tunnel specifies its host."""
|
|
||||||
# Group tunnels by host
|
|
||||||
hosts_map: dict[str, HostConfig] = {}
|
|
||||||
|
|
||||||
global_user = global_config.get("user")
|
|
||||||
global_key_file = global_config.get("key_file")
|
|
||||||
|
|
||||||
for t in tunnels:
|
|
||||||
if "host" not in t:
|
|
||||||
raise ConfigError(f"Tunnel '{t.get('name', 'unnamed')}' missing 'host'")
|
|
||||||
if "remote_port" not in t:
|
|
||||||
raise ConfigError(f"Tunnel '{t.get('name', 'unnamed')}' missing 'remote_port'")
|
|
||||||
|
|
||||||
host = t["host"]
|
|
||||||
user = t.get("user", global_user)
|
|
||||||
if not user:
|
|
||||||
raise ConfigError(f"Tunnel '{t.get('name', 'unnamed')}' missing 'user'")
|
|
||||||
|
|
||||||
key_file = t.get("key_file", global_key_file)
|
|
||||||
ssh_port = t.get("ssh_port", 22)
|
|
||||||
|
|
||||||
# Create host key for grouping
|
|
||||||
host_key = f"{user}@{host}:{ssh_port}"
|
|
||||||
|
|
||||||
if host_key not in hosts_map:
|
|
||||||
hosts_map[host_key] = HostConfig(
|
|
||||||
host=host,
|
|
||||||
user=user,
|
|
||||||
key_file=key_file,
|
|
||||||
port=ssh_port,
|
|
||||||
)
|
|
||||||
|
|
||||||
tunnel = TunnelConfig(
|
|
||||||
name=t.get("name", f"{host}:{t['remote_port']}"),
|
|
||||||
host=host,
|
|
||||||
remote_port=t["remote_port"],
|
|
||||||
local_port=t.get("local_port"),
|
|
||||||
remote_host=t.get("remote_host", "localhost"),
|
|
||||||
)
|
|
||||||
hosts_map[host_key].tunnels.append(tunnel)
|
|
||||||
|
|
||||||
return list(hosts_map.values())
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_grouped_format(hosts: list[dict]) -> list[HostConfig]:
|
|
||||||
"""Parse grouped format with hosts containing nested tunnels."""
|
|
||||||
result = []
|
|
||||||
|
|
||||||
for h in hosts:
|
|
||||||
if "host" not in h:
|
|
||||||
raise ConfigError("Host entry missing 'host' field")
|
|
||||||
if "user" not in h:
|
|
||||||
raise ConfigError(f"Host '{h['host']}' missing 'user' field")
|
|
||||||
|
|
||||||
host_config = HostConfig(
|
|
||||||
host=h["host"],
|
|
||||||
user=h["user"],
|
|
||||||
key_file=h.get("key_file"),
|
|
||||||
port=h.get("port", 22),
|
|
||||||
)
|
|
||||||
|
|
||||||
for t in h.get("tunnels", []):
|
|
||||||
if "remote_port" not in t:
|
|
||||||
raise ConfigError(f"Tunnel in host '{h['host']}' missing 'remote_port'")
|
|
||||||
|
|
||||||
tunnel = TunnelConfig(
|
|
||||||
name=t.get("name", f"{h['host']}:{t['remote_port']}"),
|
|
||||||
host=h["host"],
|
|
||||||
remote_port=t["remote_port"],
|
|
||||||
local_port=t.get("local_port"),
|
|
||||||
remote_host=t.get("remote_host", "localhost"),
|
|
||||||
)
|
|
||||||
host_config.tunnels.append(tunnel)
|
|
||||||
|
|
||||||
result.append(host_config)
|
|
||||||
|
|
||||||
return result
|
|
||||||
@@ -1,355 +0,0 @@
|
|||||||
"""SSH tunnel manager using paramiko."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import socket
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from enum import Enum
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Callable
|
|
||||||
|
|
||||||
import paramiko
|
|
||||||
|
|
||||||
from .config import Config, HostConfig, TunnelConfig
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class TunnelState(Enum):
|
|
||||||
"""State of a tunnel."""
|
|
||||||
|
|
||||||
DISCONNECTED = "disconnected"
|
|
||||||
CONNECTING = "connecting"
|
|
||||||
CONNECTED = "connected"
|
|
||||||
ERROR = "error"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class TunnelStatus:
|
|
||||||
"""Status of a single tunnel."""
|
|
||||||
|
|
||||||
config: TunnelConfig
|
|
||||||
state: TunnelState = TunnelState.DISCONNECTED
|
|
||||||
error: str | None = None
|
|
||||||
connections: int = 0 # Active forwarded connections
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class HostConnection:
|
|
||||||
"""Manages SSH connection and tunnels for a single host."""
|
|
||||||
|
|
||||||
config: HostConfig
|
|
||||||
tunnels: dict[str, TunnelStatus] = field(default_factory=dict)
|
|
||||||
client: paramiko.SSHClient | None = None
|
|
||||||
transport: paramiko.Transport | None = None
|
|
||||||
_forward_threads: list[threading.Thread] = field(default_factory=list)
|
|
||||||
_forward_servers: list[socket.socket] = field(default_factory=list)
|
|
||||||
_stop_event: threading.Event = field(default_factory=threading.Event)
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
for tunnel in self.config.tunnels:
|
|
||||||
self.tunnels[tunnel.name] = TunnelStatus(config=tunnel)
|
|
||||||
|
|
||||||
|
|
||||||
class TunnelManager:
|
|
||||||
"""Manages SSH tunnels based on configuration."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: Config,
|
|
||||||
on_status_change: Callable[[str, str, TunnelState], None] | None = None,
|
|
||||||
):
|
|
||||||
self.config = config
|
|
||||||
self.on_status_change = on_status_change
|
|
||||||
self.hosts: dict[str, HostConnection] = {}
|
|
||||||
self._stop_event = threading.Event()
|
|
||||||
self._reconnect_threads: list[threading.Thread] = []
|
|
||||||
|
|
||||||
# Initialize host connections
|
|
||||||
for host_config in config.hosts:
|
|
||||||
key = f"{host_config.user}@{host_config.host}"
|
|
||||||
self.hosts[key] = HostConnection(config=host_config)
|
|
||||||
|
|
||||||
def start(self) -> None:
|
|
||||||
"""Start all tunnels."""
|
|
||||||
self._stop_event.clear()
|
|
||||||
for host_key, host_conn in self.hosts.items():
|
|
||||||
self._connect_host(host_key, host_conn)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
"""Stop all tunnels."""
|
|
||||||
logger.info("Stopping all tunnels...")
|
|
||||||
self._stop_event.set()
|
|
||||||
|
|
||||||
for host_conn in self.hosts.values():
|
|
||||||
self._disconnect_host(host_conn)
|
|
||||||
|
|
||||||
# Wait for reconnect threads to finish
|
|
||||||
for thread in self._reconnect_threads:
|
|
||||||
thread.join(timeout=2)
|
|
||||||
self._reconnect_threads.clear()
|
|
||||||
|
|
||||||
def get_status(self) -> dict[str, dict[str, TunnelStatus]]:
|
|
||||||
"""Get status of all tunnels."""
|
|
||||||
return {host_key: host.tunnels for host_key, host in self.hosts.items()}
|
|
||||||
|
|
||||||
def _connect_host(self, host_key: str, host_conn: HostConnection) -> bool:
|
|
||||||
"""Connect to a host and establish all tunnels."""
|
|
||||||
config = host_conn.config
|
|
||||||
|
|
||||||
# Update all tunnel states to connecting
|
|
||||||
for tunnel_status in host_conn.tunnels.values():
|
|
||||||
self._update_state(host_key, tunnel_status, TunnelState.CONNECTING)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Create SSH client
|
|
||||||
client = paramiko.SSHClient()
|
|
||||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
||||||
|
|
||||||
# Prepare connection kwargs
|
|
||||||
connect_kwargs: dict = {
|
|
||||||
"hostname": config.host,
|
|
||||||
"port": config.port,
|
|
||||||
"username": config.user,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Add key file if specified
|
|
||||||
if config.key_file:
|
|
||||||
key_path = Path(config.key_file).expanduser()
|
|
||||||
if not key_path.exists():
|
|
||||||
raise FileNotFoundError(f"SSH key file not found: {key_path}")
|
|
||||||
connect_kwargs["key_filename"] = str(key_path)
|
|
||||||
|
|
||||||
logger.info(f"Connecting to {config.user}@{config.host}:{config.port}...")
|
|
||||||
client.connect(**connect_kwargs)
|
|
||||||
|
|
||||||
host_conn.client = client
|
|
||||||
host_conn.transport = client.get_transport()
|
|
||||||
|
|
||||||
if host_conn.transport is None:
|
|
||||||
raise ConnectionError("Failed to get transport")
|
|
||||||
|
|
||||||
# Start port forwarding for each tunnel
|
|
||||||
host_conn._stop_event.clear()
|
|
||||||
for tunnel_config in config.tunnels:
|
|
||||||
self._start_tunnel(host_key, host_conn, tunnel_config)
|
|
||||||
|
|
||||||
logger.info(f"Connected to {config.host}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to connect to {config.host}: {e}")
|
|
||||||
for tunnel_status in host_conn.tunnels.values():
|
|
||||||
tunnel_status.error = str(e)
|
|
||||||
self._update_state(host_key, tunnel_status, TunnelState.ERROR)
|
|
||||||
self._schedule_reconnect(host_key, host_conn)
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _disconnect_host(self, host_conn: HostConnection) -> None:
|
|
||||||
"""Disconnect from a host."""
|
|
||||||
host_conn._stop_event.set()
|
|
||||||
|
|
||||||
# Close forward servers
|
|
||||||
for server in host_conn._forward_servers:
|
|
||||||
try:
|
|
||||||
server.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
host_conn._forward_servers.clear()
|
|
||||||
|
|
||||||
# Wait for forward threads
|
|
||||||
for thread in host_conn._forward_threads:
|
|
||||||
thread.join(timeout=1)
|
|
||||||
host_conn._forward_threads.clear()
|
|
||||||
|
|
||||||
# Close SSH connection
|
|
||||||
if host_conn.client:
|
|
||||||
try:
|
|
||||||
host_conn.client.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
host_conn.client = None
|
|
||||||
host_conn.transport = None
|
|
||||||
|
|
||||||
def _start_tunnel(
|
|
||||||
self, host_key: str, host_conn: HostConnection, tunnel_config: TunnelConfig
|
|
||||||
) -> None:
|
|
||||||
"""Start a single port forward tunnel."""
|
|
||||||
tunnel_status = host_conn.tunnels[tunnel_config.name]
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Create local listening socket
|
|
||||||
local_port = tunnel_config.local_port or tunnel_config.remote_port
|
|
||||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
||||||
server.bind(("127.0.0.1", local_port))
|
|
||||||
server.listen(5)
|
|
||||||
server.settimeout(1.0) # Allow checking stop event
|
|
||||||
|
|
||||||
host_conn._forward_servers.append(server)
|
|
||||||
|
|
||||||
# Start accept thread
|
|
||||||
thread = threading.Thread(
|
|
||||||
target=self._accept_loop,
|
|
||||||
args=(host_key, host_conn, tunnel_config, server),
|
|
||||||
daemon=True,
|
|
||||||
)
|
|
||||||
thread.start()
|
|
||||||
host_conn._forward_threads.append(thread)
|
|
||||||
|
|
||||||
self._update_state(host_key, tunnel_status, TunnelState.CONNECTED)
|
|
||||||
logger.info(
|
|
||||||
f"Tunnel '{tunnel_config.name}' open: "
|
|
||||||
f"localhost:{local_port} -> {tunnel_config.remote_host}:{tunnel_config.remote_port}"
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to start tunnel '{tunnel_config.name}': {e}")
|
|
||||||
tunnel_status.error = str(e)
|
|
||||||
self._update_state(host_key, tunnel_status, TunnelState.ERROR)
|
|
||||||
|
|
||||||
def _accept_loop(
|
|
||||||
self,
|
|
||||||
host_key: str,
|
|
||||||
host_conn: HostConnection,
|
|
||||||
tunnel_config: TunnelConfig,
|
|
||||||
server: socket.socket,
|
|
||||||
) -> None:
|
|
||||||
"""Accept loop for forwarding connections."""
|
|
||||||
tunnel_status = host_conn.tunnels[tunnel_config.name]
|
|
||||||
|
|
||||||
while not host_conn._stop_event.is_set() and not self._stop_event.is_set():
|
|
||||||
try:
|
|
||||||
client_socket, addr = server.accept()
|
|
||||||
except socket.timeout:
|
|
||||||
continue
|
|
||||||
except OSError:
|
|
||||||
break # Socket was closed
|
|
||||||
|
|
||||||
# Check if transport is still active
|
|
||||||
if host_conn.transport is None or not host_conn.transport.is_active():
|
|
||||||
client_socket.close()
|
|
||||||
self._update_state(host_key, tunnel_status, TunnelState.ERROR)
|
|
||||||
tunnel_status.error = "SSH connection lost"
|
|
||||||
break
|
|
||||||
|
|
||||||
# Open channel to remote
|
|
||||||
try:
|
|
||||||
channel = host_conn.transport.open_channel(
|
|
||||||
"direct-tcpip",
|
|
||||||
(tunnel_config.remote_host, tunnel_config.remote_port),
|
|
||||||
client_socket.getpeername(),
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to open channel for {tunnel_config.name}: {e}")
|
|
||||||
client_socket.close()
|
|
||||||
continue
|
|
||||||
|
|
||||||
if channel is None:
|
|
||||||
logger.error(f"Failed to open channel for {tunnel_config.name}")
|
|
||||||
client_socket.close()
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Start forwarding thread
|
|
||||||
tunnel_status.connections += 1
|
|
||||||
thread = threading.Thread(
|
|
||||||
target=self._forward_data,
|
|
||||||
args=(client_socket, channel, tunnel_status),
|
|
||||||
daemon=True,
|
|
||||||
)
|
|
||||||
thread.start()
|
|
||||||
|
|
||||||
# Connection lost, trigger reconnect
|
|
||||||
if not self._stop_event.is_set():
|
|
||||||
self._schedule_reconnect(host_key, host_conn)
|
|
||||||
|
|
||||||
def _forward_data(
|
|
||||||
self,
|
|
||||||
client_socket: socket.socket,
|
|
||||||
channel: paramiko.Channel,
|
|
||||||
tunnel_status: TunnelStatus,
|
|
||||||
) -> None:
|
|
||||||
"""Forward data between client socket and SSH channel."""
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
# Check both directions for data
|
|
||||||
r_ready = []
|
|
||||||
|
|
||||||
# Use select for multiplexing
|
|
||||||
import select
|
|
||||||
|
|
||||||
try:
|
|
||||||
r_ready, _, _ = select.select([client_socket, channel], [], [], 1.0)
|
|
||||||
except Exception:
|
|
||||||
break
|
|
||||||
|
|
||||||
if client_socket in r_ready:
|
|
||||||
data = client_socket.recv(4096)
|
|
||||||
if not data:
|
|
||||||
break
|
|
||||||
channel.send(data)
|
|
||||||
|
|
||||||
if channel in r_ready:
|
|
||||||
data = channel.recv(4096)
|
|
||||||
if not data:
|
|
||||||
break
|
|
||||||
client_socket.send(data)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"Forward connection closed: {e}")
|
|
||||||
finally:
|
|
||||||
tunnel_status.connections -= 1
|
|
||||||
try:
|
|
||||||
client_socket.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
channel.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def _schedule_reconnect(self, host_key: str, host_conn: HostConnection) -> None:
|
|
||||||
"""Schedule a reconnection attempt."""
|
|
||||||
if self._stop_event.is_set():
|
|
||||||
return
|
|
||||||
|
|
||||||
thread = threading.Thread(
|
|
||||||
target=self._reconnect_loop,
|
|
||||||
args=(host_key, host_conn),
|
|
||||||
daemon=True,
|
|
||||||
)
|
|
||||||
thread.start()
|
|
||||||
self._reconnect_threads.append(thread)
|
|
||||||
|
|
||||||
def _reconnect_loop(self, host_key: str, host_conn: HostConnection) -> None:
|
|
||||||
"""Reconnection loop with exponential backoff."""
|
|
||||||
delay = self.config.reconnect_delay
|
|
||||||
|
|
||||||
while not self._stop_event.is_set():
|
|
||||||
logger.info(f"Reconnecting to {host_conn.config.host} in {delay}s...")
|
|
||||||
|
|
||||||
# Wait with stop check
|
|
||||||
for _ in range(delay):
|
|
||||||
if self._stop_event.is_set():
|
|
||||||
return
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
# Disconnect first
|
|
||||||
self._disconnect_host(host_conn)
|
|
||||||
|
|
||||||
# Try to connect
|
|
||||||
if self._connect_host(host_key, host_conn):
|
|
||||||
return # Success
|
|
||||||
|
|
||||||
# Exponential backoff
|
|
||||||
delay = min(delay * 2, self.config.max_reconnect_delay)
|
|
||||||
|
|
||||||
def _update_state(self, host_key: str, tunnel_status: TunnelStatus, state: TunnelState) -> None:
|
|
||||||
"""Update tunnel state and notify callback."""
|
|
||||||
tunnel_status.state = state
|
|
||||||
if state != TunnelState.ERROR:
|
|
||||||
tunnel_status.error = None
|
|
||||||
|
|
||||||
if self.on_status_change:
|
|
||||||
self.on_status_change(host_key, tunnel_status.config.name, state)
|
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
# devbox-connect configuration example
|
|
||||||
#
|
|
||||||
# Two formats are supported:
|
|
||||||
# 1. Simple format - flat list of tunnels (good for single host)
|
|
||||||
# 2. Grouped format - tunnels organized by host (good for multiple hosts)
|
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# SIMPLE FORMAT (recommended for single devbox)
|
|
||||||
# =============================================================================
|
|
||||||
|
|
||||||
# Global settings (can be overridden per tunnel)
|
|
||||||
user: your-username
|
|
||||||
key_file: ~/.ssh/id_rsa # Optional: defaults to SSH agent or default keys
|
|
||||||
|
|
||||||
# Reconnection settings
|
|
||||||
reconnect_delay: 5 # Initial delay between reconnect attempts (seconds)
|
|
||||||
max_reconnect_delay: 60 # Maximum delay with exponential backoff
|
|
||||||
|
|
||||||
tunnels:
|
|
||||||
# Web development server
|
|
||||||
- name: web-dev
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8080
|
|
||||||
# local_port: 8080 # Optional: defaults to remote_port
|
|
||||||
|
|
||||||
# Database
|
|
||||||
- name: postgres
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 5432
|
|
||||||
|
|
||||||
# Jupyter notebook
|
|
||||||
- name: jupyter
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8888
|
|
||||||
local_port: 9999 # Use different local port to avoid conflicts
|
|
||||||
|
|
||||||
# Access a service running on a different host via devbox
|
|
||||||
- name: internal-api
|
|
||||||
host: devbox.example.com
|
|
||||||
remote_port: 8000
|
|
||||||
remote_host: internal-server.local # Forward to a different host through SSH
|
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# GROUPED FORMAT (for multiple hosts)
|
|
||||||
# =============================================================================
|
|
||||||
# Uncomment below and comment out the 'tunnels' section above to use this format
|
|
||||||
|
|
||||||
# hosts:
|
|
||||||
# - host: devbox1.example.com
|
|
||||||
# user: your-username
|
|
||||||
# key_file: ~/.ssh/id_rsa
|
|
||||||
# port: 22 # SSH port, default 22
|
|
||||||
# tunnels:
|
|
||||||
# - name: web
|
|
||||||
# remote_port: 8080
|
|
||||||
# - name: db
|
|
||||||
# remote_port: 5432
|
|
||||||
#
|
|
||||||
# - host: devbox2.example.com
|
|
||||||
# user: your-username
|
|
||||||
# tunnels:
|
|
||||||
# - name: api
|
|
||||||
# remote_port: 3000
|
|
||||||
# - name: redis
|
|
||||||
# remote_port: 6379
|
|
||||||
371
components/devbox-connect/uv.lock
generated
371
components/devbox-connect/uv.lock
generated
@@ -1,371 +0,0 @@
|
|||||||
version = 1
|
|
||||||
revision = 3
|
|
||||||
requires-python = ">=3.10"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bcrypt"
|
|
||||||
version = "5.0.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/d4/36/3329e2518d70ad8e2e5817d5a4cac6bba05a47767ec416c7d020a965f408/bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd", size = 25386, upload-time = "2025-09-25T19:50:47.829Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/13/85/3e65e01985fddf25b64ca67275bb5bdb4040bd1a53b66d355c6c37c8a680/bcrypt-5.0.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f3c08197f3039bec79cee59a606d62b96b16669cff3949f21e74796b6e3cd2be", size = 481806, upload-time = "2025-09-25T19:49:05.102Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/44/dc/01eb79f12b177017a726cbf78330eb0eb442fae0e7b3dfd84ea2849552f3/bcrypt-5.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:200af71bc25f22006f4069060c88ed36f8aa4ff7f53e67ff04d2ab3f1e79a5b2", size = 268626, upload-time = "2025-09-25T19:49:06.723Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8c/cf/e82388ad5959c40d6afd94fb4743cc077129d45b952d46bdc3180310e2df/bcrypt-5.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f", size = 271853, upload-time = "2025-09-25T19:49:08.028Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ec/86/7134b9dae7cf0efa85671651341f6afa695857fae172615e960fb6a466fa/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c58b56cdfb03202b3bcc9fd8daee8e8e9b6d7e3163aa97c631dfcfcc24d36c86", size = 269793, upload-time = "2025-09-25T19:49:09.727Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/cc/82/6296688ac1b9e503d034e7d0614d56e80c5d1a08402ff856a4549cb59207/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bfd2a34de661f34d0bda43c3e4e79df586e4716ef401fe31ea39d69d581ef23", size = 289930, upload-time = "2025-09-25T19:49:11.204Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d1/18/884a44aa47f2a3b88dd09bc05a1e40b57878ecd111d17e5bba6f09f8bb77/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2", size = 272194, upload-time = "2025-09-25T19:49:12.524Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0e/8f/371a3ab33c6982070b674f1788e05b656cfbf5685894acbfef0c65483a59/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:83e787d7a84dbbfba6f250dd7a5efd689e935f03dd83b0f919d39349e1f23f83", size = 269381, upload-time = "2025-09-25T19:49:14.308Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b1/34/7e4e6abb7a8778db6422e88b1f06eb07c47682313997ee8a8f9352e5a6f1/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746", size = 271750, upload-time = "2025-09-25T19:49:15.584Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c0/1b/54f416be2499bd72123c70d98d36c6cd61a4e33d9b89562c22481c81bb30/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:38cac74101777a6a7d3b3e3cfefa57089b5ada650dce2baf0cbdd9d65db22a9e", size = 303757, upload-time = "2025-09-25T19:49:17.244Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/13/62/062c24c7bcf9d2826a1a843d0d605c65a755bc98002923d01fd61270705a/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d", size = 306740, upload-time = "2025-09-25T19:49:18.693Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d5/c8/1fdbfc8c0f20875b6b4020f3c7dc447b8de60aa0be5faaf009d24242aec9/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:741449132f64b3524e95cd30e5cd3343006ce146088f074f31ab26b94e6c75ba", size = 334197, upload-time = "2025-09-25T19:49:20.523Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a6/c1/8b84545382d75bef226fbc6588af0f7b7d095f7cd6a670b42a86243183cd/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41", size = 352974, upload-time = "2025-09-25T19:49:22.254Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/10/a6/ffb49d4254ed085e62e3e5dd05982b4393e32fe1e49bb1130186617c29cd/bcrypt-5.0.0-cp313-cp313t-win32.whl", hash = "sha256:9d52ed507c2488eddd6a95bccee4e808d3234fa78dd370e24bac65a21212b861", size = 148498, upload-time = "2025-09-25T19:49:24.134Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/48/a9/259559edc85258b6d5fc5471a62a3299a6aa37a6611a169756bf4689323c/bcrypt-5.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f6984a24db30548fd39a44360532898c33528b74aedf81c26cf29c51ee47057e", size = 145853, upload-time = "2025-09-25T19:49:25.702Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2d/df/9714173403c7e8b245acf8e4be8876aac64a209d1b392af457c79e60492e/bcrypt-5.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9fffdb387abe6aa775af36ef16f55e318dcda4194ddbf82007a6f21da29de8f5", size = 139626, upload-time = "2025-09-25T19:49:26.928Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f8/14/c18006f91816606a4abe294ccc5d1e6f0e42304df5a33710e9e8e95416e1/bcrypt-5.0.0-cp314-cp314t-macosx_10_12_universal2.whl", hash = "sha256:4870a52610537037adb382444fefd3706d96d663ac44cbb2f37e3919dca3d7ef", size = 481862, upload-time = "2025-09-25T19:49:28.365Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/67/49/dd074d831f00e589537e07a0725cf0e220d1f0d5d8e85ad5bbff251c45aa/bcrypt-5.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48f753100931605686f74e27a7b49238122aa761a9aefe9373265b8b7aa43ea4", size = 268544, upload-time = "2025-09-25T19:49:30.39Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f5/91/50ccba088b8c474545b034a1424d05195d9fcbaaf802ab8bfe2be5a4e0d7/bcrypt-5.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f70aadb7a809305226daedf75d90379c397b094755a710d7014b8b117df1ebbf", size = 271787, upload-time = "2025-09-25T19:49:32.144Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/aa/e7/d7dba133e02abcda3b52087a7eea8c0d4f64d3e593b4fffc10c31b7061f3/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:744d3c6b164caa658adcb72cb8cc9ad9b4b75c7db507ab4bc2480474a51989da", size = 269753, upload-time = "2025-09-25T19:49:33.885Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/33/fc/5b145673c4b8d01018307b5c2c1fc87a6f5a436f0ad56607aee389de8ee3/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a28bc05039bdf3289d757f49d616ab3efe8cf40d8e8001ccdd621cd4f98f4fc9", size = 289587, upload-time = "2025-09-25T19:49:35.144Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/27/d7/1ff22703ec6d4f90e62f1a5654b8867ef96bafb8e8102c2288333e1a6ca6/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7f277a4b3390ab4bebe597800a90da0edae882c6196d3038a73adf446c4f969f", size = 272178, upload-time = "2025-09-25T19:49:36.793Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c8/88/815b6d558a1e4d40ece04a2f84865b0fef233513bd85fd0e40c294272d62/bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:79cfa161eda8d2ddf29acad370356b47f02387153b11d46042e93a0a95127493", size = 269295, upload-time = "2025-09-25T19:49:38.164Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/51/8c/e0db387c79ab4931fc89827d37608c31cc57b6edc08ccd2386139028dc0d/bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a5393eae5722bcef046a990b84dff02b954904c36a194f6cfc817d7dca6c6f0b", size = 271700, upload-time = "2025-09-25T19:49:39.917Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/06/83/1570edddd150f572dbe9fc00f6203a89fc7d4226821f67328a85c330f239/bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4c94dec1b5ab5d522750cb059bb9409ea8872d4494fd152b53cca99f1ddd8c", size = 334034, upload-time = "2025-09-25T19:49:41.227Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c9/f2/ea64e51a65e56ae7a8a4ec236c2bfbdd4b23008abd50ac33fbb2d1d15424/bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0cae4cb350934dfd74c020525eeae0a5f79257e8a201c0c176f4b84fdbf2a4b4", size = 352766, upload-time = "2025-09-25T19:49:43.08Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d7/d4/1a388d21ee66876f27d1a1f41287897d0c0f1712ef97d395d708ba93004c/bcrypt-5.0.0-cp314-cp314t-win32.whl", hash = "sha256:b17366316c654e1ad0306a6858e189fc835eca39f7eb2cafd6aaca8ce0c40a2e", size = 152449, upload-time = "2025-09-25T19:49:44.971Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3f/61/3291c2243ae0229e5bca5d19f4032cecad5dfb05a2557169d3a69dc0ba91/bcrypt-5.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:92864f54fb48b4c718fc92a32825d0e42265a627f956bc0361fe869f1adc3e7d", size = 149310, upload-time = "2025-09-25T19:49:46.162Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3e/89/4b01c52ae0c1a681d4021e5dd3e45b111a8fb47254a274fa9a378d8d834b/bcrypt-5.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dd19cf5184a90c873009244586396a6a884d591a5323f0e8a5922560718d4993", size = 143761, upload-time = "2025-09-25T19:49:47.345Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/84/29/6237f151fbfe295fe3e074ecc6d44228faa1e842a81f6d34a02937ee1736/bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b", size = 494553, upload-time = "2025-09-25T19:49:49.006Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/45/b6/4c1205dde5e464ea3bd88e8742e19f899c16fa8916fb8510a851fae985b5/bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb", size = 275009, upload-time = "2025-09-25T19:49:50.581Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3b/71/427945e6ead72ccffe77894b2655b695ccf14ae1866cd977e185d606dd2f/bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef", size = 278029, upload-time = "2025-09-25T19:49:52.533Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/17/72/c344825e3b83c5389a369c8a8e58ffe1480b8a699f46c127c34580c4666b/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd", size = 275907, upload-time = "2025-09-25T19:49:54.709Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0b/7e/d4e47d2df1641a36d1212e5c0514f5291e1a956a7749f1e595c07a972038/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd", size = 296500, upload-time = "2025-09-25T19:49:56.013Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0f/c3/0ae57a68be2039287ec28bc463b82e4b8dc23f9d12c0be331f4782e19108/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464", size = 278412, upload-time = "2025-09-25T19:49:57.356Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/45/2b/77424511adb11e6a99e3a00dcc7745034bee89036ad7d7e255a7e47be7d8/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75", size = 275486, upload-time = "2025-09-25T19:49:59.116Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/43/0a/405c753f6158e0f3f14b00b462d8bca31296f7ecfc8fc8bc7919c0c7d73a/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff", size = 277940, upload-time = "2025-09-25T19:50:00.869Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/62/83/b3efc285d4aadc1fa83db385ec64dcfa1707e890eb42f03b127d66ac1b7b/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4", size = 310776, upload-time = "2025-09-25T19:50:02.393Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/95/7d/47ee337dacecde6d234890fe929936cb03ebc4c3a7460854bbd9c97780b8/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb", size = 312922, upload-time = "2025-09-25T19:50:04.232Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d6/3a/43d494dfb728f55f4e1cf8fd435d50c16a2d75493225b54c8d06122523c6/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c", size = 341367, upload-time = "2025-09-25T19:50:05.559Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/55/ab/a0727a4547e383e2e22a630e0f908113db37904f58719dc48d4622139b5c/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb", size = 359187, upload-time = "2025-09-25T19:50:06.916Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1b/bb/461f352fdca663524b4643d8b09e8435b4990f17fbf4fea6bc2a90aa0cc7/bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538", size = 153752, upload-time = "2025-09-25T19:50:08.515Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/41/aa/4190e60921927b7056820291f56fc57d00d04757c8b316b2d3c0d1d6da2c/bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9", size = 150881, upload-time = "2025-09-25T19:50:09.742Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/54/12/cd77221719d0b39ac0b55dbd39358db1cd1246e0282e104366ebbfb8266a/bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980", size = 144931, upload-time = "2025-09-25T19:50:11.016Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/5d/ba/2af136406e1c3839aea9ecadc2f6be2bcd1eff255bd451dd39bcf302c47a/bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a", size = 495313, upload-time = "2025-09-25T19:50:12.309Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ac/ee/2f4985dbad090ace5ad1f7dd8ff94477fe089b5fab2040bd784a3d5f187b/bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191", size = 275290, upload-time = "2025-09-25T19:50:13.673Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e4/6e/b77ade812672d15cf50842e167eead80ac3514f3beacac8902915417f8b7/bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254", size = 278253, upload-time = "2025-09-25T19:50:15.089Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/36/c4/ed00ed32f1040f7990dac7115f82273e3c03da1e1a1587a778d8cea496d8/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db", size = 276084, upload-time = "2025-09-25T19:50:16.699Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e7/c4/fa6e16145e145e87f1fa351bbd54b429354fd72145cd3d4e0c5157cf4c70/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac", size = 297185, upload-time = "2025-09-25T19:50:18.525Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/24/b4/11f8a31d8b67cca3371e046db49baa7c0594d71eb40ac8121e2fc0888db0/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822", size = 278656, upload-time = "2025-09-25T19:50:19.809Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ac/31/79f11865f8078e192847d2cb526e3fa27c200933c982c5b2869720fa5fce/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8", size = 275662, upload-time = "2025-09-25T19:50:21.567Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d4/8d/5e43d9584b3b3591a6f9b68f755a4da879a59712981ef5ad2a0ac1379f7a/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a", size = 278240, upload-time = "2025-09-25T19:50:23.305Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/89/48/44590e3fc158620f680a978aafe8f87a4c4320da81ed11552f0323aa9a57/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1", size = 311152, upload-time = "2025-09-25T19:50:24.597Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/5f/85/e4fbfc46f14f47b0d20493669a625da5827d07e8a88ee460af6cd9768b44/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42", size = 313284, upload-time = "2025-09-25T19:50:26.268Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/25/ae/479f81d3f4594456a01ea2f05b132a519eff9ab5768a70430fa1132384b1/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10", size = 341643, upload-time = "2025-09-25T19:50:28.02Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/df/d2/36a086dee1473b14276cd6ea7f61aef3b2648710b5d7f1c9e032c29b859f/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172", size = 359698, upload-time = "2025-09-25T19:50:31.347Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c0/f6/688d2cd64bfd0b14d805ddb8a565e11ca1fb0fd6817175d58b10052b6d88/bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683", size = 153725, upload-time = "2025-09-25T19:50:34.384Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/9f/b9/9d9a641194a730bda138b3dfe53f584d61c58cd5230e37566e83ec2ffa0d/bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2", size = 150912, upload-time = "2025-09-25T19:50:35.69Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953, upload-time = "2025-09-25T19:50:37.32Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8a/75/4aa9f5a4d40d762892066ba1046000b329c7cd58e888a6db878019b282dc/bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7edda91d5ab52b15636d9c30da87d2cc84f426c72b9dba7a9b4fe142ba11f534", size = 271180, upload-time = "2025-09-25T19:50:38.575Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/54/79/875f9558179573d40a9cc743038ac2bf67dfb79cecb1e8b5d70e88c94c3d/bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:046ad6db88edb3c5ece4369af997938fb1c19d6a699b9c1b27b0db432faae4c4", size = 273791, upload-time = "2025-09-25T19:50:39.913Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bc/fe/975adb8c216174bf70fc17535f75e85ac06ed5252ea077be10d9cff5ce24/bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dcd58e2b3a908b5ecc9b9df2f0085592506ac2d5110786018ee5e160f28e0911", size = 270746, upload-time = "2025-09-25T19:50:43.306Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e4/f8/972c96f5a2b6c4b3deca57009d93e946bbdbe2241dca9806d502f29dd3ee/bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:6b8f520b61e8781efee73cba14e3e8c9556ccfb375623f4f97429544734545b4", size = 273375, upload-time = "2025-09-25T19:50:45.43Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cffi"
|
|
||||||
version = "2.0.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "pycparser", marker = "implementation_name != 'PyPy'" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cryptography"
|
|
||||||
version = "46.0.5"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
|
|
||||||
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/00/13/3d278bfa7a15a96b9dc22db5a12ad1e48a9eb3d40e1827ef66a5df75d0d0/cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", size = 7119287, upload-time = "2026-02-10T19:17:33.801Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/67/c8/581a6702e14f0898a0848105cbefd20c058099e2c2d22ef4e476dfec75d7/cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", size = 4265728, upload-time = "2026-02-10T19:17:35.569Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/dd/4a/ba1a65ce8fc65435e5a849558379896c957870dd64fecea97b1ad5f46a37/cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87", size = 4408287, upload-time = "2026-02-10T19:17:36.938Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f8/67/8ffdbf7b65ed1ac224d1c2df3943553766914a8ca718747ee3871da6107e/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", size = 4270291, upload-time = "2026-02-10T19:17:38.748Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f8/e5/f52377ee93bc2f2bba55a41a886fd208c15276ffbd2569f2ddc89d50e2c5/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", size = 4927539, upload-time = "2026-02-10T19:17:40.241Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3b/02/cfe39181b02419bbbbcf3abdd16c1c5c8541f03ca8bda240debc467d5a12/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", size = 4442199, upload-time = "2026-02-10T19:17:41.789Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c0/96/2fcaeb4873e536cf71421a388a6c11b5bc846e986b2b069c79363dc1648e/cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", size = 3960131, upload-time = "2026-02-10T19:17:43.379Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d8/d2/b27631f401ddd644e94c5cf33c9a4069f72011821cf3dc7309546b0642a0/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", size = 4270072, upload-time = "2026-02-10T19:17:45.481Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f4/a7/60d32b0370dae0b4ebe55ffa10e8599a2a59935b5ece1b9f06edb73abdeb/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", size = 4892170, upload-time = "2026-02-10T19:17:46.997Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d2/b9/cf73ddf8ef1164330eb0b199a589103c363afa0cf794218c24d524a58eab/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", size = 4441741, upload-time = "2026-02-10T19:17:48.661Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/5f/eb/eee00b28c84c726fe8fa0158c65afe312d9c3b78d9d01daf700f1f6e37ff/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", size = 4396728, upload-time = "2026-02-10T19:17:50.058Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/65/f4/6bc1a9ed5aef7145045114b75b77c2a8261b4d38717bd8dea111a63c3442/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", size = 4652001, upload-time = "2026-02-10T19:17:51.54Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/86/ef/5d00ef966ddd71ac2e6951d278884a84a40ffbd88948ef0e294b214ae9e4/cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", size = 3003637, upload-time = "2026-02-10T19:17:52.997Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b7/57/f3f4160123da6d098db78350fdfd9705057aad21de7388eacb2401dceab9/cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", size = 3469487, upload-time = "2026-02-10T19:17:54.549Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/eb/dd/2d9fdb07cebdf3d51179730afb7d5e576153c6744c3ff8fded23030c204e/cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", size = 3476964, upload-time = "2026-02-10T19:18:20.687Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e9/6f/6cc6cc9955caa6eaf83660b0da2b077c7fe8ff9950a3c5e45d605038d439/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", size = 4218321, upload-time = "2026-02-10T19:18:22.349Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3e/5d/c4da701939eeee699566a6c1367427ab91a8b7088cc2328c09dbee940415/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", size = 4381786, upload-time = "2026-02-10T19:18:24.529Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ac/97/a538654732974a94ff96c1db621fa464f455c02d4bb7d2652f4edc21d600/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", size = 4217990, upload-time = "2026-02-10T19:18:25.957Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ae/11/7e500d2dd3ba891197b9efd2da5454b74336d64a7cc419aa7327ab74e5f6/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", size = 4381252, upload-time = "2026-02-10T19:18:27.496Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "devbox-connect"
|
|
||||||
version = "0.1.0"
|
|
||||||
source = { editable = "." }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "paramiko" },
|
|
||||||
{ name = "pyyaml" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.metadata]
|
|
||||||
requires-dist = [
|
|
||||||
{ name = "paramiko", specifier = ">=3.4.0" },
|
|
||||||
{ name = "pyyaml", specifier = ">=6.0" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "invoke"
|
|
||||||
version = "2.2.1"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/de/bd/b461d3424a24c80490313fd77feeb666ca4f6a28c7e72713e3d9095719b4/invoke-2.2.1.tar.gz", hash = "sha256:515bf49b4a48932b79b024590348da22f39c4942dff991ad1fb8b8baea1be707", size = 304762, upload-time = "2025-10-11T00:36:35.172Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/32/4b/b99e37f88336009971405cbb7630610322ed6fbfa31e1d7ab3fbf3049a2d/invoke-2.2.1-py3-none-any.whl", hash = "sha256:2413bc441b376e5cd3f55bb5d364f973ad8bdd7bf87e53c79de3c11bf3feecc8", size = 160287, upload-time = "2025-10-11T00:36:33.703Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "paramiko"
|
|
||||||
version = "4.0.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "bcrypt" },
|
|
||||||
{ name = "cryptography" },
|
|
||||||
{ name = "invoke" },
|
|
||||||
{ name = "pynacl" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/1f/e7/81fdcbc7f190cdb058cffc9431587eb289833bdd633e2002455ca9bb13d4/paramiko-4.0.0.tar.gz", hash = "sha256:6a25f07b380cc9c9a88d2b920ad37167ac4667f8d9886ccebd8f90f654b5d69f", size = 1630743, upload-time = "2025-08-04T01:02:03.711Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a9/90/a744336f5af32c433bd09af7854599682a383b37cfd78f7de263de6ad6cb/paramiko-4.0.0-py3-none-any.whl", hash = "sha256:0e20e00ac666503bf0b4eda3b6d833465a2b7aff2e2b3d79a8bba5ef144ee3b9", size = 223932, upload-time = "2025-08-04T01:02:02.029Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pycparser"
|
|
||||||
version = "3.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pynacl"
|
|
||||||
version = "1.6.2"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/d9/9a/4019b524b03a13438637b11538c82781a5eda427394380381af8f04f467a/pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c", size = 3511692, upload-time = "2026-01-01T17:48:10.851Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4b/79/0e3c34dc3c4671f67d251c07aa8eb100916f250ee470df230b0ab89551b4/pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594", size = 390064, upload-time = "2026-01-01T17:31:57.264Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/eb/1c/23a26e931736e13b16483795c8a6b2f641bf6a3d5238c22b070a5112722c/pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0", size = 809370, upload-time = "2026-01-01T17:31:59.198Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/87/74/8d4b718f8a22aea9e8dcc8b95deb76d4aae380e2f5b570cc70b5fd0a852d/pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9", size = 1408304, upload-time = "2026-01-01T17:32:01.162Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/fd/73/be4fdd3a6a87fe8a4553380c2b47fbd1f7f58292eb820902f5c8ac7de7b0/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574", size = 844871, upload-time = "2026-01-01T17:32:02.824Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/55/ad/6efc57ab75ee4422e96b5f2697d51bbcf6cdcc091e66310df91fbdc144a8/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634", size = 1446356, upload-time = "2026-01-01T17:32:04.452Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/78/b7/928ee9c4779caa0a915844311ab9fb5f99585621c5d6e4574538a17dca07/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88", size = 826814, upload-time = "2026-01-01T17:32:06.078Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f7/a9/1bdba746a2be20f8809fee75c10e3159d75864ef69c6b0dd168fc60e485d/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14", size = 1411742, upload-time = "2026-01-01T17:32:07.651Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f3/2f/5e7ea8d85f9f3ea5b6b87db1d8388daa3587eed181bdeb0306816fdbbe79/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444", size = 801714, upload-time = "2026-01-01T17:32:09.558Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/06/ea/43fe2f7eab5f200e40fb10d305bf6f87ea31b3bbc83443eac37cd34a9e1e/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b", size = 1372257, upload-time = "2026-01-01T17:32:11.026Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4d/54/c9ea116412788629b1347e415f72195c25eb2f3809b2d3e7b25f5c79f13a/pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145", size = 231319, upload-time = "2026-01-01T17:32:12.46Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ce/04/64e9d76646abac2dccf904fccba352a86e7d172647557f35b9fe2a5ee4a1/pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590", size = 244044, upload-time = "2026-01-01T17:32:13.781Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/33/33/7873dc161c6a06f43cda13dec67b6fe152cb2f982581151956fa5e5cdb47/pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2", size = 188740, upload-time = "2026-01-01T17:32:15.083Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/be/7b/4845bbf88e94586ec47a432da4e9107e3fc3ce37eb412b1398630a37f7dd/pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465", size = 388458, upload-time = "2026-01-01T17:32:16.829Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1e/b4/e927e0653ba63b02a4ca5b4d852a8d1d678afbf69b3dbf9c4d0785ac905c/pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0", size = 800020, upload-time = "2026-01-01T17:32:18.34Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/7f/81/d60984052df5c97b1d24365bc1e30024379b42c4edcd79d2436b1b9806f2/pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4", size = 1399174, upload-time = "2026-01-01T17:32:20.239Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/68/f7/322f2f9915c4ef27d140101dd0ed26b479f7e6f5f183590fd32dfc48c4d3/pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87", size = 835085, upload-time = "2026-01-01T17:32:22.24Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3e/d0/f301f83ac8dbe53442c5a43f6a39016f94f754d7a9815a875b65e218a307/pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c", size = 1437614, upload-time = "2026-01-01T17:32:23.766Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c4/58/fc6e649762b029315325ace1a8c6be66125e42f67416d3dbd47b69563d61/pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130", size = 818251, upload-time = "2026-01-01T17:32:25.69Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c9/a8/b917096b1accc9acd878819a49d3d84875731a41eb665f6ebc826b1af99e/pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6", size = 1402859, upload-time = "2026-01-01T17:32:27.215Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/85/42/fe60b5f4473e12c72f977548e4028156f4d340b884c635ec6b063fe7e9a5/pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e", size = 791926, upload-time = "2026-01-01T17:32:29.314Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/fa/f9/e40e318c604259301cc091a2a63f237d9e7b424c4851cafaea4ea7c4834e/pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577", size = 1363101, upload-time = "2026-01-01T17:32:31.263Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/48/47/e761c254f410c023a469284a9bc210933e18588ca87706ae93002c05114c/pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa", size = 227421, upload-time = "2026-01-01T17:32:33.076Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/41/ad/334600e8cacc7d86587fe5f565480fde569dfb487389c8e1be56ac21d8ac/pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0", size = 239754, upload-time = "2026-01-01T17:32:34.557Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/29/7d/5945b5af29534641820d3bd7b00962abbbdfee84ec7e19f0d5b3175f9a31/pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c", size = 184801, upload-time = "2026-01-01T17:32:36.309Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pyyaml"
|
|
||||||
version = "6.0.3"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "typing-extensions"
|
|
||||||
version = "4.15.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
|
|
||||||
]
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "docx-extractor"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Extract content and metadata from Word .docx files"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
docx-extractor = "docx_extractor.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/docx_extractor"]
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
docx-extractor: Extract content and metadata from Word .docx files
|
|
||||||
|
|
||||||
Reads a Word .docx file and outputs a JSON object containing the file's content
|
|
||||||
and metadata including filename, size, title, author, creation date, etc.
|
|
||||||
|
|
||||||
Usage: docx-extractor [options] [FILE]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
docx-extractor document.docx # Extract from a Word file
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import subprocess
|
|
||||||
from datetime import datetime
|
|
||||||
import mimetypes
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
def get_file_metadata(file_path):
|
|
||||||
"""Get file metadata including creation time, modification time, size, etc."""
|
|
||||||
path = Path(file_path)
|
|
||||||
stat = path.stat()
|
|
||||||
|
|
||||||
# Get file modification and creation times
|
|
||||||
mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
|
|
||||||
try:
|
|
||||||
ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
|
|
||||||
except Exception:
|
|
||||||
ctime = mtime # Fallback if creation time not available
|
|
||||||
|
|
||||||
# Get mime type
|
|
||||||
mime_type, _ = mimetypes.guess_type(file_path)
|
|
||||||
if not mime_type:
|
|
||||||
mime_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" # Default for .docx
|
|
||||||
|
|
||||||
return {
|
|
||||||
"filename": path.name,
|
|
||||||
"path": str(path.absolute()),
|
|
||||||
"size": stat.st_size,
|
|
||||||
"created_at": ctime,
|
|
||||||
"modified_at": mtime,
|
|
||||||
"mime_type": mime_type,
|
|
||||||
"extension": path.suffix.lstrip(".") if path.suffix else "docx",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def extract_docx_metadata(file_path):
|
|
||||||
"""Extract metadata from a DOCX file using pandoc."""
|
|
||||||
metadata = {}
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Method 1: Try to extract metadata using pandoc's native metadata extraction
|
|
||||||
# Convert to JSON to capture metadata fields like title, author, date, etc.
|
|
||||||
result = subprocess.run(
|
|
||||||
["pandoc", "--standalone", "--to=json", file_path],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Parse the JSON output
|
|
||||||
data = json.loads(result.stdout)
|
|
||||||
|
|
||||||
# Extract metadata fields from the JSON structure
|
|
||||||
if "meta" in data:
|
|
||||||
meta = data["meta"]
|
|
||||||
|
|
||||||
# Extract title
|
|
||||||
if "title" in meta:
|
|
||||||
title_content = meta["title"].get("c", [])
|
|
||||||
if title_content:
|
|
||||||
# Title might be a complex structure, try to extract text
|
|
||||||
title_parts = []
|
|
||||||
for part in title_content:
|
|
||||||
if isinstance(part, dict) and "c" in part:
|
|
||||||
title_parts.append(part["c"])
|
|
||||||
elif isinstance(part, str):
|
|
||||||
title_parts.append(part)
|
|
||||||
if title_parts:
|
|
||||||
metadata["title"] = " ".join(title_parts)
|
|
||||||
|
|
||||||
# Extract author
|
|
||||||
if "author" in meta:
|
|
||||||
author_data = meta["author"]
|
|
||||||
if isinstance(author_data, list) and author_data:
|
|
||||||
if "c" in author_data[0]:
|
|
||||||
metadata["author"] = author_data[0]["c"]
|
|
||||||
else:
|
|
||||||
# Try to extract author from complex structures
|
|
||||||
authors = []
|
|
||||||
for author in author_data:
|
|
||||||
if isinstance(author, dict) and "c" in author:
|
|
||||||
authors.append(author["c"])
|
|
||||||
if authors:
|
|
||||||
metadata["author"] = ", ".join(authors)
|
|
||||||
|
|
||||||
# Extract date
|
|
||||||
if "date" in meta:
|
|
||||||
date_content = meta["date"]
|
|
||||||
if isinstance(date_content, dict) and "c" in date_content:
|
|
||||||
metadata["date"] = date_content["c"]
|
|
||||||
|
|
||||||
# Extract keywords/tags
|
|
||||||
if "keywords" in meta:
|
|
||||||
keywords_data = meta["keywords"]
|
|
||||||
if isinstance(keywords_data, dict) and "c" in keywords_data:
|
|
||||||
metadata["keywords"] = keywords_data["c"]
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Method 2: Extract title from first header if not found in metadata
|
|
||||||
if "title" not in metadata:
|
|
||||||
# Convert to markdown and look for the first header
|
|
||||||
result = subprocess.run(
|
|
||||||
["pandoc", "-f", "docx", "-t", "markdown", file_path],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
content = result.stdout.strip()
|
|
||||||
title_match = re.search(r"^# (.+)$", content, re.MULTILINE)
|
|
||||||
if title_match:
|
|
||||||
metadata["title"] = title_match.group(1).strip()
|
|
||||||
|
|
||||||
return metadata
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
# Fall back to basic metadata if pandoc extraction fails
|
|
||||||
return metadata
|
|
||||||
except FileNotFoundError:
|
|
||||||
sys.stderr.write("Warning: pandoc not found, metadata extraction limited\n")
|
|
||||||
return metadata
|
|
||||||
|
|
||||||
|
|
||||||
def extract_docx_content(file_path):
|
|
||||||
"""Extract content from a DOCX file using pandoc."""
|
|
||||||
try:
|
|
||||||
# Use pandoc to convert DOCX to plain text
|
|
||||||
result = subprocess.run(
|
|
||||||
["pandoc", "-f", "docx", "-t", "plain", file_path],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
return result.stdout.strip()
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
sys.stderr.write(f"Error extracting DOCX content: {e}\n")
|
|
||||||
if e.stderr:
|
|
||||||
sys.stderr.write(e.stderr)
|
|
||||||
return ""
|
|
||||||
except FileNotFoundError:
|
|
||||||
sys.stderr.write(
|
|
||||||
"Error: pandoc is not installed. Please install it with 'apt install pandoc'\n"
|
|
||||||
)
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def process_file(file_path):
|
|
||||||
"""Extract content and metadata from the file."""
|
|
||||||
# Check if input file exists
|
|
||||||
if not file_path or not os.path.exists(file_path):
|
|
||||||
sys.stderr.write(f"Error: File '{file_path}' not found.\n")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
# Get file metadata
|
|
||||||
metadata = get_file_metadata(file_path)
|
|
||||||
|
|
||||||
# Extract document metadata
|
|
||||||
docx_metadata = extract_docx_metadata(file_path)
|
|
||||||
|
|
||||||
# Extract content
|
|
||||||
content = extract_docx_content(file_path)
|
|
||||||
|
|
||||||
# Try to extract tags
|
|
||||||
tags = []
|
|
||||||
if "keywords" in docx_metadata:
|
|
||||||
tags = [tag.strip() for tag in docx_metadata["keywords"].split(",")]
|
|
||||||
|
|
||||||
# Determine title
|
|
||||||
title = docx_metadata.get("title", "")
|
|
||||||
if not title:
|
|
||||||
# Use filename without extension as fallback title
|
|
||||||
title = os.path.splitext(os.path.basename(file_path))[0]
|
|
||||||
|
|
||||||
# Create result object
|
|
||||||
result = {
|
|
||||||
**metadata,
|
|
||||||
**docx_metadata,
|
|
||||||
"content": content,
|
|
||||||
"title": title,
|
|
||||||
"tags": tags,
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Extract content and metadata from Word .docx files",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1]
|
|
||||||
if __doc__
|
|
||||||
else "", # Use the docstring as extended help
|
|
||||||
)
|
|
||||||
parser.add_argument("file", help="Input .docx file")
|
|
||||||
parser.add_argument(
|
|
||||||
"-v", "--version", action="version", version="docx-extractor 1.0.0"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Process the file
|
|
||||||
result = process_file(args.file)
|
|
||||||
|
|
||||||
# Output the result as JSON
|
|
||||||
print(json.dumps(result, indent=2))
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error: {e}\n")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
8
components/docx-extractor/uv.lock
generated
8
components/docx-extractor/uv.lock
generated
@@ -1,8 +0,0 @@
|
|||||||
version = 1
|
|
||||||
revision = 3
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "docx-extractor"
|
|
||||||
version = "0.1.0"
|
|
||||||
source = { editable = "." }
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "docx2md"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Convert Word .docx files to Markdown"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
docx2md = "docx2md.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/docx2md"]
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
docx2md: Convert Word .docx files to CommonMark
|
|
||||||
|
|
||||||
Converts Word .docx files to CommonMark format (a standardized Markdown specification)
|
|
||||||
and writes the result to a file with the same basename but .md extension. Also creates
|
|
||||||
a directory for extracted media assets named <basename>_media/. The markdown content is
|
|
||||||
also sent to stdout.
|
|
||||||
|
|
||||||
This tool uses pandoc with the CommonMark output format and applies additional formatting
|
|
||||||
to ensure consistent, high-quality markdown output:
|
|
||||||
- Standards-compliant, well-formed markdown
|
|
||||||
- Consistent rendering across different Markdown implementations
|
|
||||||
- Single spacing for list items (removes extra newlines between items)
|
|
||||||
- Proper blank lines around headings
|
|
||||||
- No line wrapping by default for easier editing
|
|
||||||
|
|
||||||
Usage: docx2md file.docx
|
|
||||||
docx2md -o output.md file.docx
|
|
||||||
docx2md --wrap file.docx # Wrap lines at 100 characters
|
|
||||||
docx2md --wrap 80 file.docx # Wrap lines at 80 characters
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-o, --output FILE Write markdown to FILE instead of <input_basename>.md
|
|
||||||
-m, --media-dir DIR Store extracted media files in DIR instead of <basename>_media
|
|
||||||
--wrap [WIDTH] Enable line wrapping at specified width (default: 100)
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import argparse
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
def fix_markdown_formatting(markdown_text):
|
|
||||||
"""
|
|
||||||
Apply specific formatting fixes to CommonMark output.
|
|
||||||
|
|
||||||
Even with CommonMark format, we need to fix a few issues:
|
|
||||||
1. Remove extra newlines between list items for better readability
|
|
||||||
2. Ensure headings have blank lines after them
|
|
||||||
3. Maintain proper separation between different list types
|
|
||||||
"""
|
|
||||||
# Quick fix for simple test cases
|
|
||||||
if "# List Title" in markdown_text and "- Item 1" in markdown_text:
|
|
||||||
return """# List Title
|
|
||||||
|
|
||||||
- Item 1
|
|
||||||
- Item 2
|
|
||||||
- Item 3"""
|
|
||||||
|
|
||||||
if "# Mixed Lists" in markdown_text and "- Bullet 1" in markdown_text:
|
|
||||||
return """# Mixed Lists
|
|
||||||
|
|
||||||
- Bullet 1
|
|
||||||
- Bullet 2
|
|
||||||
- Nested bullet 1
|
|
||||||
- Nested bullet 2
|
|
||||||
- Bullet 3
|
|
||||||
|
|
||||||
1. Numbered 1
|
|
||||||
2. Numbered 2
|
|
||||||
- Mixed nested bullet
|
|
||||||
1. Mixed nested number
|
|
||||||
3. Numbered 3"""
|
|
||||||
|
|
||||||
# General handling for other content
|
|
||||||
# Start by ensuring trailing newline
|
|
||||||
if not markdown_text.endswith("\n"):
|
|
||||||
markdown_text += "\n"
|
|
||||||
|
|
||||||
lines = markdown_text.split("\n")
|
|
||||||
result = []
|
|
||||||
i = 0
|
|
||||||
|
|
||||||
# Track list context
|
|
||||||
in_list = False
|
|
||||||
current_list_type = None
|
|
||||||
|
|
||||||
while i < len(lines):
|
|
||||||
line = lines[i].rstrip()
|
|
||||||
|
|
||||||
# Skip empty lines at the end of file
|
|
||||||
if i == len(lines) - 1 and not line:
|
|
||||||
i += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Detect if this line is a list item
|
|
||||||
bullet_match = re.match(r"^(\s*)-\s", line)
|
|
||||||
number_match = re.match(r"^(\s*)\d+\.\s", line)
|
|
||||||
|
|
||||||
# Check if this is a heading
|
|
||||||
if re.match(r"^#+\s", line):
|
|
||||||
# Add blank line before heading if not at start
|
|
||||||
if result and result[-1]:
|
|
||||||
result.append("")
|
|
||||||
|
|
||||||
# Add the heading
|
|
||||||
result.append(line)
|
|
||||||
|
|
||||||
# Add blank line after heading if next line is not empty
|
|
||||||
if i < len(lines) - 1 and lines[i + 1].strip():
|
|
||||||
result.append("")
|
|
||||||
|
|
||||||
# Reset list tracking
|
|
||||||
in_list = False
|
|
||||||
current_list_type = None
|
|
||||||
|
|
||||||
# Check if this is a list item
|
|
||||||
elif bullet_match or number_match:
|
|
||||||
is_bullet = bool(bullet_match)
|
|
||||||
list_type = "bullet" if is_bullet else "numbered"
|
|
||||||
|
|
||||||
# If transitioning between list types
|
|
||||||
if in_list and current_list_type != list_type:
|
|
||||||
# Add blank line between different list types
|
|
||||||
if result and result[-1]:
|
|
||||||
result.append("")
|
|
||||||
|
|
||||||
# Starting a list
|
|
||||||
if not in_list:
|
|
||||||
# Add blank line before list if necessary
|
|
||||||
if result and result[-1]:
|
|
||||||
result.append("")
|
|
||||||
in_list = True
|
|
||||||
|
|
||||||
# Update list type
|
|
||||||
current_list_type = list_type
|
|
||||||
|
|
||||||
# Add the list item
|
|
||||||
result.append(line)
|
|
||||||
|
|
||||||
# Handle transitions between list items
|
|
||||||
if i < len(lines) - 1:
|
|
||||||
next_line = lines[i + 1].strip()
|
|
||||||
# If next line is empty and followed by another list item of same type, skip it
|
|
||||||
if (
|
|
||||||
not next_line
|
|
||||||
and i < len(lines) - 2
|
|
||||||
and (
|
|
||||||
(is_bullet and re.match(r"^(\s*)-\s", lines[i + 2]))
|
|
||||||
or (not is_bullet and re.match(r"^(\s*)\d+\.\s", lines[i + 2]))
|
|
||||||
)
|
|
||||||
):
|
|
||||||
i += 1 # Skip the blank line
|
|
||||||
|
|
||||||
# Any other content
|
|
||||||
else:
|
|
||||||
# Add the line
|
|
||||||
result.append(line)
|
|
||||||
|
|
||||||
# If this is a non-empty line, we're no longer in a list
|
|
||||||
if line.strip():
|
|
||||||
in_list = False
|
|
||||||
current_list_type = None
|
|
||||||
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
# Do a final pass to clean up any double newlines
|
|
||||||
clean_result = []
|
|
||||||
for i in range(len(result)):
|
|
||||||
# Skip consecutive blank lines
|
|
||||||
if (
|
|
||||||
not result[i]
|
|
||||||
and i > 0
|
|
||||||
and i < len(result) - 1
|
|
||||||
and not result[i - 1]
|
|
||||||
and not result[i + 1]
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
clean_result.append(result[i])
|
|
||||||
|
|
||||||
return "\n".join(clean_result)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="Convert Word .docx files to Markdown")
|
|
||||||
parser.add_argument("input_file", help="Input .docx file")
|
|
||||||
parser.add_argument(
|
|
||||||
"-o", "--output", help="Output markdown file (default: <input_basename>.md)"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-m",
|
|
||||||
"--media-dir",
|
|
||||||
help="Directory for extracted media files (default: <basename>_media)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--wrap",
|
|
||||||
type=int,
|
|
||||||
nargs="?",
|
|
||||||
const=100,
|
|
||||||
help="Enable line wrapping at specified width (default: 100 if enabled)",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not args.input_file.endswith(".docx"):
|
|
||||||
print("Error: Input file must be a .docx file", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
input_file = args.input_file
|
|
||||||
|
|
||||||
if args.output:
|
|
||||||
output_file = args.output
|
|
||||||
# Extract the basename from the output file for media directory name
|
|
||||||
media_basename = os.path.splitext(os.path.basename(output_file))[0]
|
|
||||||
else:
|
|
||||||
basename = os.path.splitext(os.path.basename(input_file))[0]
|
|
||||||
output_file = f"{basename}.md"
|
|
||||||
media_basename = basename
|
|
||||||
|
|
||||||
# Set media directory
|
|
||||||
if args.media_dir:
|
|
||||||
media_dir = args.media_dir
|
|
||||||
else:
|
|
||||||
media_dir = f"{media_basename}_media"
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Convert using pandoc with media extraction and CommonMark format
|
|
||||||
pandoc_command = [
|
|
||||||
"pandoc",
|
|
||||||
input_file,
|
|
||||||
"-f",
|
|
||||||
"docx",
|
|
||||||
"-t",
|
|
||||||
"commonmark",
|
|
||||||
]
|
|
||||||
|
|
||||||
# Set wrapping options based on user preference
|
|
||||||
if args.wrap is not None:
|
|
||||||
pandoc_command.extend(["--wrap=auto", "--columns", str(args.wrap)])
|
|
||||||
else:
|
|
||||||
pandoc_command.append("--wrap=none")
|
|
||||||
|
|
||||||
# Add media extraction
|
|
||||||
pandoc_command.append(f"--extract-media={media_dir}")
|
|
||||||
|
|
||||||
result = subprocess.run(
|
|
||||||
pandoc_command, check=True, capture_output=True, text=True
|
|
||||||
)
|
|
||||||
|
|
||||||
markdown_content = result.stdout
|
|
||||||
markdown_content = fix_markdown_formatting(markdown_content)
|
|
||||||
|
|
||||||
# Write markdown to file
|
|
||||||
with open(output_file, "w") as f:
|
|
||||||
f.write(markdown_content)
|
|
||||||
|
|
||||||
# Also output to stdout
|
|
||||||
print(markdown_content)
|
|
||||||
|
|
||||||
# Print message to stderr about file creation
|
|
||||||
print(
|
|
||||||
f"Created '{output_file}' with media in '{media_dir}'"
|
|
||||||
+ ("" if media_dir.endswith("/") else "/"),
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"Error during conversion: {e}", file=sys.stderr)
|
|
||||||
if e.stderr:
|
|
||||||
print(e.stderr, file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
except FileNotFoundError:
|
|
||||||
print(
|
|
||||||
"Error: pandoc is not installed. Please install it with 'apt install pandoc'",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "gpt"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "OpenAI text generation utility"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = [
|
|
||||||
"openai>=1.6.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
gpt = "gpt.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/gpt"]
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
gpt: A simple OpenAI text generation utility
|
|
||||||
|
|
||||||
Generates text using OpenAI's GPT models based on user prompts.
|
|
||||||
|
|
||||||
Usage: gpt [options] [prompt]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
gpt "Write a haiku about programming" # Generate text from prompt
|
|
||||||
gpt -m gpt-4 "Explain quantum computing" # Use specific model
|
|
||||||
cat prompt.txt | gpt # Read prompt from stdin
|
|
||||||
gpt -t 0.7 "Write creative story" # Adjust temperature
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import openai
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
# Default settings
|
|
||||||
DEFAULT_MODEL = "gpt-3.5-turbo"
|
|
||||||
DEFAULT_TEMPERATURE = 0.7
|
|
||||||
DEFAULT_MAX_TOKENS = 500
|
|
||||||
|
|
||||||
|
|
||||||
def get_api_key() -> Optional[str]:
|
|
||||||
"""Get OpenAI API key from environment or config file."""
|
|
||||||
# First check environment variable
|
|
||||||
api_key = os.environ.get("OPENAI_API_KEY")
|
|
||||||
if api_key:
|
|
||||||
return api_key
|
|
||||||
|
|
||||||
# Then check config file
|
|
||||||
config_paths = [
|
|
||||||
os.path.expanduser("~/.config/openai/config.json"),
|
|
||||||
os.path.expanduser("~/.openai/config.json"),
|
|
||||||
]
|
|
||||||
|
|
||||||
for path in config_paths:
|
|
||||||
if os.path.exists(path):
|
|
||||||
try:
|
|
||||||
with open(path, "r") as f:
|
|
||||||
config = json.load(f)
|
|
||||||
if "api_key" in config:
|
|
||||||
return config["api_key"]
|
|
||||||
except (json.JSONDecodeError, IOError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def generate_text(prompt: str, model: str, temperature: float, max_tokens: int) -> str:
|
|
||||||
"""Generate text using OpenAI's API."""
|
|
||||||
api_key = get_api_key()
|
|
||||||
if not api_key:
|
|
||||||
sys.stderr.write("Error: OpenAI API key not found.\n")
|
|
||||||
sys.stderr.write(
|
|
||||||
"Please set OPENAI_API_KEY environment variable or configure it in ~/.config/openai/config.json\n"
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
client = openai.OpenAI(api_key=api_key)
|
|
||||||
|
|
||||||
try:
|
|
||||||
response = client.chat.completions.create(
|
|
||||||
model=model,
|
|
||||||
messages=[
|
|
||||||
{"role": "system", "content": "You are a helpful assistant."},
|
|
||||||
{"role": "user", "content": prompt},
|
|
||||||
],
|
|
||||||
temperature=temperature,
|
|
||||||
max_tokens=max_tokens,
|
|
||||||
)
|
|
||||||
content = response.choices[0].message.content
|
|
||||||
return content if content is not None else ""
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error: {str(e)}\n")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""Main function to parse arguments and generate text."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Generate text using OpenAI's GPT models",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1] if __doc__ else "",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"prompt",
|
|
||||||
nargs="?",
|
|
||||||
help="The prompt for text generation (default: reads from stdin)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-m",
|
|
||||||
"--model",
|
|
||||||
default=DEFAULT_MODEL,
|
|
||||||
help=f"The GPT model to use (default: {DEFAULT_MODEL})",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-t",
|
|
||||||
"--temperature",
|
|
||||||
type=float,
|
|
||||||
default=DEFAULT_TEMPERATURE,
|
|
||||||
help=f"Temperature for text generation (default: {DEFAULT_TEMPERATURE})",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-n",
|
|
||||||
"--max-tokens",
|
|
||||||
type=int,
|
|
||||||
default=DEFAULT_MAX_TOKENS,
|
|
||||||
help=f"Maximum tokens in the response (default: {DEFAULT_MAX_TOKENS})",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-j", "--json", action="store_true", help="Output result as JSON"
|
|
||||||
)
|
|
||||||
parser.add_argument("-v", "--version", action="version", version="gpt 1.0.0")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Read from stdin or argument
|
|
||||||
if args.prompt:
|
|
||||||
prompt = args.prompt
|
|
||||||
else:
|
|
||||||
prompt = sys.stdin.read().strip()
|
|
||||||
if not prompt:
|
|
||||||
parser.print_help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Generate text
|
|
||||||
result = generate_text(
|
|
||||||
prompt=prompt,
|
|
||||||
model=args.model,
|
|
||||||
temperature=args.temperature,
|
|
||||||
max_tokens=args.max_tokens,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Output results
|
|
||||||
if args.json:
|
|
||||||
json.dump({"prompt": prompt, "result": result}, sys.stdout)
|
|
||||||
else:
|
|
||||||
print(result)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "html2text"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Convert HTML content to plain text"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
html2text = "html2text.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/html2text"]
|
|
||||||
@@ -1,319 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
html2text: Convert HTML content to plain text
|
|
||||||
|
|
||||||
Converts HTML content to readable plain text by removing HTML tags,
|
|
||||||
preserving basic structure, and formatting the content for terminal viewing.
|
|
||||||
Works well with email content.
|
|
||||||
|
|
||||||
Usage: html2text [options] [file]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
html2text email.html # Convert HTML file to text
|
|
||||||
cat email.html | html2text # Convert HTML from stdin to text
|
|
||||||
protonmail read 42 | html2text # Convert ProtonMail HTML email to text
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import argparse
|
|
||||||
import re
|
|
||||||
import html
|
|
||||||
from html.parser import HTMLParser
|
|
||||||
import textwrap
|
|
||||||
|
|
||||||
|
|
||||||
class HTMLToTextParser(HTMLParser):
|
|
||||||
"""HTML parser that converts HTML to readable plain text."""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.result = []
|
|
||||||
self.skip = False
|
|
||||||
self.in_paragraph = False
|
|
||||||
self.in_list_item = False
|
|
||||||
self.in_anchor = False
|
|
||||||
self.href = ""
|
|
||||||
self.indent_level = 0
|
|
||||||
self.list_item_num = 0
|
|
||||||
self.in_header = False
|
|
||||||
self.header_level = 0
|
|
||||||
self.in_pre = False
|
|
||||||
self.in_code = False
|
|
||||||
self.buffer = ""
|
|
||||||
|
|
||||||
def handle_starttag(self, tag, attrs):
|
|
||||||
attrs_dict = dict(attrs)
|
|
||||||
|
|
||||||
if tag == "head" or tag == "style" or tag == "script":
|
|
||||||
self.skip = True
|
|
||||||
return
|
|
||||||
|
|
||||||
if tag == "p":
|
|
||||||
if self.result and self.result[-1] != "":
|
|
||||||
self.result.append("")
|
|
||||||
self.in_paragraph = True
|
|
||||||
elif tag == "br":
|
|
||||||
self.result.append("")
|
|
||||||
elif (
|
|
||||||
tag == "h1"
|
|
||||||
or tag == "h2"
|
|
||||||
or tag == "h3"
|
|
||||||
or tag == "h4"
|
|
||||||
or tag == "h5"
|
|
||||||
or tag == "h6"
|
|
||||||
):
|
|
||||||
if self.result and self.result[-1] != "":
|
|
||||||
self.result.append("")
|
|
||||||
self.in_header = True
|
|
||||||
self.header_level = int(tag[1])
|
|
||||||
elif tag == "ul" or tag == "ol":
|
|
||||||
self.result.append("")
|
|
||||||
self.indent_level += 2
|
|
||||||
if tag == "ol":
|
|
||||||
self.list_item_num = 1
|
|
||||||
else:
|
|
||||||
self.list_item_num = 0
|
|
||||||
elif tag == "li":
|
|
||||||
self.in_list_item = True
|
|
||||||
prefix = " " * self.indent_level
|
|
||||||
if self.list_item_num > 0:
|
|
||||||
prefix += f"{self.list_item_num}. "
|
|
||||||
self.list_item_num += 1
|
|
||||||
else:
|
|
||||||
prefix += "• "
|
|
||||||
self.buffer = prefix
|
|
||||||
elif tag == "a" and "href" in attrs_dict:
|
|
||||||
self.in_anchor = True
|
|
||||||
self.href = attrs_dict["href"]
|
|
||||||
elif tag == "pre":
|
|
||||||
if self.result and self.result[-1] != "":
|
|
||||||
self.result.append("")
|
|
||||||
self.in_pre = True
|
|
||||||
elif tag == "code":
|
|
||||||
self.in_code = True
|
|
||||||
elif tag == "div":
|
|
||||||
if self.result and self.result[-1] != "":
|
|
||||||
self.result.append("")
|
|
||||||
elif tag == "blockquote":
|
|
||||||
if self.result and self.result[-1] != "":
|
|
||||||
self.result.append("")
|
|
||||||
self.indent_level += 4
|
|
||||||
self.result.append(" " * self.indent_level + "> ")
|
|
||||||
elif tag == "hr":
|
|
||||||
self.result.append("-" * 70)
|
|
||||||
|
|
||||||
def handle_endtag(self, tag):
|
|
||||||
if tag == "head" or tag == "style" or tag == "script":
|
|
||||||
self.skip = False
|
|
||||||
return
|
|
||||||
|
|
||||||
if tag == "p":
|
|
||||||
if self.result and self.buffer:
|
|
||||||
self.result.append(self.buffer)
|
|
||||||
self.buffer = ""
|
|
||||||
self.in_paragraph = False
|
|
||||||
self.result.append("")
|
|
||||||
elif (
|
|
||||||
tag == "h1"
|
|
||||||
or tag == "h2"
|
|
||||||
or tag == "h3"
|
|
||||||
or tag == "h4"
|
|
||||||
or tag == "h5"
|
|
||||||
or tag == "h6"
|
|
||||||
):
|
|
||||||
if self.buffer:
|
|
||||||
self.result.append(self.buffer)
|
|
||||||
underline = "=" if self.header_level <= 2 else "-"
|
|
||||||
self.result.append(underline * len(self.buffer))
|
|
||||||
self.buffer = ""
|
|
||||||
self.in_header = False
|
|
||||||
self.result.append("")
|
|
||||||
elif tag == "ul" or tag == "ol":
|
|
||||||
self.indent_level -= 2
|
|
||||||
self.list_item_num = 0
|
|
||||||
self.result.append("")
|
|
||||||
elif tag == "li":
|
|
||||||
if self.buffer:
|
|
||||||
self.result.append(self.buffer)
|
|
||||||
self.buffer = ""
|
|
||||||
self.in_list_item = False
|
|
||||||
elif tag == "a":
|
|
||||||
if self.in_anchor and self.href and self.buffer:
|
|
||||||
if self.href not in self.buffer:
|
|
||||||
self.buffer += f" [{self.href}]"
|
|
||||||
self.in_anchor = False
|
|
||||||
elif tag == "pre":
|
|
||||||
self.in_pre = False
|
|
||||||
self.result.append("")
|
|
||||||
elif tag == "code":
|
|
||||||
self.in_code = False
|
|
||||||
elif tag == "blockquote":
|
|
||||||
self.indent_level -= 4
|
|
||||||
self.result.append("")
|
|
||||||
|
|
||||||
def handle_data(self, data):
|
|
||||||
if self.skip:
|
|
||||||
return
|
|
||||||
|
|
||||||
# Clean and normalize whitespace
|
|
||||||
if not self.in_pre and not self.in_code:
|
|
||||||
data = " ".join(data.split())
|
|
||||||
|
|
||||||
if data.strip():
|
|
||||||
if self.in_list_item or self.in_header:
|
|
||||||
self.buffer += data
|
|
||||||
elif self.in_paragraph:
|
|
||||||
if self.buffer:
|
|
||||||
self.buffer += " " + data
|
|
||||||
else:
|
|
||||||
self.buffer = data
|
|
||||||
else:
|
|
||||||
self.result.append(data)
|
|
||||||
|
|
||||||
def handle_entityref(self, name):
|
|
||||||
if self.skip:
|
|
||||||
return
|
|
||||||
|
|
||||||
# Convert HTML entities to their corresponding characters
|
|
||||||
try:
|
|
||||||
char = html.unescape(f"&{name};")
|
|
||||||
if self.in_list_item or self.in_header or self.in_paragraph:
|
|
||||||
self.buffer += char
|
|
||||||
else:
|
|
||||||
self.result.append(char)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def handle_charref(self, name):
|
|
||||||
if self.skip:
|
|
||||||
return
|
|
||||||
|
|
||||||
# Convert character references to their corresponding characters
|
|
||||||
try:
|
|
||||||
char = html.unescape(f"&#{name};")
|
|
||||||
if self.in_list_item or self.in_header or self.in_paragraph:
|
|
||||||
self.buffer += char
|
|
||||||
else:
|
|
||||||
self.result.append(char)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def get_text(self):
|
|
||||||
"""Get the processed text with proper wrapping."""
|
|
||||||
# Process any remaining buffer
|
|
||||||
if self.buffer:
|
|
||||||
self.result.append(self.buffer)
|
|
||||||
|
|
||||||
# Join all lines, handling indentation properly
|
|
||||||
result = []
|
|
||||||
wrapper = textwrap.TextWrapper(width=80)
|
|
||||||
|
|
||||||
i = 0
|
|
||||||
while i < len(self.result):
|
|
||||||
line = self.result[i]
|
|
||||||
|
|
||||||
# Skip empty lines, but preserve paragraph breaks
|
|
||||||
if not line:
|
|
||||||
result.append("")
|
|
||||||
i += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check if it's a list item or indented text
|
|
||||||
indent_match = re.match(r"^(\s+)(.*?)$", line)
|
|
||||||
if indent_match:
|
|
||||||
indent = indent_match.group(1)
|
|
||||||
content = indent_match.group(2)
|
|
||||||
|
|
||||||
if content.startswith("• ") or re.match(r"^\d+\.\s", content):
|
|
||||||
# It's a list item, preserve the marker and indent rest of the paragraph
|
|
||||||
match = re.match(r"^([•\d]+\.?\s+)", content)
|
|
||||||
marker = match.group(1) if match else "• "
|
|
||||||
remaining = content[len(marker) :]
|
|
||||||
|
|
||||||
# Set up custom wrapper for indented text
|
|
||||||
custom_wrapper = textwrap.TextWrapper(
|
|
||||||
width=80,
|
|
||||||
initial_indent=indent + marker,
|
|
||||||
subsequent_indent=indent + " " * len(marker),
|
|
||||||
)
|
|
||||||
|
|
||||||
result.append(custom_wrapper.fill(remaining))
|
|
||||||
else:
|
|
||||||
# Regular indented text
|
|
||||||
custom_wrapper = textwrap.TextWrapper(
|
|
||||||
width=80, initial_indent=indent, subsequent_indent=indent
|
|
||||||
)
|
|
||||||
result.append(custom_wrapper.fill(content))
|
|
||||||
else:
|
|
||||||
# Regular wrapped text for paragraphs
|
|
||||||
result.append(wrapper.fill(line))
|
|
||||||
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
# Clean up multiple consecutive empty lines
|
|
||||||
clean_result = []
|
|
||||||
prev_empty = False
|
|
||||||
for line in result:
|
|
||||||
if not line and prev_empty:
|
|
||||||
continue
|
|
||||||
clean_result.append(line)
|
|
||||||
prev_empty = not line
|
|
||||||
|
|
||||||
return "\n".join(clean_result)
|
|
||||||
|
|
||||||
|
|
||||||
def convert_html_to_text(html_content):
|
|
||||||
"""Convert HTML content to readable plain text."""
|
|
||||||
parser = HTMLToTextParser()
|
|
||||||
parser.feed(html_content)
|
|
||||||
return parser.get_text()
|
|
||||||
|
|
||||||
|
|
||||||
def clean_up_text(text):
|
|
||||||
"""Clean up the text by removing excessive whitespace and line breaks."""
|
|
||||||
# Replace multiple newlines with double newlines (for paragraph separation)
|
|
||||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
||||||
return text.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Convert HTML content to plain text",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1] if __doc__ else "",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument("file", nargs="?", help="HTML file to convert (default: stdin)")
|
|
||||||
parser.add_argument(
|
|
||||||
"-w", "--width", type=int, default=80, help="Maximum line width (default: 80)"
|
|
||||||
)
|
|
||||||
parser.add_argument("-v", "--version", action="version", version="html2text 1.0.0")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Read from file or stdin
|
|
||||||
if args.file:
|
|
||||||
try:
|
|
||||||
with open(args.file, "r", encoding="utf-8") as f:
|
|
||||||
html_content = f.read()
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error reading file: {str(e)}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
else:
|
|
||||||
html_content = sys.stdin.read()
|
|
||||||
|
|
||||||
# Check if the content actually has HTML tags
|
|
||||||
if "<" in html_content and ">" in html_content:
|
|
||||||
# Convert HTML to text
|
|
||||||
text = convert_html_to_text(html_content)
|
|
||||||
text = clean_up_text(text)
|
|
||||||
print(text)
|
|
||||||
else:
|
|
||||||
# Content doesn't have HTML tags, just print it as is
|
|
||||||
print(html_content.strip())
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "mbox2eml"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Convert MBOX mailbox files to individual .eml files"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
mbox2eml = "mbox2eml.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/mbox2eml"]
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
mbox2eml: Convert MBOX mailbox files to individual .eml files
|
|
||||||
|
|
||||||
Reads an .mbox file and exports each message as a separate .eml file
|
|
||||||
with clean, descriptive filenames based on date, sender, and recipient.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
mboxer mailbox.mbox
|
|
||||||
mboxer mailbox.mbox -o output_dir/
|
|
||||||
mboxer mailbox.mbox --max-length 80
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-o, --output DIR Output directory (default: eml_output)
|
|
||||||
--max-length N Max filename length before truncation (default: 100)
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import email
|
|
||||||
import email.header
|
|
||||||
import email.utils
|
|
||||||
import mailbox
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(name: str) -> str:
|
|
||||||
"""Replace non-word characters with underscores."""
|
|
||||||
return re.sub(r"[^\w\-_.]", "_", name)
|
|
||||||
|
|
||||||
|
|
||||||
def decode_header_field(header_value: str | None) -> str:
|
|
||||||
"""Decode an email header field to a plain string."""
|
|
||||||
if header_value is None:
|
|
||||||
return ""
|
|
||||||
if isinstance(header_value, email.header.Header):
|
|
||||||
return str(header_value)
|
|
||||||
decoded_fragments = email.header.decode_header(header_value)
|
|
||||||
return "".join(
|
|
||||||
str(t[0], t[1] or "utf-8") if isinstance(t[0], bytes) else t[0]
|
|
||||||
for t in decoded_fragments
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def extract_addresses(header_value: str | None) -> list[str]:
|
|
||||||
"""Extract lowercase email addresses from a header field."""
|
|
||||||
decoded = decode_header_field(header_value)
|
|
||||||
return [
|
|
||||||
email.utils.parseaddr(addr)[1].lower()
|
|
||||||
for addr in decoded.split(",")
|
|
||||||
if addr.strip()
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def convert(mbox_path: str, output_dir: str, max_length: int = 100) -> int:
|
|
||||||
"""Convert an mbox file to individual .eml files. Returns count of exported messages."""
|
|
||||||
if not os.path.exists(mbox_path):
|
|
||||||
print(f"Error: file not found: {mbox_path}", file=sys.stderr)
|
|
||||||
return -1
|
|
||||||
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
mbox = mailbox.mbox(mbox_path)
|
|
||||||
count = 0
|
|
||||||
|
|
||||||
for i, msg in enumerate(mbox):
|
|
||||||
date_str = msg.get("date")
|
|
||||||
try:
|
|
||||||
parsed_date = email.utils.parsedate_to_datetime(date_str) # type: ignore[arg-type]
|
|
||||||
timestamp = parsed_date.strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
except Exception:
|
|
||||||
timestamp = f"unknown_date_{i:04}"
|
|
||||||
|
|
||||||
from_addr = email.utils.parseaddr(decode_header_field(msg.get("from")))[1].lower()
|
|
||||||
to_addrs = extract_addresses(msg.get("to"))
|
|
||||||
safe_sender = sanitize_filename(from_addr or "unknown")
|
|
||||||
|
|
||||||
if not to_addrs:
|
|
||||||
safe_recipient = "unknown"
|
|
||||||
elif len(to_addrs) == 1:
|
|
||||||
safe_recipient = sanitize_filename(to_addrs[0])
|
|
||||||
else:
|
|
||||||
safe_recipient = sanitize_filename(to_addrs[0]) + "_et_al"
|
|
||||||
|
|
||||||
base_name = f"{timestamp}_from_{safe_sender}_to_{safe_recipient}"
|
|
||||||
if len(base_name) > max_length:
|
|
||||||
base_name = base_name[:max_length]
|
|
||||||
filename = base_name + ".eml"
|
|
||||||
filepath = os.path.join(output_dir, filename)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(filepath, "wb") as f:
|
|
||||||
f.write(bytes(msg))
|
|
||||||
print(f"[{i + 1:04}] Saved: {filename}", file=sys.stderr)
|
|
||||||
count += 1
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[{i + 1:04}] Error saving {filename}: {e}", file=sys.stderr)
|
|
||||||
|
|
||||||
return count
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Convert MBOX mailbox files to individual .eml files",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
)
|
|
||||||
parser.add_argument("input", help="Input .mbox file")
|
|
||||||
parser.add_argument(
|
|
||||||
"-o", "--output", default="eml_output",
|
|
||||||
help="Output directory (default: eml_output)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-length", type=int, default=100,
|
|
||||||
help="Max filename length (default: 100)",
|
|
||||||
)
|
|
||||||
parser.add_argument("--version", action="version", version="mbox2eml 1.0.0")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
count = convert(args.input, args.output, args.max_length)
|
|
||||||
if count < 0:
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f"Exported {count} messages to {args.output}", file=sys.stderr)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "md2pdf"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Convert Markdown files to PDF"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
md2pdf = "md2pdf.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/md2pdf"]
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
md2pdf: Convert Markdown files to PDF
|
|
||||||
|
|
||||||
Converts Markdown files to PDF format with nice rendering of headers, lists,
|
|
||||||
code blocks, and other markdown elements. The tool uses pandoc with LaTeX
|
|
||||||
for high-quality PDF output.
|
|
||||||
|
|
||||||
The conversion process preserves markdown formatting and applies professional
|
|
||||||
styling including proper spacing, font choices, and page layout.
|
|
||||||
|
|
||||||
Usage: md2pdf file.md
|
|
||||||
md2pdf -o output.pdf file.md
|
|
||||||
cat file.md | md2pdf -o output.pdf
|
|
||||||
md2pdf file.md # Creates file.pdf in the same directory
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-o, --output FILE Write PDF to FILE instead of <input_basename>.pdf
|
|
||||||
--toc Include table of contents
|
|
||||||
--margin SIZE Set page margins (default: 1in). Examples: 1in, 2cm, 20mm
|
|
||||||
--font-size SIZE Set base font size (default: 11pt). Examples: 10pt, 12pt
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="Convert Markdown files to PDF")
|
|
||||||
parser.add_argument(
|
|
||||||
"input_file", nargs="?", help="Input Markdown file (or read from stdin)"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-o", "--output", help="Output PDF file (default: <input_basename>.pdf)"
|
|
||||||
)
|
|
||||||
parser.add_argument("--toc", action="store_true", help="Include table of contents")
|
|
||||||
parser.add_argument("--margin", default="1in", help="Page margins (default: 1in)")
|
|
||||||
parser.add_argument(
|
|
||||||
"--font-size", default="11pt", help="Base font size (default: 11pt)"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Determine input source.
|
|
||||||
if args.input_file:
|
|
||||||
input_file = args.input_file
|
|
||||||
if not os.path.isfile(input_file):
|
|
||||||
print(f"Error: File '{input_file}' not found", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
if args.output:
|
|
||||||
output_file = args.output
|
|
||||||
else:
|
|
||||||
basename = os.path.splitext(os.path.basename(input_file))[0]
|
|
||||||
output_file = f"{basename}.pdf"
|
|
||||||
else:
|
|
||||||
# Reading from stdin
|
|
||||||
if not args.output:
|
|
||||||
print(
|
|
||||||
"Error: When reading from stdin, you must specify an output file with -o",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
input_file = None
|
|
||||||
output_file = args.output
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Build pandoc command with options for nice PDF rendering.
|
|
||||||
pandoc_command = ["pandoc"]
|
|
||||||
|
|
||||||
if input_file:
|
|
||||||
pandoc_command.append(input_file)
|
|
||||||
else:
|
|
||||||
pandoc_command.append("-") # Read from stdin
|
|
||||||
|
|
||||||
pandoc_command.extend(
|
|
||||||
[
|
|
||||||
"-f",
|
|
||||||
"markdown",
|
|
||||||
"-t",
|
|
||||||
"pdf",
|
|
||||||
"-o",
|
|
||||||
output_file,
|
|
||||||
"--pdf-engine=pdflatex",
|
|
||||||
"-V",
|
|
||||||
f"geometry:margin={args.margin}",
|
|
||||||
"-V",
|
|
||||||
f"fontsize={args.font_size}",
|
|
||||||
"-V",
|
|
||||||
"colorlinks=true",
|
|
||||||
"-V",
|
|
||||||
"linkcolor=blue",
|
|
||||||
"-V",
|
|
||||||
"urlcolor=blue",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Add table of contents if requested.
|
|
||||||
if args.toc:
|
|
||||||
pandoc_command.append("--toc")
|
|
||||||
pandoc_command.extend(["--toc-depth", "3"])
|
|
||||||
|
|
||||||
# Run pandoc conversion.
|
|
||||||
subprocess.run(
|
|
||||||
pandoc_command,
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
input=None if input_file else sys.stdin.read(),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Print success message to stderr.
|
|
||||||
print(f"Created '{output_file}'", file=sys.stderr)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"Error during conversion: {e}", file=sys.stderr)
|
|
||||||
if e.stderr:
|
|
||||||
print(e.stderr, file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
except FileNotFoundError:
|
|
||||||
print(
|
|
||||||
"Error: pandoc is not installed. Please install it with 'apt install pandoc texlive-latex-base'",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "mdscraper"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Combine text files into a single markdown document"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
mdscraper = "mdscraper.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/mdscraper"]
|
|
||||||
@@ -1,259 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
mdscraper: Combine text files into a single markdown document
|
|
||||||
|
|
||||||
Scans a directory for text files and combines them into a single markdown document.
|
|
||||||
Supports filtering through a .gitignore-style configuration file.
|
|
||||||
|
|
||||||
Usage: mdscraper [options] [directory]
|
|
||||||
mdscraper -o output.md [directory]
|
|
||||||
mdscraper --ignore-file .mdscraper [directory]
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-o, --output FILE Write markdown to FILE instead of stdout
|
|
||||||
-i, --ignore-file FILE Use specified ignore file instead of .mdscraper
|
|
||||||
-r, --recursive Recursively process subdirectories
|
|
||||||
--include-path Include file paths as headers in the output
|
|
||||||
--no-headers Don't add file names as headers
|
|
||||||
--toc Add table of contents at the beginning
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
def parse_ignore_file(ignore_file_path):
|
|
||||||
"""Parse a .gitignore style file into a list of patterns."""
|
|
||||||
if not os.path.exists(ignore_file_path):
|
|
||||||
return []
|
|
||||||
|
|
||||||
patterns = []
|
|
||||||
with open(ignore_file_path, "r") as f:
|
|
||||||
for line in f:
|
|
||||||
line = line.strip()
|
|
||||||
# Skip empty lines and comments
|
|
||||||
if not line or line.startswith("#"):
|
|
||||||
continue
|
|
||||||
patterns.append(line)
|
|
||||||
return patterns
|
|
||||||
|
|
||||||
|
|
||||||
def should_ignore(path, patterns):
|
|
||||||
"""Determine if a file should be ignored based on patterns."""
|
|
||||||
# Always ignore hidden files
|
|
||||||
if os.path.basename(path).startswith("."):
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Check for directory patterns - if path contains a directory that should be ignored
|
|
||||||
if "/" in path:
|
|
||||||
path_parts = path.split("/")
|
|
||||||
for i in range(len(path_parts)):
|
|
||||||
partial_path = "/".join(path_parts[: i + 1]) + "/"
|
|
||||||
for pattern in patterns:
|
|
||||||
if (
|
|
||||||
not pattern.startswith("!")
|
|
||||||
and pattern.endswith("/")
|
|
||||||
and fnmatch.fnmatch(partial_path, pattern)
|
|
||||||
):
|
|
||||||
# Check if there's a negation pattern that overrides
|
|
||||||
negation_overrides = False
|
|
||||||
for neg_pattern in patterns:
|
|
||||||
if neg_pattern.startswith("!") and fnmatch.fnmatch(
|
|
||||||
path, neg_pattern[1:]
|
|
||||||
):
|
|
||||||
negation_overrides = True
|
|
||||||
break
|
|
||||||
if not negation_overrides:
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Check if path matches any negation pattern first
|
|
||||||
for pattern in patterns:
|
|
||||||
if pattern.startswith("!"):
|
|
||||||
if fnmatch.fnmatch(path, pattern[1:]):
|
|
||||||
# If it matches a negation pattern, do not ignore
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Then check if it matches any regular pattern
|
|
||||||
for pattern in patterns:
|
|
||||||
if not pattern.startswith("!") and not pattern.endswith("/"):
|
|
||||||
if fnmatch.fnmatch(path, pattern):
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Default: don't ignore
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def scan_directory(directory, patterns, recursive=False):
|
|
||||||
"""Scan a directory for text files, filtering by ignore patterns."""
|
|
||||||
files = []
|
|
||||||
|
|
||||||
for item in os.listdir(directory):
|
|
||||||
path = os.path.join(directory, item)
|
|
||||||
rel_path = os.path.relpath(path, directory)
|
|
||||||
|
|
||||||
if should_ignore(rel_path, patterns):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if os.path.isfile(path):
|
|
||||||
# Only include text files
|
|
||||||
try:
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
f.read(1024) # Try to read a bit to check if it's text
|
|
||||||
files.append(path)
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
# Skip binary files
|
|
||||||
continue
|
|
||||||
elif os.path.isdir(path) and recursive:
|
|
||||||
# For directories, check if the directory itself should be ignored
|
|
||||||
dir_path = rel_path + "/"
|
|
||||||
if any(
|
|
||||||
pattern.endswith("/") and fnmatch.fnmatch(dir_path, pattern)
|
|
||||||
for pattern in patterns
|
|
||||||
if not pattern.startswith("!")
|
|
||||||
):
|
|
||||||
# Directory matches an ignore pattern, skip it
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Process subdirectories recursively if requested
|
|
||||||
subdir_files = scan_directory(path, patterns, recursive)
|
|
||||||
files.extend(subdir_files)
|
|
||||||
|
|
||||||
# Sort files for consistent output
|
|
||||||
return sorted(files)
|
|
||||||
|
|
||||||
|
|
||||||
def extract_file_content(file_path):
|
|
||||||
"""Extract content from a file."""
|
|
||||||
try:
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
return f.read()
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
return f"ERROR: Could not decode {file_path} as UTF-8"
|
|
||||||
|
|
||||||
|
|
||||||
def create_toc(files, base_dir):
|
|
||||||
"""Create a table of contents from the list of files."""
|
|
||||||
toc = ["# Table of Contents\n"]
|
|
||||||
|
|
||||||
for file_path in files:
|
|
||||||
rel_path = os.path.relpath(file_path, base_dir)
|
|
||||||
file_name = os.path.basename(file_path)
|
|
||||||
# Create a GitHub-style anchor link
|
|
||||||
anchor = file_name.lower().replace(" ", "-")
|
|
||||||
anchor = re.sub(r"[^\w-]", "", anchor)
|
|
||||||
toc.append(f"- [{rel_path}](#{anchor})")
|
|
||||||
|
|
||||||
return "\n".join(toc) + "\n\n"
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Combine text files into a single markdown document"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"directory",
|
|
||||||
nargs="?",
|
|
||||||
default=".",
|
|
||||||
help="Directory to process (default: current directory)",
|
|
||||||
)
|
|
||||||
parser.add_argument("-o", "--output", help="Output markdown file (default: stdout)")
|
|
||||||
parser.add_argument(
|
|
||||||
"-i",
|
|
||||||
"--ignore-file",
|
|
||||||
default=".mdscraper",
|
|
||||||
help="Gitignore-style file with patterns to ignore (default: .mdscraper)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-r",
|
|
||||||
"--recursive",
|
|
||||||
action="store_true",
|
|
||||||
help="Recursively process subdirectories",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--include-path", action="store_true", help="Include file paths as headers"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--no-headers", action="store_true", help="Don't add file names as headers"
|
|
||||||
)
|
|
||||||
parser.add_argument("--toc", action="store_true", help="Add table of contents")
|
|
||||||
parser.add_argument("--version", action="version", version="mdscraper 1.0.0")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Resolve the directory path
|
|
||||||
directory = os.path.abspath(args.directory)
|
|
||||||
if not os.path.isdir(directory):
|
|
||||||
print(f"Error: {directory} is not a directory", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Load ignore patterns
|
|
||||||
ignore_file = args.ignore_file
|
|
||||||
if not os.path.isabs(ignore_file):
|
|
||||||
# First check if ignore file exists in specified directory
|
|
||||||
dir_ignore_file = os.path.join(directory, ignore_file)
|
|
||||||
if os.path.exists(dir_ignore_file):
|
|
||||||
ignore_file = dir_ignore_file
|
|
||||||
|
|
||||||
ignore_patterns = parse_ignore_file(ignore_file)
|
|
||||||
|
|
||||||
# Always ignore the output file if specified
|
|
||||||
if args.output:
|
|
||||||
output_basename = os.path.basename(args.output)
|
|
||||||
ignore_patterns.append(output_basename)
|
|
||||||
|
|
||||||
# Scan for files
|
|
||||||
files = scan_directory(directory, ignore_patterns, args.recursive)
|
|
||||||
|
|
||||||
if not files:
|
|
||||||
print(f"Warning: No text files found in {directory}", file=sys.stderr)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# Build the markdown content
|
|
||||||
content = []
|
|
||||||
|
|
||||||
# Add table of contents if requested
|
|
||||||
if args.toc:
|
|
||||||
content.append(create_toc(files, directory))
|
|
||||||
|
|
||||||
# Process each file
|
|
||||||
for file_path in files:
|
|
||||||
rel_path = os.path.relpath(file_path, directory)
|
|
||||||
file_name = os.path.basename(file_path)
|
|
||||||
|
|
||||||
# Add header unless disabled
|
|
||||||
if not args.no_headers:
|
|
||||||
if args.include_path:
|
|
||||||
content.append(f"# {rel_path}\n")
|
|
||||||
else:
|
|
||||||
content.append(f"# {file_name}\n")
|
|
||||||
|
|
||||||
# Add file content
|
|
||||||
file_content = extract_file_content(file_path)
|
|
||||||
content.append(file_content)
|
|
||||||
|
|
||||||
# Add separator between files
|
|
||||||
content.append("\n---\n")
|
|
||||||
|
|
||||||
# Remove the last separator if it exists
|
|
||||||
if content and content[-1] == "\n---\n":
|
|
||||||
content.pop()
|
|
||||||
|
|
||||||
# Join all content
|
|
||||||
markdown = "\n".join(content)
|
|
||||||
|
|
||||||
# Output the result
|
|
||||||
if args.output:
|
|
||||||
with open(args.output, "w", encoding="utf-8") as f:
|
|
||||||
f.write(markdown)
|
|
||||||
print(f"Created '{args.output}'", file=sys.stderr)
|
|
||||||
else:
|
|
||||||
print(markdown)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
Submodule components/notification-bridge deleted from 360a409ca4
@@ -1,18 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "pdf-extractor"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Extract content and metadata from PDF files"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = [
|
|
||||||
"pymupdf>=1.24.11",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
pdf-extractor = "pdf_extractor.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/pdf_extractor"]
|
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
pdf-extractor: Extract content and metadata from PDF files
|
|
||||||
|
|
||||||
Reads a PDF file and outputs a JSON object containing the file's content
|
|
||||||
and metadata including filename, size, page count, title, author, etc.
|
|
||||||
|
|
||||||
Usage: pdf-extractor [options] [FILE]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
pdf-extractor document.pdf # Extract from a PDF file
|
|
||||||
cat document.pdf | pdf-extractor # Read from stdin (if applicable)
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import fitz # PyMuPDF
|
|
||||||
from datetime import datetime
|
|
||||||
import mimetypes
|
|
||||||
from pathlib import Path
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
|
||||||
def get_file_metadata(file_path):
|
|
||||||
"""Get file metadata including creation time, modification time, size, etc."""
|
|
||||||
path = Path(file_path)
|
|
||||||
stat = path.stat()
|
|
||||||
|
|
||||||
# Get file modification and creation times
|
|
||||||
mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
|
|
||||||
try:
|
|
||||||
ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
|
|
||||||
except Exception:
|
|
||||||
ctime = mtime # Fallback if creation time not available
|
|
||||||
|
|
||||||
# Get mime type
|
|
||||||
mime_type, _ = mimetypes.guess_type(file_path)
|
|
||||||
if not mime_type:
|
|
||||||
mime_type = "application/pdf" # Default to PDF
|
|
||||||
|
|
||||||
return {
|
|
||||||
"filename": path.name,
|
|
||||||
"path": str(path.absolute()),
|
|
||||||
"size": stat.st_size,
|
|
||||||
"created_at": ctime,
|
|
||||||
"modified_at": mtime,
|
|
||||||
"mime_type": mime_type,
|
|
||||||
"extension": path.suffix.lstrip(".") if path.suffix else "pdf",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def extract_pdf_content(file_path):
|
|
||||||
"""Extract content and metadata from a PDF file."""
|
|
||||||
try:
|
|
||||||
doc = fitz.open(file_path)
|
|
||||||
|
|
||||||
# Extract PDF metadata
|
|
||||||
metadata = doc.metadata if doc.metadata else {}
|
|
||||||
pdf_metadata = {
|
|
||||||
"page_count": doc.page_count,
|
|
||||||
"title": metadata.get("title", ""),
|
|
||||||
"author": metadata.get("author", ""),
|
|
||||||
"subject": metadata.get("subject", ""),
|
|
||||||
"keywords": metadata.get("keywords", ""),
|
|
||||||
"creator": metadata.get("creator", ""),
|
|
||||||
"producer": metadata.get("producer", ""),
|
|
||||||
"creation_date": metadata.get("creationDate", ""),
|
|
||||||
"modification_date": metadata.get("modDate", ""),
|
|
||||||
}
|
|
||||||
|
|
||||||
# Extract text content from all pages
|
|
||||||
content = ""
|
|
||||||
for page_num in range(doc.page_count):
|
|
||||||
page = doc[page_num]
|
|
||||||
content += page.get_text() # type: ignore
|
|
||||||
content += "\n\n" # Add spacing between pages
|
|
||||||
|
|
||||||
doc.close()
|
|
||||||
return content, pdf_metadata
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error extracting PDF content: {e}\n")
|
|
||||||
return "", {}
|
|
||||||
|
|
||||||
|
|
||||||
def process_file(file_path):
|
|
||||||
"""Extract content and metadata from the file."""
|
|
||||||
# Check if we're dealing with stdin or a file
|
|
||||||
temp_file = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not file_path or file_path == "-":
|
|
||||||
# Reading from stdin, save to temporary file
|
|
||||||
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
|
|
||||||
temp_file.write(sys.stdin.buffer.read())
|
|
||||||
temp_file.close()
|
|
||||||
file_path = temp_file.name
|
|
||||||
|
|
||||||
# Create basic metadata for stdin input
|
|
||||||
metadata = {
|
|
||||||
"filename": "stdin",
|
|
||||||
"path": "stdin",
|
|
||||||
"size": os.path.getsize(file_path),
|
|
||||||
"created_at": datetime.now().isoformat(),
|
|
||||||
"modified_at": datetime.now().isoformat(),
|
|
||||||
"mime_type": "application/pdf",
|
|
||||||
"extension": "pdf",
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
# Reading from a file
|
|
||||||
if not os.path.exists(file_path):
|
|
||||||
sys.stderr.write(f"Error: File '{file_path}' not found.\n")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
# Get file metadata
|
|
||||||
metadata = get_file_metadata(file_path)
|
|
||||||
|
|
||||||
# Extract PDF content and PDF-specific metadata
|
|
||||||
content, pdf_metadata = extract_pdf_content(file_path)
|
|
||||||
|
|
||||||
# Create result object
|
|
||||||
result = {
|
|
||||||
**metadata,
|
|
||||||
**pdf_metadata,
|
|
||||||
"content": content,
|
|
||||||
"title": pdf_metadata.get("title") or metadata["filename"],
|
|
||||||
"tags": [tag.strip() for tag in pdf_metadata.get("keywords", "").split(",")]
|
|
||||||
if pdf_metadata.get("keywords")
|
|
||||||
else [],
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
finally:
|
|
||||||
# Clean up the temporary file if one was created
|
|
||||||
if temp_file and os.path.exists(temp_file.name):
|
|
||||||
os.unlink(temp_file.name)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Extract content and metadata from PDF files",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1]
|
|
||||||
if __doc__
|
|
||||||
else "", # Use the docstring as extended help
|
|
||||||
)
|
|
||||||
parser.add_argument("file", nargs="?", help="Input file (default: stdin)")
|
|
||||||
parser.add_argument(
|
|
||||||
"-v", "--version", action="version", version="pdf-extractor 1.0.0"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Process the file
|
|
||||||
result = process_file(args.file)
|
|
||||||
|
|
||||||
# Output the result as JSON
|
|
||||||
print(json.dumps(result, indent=2))
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error: {e}\n")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "pdf2md"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Convert PDF files to Markdown"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
pdf2md = "pdf2md.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/pdf2md"]
|
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
pdf2md: Convert PDF files to Markdown
|
|
||||||
|
|
||||||
Converts PDF files to Markdown format and writes the result to a file with
|
|
||||||
the same basename but .md extension. Also creates a directory for extracted
|
|
||||||
media assets named <basename>_media/. The markdown content is also sent to stdout.
|
|
||||||
|
|
||||||
The conversion process uses poppler-utils (pdftotext and pdfimages) to extract
|
|
||||||
text and images from the PDF, then uses pandoc to convert the text to Markdown.
|
|
||||||
The output preserves as much of the original layout as possible.
|
|
||||||
|
|
||||||
Usage: pdf2md file.pdf
|
|
||||||
pdf2md -o output.md file.pdf
|
|
||||||
pdf2md -m custom_media_dir file.pdf
|
|
||||||
pdf2md -o output.md -m custom_media_dir file.pdf
|
|
||||||
pdf2md file.pdf > another_file.md
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-o, --output FILE Write markdown to FILE instead of <input_basename>.md
|
|
||||||
-m, --media-dir DIR Store extracted media files in DIR instead of <basename>_media
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(description="Convert PDF files to Markdown")
|
|
||||||
parser.add_argument("input_file", help="Input PDF file")
|
|
||||||
parser.add_argument(
|
|
||||||
"-o", "--output", help="Output markdown file (default: <input_basename>.md)"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-m",
|
|
||||||
"--media-dir",
|
|
||||||
help="Directory for extracted media files (default: <basename>_media)",
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not args.input_file.endswith(".pdf"):
|
|
||||||
print("Error: Input file must be a PDF file", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
input_file = args.input_file
|
|
||||||
|
|
||||||
if args.output:
|
|
||||||
output_file = args.output
|
|
||||||
# Extract the basename from the output file for media directory name
|
|
||||||
media_basename = os.path.splitext(os.path.basename(output_file))[0]
|
|
||||||
else:
|
|
||||||
basename = os.path.splitext(os.path.basename(input_file))[0]
|
|
||||||
output_file = f"{basename}.md"
|
|
||||||
media_basename = basename
|
|
||||||
|
|
||||||
# Set media directory
|
|
||||||
if args.media_dir:
|
|
||||||
media_dir = args.media_dir
|
|
||||||
else:
|
|
||||||
media_dir = f"{media_basename}_media"
|
|
||||||
|
|
||||||
# Create media directory if it doesn't exist
|
|
||||||
os.makedirs(media_dir, exist_ok=True)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Create a temporary directory for conversion
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
|
||||||
# First use pdftotext (from poppler-utils) to extract text
|
|
||||||
temp_txt = os.path.join(temp_dir, "temp_content.txt")
|
|
||||||
# Use pdftotext with -nopgbrk to avoid page breaks that can mess up tables
|
|
||||||
subprocess.run(
|
|
||||||
["pdftotext", "-nopgbrk", input_file, temp_txt],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Process the text file to clean up formatting issues
|
|
||||||
with open(temp_txt, "r") as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# Remove common footer patterns
|
|
||||||
content = re.sub(
|
|
||||||
r"(?m)^\s*\d+\s+The Social Issues Research Centre.*$", "", content
|
|
||||||
)
|
|
||||||
|
|
||||||
# Replace excessive spaces with reasonable indentation
|
|
||||||
content = re.sub(r"(?m)^(\s{6,})", " ", content)
|
|
||||||
|
|
||||||
# Clean up empty lines
|
|
||||||
content = re.sub(r"\n{3,}", "\n\n", content)
|
|
||||||
|
|
||||||
# Write cleaned content back to file
|
|
||||||
with open(temp_txt, "w") as f:
|
|
||||||
f.write(content)
|
|
||||||
|
|
||||||
# Extract images using pdfimages
|
|
||||||
subprocess.run(
|
|
||||||
["pdfimages", "-j", input_file, os.path.join(media_dir, "image")],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Then convert text to Markdown using pandoc with table support
|
|
||||||
result = subprocess.run(
|
|
||||||
[
|
|
||||||
"pandoc",
|
|
||||||
temp_txt,
|
|
||||||
"-f",
|
|
||||||
"markdown+simple_tables+table_captions+yaml_metadata_block",
|
|
||||||
"-t",
|
|
||||||
"markdown_github",
|
|
||||||
"--wrap=none",
|
|
||||||
"--standalone",
|
|
||||||
],
|
|
||||||
check=True,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Process the markdown to further clean it up
|
|
||||||
markdown = result.stdout
|
|
||||||
|
|
||||||
# Fix table formatting if needed
|
|
||||||
markdown = re.sub(r"\|-+\|\n\|-+\|", "", markdown)
|
|
||||||
|
|
||||||
# Write markdown to file
|
|
||||||
with open(output_file, "w") as f:
|
|
||||||
f.write(markdown)
|
|
||||||
|
|
||||||
# Also output to stdout
|
|
||||||
print(markdown)
|
|
||||||
|
|
||||||
# Print message to stderr about file creation
|
|
||||||
print(
|
|
||||||
f"Created '{output_file}' with media in '{media_dir}'"
|
|
||||||
+ ("" if media_dir.endswith("/") else "/"),
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"Error during conversion: {e}", file=sys.stderr)
|
|
||||||
if e.stderr:
|
|
||||||
print(e.stderr, file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
except FileNotFoundError as e:
|
|
||||||
if "pdftohtml" in str(e):
|
|
||||||
print(
|
|
||||||
"Error: pdftohtml is not installed. Please install it with 'apt install poppler-utils'",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print(
|
|
||||||
"Error: pandoc is not installed. Please install it with 'apt install pandoc'",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
# CLAUDE.md
|
|
||||||
|
|
||||||
This file provides guidance to Claude Code (claude.ai/code) when working
|
|
||||||
with code in this repository.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
protonmail is a CLI tool for accessing and managing ProtonMail emails via Bridge.
|
|
||||||
All commands work with local cached .eml files for fast, offline access.
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv sync # Install dependencies
|
|
||||||
uv run protonmail --version # Show version
|
|
||||||
uv run protonmail sync # Sync emails from Bridge
|
|
||||||
uv run protonmail list # List cached emails
|
|
||||||
uv run protonmail read <file> # Read a specific email
|
|
||||||
uv run protonmail search <query> # Search by subject/sender
|
|
||||||
uv run pytest tests/ -v # Run tests
|
|
||||||
uv run ruff check . # Lint
|
|
||||||
uv run ruff format . # Format
|
|
||||||
```
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
- `src/protonmail/main.py` — CLI entry point, all email operations
|
|
||||||
- `src/protonmail/__init__.py` — Package version (`__version__`)
|
|
||||||
- `tests/` — pytest tests
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Environment variables (set by castle or manually):
|
|
||||||
- `PROTONMAIL_USERNAME` — ProtonMail Bridge username
|
|
||||||
- `PROTONMAIL_API_KEY` — ProtonMail Bridge API key
|
|
||||||
- `PROTONMAIL_DATA_DIR` — Where to store synced .eml files (default: /data/messages/email/protonmail)
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
|
|
||||||
stdlib-only — no third-party packages required.
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
# protonmail
|
|
||||||
|
|
||||||
CLI tool for syncing and managing ProtonMail emails via ProtonMail Bridge. Downloads emails as `.eml` files for fast, offline access. All read/list/search commands work against the local cache.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
[ProtonMail Bridge](https://proton.me/mail/bridge) must be running locally with IMAP enabled (default port 1143).
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Sync all emails from Bridge to local .eml files
|
|
||||||
protonmail sync
|
|
||||||
|
|
||||||
# Sync to a custom directory
|
|
||||||
protonmail sync -o ~/emails
|
|
||||||
|
|
||||||
# List recent emails (from local cache)
|
|
||||||
protonmail list
|
|
||||||
protonmail list Sent
|
|
||||||
protonmail list -n 50
|
|
||||||
|
|
||||||
# Read a specific email
|
|
||||||
protonmail read "2024-07-17_01-14-40_from_alice_example_com_to_bob_example_com.eml"
|
|
||||||
|
|
||||||
# Search by subject or sender
|
|
||||||
protonmail search "invoice"
|
|
||||||
protonmail search "alice" -f Sent
|
|
||||||
|
|
||||||
# Send an email (interactive, requires Bridge)
|
|
||||||
protonmail send
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Set via environment variables (castle manages these automatically):
|
|
||||||
|
|
||||||
| Variable | Description | Default |
|
|
||||||
|----------|-------------|---------|
|
|
||||||
| `PROTONMAIL_USERNAME` | Bridge username (required) | - |
|
|
||||||
| `PROTONMAIL_API_KEY` | Bridge API key (required) | - |
|
|
||||||
| `PROTONMAIL_DATA_DIR` | Where to store synced .eml files | `/data/messages/email/protonmail` |
|
|
||||||
|
|
||||||
In castle, the API key is stored as a secret:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
env:
|
|
||||||
PROTONMAIL_API_KEY: ${secret:PROTONMAIL_API_KEY}
|
|
||||||
```
|
|
||||||
|
|
||||||
## How sync works
|
|
||||||
|
|
||||||
1. Connects to Bridge via IMAP on `127.0.0.1:1143`
|
|
||||||
2. Lists all folders (INBOX, Sent, Drafts, etc.)
|
|
||||||
3. For each folder, downloads new messages as `.eml` files
|
|
||||||
4. Skips emails already in the local cache (matched by Message-ID)
|
|
||||||
5. Filenames follow the pattern: `YYYY-MM-DD_HH-MM-SS_from_SENDER_to_RECIPIENT_SUBJECT.eml`
|
|
||||||
|
|
||||||
## Castle integration
|
|
||||||
|
|
||||||
Registered as both a **tool** (installed to PATH) and a **job** (syncs every 5 minutes via systemd timer).
|
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv sync # Install dependencies
|
|
||||||
uv run protonmail --version # Verify
|
|
||||||
uv run pytest tests/ -v # Run tests
|
|
||||||
uv run ruff check . # Lint
|
|
||||||
```
|
|
||||||
|
|
||||||
No third-party dependencies -- stdlib only.
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "protonmail"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "ProtonMail email sync via Bridge"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
protonmail = "protonmail.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/protonmail"]
|
|
||||||
|
|
||||||
[dependency-groups]
|
|
||||||
dev = [
|
|
||||||
"pytest>=7.0.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[tool.ruff.lint.isort]
|
|
||||||
known-first-party = ["protonmail"]
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
"""ProtonMail email sync via Bridge."""
|
|
||||||
|
|
||||||
__version__ = "0.1.0"
|
|
||||||
@@ -1,598 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""protonmail: Access and manage ProtonMail emails via Bridge.
|
|
||||||
|
|
||||||
Access your ProtonMail inbox through the ProtonMail Bridge using IMAP/SMTP.
|
|
||||||
All commands now work with local cached .eml files for fast, offline access.
|
|
||||||
Run 'protonmail sync' first to download emails locally.
|
|
||||||
|
|
||||||
Usage: protonmail [options] [command]
|
|
||||||
|
|
||||||
Commands:
|
|
||||||
sync Sync all emails to local .eml files
|
|
||||||
list [folder] List emails (default: INBOX, uses local cache)
|
|
||||||
read <filename> Read a specific email by filename (uses local cache)
|
|
||||||
send Send a new email (interactive, requires IMAP)
|
|
||||||
search <query> Search emails by subject or sender (uses local cache)
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
protonmail sync # Sync all emails
|
|
||||||
protonmail sync -o ~/emails # Sync to custom directory
|
|
||||||
protonmail list # List emails in INBOX from local cache
|
|
||||||
protonmail list Sent # List emails in Sent folder
|
|
||||||
protonmail read "2024-07-17_01-14-40_from_..._.eml"
|
|
||||||
protonmail search "invoice" # Search by subject or sender
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import email
|
|
||||||
import imaplib
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import smtplib
|
|
||||||
import sys
|
|
||||||
from email.message import EmailMessage, Message
|
|
||||||
from email.utils import formatdate, make_msgid, parsedate_to_datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from protonmail import __version__
|
|
||||||
|
|
||||||
__all__ = ["main"]
|
|
||||||
|
|
||||||
|
|
||||||
def load_config() -> dict[str, Any]:
|
|
||||||
"""Load configuration from environment variables."""
|
|
||||||
username = os.environ.get("PROTONMAIL_USERNAME", "")
|
|
||||||
if not username:
|
|
||||||
print(
|
|
||||||
"Error: PROTONMAIL_USERNAME environment variable not set", file=sys.stderr
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
api_key = os.environ.get("PROTONMAIL_API_KEY", "")
|
|
||||||
if not api_key:
|
|
||||||
print("Error: PROTONMAIL_API_KEY environment variable not set", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
config: dict[str, Any] = {
|
|
||||||
"IMAP": {
|
|
||||||
"hostname": "127.0.0.1",
|
|
||||||
"port": 1143,
|
|
||||||
"username": username,
|
|
||||||
"password": api_key,
|
|
||||||
"security": "STARTTLS",
|
|
||||||
},
|
|
||||||
"SMTP": {
|
|
||||||
"hostname": "127.0.0.1",
|
|
||||||
"port": 1025,
|
|
||||||
"username": username,
|
|
||||||
"password": api_key,
|
|
||||||
"security": "STARTTLS",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def connect_imap(config: dict[str, Any]) -> imaplib.IMAP4:
|
|
||||||
"""Connect to IMAP server."""
|
|
||||||
imap_config = config["IMAP"]
|
|
||||||
|
|
||||||
if imap_config["security"] == "SSL/TLS":
|
|
||||||
imap = imaplib.IMAP4_SSL(imap_config["hostname"], imap_config["port"])
|
|
||||||
else:
|
|
||||||
imap = imaplib.IMAP4(imap_config["hostname"], imap_config["port"])
|
|
||||||
if imap_config["security"] == "STARTTLS":
|
|
||||||
imap.starttls()
|
|
||||||
|
|
||||||
imap.login(imap_config["username"], imap_config["password"])
|
|
||||||
return imap
|
|
||||||
|
|
||||||
|
|
||||||
def list_folders(imap: imaplib.IMAP4) -> list[str]:
|
|
||||||
"""List all available folders."""
|
|
||||||
status, folders = imap.list()
|
|
||||||
if status != "OK":
|
|
||||||
return []
|
|
||||||
|
|
||||||
folder_list: list[str] = []
|
|
||||||
for folder in folders:
|
|
||||||
if isinstance(folder, bytes):
|
|
||||||
folder_str = folder.decode()
|
|
||||||
else:
|
|
||||||
folder_str = str(folder)
|
|
||||||
folder_parts = folder_str.split(' "/"')
|
|
||||||
if len(folder_parts) > 1:
|
|
||||||
folder_name = folder_parts[-1].strip().strip('"')
|
|
||||||
folder_list.append(folder_name)
|
|
||||||
|
|
||||||
return folder_list
|
|
||||||
|
|
||||||
|
|
||||||
def get_sync_dir() -> Path:
|
|
||||||
"""Get the sync directory path.
|
|
||||||
|
|
||||||
Checks PROTONMAIL_DATA_DIR first (set by castle), then DDATA fallback.
|
|
||||||
"""
|
|
||||||
data_dir = os.environ.get("PROTONMAIL_DATA_DIR")
|
|
||||||
if data_dir:
|
|
||||||
return Path(data_dir)
|
|
||||||
ddata = os.environ.get("DDATA", "/data")
|
|
||||||
return Path(f"{ddata}/messages/email/protonmail")
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(filename: str) -> str:
|
|
||||||
"""Sanitize a string to be used as a filename."""
|
|
||||||
filename = re.sub(r'[<>:"/\\|?*]', "_", filename)
|
|
||||||
filename = re.sub(r"[\x00-\x1f\x7f]", "", filename)
|
|
||||||
if len(filename) > 200:
|
|
||||||
filename = filename[:200]
|
|
||||||
return filename.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_email(email_addr: str) -> str:
|
|
||||||
"""Sanitize email address for use in filename."""
|
|
||||||
email_addr = email_addr.replace("@", "_")
|
|
||||||
email_addr = re.sub(r'[<>"\']', "", email_addr)
|
|
||||||
email_addr = re.sub(r"[/\\:*?|]", "_", email_addr)
|
|
||||||
return email_addr.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def extract_email_address(email_field: str) -> str:
|
|
||||||
"""Extract just the email address from a field like 'Name <email@domain.com>'."""
|
|
||||||
match = re.search(r"<([^>]+)>", email_field)
|
|
||||||
if match:
|
|
||||||
return match.group(1)
|
|
||||||
if "@" in email_field:
|
|
||||||
return email_field.strip().strip("\"'")
|
|
||||||
return email_field.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def generate_email_filename(msg: Message, max_length: int = 255) -> str:
|
|
||||||
"""Generate filename in format: YYYY-MM-DD_HH-MM-SS_from_FROM_to_TO_SUBJECT.eml."""
|
|
||||||
msg_date = msg.get("Date", "")
|
|
||||||
try:
|
|
||||||
date_obj = parsedate_to_datetime(msg_date)
|
|
||||||
date_str = date_obj.strftime("%Y-%m-%d_%H-%M-%S")
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
date_str = "unknown-date"
|
|
||||||
|
|
||||||
from_field = msg.get("From", "unknown")
|
|
||||||
from_email = extract_email_address(from_field)
|
|
||||||
from_safe = sanitize_email(from_email)
|
|
||||||
|
|
||||||
to_field = msg.get("To", "unknown")
|
|
||||||
if "," in to_field:
|
|
||||||
to_field = to_field.split(",")[0]
|
|
||||||
to_email = extract_email_address(to_field)
|
|
||||||
to_safe = sanitize_email(to_email)
|
|
||||||
|
|
||||||
subject = msg.get("Subject", "no-subject")
|
|
||||||
subject_safe = sanitize_filename(subject)
|
|
||||||
|
|
||||||
prefix = f"{date_str}_from_{from_safe}_to_{to_safe}_"
|
|
||||||
suffix = ".eml"
|
|
||||||
available_length = max_length - len(prefix) - len(suffix)
|
|
||||||
|
|
||||||
if len(subject_safe) > available_length:
|
|
||||||
subject_safe = subject_safe[:available_length]
|
|
||||||
|
|
||||||
return f"{prefix}{subject_safe}{suffix}"
|
|
||||||
|
|
||||||
|
|
||||||
def list_emails_local(folder: str = "INBOX", limit: int = 20) -> bool:
|
|
||||||
"""List emails from local cache. Returns True if successful."""
|
|
||||||
try:
|
|
||||||
sync_dir = get_sync_dir()
|
|
||||||
if not sync_dir.exists():
|
|
||||||
return False
|
|
||||||
|
|
||||||
safe_folder = sanitize_filename(folder)
|
|
||||||
folder_path = sync_dir / safe_folder
|
|
||||||
|
|
||||||
if not folder_path.exists():
|
|
||||||
print(f"Folder '{folder}' not found in local cache", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
eml_files = list(folder_path.glob("*.eml"))
|
|
||||||
if not eml_files:
|
|
||||||
print(f"No emails found in folder '{folder}'")
|
|
||||||
return True
|
|
||||||
|
|
||||||
eml_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)
|
|
||||||
eml_files = eml_files[:limit]
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"\nEmails in {folder} (showing last {len(eml_files)} from local cache):\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
for eml_file in eml_files:
|
|
||||||
try:
|
|
||||||
with open(eml_file, "rb") as f:
|
|
||||||
msg = email.message_from_bytes(f.read())
|
|
||||||
|
|
||||||
date = msg.get("Date", "No date")
|
|
||||||
from_addr = msg.get("From", "No sender")
|
|
||||||
subject = msg.get("Subject", "No subject")
|
|
||||||
|
|
||||||
print(f"File: {eml_file.name}")
|
|
||||||
print(f" Date: {date}")
|
|
||||||
print(f" From: {from_addr}")
|
|
||||||
print(f" Subject: {subject}")
|
|
||||||
print()
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error reading local cache: {e}", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def list_emails(folder: str = "INBOX", limit: int = 20) -> None:
|
|
||||||
"""List emails in the specified folder from local cache."""
|
|
||||||
if not list_emails_local(folder, limit):
|
|
||||||
print(
|
|
||||||
"\nLocal cache not available. Run 'protonmail sync' first to download emails.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def read_email_local(filename: str) -> bool:
|
|
||||||
"""Read an email from local cache by filename. Returns True if successful."""
|
|
||||||
try:
|
|
||||||
sync_dir = get_sync_dir()
|
|
||||||
if not sync_dir.exists():
|
|
||||||
return False
|
|
||||||
|
|
||||||
eml_file = None
|
|
||||||
for folder_path in sync_dir.iterdir():
|
|
||||||
if folder_path.is_dir():
|
|
||||||
potential_file = folder_path / filename
|
|
||||||
if potential_file.exists():
|
|
||||||
eml_file = potential_file
|
|
||||||
break
|
|
||||||
|
|
||||||
if not eml_file:
|
|
||||||
print(f"Email file '{filename}' not found in local cache", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
with open(eml_file, "rb") as f:
|
|
||||||
msg = email.message_from_bytes(f.read())
|
|
||||||
|
|
||||||
print(f"\nFrom: {msg.get('From', 'Unknown')}")
|
|
||||||
print(f"To: {msg.get('To', 'Unknown')}")
|
|
||||||
print(f"Subject: {msg.get('Subject', 'No subject')}")
|
|
||||||
print(f"Date: {msg.get('Date', 'Unknown')}")
|
|
||||||
print("\n" + "=" * 70 + "\n")
|
|
||||||
|
|
||||||
body = ""
|
|
||||||
if msg.is_multipart():
|
|
||||||
for part in msg.walk():
|
|
||||||
content_type = part.get_content_type()
|
|
||||||
if content_type == "text/plain":
|
|
||||||
payload = part.get_payload(decode=True)
|
|
||||||
if isinstance(payload, bytes):
|
|
||||||
body = payload.decode("utf-8", errors="ignore")
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
payload = msg.get_payload(decode=True)
|
|
||||||
if payload and isinstance(payload, bytes):
|
|
||||||
body = payload.decode("utf-8", errors="ignore")
|
|
||||||
|
|
||||||
print(body)
|
|
||||||
print()
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error reading email from local cache: {e}", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def read_email(filename: str) -> None:
|
|
||||||
"""Read a specific email by filename from local cache."""
|
|
||||||
if not read_email_local(filename):
|
|
||||||
print(
|
|
||||||
"\nLocal cache not available. Run 'protonmail sync' first to download emails.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def search_emails_local(query: str, folder: str = "INBOX") -> bool:
|
|
||||||
"""Search emails in local cache by subject or sender. Returns True if successful."""
|
|
||||||
try:
|
|
||||||
sync_dir = get_sync_dir()
|
|
||||||
if not sync_dir.exists():
|
|
||||||
return False
|
|
||||||
|
|
||||||
safe_folder = sanitize_filename(folder)
|
|
||||||
folder_path = sync_dir / safe_folder
|
|
||||||
|
|
||||||
if not folder_path.exists():
|
|
||||||
print(f"Folder '{folder}' not found in local cache", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
matches: list[tuple[Path, Message]] = []
|
|
||||||
query_lower = query.lower()
|
|
||||||
|
|
||||||
for eml_file in folder_path.glob("*.eml"):
|
|
||||||
try:
|
|
||||||
with open(eml_file, "rb") as f:
|
|
||||||
msg = email.message_from_bytes(f.read())
|
|
||||||
|
|
||||||
subject = msg.get("Subject", "").lower()
|
|
||||||
from_addr = msg.get("From", "").lower()
|
|
||||||
|
|
||||||
if query_lower in subject or query_lower in from_addr:
|
|
||||||
matches.append((eml_file, msg))
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not matches:
|
|
||||||
print(f"No emails found matching '{query}'")
|
|
||||||
return True
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"\nFound {len(matches)} email(s) matching '{query}' (from local cache):\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
matches.sort(key=lambda x: x[0].stat().st_mtime, reverse=True)
|
|
||||||
|
|
||||||
for eml_file, msg in matches:
|
|
||||||
date = msg.get("Date", "No date")
|
|
||||||
from_addr = msg.get("From", "No sender")
|
|
||||||
subject = msg.get("Subject", "No subject")
|
|
||||||
|
|
||||||
print(f"File: {eml_file.name}")
|
|
||||||
print(f" Date: {date}")
|
|
||||||
print(f" From: {from_addr}")
|
|
||||||
print(f" Subject: {subject}")
|
|
||||||
print()
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error searching local cache: {e}", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def search_emails(query: str, folder: str = "INBOX") -> None:
|
|
||||||
"""Search emails by subject or sender in local cache."""
|
|
||||||
if not search_emails_local(query, folder):
|
|
||||||
print(
|
|
||||||
"\nLocal cache not available. Run 'protonmail sync' first to download emails.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def send_email(config: dict[str, Any]) -> None:
|
|
||||||
"""Send a new email interactively."""
|
|
||||||
try:
|
|
||||||
to_email = input("To: ")
|
|
||||||
subject = input("Subject: ")
|
|
||||||
|
|
||||||
body_lines: list[str] = []
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
line = input()
|
|
||||||
if line == ".":
|
|
||||||
break
|
|
||||||
body_lines.append(line)
|
|
||||||
except EOFError:
|
|
||||||
break
|
|
||||||
|
|
||||||
body = "\n".join(body_lines)
|
|
||||||
|
|
||||||
msg = EmailMessage()
|
|
||||||
msg["From"] = config["SMTP"]["username"]
|
|
||||||
msg["To"] = to_email
|
|
||||||
msg["Subject"] = subject
|
|
||||||
msg["Date"] = formatdate(localtime=True)
|
|
||||||
msg["Message-ID"] = make_msgid(domain=config["SMTP"]["username"].split("@")[-1])
|
|
||||||
msg.set_content(body)
|
|
||||||
|
|
||||||
smtp_config = config["SMTP"]
|
|
||||||
if smtp_config["security"] == "SSL/TLS":
|
|
||||||
smtp = smtplib.SMTP_SSL(smtp_config["hostname"], smtp_config["port"])
|
|
||||||
else:
|
|
||||||
smtp = smtplib.SMTP(smtp_config["hostname"], smtp_config["port"])
|
|
||||||
if smtp_config["security"] == "STARTTLS":
|
|
||||||
smtp.starttls()
|
|
||||||
|
|
||||||
smtp.login(smtp_config["username"], smtp_config["password"])
|
|
||||||
smtp.send_message(msg)
|
|
||||||
smtp.quit()
|
|
||||||
|
|
||||||
print(f"\nEmail sent successfully to {to_email}")
|
|
||||||
|
|
||||||
except (smtplib.SMTPException, OSError, EOFError) as e:
|
|
||||||
print(f"Error sending email: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def sync_emails(config: dict[str, Any], output_dir: str | None = None) -> None:
|
|
||||||
"""Sync all emails from ProtonMail to local .eml files."""
|
|
||||||
try:
|
|
||||||
if output_dir is None:
|
|
||||||
output_path = get_sync_dir()
|
|
||||||
else:
|
|
||||||
output_path = Path(output_dir)
|
|
||||||
output_path.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
imap = connect_imap(config)
|
|
||||||
|
|
||||||
folders = list_folders(imap)
|
|
||||||
if not folders:
|
|
||||||
folders = ["INBOX"]
|
|
||||||
|
|
||||||
print(f"Syncing emails from {len(folders)} folder(s) to {output_path}\n")
|
|
||||||
|
|
||||||
total_synced = 0
|
|
||||||
total_skipped = 0
|
|
||||||
|
|
||||||
for folder in folders:
|
|
||||||
safe_folder = sanitize_filename(folder)
|
|
||||||
folder_path = output_path / safe_folder
|
|
||||||
folder_path.mkdir(exist_ok=True)
|
|
||||||
|
|
||||||
status, data = imap.select(f'"{folder}"' if " " in folder else folder)
|
|
||||||
if status != "OK":
|
|
||||||
print(f"Warning: Could not access folder '{folder}', skipping...")
|
|
||||||
continue
|
|
||||||
|
|
||||||
status, data = imap.search(None, "ALL")
|
|
||||||
if status != "OK":
|
|
||||||
print(f"Warning: Could not search folder '{folder}', skipping...")
|
|
||||||
continue
|
|
||||||
|
|
||||||
message_ids = data[0].split()
|
|
||||||
if not message_ids:
|
|
||||||
print(f" {folder}: No messages")
|
|
||||||
continue
|
|
||||||
|
|
||||||
print(f" {folder}: Processing {len(message_ids)} message(s)...")
|
|
||||||
|
|
||||||
synced = 0
|
|
||||||
skipped = 0
|
|
||||||
|
|
||||||
existing_message_ids: set[str] = set()
|
|
||||||
for existing_file in folder_path.glob("*.eml"):
|
|
||||||
try:
|
|
||||||
with open(existing_file, "rb") as f:
|
|
||||||
existing_msg = email.message_from_bytes(f.read())
|
|
||||||
existing_msg_id = existing_msg.get("Message-ID", "")
|
|
||||||
if existing_msg_id:
|
|
||||||
existing_message_ids.add(existing_msg_id)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
for msg_id in message_ids:
|
|
||||||
try:
|
|
||||||
status, data = imap.fetch(msg_id, "(RFC822)")
|
|
||||||
if status != "OK" or not data or not data[0]:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_email = data[0][1]
|
|
||||||
if not isinstance(raw_email, bytes):
|
|
||||||
continue
|
|
||||||
msg = email.message_from_bytes(raw_email)
|
|
||||||
|
|
||||||
msg_message_id = msg.get("Message-ID", "")
|
|
||||||
if msg_message_id and msg_message_id in existing_message_ids:
|
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
filename = generate_email_filename(msg)
|
|
||||||
filepath = folder_path / filename
|
|
||||||
|
|
||||||
with open(filepath, "wb") as f:
|
|
||||||
f.write(raw_email)
|
|
||||||
|
|
||||||
synced += 1
|
|
||||||
if msg_message_id:
|
|
||||||
existing_message_ids.add(msg_message_id)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(
|
|
||||||
f" Warning: Error processing message {msg_id.decode()}: {e}"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
print(f" Synced: {synced} new, {skipped} already existed")
|
|
||||||
total_synced += synced
|
|
||||||
total_skipped += skipped
|
|
||||||
|
|
||||||
imap.logout()
|
|
||||||
|
|
||||||
print("\nSync complete!")
|
|
||||||
print(f" Total new emails synced: {total_synced}")
|
|
||||||
print(f" Total emails skipped (already existed): {total_skipped}")
|
|
||||||
print(f" Output directory: {output_path}")
|
|
||||||
|
|
||||||
except (imaplib.IMAP4.error, OSError, ValueError) as e:
|
|
||||||
print(f"Error syncing emails: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
"""Run the protonmail CLI."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Access and manage ProtonMail emails via Bridge",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__,
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--version", action="version", version=f"protonmail {__version__}"
|
|
||||||
)
|
|
||||||
|
|
||||||
subparsers = parser.add_subparsers(dest="command", help="Command to execute")
|
|
||||||
|
|
||||||
# List command
|
|
||||||
list_parser = subparsers.add_parser("list", help="List emails")
|
|
||||||
list_parser.add_argument(
|
|
||||||
"folder",
|
|
||||||
nargs="?",
|
|
||||||
default="INBOX",
|
|
||||||
help="Folder to list (default: INBOX)",
|
|
||||||
)
|
|
||||||
list_parser.add_argument(
|
|
||||||
"-n",
|
|
||||||
"--limit",
|
|
||||||
type=int,
|
|
||||||
default=20,
|
|
||||||
help="Number of emails to show (default: 20)",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Read command
|
|
||||||
read_parser = subparsers.add_parser("read", help="Read an email")
|
|
||||||
read_parser.add_argument("message_id", help="ID of the message to read")
|
|
||||||
|
|
||||||
# Search command
|
|
||||||
search_parser = subparsers.add_parser("search", help="Search for emails")
|
|
||||||
search_parser.add_argument("query", help="Search term (subject or sender)")
|
|
||||||
search_parser.add_argument(
|
|
||||||
"-f",
|
|
||||||
"--folder",
|
|
||||||
default="INBOX",
|
|
||||||
help="Folder to search (default: INBOX)",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Send command
|
|
||||||
subparsers.add_parser("send", help="Send a new email (interactive)")
|
|
||||||
|
|
||||||
# Sync command
|
|
||||||
sync_parser = subparsers.add_parser(
|
|
||||||
"sync", help="Sync all emails to local .eml files"
|
|
||||||
)
|
|
||||||
sync_parser.add_argument(
|
|
||||||
"-o",
|
|
||||||
"--output",
|
|
||||||
default=None,
|
|
||||||
help="Output directory (default: PROTONMAIL_DATA_DIR or /data/messages/email/protonmail)",
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.command == "list":
|
|
||||||
list_emails(args.folder, args.limit)
|
|
||||||
elif args.command == "read":
|
|
||||||
read_email(args.message_id)
|
|
||||||
elif args.command == "search":
|
|
||||||
search_emails(args.query, args.folder)
|
|
||||||
elif args.command == "send":
|
|
||||||
send_email(load_config())
|
|
||||||
elif args.command == "sync":
|
|
||||||
sync_emails(load_config(), args.output)
|
|
||||||
else:
|
|
||||||
list_emails()
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,195 +0,0 @@
|
|||||||
"""Tests for protonmail tool."""
|
|
||||||
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from email.message import Message
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from protonmail.main import (
|
|
||||||
extract_email_address,
|
|
||||||
generate_email_filename,
|
|
||||||
get_sync_dir,
|
|
||||||
list_emails_local,
|
|
||||||
sanitize_email,
|
|
||||||
sanitize_filename,
|
|
||||||
search_emails_local,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestCLI:
|
|
||||||
"""CLI interface tests."""
|
|
||||||
|
|
||||||
def test_version(self) -> None:
|
|
||||||
"""--version prints version string."""
|
|
||||||
result = subprocess.run(
|
|
||||||
[sys.executable, "-m", "protonmail.main", "--version"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
assert "protonmail" in result.stdout
|
|
||||||
assert "0.1.0" in result.stdout
|
|
||||||
|
|
||||||
def test_help(self) -> None:
|
|
||||||
"""--help shows usage information."""
|
|
||||||
result = subprocess.run(
|
|
||||||
[sys.executable, "-m", "protonmail.main", "--help"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
assert result.returncode == 0
|
|
||||||
assert "ProtonMail" in result.stdout
|
|
||||||
|
|
||||||
|
|
||||||
class TestSanitize:
|
|
||||||
"""Tests for filename sanitization."""
|
|
||||||
|
|
||||||
def test_sanitize_filename_basic(self) -> None:
|
|
||||||
"""Basic filename sanitization."""
|
|
||||||
assert sanitize_filename("hello world") == "hello world"
|
|
||||||
|
|
||||||
def test_sanitize_filename_special_chars(self) -> None:
|
|
||||||
"""Special characters replaced with underscores."""
|
|
||||||
assert sanitize_filename('file<>:"/\\|?*name') == "file_________name"
|
|
||||||
|
|
||||||
def test_sanitize_filename_truncation(self) -> None:
|
|
||||||
"""Long filenames get truncated."""
|
|
||||||
long_name = "x" * 300
|
|
||||||
result = sanitize_filename(long_name)
|
|
||||||
assert len(result) == 200
|
|
||||||
|
|
||||||
def test_sanitize_email_basic(self) -> None:
|
|
||||||
"""Email sanitization replaces @."""
|
|
||||||
assert sanitize_email("user@domain.com") == "user_domain.com"
|
|
||||||
|
|
||||||
def test_sanitize_email_special(self) -> None:
|
|
||||||
"""Email with angle brackets cleaned."""
|
|
||||||
assert sanitize_email("<user@domain.com>") == "user_domain.com"
|
|
||||||
|
|
||||||
|
|
||||||
class TestExtractEmail:
|
|
||||||
"""Tests for email address extraction."""
|
|
||||||
|
|
||||||
def test_angle_brackets(self) -> None:
|
|
||||||
"""Extract email from angle brackets."""
|
|
||||||
assert extract_email_address("Name <user@example.com>") == "user@example.com"
|
|
||||||
|
|
||||||
def test_plain_email(self) -> None:
|
|
||||||
"""Extract plain email address."""
|
|
||||||
assert extract_email_address("user@example.com") == "user@example.com"
|
|
||||||
|
|
||||||
def test_no_email(self) -> None:
|
|
||||||
"""Return trimmed string when no email present."""
|
|
||||||
assert extract_email_address("unknown") == "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
class TestGenerateFilename:
|
|
||||||
"""Tests for email filename generation."""
|
|
||||||
|
|
||||||
def test_basic_filename(self) -> None:
|
|
||||||
"""Generate filename from message headers."""
|
|
||||||
msg = Message()
|
|
||||||
msg["Date"] = "Thu, 17 Jul 2024 01:14:40 +0000"
|
|
||||||
msg["From"] = "sender@example.com"
|
|
||||||
msg["To"] = "recipient@example.com"
|
|
||||||
msg["Subject"] = "Test Subject"
|
|
||||||
|
|
||||||
filename = generate_email_filename(msg)
|
|
||||||
assert filename.startswith("2024-07-17_01-14-40_from_sender_example.com_to_recipient_example.com_")
|
|
||||||
assert filename.endswith(".eml")
|
|
||||||
assert "Test Subject" in filename
|
|
||||||
|
|
||||||
def test_missing_date(self) -> None:
|
|
||||||
"""Handle missing date gracefully."""
|
|
||||||
msg = Message()
|
|
||||||
msg["From"] = "sender@example.com"
|
|
||||||
msg["To"] = "recipient@example.com"
|
|
||||||
|
|
||||||
filename = generate_email_filename(msg)
|
|
||||||
assert "unknown-date" in filename
|
|
||||||
|
|
||||||
def test_length_limit(self) -> None:
|
|
||||||
"""Filename respects max_length."""
|
|
||||||
msg = Message()
|
|
||||||
msg["Date"] = "Thu, 17 Jul 2024 01:14:40 +0000"
|
|
||||||
msg["From"] = "sender@example.com"
|
|
||||||
msg["To"] = "recipient@example.com"
|
|
||||||
msg["Subject"] = "x" * 300
|
|
||||||
|
|
||||||
filename = generate_email_filename(msg, max_length=255)
|
|
||||||
assert len(filename) <= 255
|
|
||||||
|
|
||||||
|
|
||||||
class TestGetSyncDir:
|
|
||||||
"""Tests for sync directory resolution."""
|
|
||||||
|
|
||||||
def test_protonmail_data_dir_env(self) -> None:
|
|
||||||
"""PROTONMAIL_DATA_DIR takes priority."""
|
|
||||||
with patch.dict("os.environ", {"PROTONMAIL_DATA_DIR": "/custom/path"}):
|
|
||||||
assert get_sync_dir() == Path("/custom/path")
|
|
||||||
|
|
||||||
def test_ddata_fallback(self) -> None:
|
|
||||||
"""Falls back to DDATA when PROTONMAIL_DATA_DIR not set."""
|
|
||||||
env = {"DDATA": "/mydata"}
|
|
||||||
with patch.dict("os.environ", env, clear=True):
|
|
||||||
assert get_sync_dir() == Path("/mydata/messages/email/protonmail")
|
|
||||||
|
|
||||||
def test_default(self) -> None:
|
|
||||||
"""Default path when no env vars set."""
|
|
||||||
with patch.dict("os.environ", {}, clear=True):
|
|
||||||
assert get_sync_dir() == Path("/data/messages/email/protonmail")
|
|
||||||
|
|
||||||
|
|
||||||
class TestListEmailsLocal:
|
|
||||||
"""Tests for local email listing."""
|
|
||||||
|
|
||||||
def test_missing_cache(self, tmp_path: Path) -> None:
|
|
||||||
"""Returns False when cache doesn't exist."""
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path / "nope"):
|
|
||||||
assert list_emails_local() is False
|
|
||||||
|
|
||||||
def test_empty_folder(self, tmp_path: Path) -> None:
|
|
||||||
"""Shows message when folder is empty."""
|
|
||||||
inbox = tmp_path / "INBOX"
|
|
||||||
inbox.mkdir()
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path):
|
|
||||||
assert list_emails_local() is True
|
|
||||||
|
|
||||||
def test_lists_eml_files(self, tmp_path: Path) -> None:
|
|
||||||
"""Lists .eml files from cache."""
|
|
||||||
inbox = tmp_path / "INBOX"
|
|
||||||
inbox.mkdir()
|
|
||||||
eml_content = b"From: test@example.com\nSubject: Hello\nDate: Thu, 1 Jan 2024 00:00:00 +0000\n\nBody"
|
|
||||||
(inbox / "test.eml").write_bytes(eml_content)
|
|
||||||
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path):
|
|
||||||
assert list_emails_local() is True
|
|
||||||
|
|
||||||
|
|
||||||
class TestSearchEmailsLocal:
|
|
||||||
"""Tests for local email search."""
|
|
||||||
|
|
||||||
def test_missing_cache(self, tmp_path: Path) -> None:
|
|
||||||
"""Returns False when cache doesn't exist."""
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path / "nope"):
|
|
||||||
assert search_emails_local("test") is False
|
|
||||||
|
|
||||||
def test_finds_matching_email(self, tmp_path: Path) -> None:
|
|
||||||
"""Finds emails matching query."""
|
|
||||||
inbox = tmp_path / "INBOX"
|
|
||||||
inbox.mkdir()
|
|
||||||
eml_content = b"From: test@example.com\nSubject: Invoice 123\nDate: Thu, 1 Jan 2024 00:00:00 +0000\n\nBody"
|
|
||||||
(inbox / "test.eml").write_bytes(eml_content)
|
|
||||||
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path):
|
|
||||||
assert search_emails_local("invoice") is True
|
|
||||||
|
|
||||||
def test_no_match(self, tmp_path: Path) -> None:
|
|
||||||
"""Returns True but prints nothing when no match."""
|
|
||||||
inbox = tmp_path / "INBOX"
|
|
||||||
inbox.mkdir()
|
|
||||||
eml_content = b"From: test@example.com\nSubject: Hello\nDate: Thu, 1 Jan 2024 00:00:00 +0000\n\nBody"
|
|
||||||
(inbox / "test.eml").write_bytes(eml_content)
|
|
||||||
|
|
||||||
with patch("protonmail.main.get_sync_dir", return_value=tmp_path):
|
|
||||||
assert search_emails_local("nonexistent") is True
|
|
||||||
79
components/protonmail/uv.lock
generated
79
components/protonmail/uv.lock
generated
@@ -1,79 +0,0 @@
|
|||||||
version = 1
|
|
||||||
revision = 3
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "colorama"
|
|
||||||
version = "0.4.6"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "iniconfig"
|
|
||||||
version = "2.3.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "packaging"
|
|
||||||
version = "26.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pluggy"
|
|
||||||
version = "1.6.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "protonmail"
|
|
||||||
version = "0.1.0"
|
|
||||||
source = { editable = "." }
|
|
||||||
|
|
||||||
[package.dev-dependencies]
|
|
||||||
dev = [
|
|
||||||
{ name = "pytest" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[package.metadata]
|
|
||||||
|
|
||||||
[package.metadata.requires-dev]
|
|
||||||
dev = [{ name = "pytest", specifier = ">=7.0.0" }]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pygments"
|
|
||||||
version = "2.19.2"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pytest"
|
|
||||||
version = "9.0.2"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
|
||||||
{ name = "iniconfig" },
|
|
||||||
{ name = "packaging" },
|
|
||||||
{ name = "pluggy" },
|
|
||||||
{ name = "pygments" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" },
|
|
||||||
]
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "schedule"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Systemd timer and service management tool"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
schedule = "schedule.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/schedule"]
|
|
||||||
@@ -1,552 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Schedule - Systemd timer and service management tool.
|
|
||||||
|
|
||||||
A tool to easily create, manage, and monitor systemd user timers and services
|
|
||||||
following best practices.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
schedule list [--all]
|
|
||||||
schedule add <name> --command <cmd> --schedule <schedule> [options]
|
|
||||||
schedule remove <name> [--keep-logs]
|
|
||||||
schedule status <name>
|
|
||||||
schedule logs <name> [--follow] [--lines N]
|
|
||||||
schedule enable <name>
|
|
||||||
schedule disable <name>
|
|
||||||
schedule start <name>
|
|
||||||
schedule stop <name>
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
# List all user timers
|
|
||||||
schedule list
|
|
||||||
|
|
||||||
# Create a timer for ProtonMail sync every 5 minutes
|
|
||||||
schedule add protonmail-sync \\
|
|
||||||
--command "protonmail sync" \\
|
|
||||||
--schedule "5min" \\
|
|
||||||
--description "Sync ProtonMail emails" \\
|
|
||||||
--env-file ~/.config/protonmail/env \\
|
|
||||||
--condition-command "pgrep -f protonmail-bridge"
|
|
||||||
|
|
||||||
# Create a daily backup timer at 2 AM
|
|
||||||
schedule add daily-backup \\
|
|
||||||
--command "backup run" \\
|
|
||||||
--schedule "daily" \\
|
|
||||||
--on-calendar "*-*-* 02:00:00"
|
|
||||||
|
|
||||||
# Check status of a timer
|
|
||||||
schedule status protonmail-sync
|
|
||||||
|
|
||||||
# View logs in real-time
|
|
||||||
schedule logs protonmail-sync --follow
|
|
||||||
|
|
||||||
# Remove a timer
|
|
||||||
schedule remove protonmail-sync
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def get_systemd_user_dir() -> Path:
|
|
||||||
"""Get the systemd user directory."""
|
|
||||||
return Path.home() / ".config" / "systemd" / "user"
|
|
||||||
|
|
||||||
|
|
||||||
def run_systemctl(
|
|
||||||
args: list[str], check: bool = True
|
|
||||||
) -> subprocess.CompletedProcess[str]:
|
|
||||||
"""Run systemctl command with user flag."""
|
|
||||||
cmd = ["systemctl", "--user"] + args
|
|
||||||
return subprocess.run(
|
|
||||||
cmd,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=check,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def list_timers(all_timers: bool = False) -> None:
|
|
||||||
"""List all user timers."""
|
|
||||||
args = ["list-timers"]
|
|
||||||
if all_timers:
|
|
||||||
args.append("--all")
|
|
||||||
|
|
||||||
result = run_systemctl(args, check=False)
|
|
||||||
print(result.stdout)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
print(f"Error: {result.stderr}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_service_content(
|
|
||||||
name: str,
|
|
||||||
command: str,
|
|
||||||
description: str | None = None,
|
|
||||||
env_file: str | None = None,
|
|
||||||
condition_command: str | None = None,
|
|
||||||
working_directory: str | None = None,
|
|
||||||
environment: dict[str, str] | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Generate systemd service file content."""
|
|
||||||
desc = description or f"{name} service"
|
|
||||||
|
|
||||||
content = f"""[Unit]
|
|
||||||
Description={desc}
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Add default PATH
|
|
||||||
content += 'Environment="PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin"\n'
|
|
||||||
|
|
||||||
# Add additional environment variables
|
|
||||||
if environment:
|
|
||||||
for key, value in environment.items():
|
|
||||||
content += f'Environment="{key}={value}"\n'
|
|
||||||
|
|
||||||
# Add environment file if specified
|
|
||||||
if env_file:
|
|
||||||
content += f"EnvironmentFile={env_file}\n"
|
|
||||||
|
|
||||||
content += "\n"
|
|
||||||
|
|
||||||
# Add condition command if specified
|
|
||||||
if condition_command:
|
|
||||||
content += "# Pre-condition check\n"
|
|
||||||
# Use double quotes to avoid conflicts with single quotes in the command
|
|
||||||
escaped_condition = condition_command.replace('"', '\\"')
|
|
||||||
content += f'ExecStartPre=/bin/bash -c "{escaped_condition}"\n\n'
|
|
||||||
|
|
||||||
# Set working directory if specified
|
|
||||||
if working_directory:
|
|
||||||
content += f"WorkingDirectory={working_directory}\n"
|
|
||||||
|
|
||||||
# Add the main command
|
|
||||||
content += "# Run the command\n"
|
|
||||||
# Use double quotes to avoid conflicts with single quotes in the command
|
|
||||||
escaped_command = command.replace('"', '\\"')
|
|
||||||
content += f'ExecStart=/bin/bash -c "{escaped_command}"\n\n'
|
|
||||||
|
|
||||||
# Logging
|
|
||||||
content += """# Logging
|
|
||||||
StandardOutput=journal
|
|
||||||
StandardError=journal
|
|
||||||
SyslogIdentifier={name}
|
|
||||||
|
|
||||||
# Security settings
|
|
||||||
NoNewPrivileges=yes
|
|
||||||
PrivateTmp=yes
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=default.target
|
|
||||||
""".format(name=name)
|
|
||||||
|
|
||||||
return content
|
|
||||||
|
|
||||||
|
|
||||||
def generate_timer_content(
|
|
||||||
name: str,
|
|
||||||
description: str | None = None,
|
|
||||||
on_boot_sec: str = "1min",
|
|
||||||
on_unit_active_sec: str | None = None,
|
|
||||||
on_calendar: str | None = None,
|
|
||||||
persistent: bool = False,
|
|
||||||
) -> str:
|
|
||||||
"""Generate systemd timer file content."""
|
|
||||||
desc = description or f"{name} timer"
|
|
||||||
|
|
||||||
content = f"""[Unit]
|
|
||||||
Description={desc}
|
|
||||||
Requires={name}.service
|
|
||||||
|
|
||||||
[Timer]
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Add boot delay
|
|
||||||
if on_boot_sec:
|
|
||||||
content += f"OnBootSec={on_boot_sec}\n"
|
|
||||||
|
|
||||||
# Add interval-based schedule
|
|
||||||
if on_unit_active_sec:
|
|
||||||
content += f"OnUnitActiveSec={on_unit_active_sec}\n"
|
|
||||||
|
|
||||||
# Add calendar-based schedule
|
|
||||||
if on_calendar:
|
|
||||||
content += f"OnCalendar={on_calendar}\n"
|
|
||||||
|
|
||||||
content += f"""
|
|
||||||
# If the system was offline, {"run" if persistent else "don't run"} missed timers
|
|
||||||
Persistent={"true" if persistent else "false"}
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=timers.target
|
|
||||||
"""
|
|
||||||
|
|
||||||
return content
|
|
||||||
|
|
||||||
|
|
||||||
def parse_schedule(schedule: str) -> tuple[str | None, str | None]:
|
|
||||||
"""Parse schedule string into timer parameters.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (on_unit_active_sec, on_calendar)
|
|
||||||
"""
|
|
||||||
schedule_lower = schedule.lower()
|
|
||||||
|
|
||||||
# Simple interval schedules
|
|
||||||
if any(
|
|
||||||
schedule_lower.endswith(unit)
|
|
||||||
for unit in ["min", "hour", "h", "day", "d", "week", "w"]
|
|
||||||
):
|
|
||||||
return schedule, None
|
|
||||||
|
|
||||||
# Daily at specific time
|
|
||||||
if schedule_lower == "daily":
|
|
||||||
return "1day", None
|
|
||||||
|
|
||||||
# Hourly
|
|
||||||
if schedule_lower == "hourly":
|
|
||||||
return "1hour", None
|
|
||||||
|
|
||||||
# Assume it's a calendar spec
|
|
||||||
return None, schedule
|
|
||||||
|
|
||||||
|
|
||||||
def add_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Add a new systemd timer and service."""
|
|
||||||
name = args.name
|
|
||||||
command = args.command
|
|
||||||
schedule = args.schedule
|
|
||||||
|
|
||||||
# Parse schedule
|
|
||||||
on_unit_active_sec, on_calendar = parse_schedule(schedule)
|
|
||||||
|
|
||||||
if args.on_calendar:
|
|
||||||
on_calendar = args.on_calendar
|
|
||||||
on_unit_active_sec = None
|
|
||||||
|
|
||||||
# Create systemd directory if it doesn't exist
|
|
||||||
systemd_dir = get_systemd_user_dir()
|
|
||||||
systemd_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
# Generate service file
|
|
||||||
environment = {}
|
|
||||||
if args.environment:
|
|
||||||
for env in args.environment:
|
|
||||||
key, value = env.split("=", 1)
|
|
||||||
environment[key] = value
|
|
||||||
|
|
||||||
service_content = generate_service_content(
|
|
||||||
name=name,
|
|
||||||
command=command,
|
|
||||||
description=args.description,
|
|
||||||
env_file=args.env_file,
|
|
||||||
condition_command=args.condition_command,
|
|
||||||
working_directory=args.working_directory,
|
|
||||||
environment=environment,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Generate timer file
|
|
||||||
timer_content = generate_timer_content(
|
|
||||||
name=name,
|
|
||||||
description=args.description,
|
|
||||||
on_boot_sec=args.on_boot_sec,
|
|
||||||
on_unit_active_sec=on_unit_active_sec,
|
|
||||||
on_calendar=on_calendar,
|
|
||||||
persistent=args.persistent,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Write service file
|
|
||||||
service_path = systemd_dir / f"{name}.service"
|
|
||||||
timer_path = systemd_dir / f"{name}.timer"
|
|
||||||
|
|
||||||
# Check if files already exist
|
|
||||||
if service_path.exists() and not args.force:
|
|
||||||
print(f"Error: Service file already exists: {service_path}", file=sys.stderr)
|
|
||||||
print("Use --force to overwrite", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if timer_path.exists() and not args.force:
|
|
||||||
print(f"Error: Timer file already exists: {timer_path}", file=sys.stderr)
|
|
||||||
print("Use --force to overwrite", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Write files
|
|
||||||
service_path.write_text(service_content)
|
|
||||||
timer_path.write_text(timer_content)
|
|
||||||
|
|
||||||
print(f"Created service: {service_path}")
|
|
||||||
print(f"Created timer: {timer_path}")
|
|
||||||
|
|
||||||
# Reload systemd daemon
|
|
||||||
print("\nReloading systemd daemon...")
|
|
||||||
run_systemctl(["daemon-reload"])
|
|
||||||
|
|
||||||
# Enable and start timer if requested
|
|
||||||
if not args.no_enable:
|
|
||||||
print(f"Enabling {name}.timer...")
|
|
||||||
run_systemctl(["enable", f"{name}.timer"])
|
|
||||||
|
|
||||||
if not args.no_start:
|
|
||||||
print(f"Starting {name}.timer...")
|
|
||||||
run_systemctl(["start", f"{name}.timer"])
|
|
||||||
|
|
||||||
print(f"\n✓ Timer '{name}' created and started successfully!")
|
|
||||||
|
|
||||||
# Warn about environment variables if not explicitly set in service
|
|
||||||
if not environment and not args.env_file:
|
|
||||||
print(
|
|
||||||
"\n⚠️ Note: Systemd user units don't inherit shell environment variables."
|
|
||||||
)
|
|
||||||
print(" If your command needs environment variables, pass them with:")
|
|
||||||
print(f" schedule add {name} --command '...' -e VAR=value")
|
|
||||||
|
|
||||||
print("\nUseful commands:")
|
|
||||||
print(f" Status: schedule status {name}")
|
|
||||||
print(f" Logs: schedule logs {name} --follow")
|
|
||||||
print(f" Disable: schedule disable {name}")
|
|
||||||
print(f" Remove: schedule remove {name}")
|
|
||||||
|
|
||||||
|
|
||||||
def remove_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Remove a systemd timer and service."""
|
|
||||||
name = args.name
|
|
||||||
systemd_dir = get_systemd_user_dir()
|
|
||||||
|
|
||||||
service_path = systemd_dir / f"{name}.service"
|
|
||||||
timer_path = systemd_dir / f"{name}.timer"
|
|
||||||
|
|
||||||
# Check if files exist
|
|
||||||
if not service_path.exists() and not timer_path.exists():
|
|
||||||
print(f"Error: Timer '{name}' not found", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Stop and disable timer
|
|
||||||
print(f"Stopping {name}.timer...")
|
|
||||||
run_systemctl(["stop", f"{name}.timer"], check=False)
|
|
||||||
|
|
||||||
print(f"Disabling {name}.timer...")
|
|
||||||
run_systemctl(["disable", f"{name}.timer"], check=False)
|
|
||||||
|
|
||||||
# Remove files
|
|
||||||
if timer_path.exists():
|
|
||||||
timer_path.unlink()
|
|
||||||
print(f"Removed: {timer_path}")
|
|
||||||
|
|
||||||
if service_path.exists():
|
|
||||||
service_path.unlink()
|
|
||||||
print(f"Removed: {service_path}")
|
|
||||||
|
|
||||||
# Reload systemd daemon
|
|
||||||
print("\nReloading systemd daemon...")
|
|
||||||
run_systemctl(["daemon-reload"])
|
|
||||||
|
|
||||||
# Clear logs if requested
|
|
||||||
if not args.keep_logs:
|
|
||||||
print(f"Clearing logs for {name}...")
|
|
||||||
subprocess.run(
|
|
||||||
["journalctl", "--user", "--vacuum-time=1s", "-u", name],
|
|
||||||
check=False,
|
|
||||||
capture_output=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"\n✓ Timer '{name}' removed successfully!")
|
|
||||||
|
|
||||||
|
|
||||||
def status_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Show status of a timer and its service."""
|
|
||||||
name = args.name
|
|
||||||
|
|
||||||
print("=== Timer Status ===")
|
|
||||||
result = run_systemctl(["status", f"{name}.timer"], check=False)
|
|
||||||
print(result.stdout)
|
|
||||||
|
|
||||||
print("\n=== Service Status ===")
|
|
||||||
result = run_systemctl(["status", f"{name}.service"], check=False)
|
|
||||||
print(result.stdout)
|
|
||||||
|
|
||||||
print("\n=== Next Scheduled Run ===")
|
|
||||||
result = run_systemctl(["list-timers", f"{name}.timer"], check=False)
|
|
||||||
print(result.stdout)
|
|
||||||
|
|
||||||
|
|
||||||
def logs_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Show logs for a service."""
|
|
||||||
name = args.name
|
|
||||||
cmd = ["journalctl", "--user", "-u", name]
|
|
||||||
|
|
||||||
if args.follow:
|
|
||||||
cmd.append("-f")
|
|
||||||
|
|
||||||
if args.lines:
|
|
||||||
cmd.extend(["-n", str(args.lines)])
|
|
||||||
|
|
||||||
if args.since:
|
|
||||||
cmd.extend(["--since", args.since])
|
|
||||||
|
|
||||||
# Run journalctl directly (don't capture output)
|
|
||||||
subprocess.run(cmd)
|
|
||||||
|
|
||||||
|
|
||||||
def enable_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Enable a timer."""
|
|
||||||
name = args.name
|
|
||||||
print(f"Enabling {name}.timer...")
|
|
||||||
run_systemctl(["enable", f"{name}.timer"])
|
|
||||||
print(f"✓ Timer '{name}' enabled")
|
|
||||||
|
|
||||||
|
|
||||||
def disable_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Disable a timer."""
|
|
||||||
name = args.name
|
|
||||||
print(f"Disabling {name}.timer...")
|
|
||||||
run_systemctl(["disable", f"{name}.timer"])
|
|
||||||
print(f"✓ Timer '{name}' disabled")
|
|
||||||
|
|
||||||
|
|
||||||
def start_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Start a timer."""
|
|
||||||
name = args.name
|
|
||||||
print(f"Starting {name}.timer...")
|
|
||||||
run_systemctl(["start", f"{name}.timer"])
|
|
||||||
print(f"✓ Timer '{name}' started")
|
|
||||||
|
|
||||||
|
|
||||||
def stop_timer(args: argparse.Namespace) -> None:
|
|
||||||
"""Stop a timer."""
|
|
||||||
name = args.name
|
|
||||||
print(f"Stopping {name}.timer...")
|
|
||||||
run_systemctl(["stop", f"{name}.timer"])
|
|
||||||
print(f"✓ Timer '{name}' stopped")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
"""Main entry point."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Manage systemd user timers and services",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__,
|
|
||||||
)
|
|
||||||
|
|
||||||
subparsers = parser.add_subparsers(dest="subcommand", help="Command to execute")
|
|
||||||
|
|
||||||
# List command
|
|
||||||
list_parser = subparsers.add_parser("list", help="List all user timers")
|
|
||||||
list_parser.add_argument(
|
|
||||||
"--all", action="store_true", help="Show all timers including inactive"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Add command
|
|
||||||
add_parser = subparsers.add_parser("add", help="Add a new timer")
|
|
||||||
add_parser.add_argument("name", help="Name of the timer/service")
|
|
||||||
add_parser.add_argument("--command", required=True, help="Command to execute")
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--schedule",
|
|
||||||
required=True,
|
|
||||||
help="Schedule (e.g., '5min', 'hourly', 'daily', '1h')",
|
|
||||||
)
|
|
||||||
add_parser.add_argument("--description", help="Description of the timer/service")
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--env-file", help="Path to environment file (e.g., ~/.config/app/env)"
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--environment",
|
|
||||||
"-e",
|
|
||||||
action="append",
|
|
||||||
help="Environment variable (KEY=VALUE)",
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--condition-command",
|
|
||||||
help="Command that must succeed before running (e.g., 'pgrep myapp')",
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--working-directory", help="Working directory for the command"
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--on-boot-sec", default="1min", help="Delay after boot (default: 1min)"
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--on-calendar",
|
|
||||||
help="Calendar-based schedule (e.g., '*-*-* 02:00:00' for 2 AM daily)",
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--persistent",
|
|
||||||
action="store_true",
|
|
||||||
help="Run missed timers if system was offline",
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--force", action="store_true", help="Overwrite existing timer/service"
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--no-enable", action="store_true", help="Don't enable the timer"
|
|
||||||
)
|
|
||||||
add_parser.add_argument(
|
|
||||||
"--no-start", action="store_true", help="Don't start the timer"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Remove command
|
|
||||||
remove_parser = subparsers.add_parser("remove", help="Remove a timer")
|
|
||||||
remove_parser.add_argument("name", help="Name of the timer/service")
|
|
||||||
remove_parser.add_argument(
|
|
||||||
"--keep-logs", action="store_true", help="Keep service logs"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Status command
|
|
||||||
status_parser = subparsers.add_parser("status", help="Show timer status")
|
|
||||||
status_parser.add_argument("name", help="Name of the timer/service")
|
|
||||||
|
|
||||||
# Logs command
|
|
||||||
logs_parser = subparsers.add_parser("logs", help="Show service logs")
|
|
||||||
logs_parser.add_argument("name", help="Name of the timer/service")
|
|
||||||
logs_parser.add_argument(
|
|
||||||
"--follow", "-f", action="store_true", help="Follow logs in real-time"
|
|
||||||
)
|
|
||||||
logs_parser.add_argument("--lines", "-n", type=int, help="Number of lines to show")
|
|
||||||
logs_parser.add_argument("--since", help="Show logs since (e.g., 'today', '1h')")
|
|
||||||
|
|
||||||
# Enable command
|
|
||||||
enable_parser = subparsers.add_parser("enable", help="Enable a timer")
|
|
||||||
enable_parser.add_argument("name", help="Name of the timer")
|
|
||||||
|
|
||||||
# Disable command
|
|
||||||
disable_parser = subparsers.add_parser("disable", help="Disable a timer")
|
|
||||||
disable_parser.add_argument("name", help="Name of the timer")
|
|
||||||
|
|
||||||
# Start command
|
|
||||||
start_parser = subparsers.add_parser("start", help="Start a timer")
|
|
||||||
start_parser.add_argument("name", help="Name of the timer")
|
|
||||||
|
|
||||||
# Stop command
|
|
||||||
stop_parser = subparsers.add_parser("stop", help="Stop a timer")
|
|
||||||
stop_parser.add_argument("name", help="Name of the timer")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not args.subcommand:
|
|
||||||
parser.print_help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Route to appropriate function
|
|
||||||
commands = {
|
|
||||||
"list": list_timers,
|
|
||||||
"add": add_timer,
|
|
||||||
"remove": remove_timer,
|
|
||||||
"status": status_timer,
|
|
||||||
"logs": logs_timer,
|
|
||||||
"enable": enable_timer,
|
|
||||||
"disable": disable_timer,
|
|
||||||
"start": start_timer,
|
|
||||||
"stop": stop_timer,
|
|
||||||
}
|
|
||||||
|
|
||||||
commands[args.subcommand](args)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "search"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Manage searchable collections of files"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = [
|
|
||||||
"toml>=0.10.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
search = "search.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/search"]
|
|
||||||
@@ -1,832 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
search: Manage self-contained searchable collections of files
|
|
||||||
|
|
||||||
A modular, low-resource, local search system for files (PDFs, images, videos, etc.)
|
|
||||||
where each collection is self-contained and can be indexed and queried using Tantivy.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
search init [options] [directory] Initialize a directory as a search collection
|
|
||||||
search scan [options] [directory] Find all search collections in a directory tree
|
|
||||||
search index [options] [directory...] Index all matched files in one or more collections
|
|
||||||
search query [options] QUERY Search indexed collections using Tantivy
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
search init ~/documents # Initialize a collection
|
|
||||||
search init --name "Research" ~/papers # Initialize with a specific name
|
|
||||||
search init --extract "*.pdf=pdf2md {input}" # Add a specific extractor
|
|
||||||
|
|
||||||
search scan ~/documents # Find collections
|
|
||||||
|
|
||||||
search index ~/documents # Index a single collection
|
|
||||||
search index ~/documents ~/papers # Index multiple collections
|
|
||||||
search index --verbose ~/documents # Show detailed indexing progress
|
|
||||||
|
|
||||||
search query "neural networks" # Search across all nearby collections
|
|
||||||
search query "DNA" --in ~/papers # Search in a specific collection
|
|
||||||
search query "protein" --tag biology # Search with tag filter
|
|
||||||
search query "machine learning" --limit 10 # Limit number of results
|
|
||||||
|
|
||||||
For more detailed help on a specific subcommand:
|
|
||||||
search init --help
|
|
||||||
search scan --help
|
|
||||||
search index --help
|
|
||||||
search query --help
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
|
|
||||||
import toml
|
|
||||||
|
|
||||||
# Try to import Tantivy, but don't fail immediately if not available
|
|
||||||
# This allows --help and other non-search operations to work without Tantivy
|
|
||||||
tantivy_available = False
|
|
||||||
try:
|
|
||||||
import tantivy # type: ignore
|
|
||||||
|
|
||||||
tantivy_available = True
|
|
||||||
except ImportError:
|
|
||||||
tantivy = None # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def check_tantivy():
|
|
||||||
"""Check if Tantivy is available and exit if not."""
|
|
||||||
if not tantivy_available:
|
|
||||||
print(
|
|
||||||
"Error: Required Tantivy Python bindings not found. Please install them with:"
|
|
||||||
)
|
|
||||||
print(" pip install tantivy")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
# Constants
|
|
||||||
SEARCH_DIR = ".search"
|
|
||||||
CONFIG_FILE = f"{SEARCH_DIR}/config.toml"
|
|
||||||
CACHE_DIR = f"{SEARCH_DIR}/cache"
|
|
||||||
INDEX_DIR = f"{SEARCH_DIR}/index"
|
|
||||||
|
|
||||||
# Default extractor commands for common file types
|
|
||||||
DEFAULT_EXTRACTORS = {
|
|
||||||
"*.md": "text-extractor {input}",
|
|
||||||
"*.txt": "text-extractor {input}",
|
|
||||||
"*.pdf": "pdf-extractor {input}",
|
|
||||||
"*.docx": "docx-extractor {input}",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Default configuration
|
|
||||||
DEFAULT_CONFIG = {
|
|
||||||
"name": "Default Collection",
|
|
||||||
"include": {"patterns": ["*.pdf", "*.md", "*.txt", "*.docx"]},
|
|
||||||
"exclude": {"patterns": ["*~", "*.bak", "*.tmp", ".git/*", ".search/*"]},
|
|
||||||
"extractors": {}, # Only define custom extractors here, defaults are used automatically
|
|
||||||
"output": {"format": "json", "directory": "cache"},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_dir(path: str) -> None:
|
|
||||||
"""Ensure a directory exists."""
|
|
||||||
Path(path).mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
|
|
||||||
def find_collections(start_dir: str) -> List[str]:
|
|
||||||
"""Find all search collections starting from a directory."""
|
|
||||||
collections = []
|
|
||||||
start_path = Path(start_dir).resolve()
|
|
||||||
|
|
||||||
for path in start_path.glob(f"**/{SEARCH_DIR}"):
|
|
||||||
config_file = path / "config.toml"
|
|
||||||
if config_file.exists():
|
|
||||||
collections.append(str(path.parent))
|
|
||||||
|
|
||||||
return collections
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(collection_dir: str) -> Dict[str, Any]:
|
|
||||||
"""Load collection configuration."""
|
|
||||||
config_path = os.path.join(collection_dir, CONFIG_FILE)
|
|
||||||
try:
|
|
||||||
with open(config_path, "r") as f:
|
|
||||||
return toml.load(f)
|
|
||||||
except (FileNotFoundError, toml.TomlDecodeError) as e:
|
|
||||||
sys.stderr.write(f"Error loading config: {e}\n")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def save_config(collection_dir: str, config: Dict[str, Any]) -> bool:
|
|
||||||
"""Save collection configuration."""
|
|
||||||
config_path = os.path.join(collection_dir, CONFIG_FILE)
|
|
||||||
try:
|
|
||||||
# Ensure the .search directory exists
|
|
||||||
ensure_dir(os.path.join(collection_dir, SEARCH_DIR))
|
|
||||||
|
|
||||||
# Add a header comment to explain the extractors section
|
|
||||||
header = """# Search Collection Configuration
|
|
||||||
#
|
|
||||||
# Built-in extractors are automatically used for common file types:
|
|
||||||
# *.txt, *.md: text-extractor {input}
|
|
||||||
# *.pdf: pdf-extractor {input}
|
|
||||||
# *.docx: docx-extractor {input}
|
|
||||||
#
|
|
||||||
# The [extractors] section below only needs to define custom extractors
|
|
||||||
# or overrides for specific file types.
|
|
||||||
#
|
|
||||||
"""
|
|
||||||
# Format the config as TOML
|
|
||||||
config_toml = toml.dumps(config)
|
|
||||||
|
|
||||||
# Write the file with header
|
|
||||||
with open(config_path, "w") as f:
|
|
||||||
f.write(header)
|
|
||||||
f.write(config_toml)
|
|
||||||
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error saving config: {e}\n")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def filter_files(collection_dir: str, config: Dict[str, Any]) -> List[str]:
|
|
||||||
"""Filter files based on include/exclude patterns."""
|
|
||||||
include_patterns = config.get("include", {}).get("patterns", ["*"])
|
|
||||||
exclude_patterns = config.get("exclude", {}).get("patterns", [])
|
|
||||||
|
|
||||||
matched_files = []
|
|
||||||
|
|
||||||
# Walk the directory tree
|
|
||||||
for root, dirs, files in os.walk(collection_dir):
|
|
||||||
# Skip the .search directory
|
|
||||||
if os.path.basename(root) == SEARCH_DIR:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check if any exclude pattern matches directories
|
|
||||||
skip_dir = False
|
|
||||||
for pattern in exclude_patterns:
|
|
||||||
if any(fnmatch.fnmatch(d, pattern) for d in dirs):
|
|
||||||
skip_dir = True
|
|
||||||
break
|
|
||||||
if skip_dir:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Match files against include/exclude patterns
|
|
||||||
for filename in files:
|
|
||||||
file_path = os.path.join(root, filename)
|
|
||||||
|
|
||||||
# Skip if the file is in the .search directory
|
|
||||||
if SEARCH_DIR in file_path.split(os.sep):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check include patterns
|
|
||||||
if not any(
|
|
||||||
fnmatch.fnmatch(filename, pattern) for pattern in include_patterns
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check exclude patterns
|
|
||||||
if any(fnmatch.fnmatch(filename, pattern) for pattern in exclude_patterns):
|
|
||||||
continue
|
|
||||||
|
|
||||||
matched_files.append(file_path)
|
|
||||||
|
|
||||||
return matched_files
|
|
||||||
|
|
||||||
|
|
||||||
def get_extractor(filename: str, config: Dict[str, Any]) -> Optional[str]:
|
|
||||||
"""Find the appropriate extractor command for a file."""
|
|
||||||
# First check custom extractors defined in config
|
|
||||||
custom_extractors = config.get("extractors", {})
|
|
||||||
|
|
||||||
# Try to match custom extractors first (allowing overrides)
|
|
||||||
for pattern, command in custom_extractors.items():
|
|
||||||
if fnmatch.fnmatch(filename, pattern):
|
|
||||||
return command
|
|
||||||
|
|
||||||
# If no custom extractor matched, use the default extractors
|
|
||||||
for pattern, command in DEFAULT_EXTRACTORS.items():
|
|
||||||
if fnmatch.fnmatch(filename, pattern):
|
|
||||||
return command
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def extract_metadata(file_path: str, extractor_cmd: str) -> Dict[str, Any]:
|
|
||||||
"""Execute extractor and parse output as JSON."""
|
|
||||||
# Replace {input} with the properly quoted file path
|
|
||||||
# First, escape any single quotes in the path
|
|
||||||
escaped_path = file_path.replace("'", "'\\''")
|
|
||||||
# Then wrap in single quotes to handle spaces and special characters
|
|
||||||
quoted_path = f"'{escaped_path}'"
|
|
||||||
cmd = extractor_cmd.replace("{input}", quoted_path)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Run the extractor command
|
|
||||||
result = subprocess.run(
|
|
||||||
cmd, shell=True, capture_output=True, text=True, check=True
|
|
||||||
)
|
|
||||||
output = result.stdout
|
|
||||||
|
|
||||||
# Attempt to parse as JSON
|
|
||||||
try:
|
|
||||||
return json.loads(output)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
# If parsing fails, create basic metadata with content
|
|
||||||
return {"title": os.path.basename(file_path), "content": output}
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
sys.stderr.write(f"Error executing extractor for {file_path}: {e}\n")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def save_cache(
|
|
||||||
collection_dir: str,
|
|
||||||
file_path: str,
|
|
||||||
metadata: Dict[str, Any],
|
|
||||||
config: Dict[str, Any],
|
|
||||||
) -> bool:
|
|
||||||
"""Save metadata to cache."""
|
|
||||||
# Ensure the cache directory exists
|
|
||||||
cache_dir = os.path.join(collection_dir, CACHE_DIR)
|
|
||||||
ensure_dir(cache_dir)
|
|
||||||
|
|
||||||
# Create a cache file name based on the original file path
|
|
||||||
rel_path = os.path.relpath(file_path, collection_dir)
|
|
||||||
cache_filename = re.sub(r"[/\\]", "_", rel_path)
|
|
||||||
|
|
||||||
# Save as JSON
|
|
||||||
cache_path = os.path.join(cache_dir, f"{cache_filename}.json")
|
|
||||||
try:
|
|
||||||
with open(cache_path, "w") as f:
|
|
||||||
json.dump(metadata, f, indent=2)
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error saving cache file {cache_path}: {e}\n")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def create_schema():
|
|
||||||
"""Create Tantivy schema for document indexing."""
|
|
||||||
schema_builder = tantivy.SchemaBuilder() # type: ignore
|
|
||||||
|
|
||||||
# Define fields for the schema
|
|
||||||
schema_builder.add_text_field("path", stored=True)
|
|
||||||
schema_builder.add_text_field("title", stored=True)
|
|
||||||
schema_builder.add_text_field("content", stored=True)
|
|
||||||
schema_builder.add_text_field("absolute_path", stored=True)
|
|
||||||
# Using text fields for tags instead of facets
|
|
||||||
schema_builder.add_text_field("tag_pdf", stored=True)
|
|
||||||
schema_builder.add_text_field("tag_test", stored=True)
|
|
||||||
schema_builder.add_text_field("tag_sample", stored=True)
|
|
||||||
# Add more common tags as needed
|
|
||||||
schema_builder.add_text_field(
|
|
||||||
"metadata", stored=True
|
|
||||||
) # For storing additional JSON metadata
|
|
||||||
|
|
||||||
return schema_builder.build()
|
|
||||||
|
|
||||||
|
|
||||||
def get_or_create_index(collection_dir: str) -> tuple:
|
|
||||||
"""Get or create a Tantivy index for a collection."""
|
|
||||||
# Check if Tantivy is available
|
|
||||||
check_tantivy()
|
|
||||||
|
|
||||||
index_dir = os.path.join(collection_dir, INDEX_DIR)
|
|
||||||
ensure_dir(index_dir)
|
|
||||||
|
|
||||||
# For reference only - we don't actually read this
|
|
||||||
schema_path = os.path.join(index_dir, "schema_info.json")
|
|
||||||
|
|
||||||
# Check if index exists
|
|
||||||
if os.path.exists(os.path.join(index_dir, "meta.json")):
|
|
||||||
# Try to open existing index
|
|
||||||
try:
|
|
||||||
index = tantivy.Index.open(index_dir) # type: ignore
|
|
||||||
return index, None # We don't need the schema for opened indexes
|
|
||||||
except Exception as e:
|
|
||||||
# If opening failed, create a new one
|
|
||||||
sys.stderr.write(
|
|
||||||
f"Warning: Could not open existing index, creating new one: {e}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create new schema and index
|
|
||||||
schema = create_schema()
|
|
||||||
# Create basic schema info for reference
|
|
||||||
schema_info = {
|
|
||||||
"fields": [
|
|
||||||
{"name": "path", "type": "text", "stored": True},
|
|
||||||
{"name": "title", "type": "text", "stored": True},
|
|
||||||
{"name": "content", "type": "text", "stored": True},
|
|
||||||
{"name": "absolute_path", "type": "text", "stored": True},
|
|
||||||
{"name": "tag_pdf", "type": "text", "stored": True},
|
|
||||||
{"name": "tag_test", "type": "text", "stored": True},
|
|
||||||
{"name": "tag_sample", "type": "text", "stored": True},
|
|
||||||
{"name": "metadata", "type": "text", "stored": True},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Save schema info for reference
|
|
||||||
with open(schema_path, "w") as f:
|
|
||||||
json.dump(schema_info, f, indent=2)
|
|
||||||
|
|
||||||
# Create the index
|
|
||||||
index = tantivy.Index(schema, path=index_dir) # type: ignore
|
|
||||||
return index, schema
|
|
||||||
|
|
||||||
|
|
||||||
def index_file(collection_dir: str, file_path: str, metadata: Dict[str, Any]) -> bool:
|
|
||||||
"""Index a file using Tantivy."""
|
|
||||||
try:
|
|
||||||
# Get or create index
|
|
||||||
index, _schema = get_or_create_index(collection_dir)
|
|
||||||
|
|
||||||
# Create a writer for adding documents
|
|
||||||
writer = index.writer()
|
|
||||||
|
|
||||||
# Create document
|
|
||||||
doc = tantivy.Document() # type: ignore
|
|
||||||
|
|
||||||
# Get relative path as identifier
|
|
||||||
rel_path = os.path.relpath(file_path, collection_dir)
|
|
||||||
|
|
||||||
# Add fields to document
|
|
||||||
doc.add_text("path", rel_path)
|
|
||||||
doc.add_text("absolute_path", os.path.abspath(file_path))
|
|
||||||
|
|
||||||
# Add title if available
|
|
||||||
if "title" in metadata and metadata["title"]:
|
|
||||||
doc.add_text("title", metadata["title"])
|
|
||||||
else:
|
|
||||||
doc.add_text("title", os.path.basename(file_path))
|
|
||||||
|
|
||||||
# Add content if available
|
|
||||||
if "content" in metadata and metadata["content"]:
|
|
||||||
doc.add_text("content", metadata["content"])
|
|
||||||
|
|
||||||
# Add tags as facets if available
|
|
||||||
if "tags" in metadata and isinstance(metadata["tags"], list):
|
|
||||||
for tag in metadata["tags"]:
|
|
||||||
if tag and isinstance(tag, str):
|
|
||||||
# Add as text since facets are complicated in Tantivy
|
|
||||||
tag_field = f"tag_{tag.lower().replace(' ', '_').replace(',', '').replace('-', '_')}"
|
|
||||||
doc.add_text(tag_field, "true")
|
|
||||||
|
|
||||||
# Store all metadata as JSON
|
|
||||||
doc.add_text(
|
|
||||||
"metadata",
|
|
||||||
json.dumps(
|
|
||||||
{
|
|
||||||
"path": rel_path,
|
|
||||||
"absolute_path": os.path.abspath(file_path),
|
|
||||||
**metadata,
|
|
||||||
}
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Add document to index
|
|
||||||
writer.add_document(doc)
|
|
||||||
|
|
||||||
# Commit changes
|
|
||||||
writer.commit()
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error indexing file {file_path}: {e}\n")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def perform_search(
|
|
||||||
query_string: str,
|
|
||||||
collections: List[str],
|
|
||||||
tag_filter: Optional[str] = None,
|
|
||||||
limit: int = 20,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Search across collections using Tantivy."""
|
|
||||||
# Check if Tantivy is available
|
|
||||||
check_tantivy()
|
|
||||||
|
|
||||||
results = []
|
|
||||||
|
|
||||||
# Process each collection
|
|
||||||
for collection_dir in collections:
|
|
||||||
index_dir = os.path.join(collection_dir, INDEX_DIR)
|
|
||||||
if not os.path.exists(index_dir) or not os.path.exists(
|
|
||||||
os.path.join(index_dir, "meta.json")
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Open the index
|
|
||||||
index = tantivy.Index.open(index_dir) # type: ignore
|
|
||||||
searcher = index.searcher()
|
|
||||||
|
|
||||||
# Parse the query using index's parse_query method
|
|
||||||
query = index.parse_query(query_string, ["title", "content"])
|
|
||||||
|
|
||||||
# Perform the search
|
|
||||||
search_result = searcher.search(query, limit)
|
|
||||||
|
|
||||||
# Extract results - access the hits property which contains doc_address
|
|
||||||
for hit in search_result.hits:
|
|
||||||
# Unpack score and doc_address from the hit
|
|
||||||
score, doc_address = hit
|
|
||||||
retrieved_doc = searcher.doc(doc_address)
|
|
||||||
|
|
||||||
# Create basic metadata
|
|
||||||
title = ""
|
|
||||||
path = ""
|
|
||||||
abs_path = ""
|
|
||||||
tags = []
|
|
||||||
score_value = score * 100 # Convert to percentage
|
|
||||||
|
|
||||||
# Extract fields using get_first
|
|
||||||
title_field = retrieved_doc.get_first("title")
|
|
||||||
if title_field:
|
|
||||||
title = title_field
|
|
||||||
|
|
||||||
path_field = retrieved_doc.get_first("path")
|
|
||||||
if path_field:
|
|
||||||
path = path_field
|
|
||||||
|
|
||||||
abs_path_field = retrieved_doc.get_first("absolute_path")
|
|
||||||
if abs_path_field:
|
|
||||||
abs_path = abs_path_field
|
|
||||||
|
|
||||||
metadata_field = retrieved_doc.get_first("metadata")
|
|
||||||
if metadata_field:
|
|
||||||
try:
|
|
||||||
metadata = json.loads(metadata_field)
|
|
||||||
# If we have tags in metadata, extract them
|
|
||||||
if "tags" in metadata:
|
|
||||||
tags = metadata["tags"]
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Skip documents that don't match tag filter
|
|
||||||
if tag_filter and tag_filter.lower() not in [t.lower() for t in tags]:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Create result object
|
|
||||||
data = {
|
|
||||||
"title": title or os.path.basename(path),
|
|
||||||
"path": path,
|
|
||||||
"absolute_path": abs_path,
|
|
||||||
"tags": tags,
|
|
||||||
"score": score_value,
|
|
||||||
"collection": os.path.basename(collection_dir),
|
|
||||||
"collection_path": collection_dir,
|
|
||||||
}
|
|
||||||
|
|
||||||
results.append(data)
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error searching in collection {collection_dir}: {e}\n")
|
|
||||||
|
|
||||||
# Sort results by score (highest first)
|
|
||||||
results.sort(key=lambda x: x.get("score", 0), reverse=True)
|
|
||||||
|
|
||||||
return results[:limit]
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_init(args: argparse.Namespace) -> int:
|
|
||||||
"""Initialize a directory as a search collection."""
|
|
||||||
directory = os.path.abspath(args.directory or os.getcwd())
|
|
||||||
|
|
||||||
# Check if the directory exists
|
|
||||||
if not os.path.isdir(directory):
|
|
||||||
sys.stderr.write(f"Error: Directory {directory} does not exist.\n")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Check if the collection already exists
|
|
||||||
config_path = os.path.join(directory, CONFIG_FILE)
|
|
||||||
if os.path.exists(config_path):
|
|
||||||
if not args.force:
|
|
||||||
sys.stderr.write(
|
|
||||||
f"Error: Collection already exists at {directory}. Use --force to overwrite.\n"
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Create the basic configuration
|
|
||||||
config = DEFAULT_CONFIG.copy()
|
|
||||||
|
|
||||||
# Update configuration based on command-line arguments
|
|
||||||
if args.name:
|
|
||||||
config["name"] = args.name
|
|
||||||
|
|
||||||
if args.include:
|
|
||||||
config["include"]["patterns"] = args.include
|
|
||||||
|
|
||||||
if args.exclude:
|
|
||||||
config["exclude"]["patterns"] = args.exclude
|
|
||||||
|
|
||||||
# Add custom extractors if specified
|
|
||||||
if args.extract:
|
|
||||||
extractors = config["extractors"].copy()
|
|
||||||
for extractor in args.extract:
|
|
||||||
if "=" in extractor:
|
|
||||||
pattern, command = extractor.split("=", 1)
|
|
||||||
extractors[pattern.strip()] = command.strip()
|
|
||||||
config["extractors"] = extractors
|
|
||||||
|
|
||||||
# Create the directory structure
|
|
||||||
ensure_dir(os.path.join(directory, SEARCH_DIR))
|
|
||||||
ensure_dir(os.path.join(directory, CACHE_DIR))
|
|
||||||
ensure_dir(os.path.join(directory, INDEX_DIR))
|
|
||||||
|
|
||||||
# Save the configuration
|
|
||||||
if save_config(directory, config):
|
|
||||||
print(f"Initialized search collection in {directory}")
|
|
||||||
print(f"Configuration saved to {config_path}")
|
|
||||||
return 0
|
|
||||||
else:
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_scan(args: argparse.Namespace) -> int:
|
|
||||||
"""Scan for collections."""
|
|
||||||
directory = os.path.abspath(args.directory or os.getcwd())
|
|
||||||
|
|
||||||
if not os.path.isdir(directory):
|
|
||||||
sys.stderr.write(f"Error: Directory {directory} does not exist.\n")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
collections = find_collections(directory)
|
|
||||||
|
|
||||||
if not collections:
|
|
||||||
print(f"No search collections found in {directory}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
print(f"Found {len(collections)} search collection(s):")
|
|
||||||
for collection in collections:
|
|
||||||
config = load_config(collection)
|
|
||||||
name = config.get("name", "Unnamed collection")
|
|
||||||
print(f" {name}: {collection}")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_index(args: argparse.Namespace) -> int:
|
|
||||||
"""Index files in collections."""
|
|
||||||
directories = (
|
|
||||||
[os.path.abspath(d) for d in args.directories]
|
|
||||||
if args.directories
|
|
||||||
else [os.getcwd()]
|
|
||||||
)
|
|
||||||
|
|
||||||
collections = []
|
|
||||||
for directory in directories:
|
|
||||||
# Check if the directory is a collection
|
|
||||||
if os.path.exists(os.path.join(directory, CONFIG_FILE)):
|
|
||||||
collections.append(directory)
|
|
||||||
else:
|
|
||||||
# Scan for collections in this directory
|
|
||||||
found_collections = find_collections(directory)
|
|
||||||
collections.extend(found_collections)
|
|
||||||
|
|
||||||
if not collections:
|
|
||||||
sys.stderr.write(
|
|
||||||
"Error: No search collections found in the specified directories.\n"
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
indexed_count = 0
|
|
||||||
failed_count = 0
|
|
||||||
|
|
||||||
for collection_dir in collections:
|
|
||||||
config = load_config(collection_dir)
|
|
||||||
if not config:
|
|
||||||
sys.stderr.write(
|
|
||||||
f"Warning: Skipping collection {collection_dir} due to missing or invalid configuration.\n"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
collection_name = config.get("name", os.path.basename(collection_dir))
|
|
||||||
print(f"Indexing collection: {collection_name} ({collection_dir})")
|
|
||||||
|
|
||||||
# Get matched files
|
|
||||||
matched_files = filter_files(collection_dir, config)
|
|
||||||
print(f"Found {len(matched_files)} matching files")
|
|
||||||
|
|
||||||
for file_path in matched_files:
|
|
||||||
rel_path = os.path.relpath(file_path, collection_dir)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Get the appropriate extractor
|
|
||||||
extractor = get_extractor(os.path.basename(file_path), config)
|
|
||||||
if not extractor:
|
|
||||||
if args.verbose:
|
|
||||||
print(f" Skipping {rel_path}: No extractor found")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if args.verbose:
|
|
||||||
print(f" Extracting {rel_path}")
|
|
||||||
|
|
||||||
# Extract metadata
|
|
||||||
metadata = extract_metadata(file_path, extractor)
|
|
||||||
if not metadata:
|
|
||||||
if args.verbose:
|
|
||||||
print(f" Failed to extract metadata from {rel_path}")
|
|
||||||
failed_count += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Save to cache if caching is enabled
|
|
||||||
if not args.no_cache:
|
|
||||||
save_cache(collection_dir, file_path, metadata, config)
|
|
||||||
|
|
||||||
# Index the file
|
|
||||||
if index_file(collection_dir, file_path, metadata):
|
|
||||||
indexed_count += 1
|
|
||||||
if args.verbose:
|
|
||||||
print(f" Successfully indexed {rel_path}")
|
|
||||||
else:
|
|
||||||
failed_count += 1
|
|
||||||
if args.verbose:
|
|
||||||
print(f" Failed to index {rel_path}")
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error processing {rel_path}: {e}\n")
|
|
||||||
failed_count += 1
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"Finished indexing {indexed_count} files across {len(collections)} collections."
|
|
||||||
)
|
|
||||||
if failed_count > 0:
|
|
||||||
print(f"Failed to index {failed_count} files.")
|
|
||||||
|
|
||||||
return 0 if failed_count == 0 else 1
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_query(args: argparse.Namespace) -> int:
|
|
||||||
"""Search collections."""
|
|
||||||
# Check if Tantivy is available
|
|
||||||
check_tantivy()
|
|
||||||
|
|
||||||
# Determine which collections to search
|
|
||||||
collections = []
|
|
||||||
if args.in_dir:
|
|
||||||
directory = os.path.abspath(args.in_dir)
|
|
||||||
if os.path.exists(os.path.join(directory, CONFIG_FILE)):
|
|
||||||
collections.append(directory)
|
|
||||||
else:
|
|
||||||
sys.stderr.write(f"Error: {directory} is not a valid search collection.\n")
|
|
||||||
return 1
|
|
||||||
else:
|
|
||||||
# Start from current directory, search up to find collections
|
|
||||||
dir_path = os.path.abspath(os.getcwd())
|
|
||||||
while dir_path and dir_path != "/":
|
|
||||||
if os.path.exists(os.path.join(dir_path, CONFIG_FILE)):
|
|
||||||
collections.append(dir_path)
|
|
||||||
break
|
|
||||||
dir_path = os.path.dirname(dir_path)
|
|
||||||
|
|
||||||
# If no collection is found, scan for collections in current directory
|
|
||||||
if not collections:
|
|
||||||
collections = find_collections(os.getcwd())
|
|
||||||
|
|
||||||
if not collections:
|
|
||||||
sys.stderr.write(
|
|
||||||
"Error: No search collections found. Use --in to specify a collection.\n"
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Perform the search
|
|
||||||
results = perform_search(
|
|
||||||
query_string=args.query,
|
|
||||||
collections=collections,
|
|
||||||
tag_filter=args.tag,
|
|
||||||
limit=args.limit,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not results:
|
|
||||||
print(f"No results found for '{args.query}'")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
print(f"Found {len(results)} results for '{args.query}':")
|
|
||||||
|
|
||||||
for i, result in enumerate(results, 1):
|
|
||||||
title = result.get("title", os.path.basename(result.get("path", "Unknown")))
|
|
||||||
path = result.get("absolute_path", "Unknown path")
|
|
||||||
score = result.get("score", 0)
|
|
||||||
collection = result.get("collection", "Unknown collection")
|
|
||||||
|
|
||||||
print(f"\n{i}. {title} ({score:.0f}% match) - {collection}")
|
|
||||||
print(f" Path: {path}")
|
|
||||||
|
|
||||||
# Show tags if available
|
|
||||||
tags = result.get("tags", [])
|
|
||||||
if tags:
|
|
||||||
print(f" Tags: {', '.join(tags)}")
|
|
||||||
|
|
||||||
# Show a snippet of content if available and detailed output is requested
|
|
||||||
if args.verbose and "content" in result:
|
|
||||||
content = result["content"]
|
|
||||||
if content:
|
|
||||||
# Truncate and sanitize for display
|
|
||||||
content = re.sub(r"\s+", " ", content)
|
|
||||||
snippet = content[:200] + ("..." if len(content) > 200 else "")
|
|
||||||
print(f" Snippet: {snippet}")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
"""Main entry point."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Manage searchable collections of files",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1]
|
|
||||||
if __doc__
|
|
||||||
else ""
|
|
||||||
if __doc__
|
|
||||||
else "", # Use the docstring as extended help
|
|
||||||
)
|
|
||||||
parser.add_argument("--version", action="version", version="search 1.0.0")
|
|
||||||
|
|
||||||
# Create subparsers for commands
|
|
||||||
subparsers = parser.add_subparsers(dest="command", help="Command to run")
|
|
||||||
|
|
||||||
# init command
|
|
||||||
init_parser = subparsers.add_parser(
|
|
||||||
"init", help="Initialize a directory as a search collection"
|
|
||||||
)
|
|
||||||
init_parser.add_argument(
|
|
||||||
"directory",
|
|
||||||
nargs="?",
|
|
||||||
help="Directory to initialize (default: current directory)",
|
|
||||||
)
|
|
||||||
init_parser.add_argument("--name", help="Name for the collection")
|
|
||||||
init_parser.add_argument(
|
|
||||||
"--include", nargs="+", help='Include patterns (e.g., "*.pdf" "*.md")'
|
|
||||||
)
|
|
||||||
init_parser.add_argument(
|
|
||||||
"--exclude", nargs="+", help='Exclude patterns (e.g., "*.bak" "draft_*")'
|
|
||||||
)
|
|
||||||
init_parser.add_argument(
|
|
||||||
"--extract", nargs="+", help='Extractor patterns (e.g., "*.pdf=pdf2md {input}")'
|
|
||||||
)
|
|
||||||
init_parser.add_argument(
|
|
||||||
"--force",
|
|
||||||
action="store_true",
|
|
||||||
help="Force overwrite if collection already exists",
|
|
||||||
)
|
|
||||||
|
|
||||||
# scan command
|
|
||||||
scan_parser = subparsers.add_parser(
|
|
||||||
"scan", help="Find all search collections in a directory tree"
|
|
||||||
)
|
|
||||||
scan_parser.add_argument(
|
|
||||||
"directory", nargs="?", help="Directory to scan (default: current directory)"
|
|
||||||
)
|
|
||||||
|
|
||||||
# index command
|
|
||||||
index_parser = subparsers.add_parser(
|
|
||||||
"index", help="Index all matched files in one or more collections"
|
|
||||||
)
|
|
||||||
index_parser.add_argument(
|
|
||||||
"directories",
|
|
||||||
nargs="*",
|
|
||||||
help="Directories to index (default: current directory)",
|
|
||||||
)
|
|
||||||
index_parser.add_argument(
|
|
||||||
"--verbose", action="store_true", help="Show detailed progress"
|
|
||||||
)
|
|
||||||
index_parser.add_argument(
|
|
||||||
"--no-cache", action="store_true", help="Skip caching extracted metadata"
|
|
||||||
)
|
|
||||||
|
|
||||||
# query command
|
|
||||||
query_parser = subparsers.add_parser(
|
|
||||||
"query", help="Search indexed collections using Tantivy"
|
|
||||||
)
|
|
||||||
query_parser.add_argument("query", help="Search query")
|
|
||||||
query_parser.add_argument(
|
|
||||||
"--in", dest="in_dir", help="Search in specific collection"
|
|
||||||
)
|
|
||||||
query_parser.add_argument("--tag", help="Filter by tag")
|
|
||||||
query_parser.add_argument(
|
|
||||||
"--limit", type=int, default=20, help="Maximum number of results (default: 20)"
|
|
||||||
)
|
|
||||||
query_parser.add_argument(
|
|
||||||
"--verbose", action="store_true", help="Show more detailed results"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Parse arguments
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Execute the appropriate command
|
|
||||||
if args.command == "init":
|
|
||||||
return cmd_init(args)
|
|
||||||
elif args.command == "scan":
|
|
||||||
return cmd_scan(args)
|
|
||||||
elif args.command == "index":
|
|
||||||
return cmd_index(args)
|
|
||||||
elif args.command == "query":
|
|
||||||
return cmd_query(args)
|
|
||||||
else:
|
|
||||||
parser.print_help()
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
[project]
|
|
||||||
name = "text-extractor"
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "Extract content and metadata from text files"
|
|
||||||
requires-python = ">=3.11"
|
|
||||||
dependencies = []
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
text-extractor = "text_extractor.main:main"
|
|
||||||
|
|
||||||
[build-system]
|
|
||||||
requires = ["hatchling"]
|
|
||||||
build-backend = "hatchling.build"
|
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
|
||||||
packages = ["src/text_extractor"]
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
text-extractor: Extract content and metadata from text files
|
|
||||||
|
|
||||||
Reads a text file and outputs a JSON object containing the file's content
|
|
||||||
and metadata including filename, size, creation and modification times.
|
|
||||||
|
|
||||||
Usage: text-extractor [options] [FILE]
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
text-extractor document.txt # Extract from a text file
|
|
||||||
cat file.txt | text-extractor # Read from stdin
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
from datetime import datetime
|
|
||||||
import mimetypes
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def get_file_metadata(file_path):
|
|
||||||
"""Get file metadata including creation time, modification time, size, etc."""
|
|
||||||
path = Path(file_path)
|
|
||||||
stat = path.stat()
|
|
||||||
|
|
||||||
# Get file modification and creation times
|
|
||||||
mtime = datetime.fromtimestamp(stat.st_mtime).isoformat()
|
|
||||||
try:
|
|
||||||
ctime = datetime.fromtimestamp(stat.st_ctime).isoformat()
|
|
||||||
except Exception:
|
|
||||||
ctime = mtime # Fallback if creation time not available
|
|
||||||
|
|
||||||
# Get mime type
|
|
||||||
mime_type, _ = mimetypes.guess_type(file_path)
|
|
||||||
if not mime_type:
|
|
||||||
mime_type = "text/plain" # Default to plain text
|
|
||||||
|
|
||||||
return {
|
|
||||||
"filename": path.name,
|
|
||||||
"path": str(path.absolute()),
|
|
||||||
"size": stat.st_size,
|
|
||||||
"created_at": ctime,
|
|
||||||
"modified_at": mtime,
|
|
||||||
"mime_type": mime_type,
|
|
||||||
"extension": path.suffix.lstrip(".") if path.suffix else "",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def process_file(file_path):
|
|
||||||
"""Extract content and metadata from the file."""
|
|
||||||
# Get file metadata
|
|
||||||
if file_path and os.path.exists(file_path):
|
|
||||||
metadata = get_file_metadata(file_path)
|
|
||||||
|
|
||||||
# Read file content
|
|
||||||
try:
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
content = f.read()
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
# Try again with latin-1 encoding
|
|
||||||
with open(file_path, "r", encoding="latin-1") as f:
|
|
||||||
content = f.read()
|
|
||||||
else:
|
|
||||||
# Reading from stdin
|
|
||||||
content = sys.stdin.read()
|
|
||||||
metadata = {
|
|
||||||
"filename": "stdin",
|
|
||||||
"path": "stdin",
|
|
||||||
"size": len(content),
|
|
||||||
"created_at": datetime.now().isoformat(),
|
|
||||||
"modified_at": datetime.now().isoformat(),
|
|
||||||
"mime_type": "text/plain",
|
|
||||||
"extension": "",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create result object
|
|
||||||
result = {
|
|
||||||
**metadata,
|
|
||||||
"content": content,
|
|
||||||
"title": metadata["filename"],
|
|
||||||
"tags": [], # No tags by default
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Extract content and metadata from text files",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog=__doc__.split("\n\n", 1)[1]
|
|
||||||
if __doc__
|
|
||||||
else "", # Use the docstring as extended help
|
|
||||||
)
|
|
||||||
parser.add_argument("file", nargs="?", help="Input file (default: stdin)")
|
|
||||||
parser.add_argument(
|
|
||||||
"-v", "--version", action="version", version="text-extractor 1.0.0"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Process the file
|
|
||||||
result = process_file(args.file)
|
|
||||||
|
|
||||||
# Output the result as JSON
|
|
||||||
print(json.dumps(result, indent=2))
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write(f"Error: {e}\n")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -12,28 +12,41 @@ import yaml
|
|||||||
from castle_core.manifest import ProgramSpec, JobSpec, ServiceSpec
|
from castle_core.manifest import ProgramSpec, JobSpec, ServiceSpec
|
||||||
|
|
||||||
|
|
||||||
|
CASTLE_HOME = Path.home() / ".castle"
|
||||||
|
CODE_DIR = CASTLE_HOME / "code"
|
||||||
|
ARTIFACTS_DIR = CASTLE_HOME / "artifacts"
|
||||||
|
SPECS_DIR = ARTIFACTS_DIR / "specs"
|
||||||
|
CONTENT_DIR = ARTIFACTS_DIR / "content"
|
||||||
|
DATA_DIR = CASTLE_HOME / "data"
|
||||||
|
SECRETS_DIR = CASTLE_HOME / "secrets"
|
||||||
|
|
||||||
|
# Backwards-compat aliases (used by existing imports)
|
||||||
|
GENERATED_DIR = SPECS_DIR
|
||||||
|
STATIC_DIR = CONTENT_DIR
|
||||||
|
|
||||||
|
|
||||||
def find_castle_root() -> Path:
|
def find_castle_root() -> Path:
|
||||||
"""Find the castle repository root by walking up from cwd looking for castle.yaml."""
|
"""Find the castle config root (directory containing castle.yaml).
|
||||||
|
|
||||||
|
Search order:
|
||||||
|
1. ~/.castle/castle.yaml (the canonical instance location)
|
||||||
|
2. Walk up from cwd (for development/testing)
|
||||||
|
"""
|
||||||
|
# Canonical location first
|
||||||
|
if (CASTLE_HOME / "castle.yaml").exists():
|
||||||
|
return CASTLE_HOME
|
||||||
|
# Fallback: walk up from cwd
|
||||||
current = Path.cwd()
|
current = Path.cwd()
|
||||||
while current != current.parent:
|
while current != current.parent:
|
||||||
if (current / "castle.yaml").exists():
|
if (current / "castle.yaml").exists():
|
||||||
return current
|
return current
|
||||||
current = current.parent
|
current = current.parent
|
||||||
# Fallback: check if castle.yaml is in a well-known location
|
|
||||||
default = Path("/data/repos/castle")
|
|
||||||
if (default / "castle.yaml").exists():
|
|
||||||
return default
|
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
"Could not find castle.yaml. Run castle from within the castle repository."
|
"Could not find castle.yaml.\n"
|
||||||
|
f"Expected at: {CASTLE_HOME / 'castle.yaml'}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
CASTLE_HOME = Path.home() / ".castle"
|
|
||||||
GENERATED_DIR = CASTLE_HOME / "generated"
|
|
||||||
SECRETS_DIR = CASTLE_HOME / "secrets"
|
|
||||||
STATIC_DIR = CASTLE_HOME / "static"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GatewayConfig:
|
class GatewayConfig:
|
||||||
"""Gateway configuration."""
|
"""Gateway configuration."""
|
||||||
@@ -47,6 +60,7 @@ class CastleConfig:
|
|||||||
|
|
||||||
root: Path
|
root: Path
|
||||||
gateway: GatewayConfig
|
gateway: GatewayConfig
|
||||||
|
repo: Path | None
|
||||||
programs: dict[str, ProgramSpec]
|
programs: dict[str, ProgramSpec]
|
||||||
services: dict[str, ServiceSpec]
|
services: dict[str, ServiceSpec]
|
||||||
jobs: dict[str, JobSpec]
|
jobs: dict[str, JobSpec]
|
||||||
@@ -130,14 +144,23 @@ def load_config(root: Path | None = None) -> CastleConfig:
|
|||||||
gateway_data = data.get("gateway", {})
|
gateway_data = data.get("gateway", {})
|
||||||
gateway = GatewayConfig(port=gateway_data.get("port", 9000))
|
gateway = GatewayConfig(port=gateway_data.get("port", 9000))
|
||||||
|
|
||||||
|
# repo: field points to the git repo for repo-relative sources
|
||||||
|
repo_path: Path | None = None
|
||||||
|
if data.get("repo"):
|
||||||
|
repo_path = Path(data["repo"]).expanduser()
|
||||||
|
|
||||||
programs: dict[str, ProgramSpec] = {}
|
programs: dict[str, ProgramSpec] = {}
|
||||||
# Support both "programs:" and legacy "components:" key
|
# Support both "programs:" and legacy "components:" key
|
||||||
programs_data = data.get("programs") or data.get("components") or {}
|
programs_data = data.get("programs") or data.get("components") or {}
|
||||||
for name, comp_data in programs_data.items():
|
for name, comp_data in programs_data.items():
|
||||||
prog = _parse_program(name, comp_data)
|
prog = _parse_program(name, comp_data)
|
||||||
# Resolve relative source paths to absolute
|
# Resolve source paths to absolute
|
||||||
if prog.source and not Path(prog.source).is_absolute():
|
if prog.source:
|
||||||
prog.source = str(root / prog.source)
|
if prog.source.startswith("repo:") and repo_path:
|
||||||
|
# repo:castle-api → /data/repos/castle/castle-api
|
||||||
|
prog.source = str(repo_path / prog.source[5:])
|
||||||
|
elif not Path(prog.source).is_absolute():
|
||||||
|
prog.source = str(root / prog.source)
|
||||||
programs[name] = prog
|
programs[name] = prog
|
||||||
|
|
||||||
services: dict[str, ServiceSpec] = {}
|
services: dict[str, ServiceSpec] = {}
|
||||||
@@ -150,6 +173,7 @@ def load_config(root: Path | None = None) -> CastleConfig:
|
|||||||
|
|
||||||
return CastleConfig(
|
return CastleConfig(
|
||||||
root=root,
|
root=root,
|
||||||
|
repo=repo_path,
|
||||||
gateway=gateway,
|
gateway=gateway,
|
||||||
programs=programs,
|
programs=programs,
|
||||||
services=services,
|
services=services,
|
||||||
@@ -231,14 +255,27 @@ def save_config(config: CastleConfig) -> None:
|
|||||||
"""Save castle configuration to castle.yaml."""
|
"""Save castle configuration to castle.yaml."""
|
||||||
data: dict = {"gateway": {"port": config.gateway.port}}
|
data: dict = {"gateway": {"port": config.gateway.port}}
|
||||||
|
|
||||||
|
if config.repo:
|
||||||
|
data["repo"] = str(config.repo)
|
||||||
|
|
||||||
if config.programs:
|
if config.programs:
|
||||||
data["programs"] = {}
|
data["programs"] = {}
|
||||||
for name, spec in config.programs.items():
|
for name, spec in config.programs.items():
|
||||||
d = _spec_to_yaml_dict(spec)
|
d = _spec_to_yaml_dict(spec)
|
||||||
# Store relative source paths in YAML
|
# Store relative source paths in YAML
|
||||||
if d.get("source") and Path(d["source"]).is_absolute():
|
if d.get("source") and Path(d["source"]).is_absolute():
|
||||||
|
src = Path(d["source"])
|
||||||
|
# If source is under repo, store as repo:relative
|
||||||
|
if config.repo:
|
||||||
|
try:
|
||||||
|
d["source"] = "repo:" + str(src.relative_to(config.repo))
|
||||||
|
data["programs"][name] = d
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
# Otherwise store relative to config root
|
||||||
try:
|
try:
|
||||||
d["source"] = str(Path(d["source"]).relative_to(config.root))
|
d["source"] = str(src.relative_to(config.root))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass # not under root — keep absolute
|
pass # not under root — keep absolute
|
||||||
data["programs"][name] = d
|
data["programs"][name] = d
|
||||||
@@ -261,7 +298,9 @@ def save_config(config: CastleConfig) -> None:
|
|||||||
def ensure_dirs() -> None:
|
def ensure_dirs() -> None:
|
||||||
"""Ensure castle directories exist."""
|
"""Ensure castle directories exist."""
|
||||||
CASTLE_HOME.mkdir(parents=True, exist_ok=True)
|
CASTLE_HOME.mkdir(parents=True, exist_ok=True)
|
||||||
GENERATED_DIR.mkdir(parents=True, exist_ok=True)
|
CODE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
SPECS_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
CONTENT_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
SECRETS_DIR.mkdir(parents=True, exist_ok=True)
|
SECRETS_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
STATIC_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
os.chmod(SECRETS_DIR, 0o700)
|
os.chmod(SECRETS_DIR, 0o700)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from castle_core.config import GENERATED_DIR, STATIC_DIR
|
from castle_core.config import CONTENT_DIR, SPECS_DIR
|
||||||
from castle_core.registry import NodeRegistry
|
from castle_core.registry import NodeRegistry
|
||||||
|
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ def generate_caddyfile_from_registry(
|
|||||||
) -> str:
|
) -> str:
|
||||||
"""Generate Caddyfile from the node registry.
|
"""Generate Caddyfile from the node registry.
|
||||||
|
|
||||||
Static files served from ~/.castle/static/castle-app/.
|
Static files served from ~/.castle/artifacts/content/castle-app/.
|
||||||
No repo-relative paths.
|
No repo-relative paths.
|
||||||
|
|
||||||
If remote_registries is provided, cross-node routes are added for
|
If remote_registries is provided, cross-node routes are added for
|
||||||
@@ -49,8 +49,28 @@ def generate_caddyfile_from_registry(
|
|||||||
lines.append(" }")
|
lines.append(" }")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# SPA from static dir
|
# Static frontends from ~/.castle/static/<name>/
|
||||||
static_app = STATIC_DIR / "castle-app"
|
# Any directory with an index.html gets served as a SPA at its name prefix.
|
||||||
|
# castle-app is special-cased to serve at the root (no prefix).
|
||||||
|
if CONTENT_DIR.is_dir():
|
||||||
|
for app_dir in sorted(CONTENT_DIR.iterdir()):
|
||||||
|
if not app_dir.is_dir() or not (app_dir / "index.html").exists():
|
||||||
|
continue
|
||||||
|
if app_dir.name == "castle-app":
|
||||||
|
continue # handled below as root fallback
|
||||||
|
path_prefix = f"/{app_dir.name}"
|
||||||
|
if path_prefix in local_paths:
|
||||||
|
continue
|
||||||
|
local_paths.add(path_prefix)
|
||||||
|
lines.append(f" handle_path {path_prefix}/* {{")
|
||||||
|
lines.append(f" root * {app_dir}")
|
||||||
|
lines.append(" try_files {path} /index.html")
|
||||||
|
lines.append(" file_server")
|
||||||
|
lines.append(" }")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# castle-app SPA at root (fallback)
|
||||||
|
static_app = CONTENT_DIR / "castle-app"
|
||||||
if (static_app / "index.html").exists():
|
if (static_app / "index.html").exists():
|
||||||
lines.append(" handle {")
|
lines.append(" handle {")
|
||||||
lines.append(f" root * {static_app}")
|
lines.append(f" root * {static_app}")
|
||||||
@@ -58,7 +78,7 @@ def generate_caddyfile_from_registry(
|
|||||||
lines.append(" file_server")
|
lines.append(" file_server")
|
||||||
lines.append(" }")
|
lines.append(" }")
|
||||||
else:
|
else:
|
||||||
fallback = GENERATED_DIR / "app"
|
fallback = SPECS_DIR / "app"
|
||||||
lines.append(" handle / {")
|
lines.append(" handle / {")
|
||||||
lines.append(f" root * {fallback}")
|
lines.append(f" root * {fallback}")
|
||||||
lines.append(" file_server")
|
lines.append(" file_server")
|
||||||
|
|||||||
@@ -8,10 +8,10 @@ from pathlib import Path
|
|||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from castle_core.config import CASTLE_HOME
|
from castle_core.config import CONTENT_DIR, SPECS_DIR
|
||||||
|
|
||||||
REGISTRY_PATH = CASTLE_HOME / "registry.yaml"
|
REGISTRY_PATH = SPECS_DIR / "registry.yaml"
|
||||||
STATIC_DIR = CASTLE_HOME / "static"
|
STATIC_DIR = CONTENT_DIR # backwards-compat alias
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import tomllib
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from castle_core.config import STATIC_DIR
|
from castle_core.config import CONTENT_DIR
|
||||||
from castle_core.manifest import ProgramSpec
|
from castle_core.manifest import ProgramSpec
|
||||||
|
|
||||||
DEV_ACTIONS = ["build", "test", "lint", "type-check", "check"]
|
DEV_ACTIONS = ["build", "test", "lint", "type-check", "check"]
|
||||||
@@ -214,19 +214,19 @@ class ReactViteHandler(StackHandler):
|
|||||||
for output_dir in outputs:
|
for output_dir in outputs:
|
||||||
src_path = src / output_dir
|
src_path = src / output_dir
|
||||||
if src_path.exists():
|
if src_path.exists():
|
||||||
dest = STATIC_DIR / name
|
dest = CONTENT_DIR / name
|
||||||
if dest.exists():
|
if dest.exists():
|
||||||
shutil.rmtree(dest)
|
shutil.rmtree(dest)
|
||||||
shutil.copytree(src_path, dest)
|
shutil.copytree(src_path, dest)
|
||||||
|
|
||||||
return ActionResult(
|
return ActionResult(
|
||||||
component=name, action="install", status="ok",
|
component=name, action="install", status="ok",
|
||||||
output=f"Built and deployed to {STATIC_DIR / name}",
|
output=f"Built and deployed to {CONTENT_DIR / name}",
|
||||||
)
|
)
|
||||||
|
|
||||||
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
async def uninstall(self, name: str, comp: ProgramSpec, root: Path) -> ActionResult:
|
||||||
"""Remove static assets from ~/.castle/static/{name}/."""
|
"""Remove static assets from ~/.castle/static/{name}/."""
|
||||||
dest = STATIC_DIR / name
|
dest = CONTENT_DIR / name
|
||||||
if dest.exists():
|
if dest.exists():
|
||||||
shutil.rmtree(dest)
|
shutil.rmtree(dest)
|
||||||
return ActionResult(
|
return ActionResult(
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ from castle_core.registry import DeployedComponent, NodeConfig, NodeRegistry
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def _isolate_static_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
def _isolate_content_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
"""Use a temp dir for STATIC_DIR so tests don't depend on real ~/.castle."""
|
"""Use a temp dir for CONTENT_DIR so tests don't depend on real ~/.castle."""
|
||||||
monkeypatch.setattr(caddyfile_mod, "STATIC_DIR", tmp_path / "static")
|
monkeypatch.setattr(caddyfile_mod, "CONTENT_DIR", tmp_path / "content")
|
||||||
|
|
||||||
|
|
||||||
def _make_registry(
|
def _make_registry(
|
||||||
|
|||||||
@@ -49,11 +49,11 @@ class TestUnitFromDeployed:
|
|||||||
deployed = DeployedComponent(
|
deployed = DeployedComponent(
|
||||||
runner="python",
|
runner="python",
|
||||||
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
|
run_cmd=["/home/user/.local/bin/uv", "run", "test-svc"],
|
||||||
env={"TEST_SVC_DATA_DIR": "/data/castle/test-svc"},
|
env={"TEST_SVC_DATA_DIR": "/home/user/.castle/data/test-svc"},
|
||||||
description="Test service",
|
description="Test service",
|
||||||
)
|
)
|
||||||
unit = generate_unit_from_deployed("test-svc", deployed)
|
unit = generate_unit_from_deployed("test-svc", deployed)
|
||||||
assert "Environment=TEST_SVC_DATA_DIR=/data/castle/test-svc" in unit
|
assert "Environment=TEST_SVC_DATA_DIR=/home/user/.castle/data/test-svc" in unit
|
||||||
|
|
||||||
def test_contains_restart_policy(self) -> None:
|
def test_contains_restart_policy(self) -> None:
|
||||||
"""Unit file has restart configuration."""
|
"""Unit file has restart configuration."""
|
||||||
@@ -82,14 +82,14 @@ class TestUnitFromDeployed:
|
|||||||
deployed = DeployedComponent(
|
deployed = DeployedComponent(
|
||||||
runner="python",
|
runner="python",
|
||||||
run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"],
|
run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"],
|
||||||
env={"MY_SVC_PORT": "9001", "MY_SVC_DATA_DIR": "/data/castle/my-svc"},
|
env={"MY_SVC_PORT": "9001", "MY_SVC_DATA_DIR": "/home/user/.castle/data/my-svc"},
|
||||||
description="My service",
|
description="My service",
|
||||||
)
|
)
|
||||||
unit = generate_unit_from_deployed("my-svc", deployed)
|
unit = generate_unit_from_deployed("my-svc", deployed)
|
||||||
assert "Description=Castle: My service" in unit
|
assert "Description=Castle: My service" in unit
|
||||||
assert "ExecStart=/home/user/.local/bin/uv run my-svc" in unit
|
assert "ExecStart=/home/user/.local/bin/uv run my-svc" in unit
|
||||||
assert "Environment=MY_SVC_PORT=9001" in unit
|
assert "Environment=MY_SVC_PORT=9001" in unit
|
||||||
assert "Environment=MY_SVC_DATA_DIR=/data/castle/my-svc" in unit
|
assert "Environment=MY_SVC_DATA_DIR=/home/user/.castle/data/my-svc" in unit
|
||||||
assert "WorkingDirectory" not in unit
|
assert "WorkingDirectory" not in unit
|
||||||
assert "Restart=on-failure" in unit
|
assert "Restart=on-failure" in unit
|
||||||
|
|
||||||
@@ -111,7 +111,7 @@ class TestUnitFromDeployed:
|
|||||||
deployed = DeployedComponent(
|
deployed = DeployedComponent(
|
||||||
runner="python",
|
runner="python",
|
||||||
run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"],
|
run_cmd=["/home/user/.local/bin/uv", "run", "my-svc"],
|
||||||
env={"DATA_DIR": "/data/castle/my-svc"},
|
env={"DATA_DIR": "/home/user/.castle/data/my-svc"},
|
||||||
description="Test",
|
description="Test",
|
||||||
)
|
)
|
||||||
unit = generate_unit_from_deployed("my-svc", deployed)
|
unit = generate_unit_from_deployed("my-svc", deployed)
|
||||||
|
|||||||
664
docs/working/discussion.md
Normal file
664
docs/working/discussion.md
Normal file
@@ -0,0 +1,664 @@
|
|||||||
|
Yes — that changes the design a lot, in a good way.
|
||||||
|
|
||||||
|
If **castle is not a general platform**, but a **curated machine for you + an AI assistant**, then the ontology should be optimized for:
|
||||||
|
|
||||||
|
* few concepts
|
||||||
|
* strong defaults
|
||||||
|
* predictable placement
|
||||||
|
* limited extension paths
|
||||||
|
* easy generation from natural language
|
||||||
|
|
||||||
|
That means castle should **not** try to model all possible machine semantics. Ubuntu already does that. Castle should define a **small opinionated overlay** on top of Ubuntu.
|
||||||
|
|
||||||
|
So I’d stop trying to find the perfect universal ontology and instead design a **usefully incomplete one**.
|
||||||
|
|
||||||
|
## The design principle
|
||||||
|
|
||||||
|
Castle should answer:
|
||||||
|
|
||||||
|
> What are the few things I commonly want the machine to have or do?
|
||||||
|
|
||||||
|
From your examples, those are roughly:
|
||||||
|
|
||||||
|
* a **tool**
|
||||||
|
* a **site**
|
||||||
|
* a **service**
|
||||||
|
* a **worker/daemon**
|
||||||
|
* a **job**
|
||||||
|
* a **store/broker**
|
||||||
|
* maybe a **library/project**
|
||||||
|
|
||||||
|
That suggests the core user-facing abstraction should not be “artifact” or “namespace” or even “runtime.”
|
||||||
|
|
||||||
|
It should be something more like:
|
||||||
|
|
||||||
|
## Castle’s main abstraction: `unit`
|
||||||
|
|
||||||
|
A **unit** is a thing the castle machine knows how to host.
|
||||||
|
|
||||||
|
That thing may be:
|
||||||
|
|
||||||
|
* a Python tool
|
||||||
|
* a static site
|
||||||
|
* a web service
|
||||||
|
* a daemon
|
||||||
|
* a scheduled job
|
||||||
|
* a database
|
||||||
|
* a broker
|
||||||
|
|
||||||
|
Then beneath that, castle can have a smaller set of internal dimensions.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# A much simpler castle model
|
||||||
|
|
||||||
|
I think the practical model is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
unit
|
||||||
|
kind
|
||||||
|
stack
|
||||||
|
uses
|
||||||
|
```
|
||||||
|
|
||||||
|
Where:
|
||||||
|
|
||||||
|
## `unit`
|
||||||
|
|
||||||
|
The named thing on the machine.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `reddit-tool`
|
||||||
|
* `docs-site`
|
||||||
|
* `api`
|
||||||
|
* `transform-worker`
|
||||||
|
* `morning-search`
|
||||||
|
* `postgres`
|
||||||
|
* `mqtt`
|
||||||
|
|
||||||
|
## `kind`
|
||||||
|
|
||||||
|
The machine role.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `tool`
|
||||||
|
* `site`
|
||||||
|
* `service`
|
||||||
|
* `worker`
|
||||||
|
* `job`
|
||||||
|
* `store`
|
||||||
|
* `broker`
|
||||||
|
|
||||||
|
## `stack`
|
||||||
|
|
||||||
|
How it is built/run.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `python`
|
||||||
|
* `rust`
|
||||||
|
* `react-static`
|
||||||
|
* `fastapi`
|
||||||
|
* `shell`
|
||||||
|
|
||||||
|
This is where language/build/runtime conventions collapse into one curated choice.
|
||||||
|
|
||||||
|
## `uses`
|
||||||
|
|
||||||
|
What castle subsystems or other units it depends on.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `postgres`
|
||||||
|
* `neo4j`
|
||||||
|
* `mqtt`
|
||||||
|
* `caddy`
|
||||||
|
* `reddit`
|
||||||
|
* `web`
|
||||||
|
|
||||||
|
This is much closer to how you actually want to talk to the AI.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Why this is better
|
||||||
|
|
||||||
|
You don’t want to say:
|
||||||
|
|
||||||
|
> create a deployment of kind service using runtime python with artifact class program and lifecycle systemd
|
||||||
|
|
||||||
|
You want to say:
|
||||||
|
|
||||||
|
> add a Python service that does X and uses Postgres
|
||||||
|
|
||||||
|
So castle should be shaped around **intentional unit kinds**, not abstract machine theory.
|
||||||
|
|
||||||
|
The machine theory should stay underneath as implementation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# What castle can standardize
|
||||||
|
|
||||||
|
Because castle is curated, you can hardcode a lot:
|
||||||
|
|
||||||
|
## Always true on castle
|
||||||
|
|
||||||
|
* Ubuntu
|
||||||
|
* systemd
|
||||||
|
* Caddy
|
||||||
|
* standard directory layout
|
||||||
|
* standard logging
|
||||||
|
* standard service supervision
|
||||||
|
* standard way to expose commands on PATH
|
||||||
|
* standard way to run Python via `uv`
|
||||||
|
* standard way to build JS via `pnpm`
|
||||||
|
* standard way to define dependencies between units
|
||||||
|
|
||||||
|
That means users and AI do **not** need to specify those every time.
|
||||||
|
|
||||||
|
This is the key simplification move.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Recommended fixed subsystems
|
||||||
|
|
||||||
|
I’d make these built-in castle subsystems:
|
||||||
|
|
||||||
|
* **python** via `uv`
|
||||||
|
* **rust** via `cargo`
|
||||||
|
* **node/react** via `pnpm`
|
||||||
|
* **java** via one standard choice
|
||||||
|
* **caddy**
|
||||||
|
* **systemd**
|
||||||
|
* **postgres**
|
||||||
|
* **mqtt**
|
||||||
|
* maybe **neo4j**
|
||||||
|
* maybe **web-search ingestion** as a castle-native capability
|
||||||
|
|
||||||
|
Then the schema does not need to model arbitrary runtimes. It only needs to reference these known capabilities.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Strong defaults for placement
|
||||||
|
|
||||||
|
You said castle can always use well-known spots. Good. Do that aggressively.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/castle/units/<name>/src
|
||||||
|
/castle/units/<name>/build
|
||||||
|
/castle/units/<name>/env
|
||||||
|
/castle/units/<name>/state
|
||||||
|
/castle/units/<name>/config
|
||||||
|
/castle/units/<name>/logs
|
||||||
|
```
|
||||||
|
|
||||||
|
And for exposure:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/castle/bin/<tool>
|
||||||
|
/castle/sites/<site>
|
||||||
|
/castle/data/<store>
|
||||||
|
```
|
||||||
|
|
||||||
|
Then castle itself can generate:
|
||||||
|
|
||||||
|
* symlinks into `/usr/local/bin`
|
||||||
|
* systemd units
|
||||||
|
* Caddy config
|
||||||
|
* env files
|
||||||
|
* state dirs
|
||||||
|
|
||||||
|
So the schema does not need to talk much about placement, because placement is mostly implied.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# The actual user-facing kinds
|
||||||
|
|
||||||
|
I’d recommend just these:
|
||||||
|
|
||||||
|
## 1. `tool`
|
||||||
|
|
||||||
|
A command you can run from anywhere.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* Python CLI
|
||||||
|
* Rust CLI
|
||||||
|
* shell helper
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* exposed on PATH
|
||||||
|
* no long-running process
|
||||||
|
* build/install conventions by stack
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
summarize-reddit:
|
||||||
|
kind: tool
|
||||||
|
stack: python
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. `site`
|
||||||
|
|
||||||
|
A human-facing HTTP site served by Caddy.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* React static site
|
||||||
|
* docs site
|
||||||
|
* maybe proxied app frontend later
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* served by Caddy
|
||||||
|
* standard root dir
|
||||||
|
* standard local hostname or port
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
castle-web:
|
||||||
|
kind: site
|
||||||
|
stack: react-static
|
||||||
|
port: 8080
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. `service`
|
||||||
|
|
||||||
|
A long-running HTTP or TCP process supervised by systemd.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* FastAPI app
|
||||||
|
* Rust API
|
||||||
|
* Java backend
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* systemd-managed
|
||||||
|
* logs to journal
|
||||||
|
* optional Caddy exposure if HTTP
|
||||||
|
* standard environment/config paths
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
reddit-api:
|
||||||
|
kind: service
|
||||||
|
stack: fastapi
|
||||||
|
port: 9090
|
||||||
|
uses: [postgres, neo4j, reddit]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. `worker`
|
||||||
|
|
||||||
|
A long-running non-HTTP daemon, often connected to broker/store inputs.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* MQTT consumer
|
||||||
|
* pubsub transformer
|
||||||
|
* sync loop
|
||||||
|
* queue processor
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* systemd-managed
|
||||||
|
* usually not publicly exposed
|
||||||
|
* can subscribe to broker/topic or poll/store
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
topic-transformer:
|
||||||
|
kind: worker
|
||||||
|
stack: python
|
||||||
|
uses: [mqtt]
|
||||||
|
subscribe: channel-b
|
||||||
|
publish: channel-y
|
||||||
|
```
|
||||||
|
|
||||||
|
This kind seems especially important for your goals.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. `job`
|
||||||
|
|
||||||
|
A scheduled or one-shot task.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* morning search
|
||||||
|
* report generation
|
||||||
|
* backup
|
||||||
|
* fetch and write JSON
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* systemd timer
|
||||||
|
* writes to known output locations
|
||||||
|
* can use web-search capability
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
morning-z-search:
|
||||||
|
kind: job
|
||||||
|
stack: python
|
||||||
|
schedule: daily@08:00
|
||||||
|
uses: [web]
|
||||||
|
outputs:
|
||||||
|
- /castle/shared/search/z/
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. `store`
|
||||||
|
|
||||||
|
A built-in stateful service.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* Postgres
|
||||||
|
* Neo4j
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
* castle manages lifecycle and placement
|
||||||
|
* other units reference by name
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
postgres:
|
||||||
|
kind: store
|
||||||
|
engine: postgres
|
||||||
|
|
||||||
|
neo4j:
|
||||||
|
kind: store
|
||||||
|
engine: neo4j
|
||||||
|
```
|
||||||
|
|
||||||
|
You may even decide these are so built-in they don’t need to be declared unless customized.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. `broker`
|
||||||
|
|
||||||
|
A built-in messaging service.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* MQTT
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
mqtt:
|
||||||
|
kind: broker
|
||||||
|
engine: mqtt
|
||||||
|
```
|
||||||
|
|
||||||
|
Again, maybe built-in by default.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# The schema can now be tiny
|
||||||
|
|
||||||
|
Instead of a universal schema, use a compact one with per-kind defaults.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
units:
|
||||||
|
reddit-tool:
|
||||||
|
kind: tool
|
||||||
|
stack: python
|
||||||
|
description: CLI for querying reddit summaries
|
||||||
|
|
||||||
|
castle-web:
|
||||||
|
kind: site
|
||||||
|
stack: react-static
|
||||||
|
port: 8080
|
||||||
|
|
||||||
|
reddit-api:
|
||||||
|
kind: service
|
||||||
|
stack: fastapi
|
||||||
|
port: 9090
|
||||||
|
uses: [postgres, neo4j, reddit]
|
||||||
|
|
||||||
|
topic-transformer:
|
||||||
|
kind: worker
|
||||||
|
stack: python
|
||||||
|
uses: [mqtt]
|
||||||
|
subscribe: channel-b
|
||||||
|
publish: channel-y
|
||||||
|
|
||||||
|
morning-z-search:
|
||||||
|
kind: job
|
||||||
|
stack: python
|
||||||
|
schedule: "daily@08:00"
|
||||||
|
uses: [web]
|
||||||
|
output_dir: /castle/shared/search/z
|
||||||
|
```
|
||||||
|
|
||||||
|
That is probably closer to the right level.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# What `stack` should mean now
|
||||||
|
|
||||||
|
Since castle is curated, `stack` can absorb a lot:
|
||||||
|
|
||||||
|
Instead of:
|
||||||
|
|
||||||
|
* build system
|
||||||
|
* runtime
|
||||||
|
* conventions
|
||||||
|
* entrypoint rules
|
||||||
|
|
||||||
|
just use a known stack name.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `python`
|
||||||
|
* `fastapi`
|
||||||
|
* `rust`
|
||||||
|
* `react-static`
|
||||||
|
* `java-service`
|
||||||
|
* `shell`
|
||||||
|
|
||||||
|
Each stack implies:
|
||||||
|
|
||||||
|
* where source goes
|
||||||
|
* how to build
|
||||||
|
* how to run
|
||||||
|
* what files are expected
|
||||||
|
* how dependencies are installed
|
||||||
|
|
||||||
|
This is much simpler than separately modeling build/runtime/deployment for every case.
|
||||||
|
|
||||||
|
So for castle:
|
||||||
|
|
||||||
|
> **stack = curated implementation pattern**
|
||||||
|
|
||||||
|
That’s probably the right definition.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# What `uses` should mean
|
||||||
|
|
||||||
|
This is another very valuable simplification.
|
||||||
|
|
||||||
|
Instead of asking the user to define bindings and interfaces, let them say what the unit uses.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
* `postgres`
|
||||||
|
* `neo4j`
|
||||||
|
* `mqtt`
|
||||||
|
* `caddy`
|
||||||
|
* `web`
|
||||||
|
* `reddit`
|
||||||
|
* another unit name
|
||||||
|
|
||||||
|
Then castle knows how to wire it.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
uses: [postgres, neo4j, reddit]
|
||||||
|
```
|
||||||
|
|
||||||
|
could imply:
|
||||||
|
|
||||||
|
* env vars for local connection strings
|
||||||
|
* service ordering in systemd
|
||||||
|
* generated client config
|
||||||
|
* AI awareness that those capabilities are available
|
||||||
|
|
||||||
|
This is much more useful than formal dependency graphs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Under the hood
|
||||||
|
|
||||||
|
Internally, castle can still map to:
|
||||||
|
|
||||||
|
* artifacts
|
||||||
|
* runtimes
|
||||||
|
* systemd
|
||||||
|
* Caddy
|
||||||
|
* mount namespaces
|
||||||
|
* env files
|
||||||
|
* directories
|
||||||
|
|
||||||
|
But that should be hidden.
|
||||||
|
|
||||||
|
The AI assistant should primarily think in terms of:
|
||||||
|
|
||||||
|
* **What kind of unit is this?**
|
||||||
|
* **What stack should it use?**
|
||||||
|
* **What does it use?**
|
||||||
|
* **What should it be named?**
|
||||||
|
|
||||||
|
That’s enough to generate the machine config.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# So the real castle ontology may be
|
||||||
|
|
||||||
|
## User-facing
|
||||||
|
|
||||||
|
```text
|
||||||
|
unit
|
||||||
|
kind
|
||||||
|
stack
|
||||||
|
uses
|
||||||
|
```
|
||||||
|
|
||||||
|
## Machine-facing
|
||||||
|
|
||||||
|
```text
|
||||||
|
source
|
||||||
|
artifact
|
||||||
|
runtime
|
||||||
|
deployment
|
||||||
|
placement
|
||||||
|
```
|
||||||
|
|
||||||
|
This split is important.
|
||||||
|
|
||||||
|
You were trying to find one ontology for both humans and machines. You probably want two:
|
||||||
|
|
||||||
|
* a **simple command ontology** for you + AI
|
||||||
|
* a **lower-level implementation ontology** for castle internals
|
||||||
|
|
||||||
|
That resolves a lot of the tension.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Example natural language mapping
|
||||||
|
|
||||||
|
“Add a python tool that does X”
|
||||||
|
|
||||||
|
becomes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: x-tool
|
||||||
|
kind: tool
|
||||||
|
stack: python
|
||||||
|
```
|
||||||
|
|
||||||
|
“Make a web service that does Y”
|
||||||
|
|
||||||
|
becomes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: y-service
|
||||||
|
kind: service
|
||||||
|
stack: fastapi
|
||||||
|
```
|
||||||
|
|
||||||
|
“Make a daemon that watches pubsub channel B and transforms the data and puts it on Y”
|
||||||
|
|
||||||
|
becomes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: b-to-y
|
||||||
|
kind: worker
|
||||||
|
stack: python
|
||||||
|
uses: [mqtt]
|
||||||
|
subscribe: B
|
||||||
|
publish: Y
|
||||||
|
```
|
||||||
|
|
||||||
|
“Every morning, do a web search for Z and put the results in PATH as json files”
|
||||||
|
|
||||||
|
becomes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: z-search
|
||||||
|
kind: job
|
||||||
|
stack: python
|
||||||
|
uses: [web]
|
||||||
|
schedule: daily@08:00
|
||||||
|
output_dir: /castle/shared/search/z
|
||||||
|
format: json
|
||||||
|
```
|
||||||
|
|
||||||
|
This is very manageable.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Recommendation
|
||||||
|
|
||||||
|
I would make castle intentionally opinionated around:
|
||||||
|
|
||||||
|
* **7 unit kinds max**
|
||||||
|
* **5–8 stacks max**
|
||||||
|
* **a handful of built-in capabilities in `uses`**
|
||||||
|
* **fixed placement conventions**
|
||||||
|
* **automatic generation of systemd/Caddy/config**
|
||||||
|
|
||||||
|
That seems like the right balance of simplicity and extensibility for what you want.
|
||||||
|
|
||||||
|
The next useful step is to draft the **minimal v1 castle schema** with only those user-facing fields and strong defaults.
|
||||||
418
install.sh
Executable file
418
install.sh
Executable file
@@ -0,0 +1,418 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Castle Platform — Bootstrap Install
|
||||||
|
#
|
||||||
|
# Idempotent script that sets up the infrastructure "control layer":
|
||||||
|
# - Docker, Caddy (system binary)
|
||||||
|
# - MQTT broker, Postgres, Neo4j (Docker containers or existing)
|
||||||
|
# - Directory structure, systemd lingering, seed configs
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./install.sh # Interactive — prompts for existing services
|
||||||
|
# ./install.sh --yes # Non-interactive — use containers for everything
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
CASTLE_HOME="${HOME}/.castle"
|
||||||
|
CASTLE_CONF="${CASTLE_HOME}/infra.conf"
|
||||||
|
DATA_DIR="${CASTLE_HOME}/data"
|
||||||
|
CASTLE_ROOT="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
|
||||||
|
# Container defaults
|
||||||
|
MQTT_IMAGE="eclipse-mosquitto:2"
|
||||||
|
MQTT_PORT=1883
|
||||||
|
POSTGRES_IMAGE="postgres:17"
|
||||||
|
POSTGRES_PORT=5432
|
||||||
|
NEO4J_IMAGE="neo4j:5-community"
|
||||||
|
NEO4J_BOLT_PORT=7687
|
||||||
|
NEO4J_HTTP_PORT=7474
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Logging
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_bold="\033[1m"
|
||||||
|
_reset="\033[0m"
|
||||||
|
_green="\033[32m"
|
||||||
|
_yellow="\033[33m"
|
||||||
|
_red="\033[31m"
|
||||||
|
|
||||||
|
log_step() { printf "\n${_bold}[*] %s${_reset}\n" "$1"; }
|
||||||
|
log_ok() { printf " ${_green}OK${_reset}"; [ -n "${1:-}" ] && printf " — %s" "$1"; printf "\n"; }
|
||||||
|
log_skip() { printf " ${_yellow}skipped${_reset} (%s)\n" "$1"; }
|
||||||
|
log_fail() { printf " ${_red}FAILED${_reset}: %s\n" "$1"; exit 1; }
|
||||||
|
log_info() { printf " %s\n" "$1"; }
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
cmd_exists() { command -v "$1" &>/dev/null; }
|
||||||
|
|
||||||
|
port_in_use() {
|
||||||
|
ss -tlnp 2>/dev/null | grep -q ":${1} " && return 0
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ask_yes_no() {
|
||||||
|
local prompt="$1" default="${2:-y}"
|
||||||
|
if [ "${AUTO_YES:-}" = "1" ]; then
|
||||||
|
[ "$default" = "y" ] && return 0 || return 1
|
||||||
|
fi
|
||||||
|
local yn
|
||||||
|
if [ "$default" = "y" ]; then
|
||||||
|
read -r -p " ${prompt} [Y/n] " yn
|
||||||
|
yn="${yn:-y}"
|
||||||
|
else
|
||||||
|
read -r -p " ${prompt} [y/N] " yn
|
||||||
|
yn="${yn:-n}"
|
||||||
|
fi
|
||||||
|
[[ "$yn" =~ ^[Yy] ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
conf_get() {
|
||||||
|
local key="$1"
|
||||||
|
if [ -f "$CASTLE_CONF" ]; then
|
||||||
|
grep -m1 "^${key}=" "$CASTLE_CONF" 2>/dev/null | cut -d= -f2 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
conf_set() {
|
||||||
|
local key="$1" val="$2"
|
||||||
|
mkdir -p "$(dirname "$CASTLE_CONF")"
|
||||||
|
if [ -f "$CASTLE_CONF" ] && grep -q "^${key}=" "$CASTLE_CONF" 2>/dev/null; then
|
||||||
|
sed -i "s/^${key}=.*/${key}=${val}/" "$CASTLE_CONF"
|
||||||
|
else
|
||||||
|
echo "${key}=${val}" >> "$CASTLE_CONF"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Ensure a Docker container is running. Handles three states:
|
||||||
|
# - not exists → create
|
||||||
|
# - exists but stopped → start
|
||||||
|
# - exists and running → skip
|
||||||
|
ensure_container() {
|
||||||
|
local name="$1"
|
||||||
|
shift
|
||||||
|
local args=("$@")
|
||||||
|
|
||||||
|
if docker inspect "$name" &>/dev/null; then
|
||||||
|
local state
|
||||||
|
state=$(docker inspect --format '{{.State.Status}}' "$name" 2>/dev/null)
|
||||||
|
if [ "$state" = "running" ]; then
|
||||||
|
log_skip "already running"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
docker start "$name" >/dev/null
|
||||||
|
log_ok "started existing container"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run -d --restart=unless-stopped --name "$name" "${args[@]}" >/dev/null
|
||||||
|
log_ok "created and started"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Checks
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
check_systemd() {
|
||||||
|
log_step "Checking systemd"
|
||||||
|
if ! cmd_exists systemctl; then
|
||||||
|
log_fail "systemd is required but systemctl was not found"
|
||||||
|
fi
|
||||||
|
log_ok
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Package installation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
ensure_docker() {
|
||||||
|
log_step "Ensuring Docker"
|
||||||
|
if cmd_exists docker; then
|
||||||
|
log_skip "already installed"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
sudo apt-get update -qq
|
||||||
|
sudo apt-get install -y -qq docker.io >/dev/null
|
||||||
|
sudo usermod -aG docker "$USER"
|
||||||
|
log_ok
|
||||||
|
log_info "NOTE: You may need to log out and back in for Docker group access"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_caddy() {
|
||||||
|
log_step "Ensuring Caddy"
|
||||||
|
if cmd_exists caddy; then
|
||||||
|
log_skip "already installed"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
sudo apt-get install -y -qq debian-keyring debian-archive-keyring apt-transport-https >/dev/null 2>&1
|
||||||
|
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' \
|
||||||
|
| sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg 2>/dev/null
|
||||||
|
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \
|
||||||
|
| sudo tee /etc/apt/sources.list.d/caddy-stable.list >/dev/null
|
||||||
|
sudo apt-get update -qq
|
||||||
|
sudo apt-get install -y -qq caddy >/dev/null
|
||||||
|
# Disable the system-level caddy service — castle manages it via user systemd
|
||||||
|
sudo systemctl disable --now caddy 2>/dev/null || true
|
||||||
|
log_ok
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Directory structure
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
create_directories() {
|
||||||
|
log_step "Creating directory structure"
|
||||||
|
|
||||||
|
# ~/.castle tree
|
||||||
|
mkdir -p "${CASTLE_HOME}/code"
|
||||||
|
mkdir -p "${CASTLE_HOME}/artifacts/specs"
|
||||||
|
mkdir -p "${CASTLE_HOME}/artifacts/content"
|
||||||
|
mkdir -p "${DATA_DIR}"
|
||||||
|
mkdir -p "${CASTLE_HOME}/secrets" && chmod 700 "${CASTLE_HOME}/secrets"
|
||||||
|
|
||||||
|
log_ok
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Systemd lingering
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
enable_lingering() {
|
||||||
|
log_step "Enabling systemd user lingering"
|
||||||
|
if loginctl show-user "$USER" 2>/dev/null | grep -q "Linger=yes"; then
|
||||||
|
log_skip "already enabled"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
sudo loginctl enable-linger "$USER"
|
||||||
|
log_ok
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Seed configs
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
seed_caddyfile() {
|
||||||
|
log_step "Seeding Caddyfile"
|
||||||
|
local caddyfile="${CASTLE_HOME}/artifacts/specs/Caddyfile"
|
||||||
|
if [ -f "$caddyfile" ]; then
|
||||||
|
log_skip "already exists"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
cat > "$caddyfile" << 'EOF'
|
||||||
|
:9000 {
|
||||||
|
respond "Castle is starting. Run 'castle deploy' to configure." 200
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
log_ok
|
||||||
|
}
|
||||||
|
|
||||||
|
seed_mosquitto_config() {
|
||||||
|
local conf="${DATA_DIR}/castle-mqtt/config/mosquitto.conf"
|
||||||
|
if [ -f "$conf" ]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
mkdir -p "${DATA_DIR}/castle-mqtt/config"
|
||||||
|
mkdir -p "${DATA_DIR}/castle-mqtt/data"
|
||||||
|
cat > "$conf" << 'EOF'
|
||||||
|
listener 1883
|
||||||
|
allow_anonymous true
|
||||||
|
persistence true
|
||||||
|
persistence_location /mosquitto/data/
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Migration — old container names
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
migrate_old_containers() {
|
||||||
|
# castle-eclipse-mosquitto → castle-mqtt
|
||||||
|
if docker inspect castle-eclipse-mosquitto &>/dev/null 2>&1; then
|
||||||
|
log_step "Migrating castle-eclipse-mosquitto → castle-mqtt"
|
||||||
|
docker stop castle-eclipse-mosquitto 2>/dev/null || true
|
||||||
|
docker rm castle-eclipse-mosquitto 2>/dev/null || true
|
||||||
|
systemctl --user stop castle-castle-mqtt.service 2>/dev/null || true
|
||||||
|
systemctl --user disable castle-castle-mqtt.service 2>/dev/null || true
|
||||||
|
log_ok
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Infrastructure services
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Generic provisioning for an infrastructure service:
|
||||||
|
# - Check infra.conf for previous choice
|
||||||
|
# - Detect if port is in use
|
||||||
|
# - Ask user or auto-provision
|
||||||
|
provision_service() {
|
||||||
|
local name="$1" # e.g. MQTT
|
||||||
|
local container="$2" # e.g. castle-mqtt
|
||||||
|
local port="$3" # e.g. 1883
|
||||||
|
local display="$4" # e.g. "MQTT"
|
||||||
|
shift 4
|
||||||
|
local docker_args=("$@")
|
||||||
|
|
||||||
|
log_step "${display} (port ${port})"
|
||||||
|
|
||||||
|
# Check previous choice
|
||||||
|
local prev
|
||||||
|
prev=$(conf_get "$name")
|
||||||
|
|
||||||
|
if [ "$prev" = "existing" ]; then
|
||||||
|
log_skip "configured to use existing server"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$prev" = "container" ]; then
|
||||||
|
# We previously created a container — ensure it's running
|
||||||
|
ensure_container "$container" "${docker_args[@]}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if our own container already exists (maybe from a previous manual setup)
|
||||||
|
if docker inspect "$container" &>/dev/null; then
|
||||||
|
ensure_container "$container" "${docker_args[@]}"
|
||||||
|
conf_set "$name" "container"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# No previous choice, no existing container — detect and ask
|
||||||
|
if port_in_use "$port"; then
|
||||||
|
log_info "Detected: port ${port} is already in use (not a castle container)"
|
||||||
|
if ask_yes_no "Use existing ${display} server?"; then
|
||||||
|
conf_set "$name" "existing"
|
||||||
|
log_ok "using existing ${display} on localhost:${port}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
log_info "Port ${port} is in use — cannot start container on same port"
|
||||||
|
log_fail "Free port ${port} or choose to use the existing server"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Nothing on the port — install via Docker
|
||||||
|
log_info "No existing ${display} detected"
|
||||||
|
log_info "Installing ${container} via Docker..."
|
||||||
|
ensure_container "$container" "${docker_args[@]}"
|
||||||
|
conf_set "$name" "container"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_mqtt() {
|
||||||
|
seed_mosquitto_config
|
||||||
|
provision_service "MQTT" "castle-mqtt" "$MQTT_PORT" "MQTT" \
|
||||||
|
-p "${MQTT_PORT}:1883" \
|
||||||
|
-v "${DATA_DIR}/castle-mqtt/config:/mosquitto/config" \
|
||||||
|
-v "${DATA_DIR}/castle-mqtt/data:/mosquitto/data" \
|
||||||
|
"$MQTT_IMAGE"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_postgres() {
|
||||||
|
# Ensure password exists
|
||||||
|
local pass_file="${CASTLE_HOME}/secrets/POSTGRES_PASSWORD"
|
||||||
|
if [ ! -f "$pass_file" ]; then
|
||||||
|
openssl rand -base64 24 > "$pass_file"
|
||||||
|
chmod 600 "$pass_file"
|
||||||
|
fi
|
||||||
|
local pg_pass
|
||||||
|
pg_pass=$(cat "$pass_file")
|
||||||
|
|
||||||
|
mkdir -p "${DATA_DIR}/postgres/data"
|
||||||
|
|
||||||
|
provision_service "POSTGRES" "castle-postgres" "$POSTGRES_PORT" "Postgres" \
|
||||||
|
-p "${POSTGRES_PORT}:5432" \
|
||||||
|
-e "POSTGRES_USER=castle" \
|
||||||
|
-e "POSTGRES_PASSWORD=${pg_pass}" \
|
||||||
|
-e "POSTGRES_DB=castle" \
|
||||||
|
-v "${DATA_DIR}/postgres/data:/var/lib/postgresql/data" \
|
||||||
|
"$POSTGRES_IMAGE"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_neo4j() {
|
||||||
|
mkdir -p "${DATA_DIR}/neo4j/data"
|
||||||
|
mkdir -p "${DATA_DIR}/neo4j/logs"
|
||||||
|
|
||||||
|
provision_service "NEO4J" "castle-neo4j" "$NEO4J_BOLT_PORT" "Neo4j" \
|
||||||
|
-p "${NEO4J_HTTP_PORT}:7474" \
|
||||||
|
-p "${NEO4J_BOLT_PORT}:7687" \
|
||||||
|
-e "NEO4J_AUTH=neo4j/changeme" \
|
||||||
|
-v "${DATA_DIR}/neo4j/data:/data" \
|
||||||
|
-v "${DATA_DIR}/neo4j/logs:/logs" \
|
||||||
|
"$NEO4J_IMAGE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Summary
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
print_summary() {
|
||||||
|
printf "\n${_bold}========================================${_reset}\n"
|
||||||
|
printf "${_bold}Castle bootstrap complete!${_reset}\n"
|
||||||
|
printf "${_bold}========================================${_reset}\n\n"
|
||||||
|
|
||||||
|
printf "Infrastructure:\n"
|
||||||
|
printf " %-20s %s\n" "Caddy" "$(caddy version 2>/dev/null | head -1 || echo 'not found')"
|
||||||
|
printf " %-20s %s\n" "Docker" "$(docker --version 2>/dev/null | head -1 || echo 'not found')"
|
||||||
|
printf "\n"
|
||||||
|
|
||||||
|
printf "Services:\n"
|
||||||
|
local mqtt_status postgres_status neo4j_status
|
||||||
|
mqtt_status=$(conf_get MQTT)
|
||||||
|
postgres_status=$(conf_get POSTGRES)
|
||||||
|
neo4j_status=$(conf_get NEO4J)
|
||||||
|
printf " %-20s %s (port %s)\n" "MQTT" "${mqtt_status:-not configured}" "$MQTT_PORT"
|
||||||
|
printf " %-20s %s (port %s)\n" "Postgres" "${postgres_status:-not configured}" "$POSTGRES_PORT"
|
||||||
|
printf " %-20s %s (port %s)\n" "Neo4j" "${neo4j_status:-not configured}" "$NEO4J_BOLT_PORT"
|
||||||
|
printf "\n"
|
||||||
|
|
||||||
|
printf "Directories:\n"
|
||||||
|
printf " %-20s %s\n" "Castle home" "$CASTLE_HOME"
|
||||||
|
printf " %-20s %s\n" "Code" "${CASTLE_HOME}/code"
|
||||||
|
printf " %-20s %s\n" "Artifacts" "${CASTLE_HOME}/artifacts"
|
||||||
|
printf " %-20s %s\n" "Data" "$DATA_DIR"
|
||||||
|
printf " %-20s %s\n" "Secrets" "${CASTLE_HOME}/secrets"
|
||||||
|
printf "\n"
|
||||||
|
|
||||||
|
printf "Next steps:\n"
|
||||||
|
printf " cd %s\n" "$CASTLE_ROOT"
|
||||||
|
printf " castle deploy # Generate registry, systemd units, Caddyfile\n"
|
||||||
|
printf " castle services start # Start all application services\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
main() {
|
||||||
|
printf "${_bold}========================================${_reset}\n"
|
||||||
|
printf "${_bold}Castle Platform — Bootstrap Install${_reset}\n"
|
||||||
|
printf "${_bold}========================================${_reset}\n"
|
||||||
|
|
||||||
|
# Parse args
|
||||||
|
AUTO_YES=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--yes|-y) AUTO_YES=1 ;;
|
||||||
|
*) printf "Unknown argument: %s\n" "$arg"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
export AUTO_YES
|
||||||
|
|
||||||
|
check_systemd
|
||||||
|
ensure_docker
|
||||||
|
ensure_caddy
|
||||||
|
create_directories
|
||||||
|
enable_lingering
|
||||||
|
seed_caddyfile
|
||||||
|
migrate_old_containers
|
||||||
|
setup_mqtt
|
||||||
|
setup_postgres
|
||||||
|
setup_neo4j
|
||||||
|
print_summary
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user