Compare commits
144 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d03065f6be | ||
|
|
c73b721ca2 | ||
|
|
dd0166b5d5 | ||
|
|
1d75970dd5 | ||
|
|
dd417fe8b1 | ||
|
|
e83b17d495 | ||
|
|
ee14bd357b | ||
|
|
35a1357cd7 | ||
|
|
9ea98c5036 | ||
|
|
809abccb40 | ||
|
|
e5c8d4f0c5 | ||
|
|
43750e1a90 | ||
|
|
6c3f7e650e | ||
|
|
123342b34c | ||
|
|
df1ff70a01 | ||
|
|
c666ce961b | ||
|
|
baf174702e | ||
|
|
3774ead167 | ||
|
|
ef2399ef64 | ||
|
|
84520628c3 | ||
|
|
2935946f28 | ||
|
|
d240a8194c | ||
|
|
c8c60fb11b | ||
|
|
c1ddf46f44 | ||
|
|
a533082388 | ||
|
|
658c7ab24c | ||
|
|
288b448e48 | ||
|
|
ce5ca426d6 | ||
|
|
fa59b5d8ad | ||
|
|
60fc76e9a4 | ||
|
|
d38ed94d12 | ||
|
|
e2144412ce | ||
|
|
e93a14aa92 | ||
|
|
374bcb3bd0 | ||
|
|
e82c92b72e | ||
|
|
270fbeabef | ||
|
|
fdab9484a6 | ||
|
|
322492a85f | ||
|
|
3f97dce86a | ||
|
|
11c875a513 | ||
|
|
e051e80601 | ||
|
|
fd58c7b694 | ||
|
|
3e9aa153e2 | ||
|
|
7cad37db07 | ||
|
|
eff5246144 | ||
|
|
81604879dc | ||
|
|
44c7cb6f72 | ||
|
|
7a3ef65683 | ||
|
|
9ac643a50f | ||
|
|
8e55a589fb | ||
|
|
5e591b62d1 | ||
|
|
7c1821a2f8 | ||
|
|
cd31e6a365 | ||
|
|
d15ce9fcbe | ||
|
|
d185f9cf10 | ||
|
|
164a44192a | ||
|
|
92364fbe96 | ||
|
|
a0dcd91857 | ||
|
|
313f788134 | ||
|
|
771cabad1c | ||
|
|
c51536eed1 | ||
|
|
fad3946f6d | ||
|
|
569044d8f9 | ||
|
|
b73b7c25e5 | ||
|
|
b393c29fc9 | ||
|
|
548f210849 | ||
|
|
f93bb6ae47 | ||
|
|
3621c92194 | ||
|
|
8d8bfed515 | ||
|
|
66389eebf3 | ||
|
|
fff321b05c | ||
|
|
bdbc26d892 | ||
|
|
0dabb8b824 | ||
|
|
7ddebaa80c | ||
|
|
18259034ea | ||
|
|
48f8809587 | ||
|
|
da7a165447 | ||
|
|
2b8ef8e2b6 | ||
|
|
b1f0ba07d8 | ||
|
|
f37cb458f8 | ||
|
|
2dd661118c | ||
|
|
7e017eaec7 | ||
|
|
f1a6a70bf8 | ||
|
|
5804c5fdd0 | ||
|
|
42a58d383f | ||
|
|
10515f2c92 | ||
|
|
773d2e88c8 | ||
|
|
b001b762f3 | ||
|
|
86cf4443d6 | ||
|
|
e00ea0503b | ||
|
|
5f305672ac | ||
|
|
26c88decd8 | ||
|
|
aa528a2b01 | ||
|
|
395b740d78 | ||
|
|
07bb8f83ef | ||
|
|
e78352e87f | ||
|
|
09171b7d85 | ||
|
|
f8e826eade | ||
|
|
591c58a1a5 | ||
|
|
afa39a9786 | ||
|
|
4f08487abe | ||
|
|
f8ca59f6ef | ||
|
|
982933e802 | ||
|
|
c7a6a0e6ae | ||
|
|
2ed8c94d2a | ||
|
|
7e04f463a2 | ||
|
|
326a6bf2b6 | ||
|
|
5febd84276 | ||
|
|
d0f03dd644 | ||
|
|
6f221978fa | ||
|
|
457a912c5c | ||
|
|
088fd4903f | ||
|
|
543967ae81 | ||
|
|
9e97ec6aa9 | ||
|
|
b7da8133e1 | ||
|
|
7c670bade2 | ||
|
|
7c8c182a9c | ||
|
|
96dfaaf07c | ||
|
|
5d8129dad8 | ||
|
|
c054dbff39 | ||
|
|
3531daf447 | ||
|
|
f8974d9787 | ||
|
|
9295804034 | ||
|
|
7ba269ae0c | ||
|
|
b27eef2592 | ||
|
|
d3e308b5dc | ||
|
|
01bea32b9c | ||
|
|
9ae8d446b7 | ||
|
|
576e134095 | ||
|
|
654fcc94a2 | ||
|
|
2199e90691 | ||
|
|
9b5cb77618 | ||
|
|
8c5e83f667 | ||
|
|
f6864bb2a7 | ||
|
|
2d65569a28 | ||
|
|
0d4fb41e7e | ||
|
|
4e49c435a9 | ||
|
|
c20006192e | ||
|
|
d83d2976fa | ||
|
|
fac866ffa1 | ||
|
|
b68abd2f08 | ||
|
|
bfc9d635e6 | ||
|
|
d0fdb2daf6 | ||
|
|
17d5758421 |
65
CHANGELOG.md
Normal file
65
CHANGELOG.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to Wild Cloud Central are documented here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.2.0] - 2026-06-21
|
||||
|
||||
### Added
|
||||
- **Talosconfig renewal**: new `wild cluster talosconfig renew` CLI command and API endpoint to regenerate an expired talosconfig client certificate from the existing PKI root — no cluster access required
|
||||
- **Multi-file config editor**: the configuration editor now shows a dropdown to select and edit any instance config file (instance.yaml, per-node configs, per-app configs) instead of a single fixed file
|
||||
- **Node upgrade and rollback**: `wild node upgrade` and `wild node rollback` CLI commands for managing Talos node versions
|
||||
- **Operation cancellation**: long-running operations can now be cancelled from the web UI
|
||||
- **App restart**: deployed apps can be restarted directly from the app detail page
|
||||
- **App script parameters**: app maintenance scripts now support parameters
|
||||
- **NFS server setup**: improved NFS host configuration scripts
|
||||
|
||||
### Changed
|
||||
- **Instance directory layout**: instance data directories now follow an XDG-inspired structure — `config/` (operator-authored), `data/` (durable artifacts), `state/` (runtime state), `cache/` (regeneratable). Instance data files moved to their appropriate locations under this layout
|
||||
- **Services renamed**: "infrastructure" category renamed to "services" for clarity
|
||||
- **In-place backup/restore**: backup and restore operations now work in-place without requiring a separate staging area
|
||||
|
||||
### Removed
|
||||
- Migration code for the XDG directory layout — both instances are now migrated; the `/migrate` and `/migrate/schedules` API endpoints have been removed
|
||||
|
||||
## [0.1.2] - 2026-01-31
|
||||
|
||||
### Added
|
||||
- **App upgrades**: update installed apps to newer versions from the web UI
|
||||
- **Backup and restore**: back up and restore individual apps from the web UI; per-app backup schedules
|
||||
- **Run app scripts**: execute maintenance scripts defined by apps directly from the app detail page
|
||||
- **Live updates**: the web UI now updates in real-time without page refreshes — app status, node state, and long-running operations all update automatically
|
||||
- **Node reboot and health monitoring**: reboot nodes and view their health status from the Cluster page
|
||||
- **Cluster config backup**
|
||||
- **Headlamp** replaces the Kubernetes Dashboard for cluster visibility
|
||||
- **APT repository**: Wild Cloud packages are now published to an APT repository — future installs and updates can use `apt` instead of manual `.deb` downloads
|
||||
- **Aptly** added to the Wild Directory for self-hosted APT repository management
|
||||
|
||||
### Fixed
|
||||
- App status indicators no longer show false warnings for configuration-only services
|
||||
- Load balancer IP was being read from the wrong config location
|
||||
- NFS configuration issues
|
||||
- Various UI polish: navbar highlighting, alignment, node status colors
|
||||
|
||||
## [0.1.1] - 2026-01-28
|
||||
|
||||
### Added
|
||||
- **Setup wizard**: step-by-step guided setup for configuring a new Wild Central device and cluster for the first time
|
||||
|
||||
## [0.1.0] - 2026-01-27
|
||||
|
||||
Initial release — primarily to validate the release and packaging pipeline.
|
||||
|
||||
### Added
|
||||
- Web interface for managing Wild Cloud instances
|
||||
- `wild` CLI for terminal-based management
|
||||
- Deploy and manage apps from the Wild Directory
|
||||
- Deploy and manage infrastructure services (cert-manager, ExternalDNS, MetalLB, Traefik, etc.)
|
||||
- Add, configure, and provision cluster nodes
|
||||
- Bootstrap a Talos/Kubernetes cluster
|
||||
- Local DNS management via dnsmasq
|
||||
- PXE boot support for network node installation
|
||||
- In-browser terminal
|
||||
- App and pod log streaming
|
||||
- `.deb` packages for amd64 and arm64
|
||||
202
CLAUDE.md
Normal file
202
CLAUDE.md
Normal file
@@ -0,0 +1,202 @@
|
||||
# CLAUDE.md
|
||||
|
||||
## Project Overview
|
||||
|
||||
This project is called "Wild Cloud". Wild Cloud is a platform for managing and orchestrating cloud-native applications on local networks using a network appliance called "Wild Central".
|
||||
|
||||
Wild Central is a lightweight server that runs on a local machine (e.g., a Raspberry Pi) and provides an API for users to manage their Wild Cloud instances. The core Wild Cloud components are:
|
||||
|
||||
- **API** (`wild-cloud/api`): The Wild Cloud API that runs on Wild Central. @api/CLAUDE.md
|
||||
- **CLI** (`wild-cloud/cli`): Command-line interface for managing Wild Cloud instances. @cli/README.md
|
||||
- **Web App** (`wild-cloud/web`): Web-based interface for managing Wild Cloud instances. @web/CLAUDE.md
|
||||
- **Dist** (`wild-cloud/dist`): Distribution package for setting up Wild Central devices via apt. @dist/README.md
|
||||
|
||||
A Wild Cloud instance is a kubernetes (k8s) environment that runs Wild Cloud services and applications. Wild Cloud instances can be created, managed, and monitored using the Wild Cloud API running on a Wild Central device.
|
||||
|
||||
Wild Cloud applications are custom packages designed to be deployed to Wild Cloud instances. They consist of kustomize templates and a Wild Cloud app manifest file that describes the application and how it should be deployed configured and deployed in a Wild Cloud instance. Wild Cloud applications are stored in a "Wild Directory".
|
||||
|
||||
The Wild Cloud API maintains data for each Wild Cloud instance in its configured WILD_API_DATA_DIR. A data directory is intended to be checked into version control (e.g., git) to track changes to the configuration of Wild Cloud instances and their deployed applications over time. These are designed to follow infrastructure-as-code principles, allowing experienced devops users to manage their Wild Cloud instances using familiar tools and workflows.
|
||||
|
||||
## Additional Documentation
|
||||
|
||||
- @docs/services.md
|
||||
- @docs/app-states.md
|
||||
|
||||
### Info about Talos
|
||||
|
||||
- docs/talos-v1.11/README.md
|
||||
- docs/talos-v1.11/architecture-and-components.md
|
||||
- docs/talos-v1.11/cli-essentials.md
|
||||
- docs/talos-v1.11/cluster-operations.md
|
||||
- docs/talos-v1.11/discovery-and-networking.md
|
||||
- docs/talos-v1.11/etcd-management.md
|
||||
- docs/talos-v1.11/bare-metal-administration.md
|
||||
- docs/talos-v1.11/troubleshooting-guide.md
|
||||
|
||||
## Implementation Philosophy
|
||||
|
||||
## Core Philosophy
|
||||
|
||||
Embodies a Zen-like minimalism that values simplicity and clarity above all. This approach reflects:
|
||||
|
||||
- **Wabi-sabi philosophy**: Embracing simplicity and the essential. Each line serves a clear purpose without unnecessary embellishment.
|
||||
- **KISS**: The solution should be as simple as possible, but no simpler.
|
||||
- **YAGNI**: Avoid building features or abstractions that aren't immediately needed. The code handles what's needed now rather than anticipating every possible future scenario.
|
||||
- **Trust in emergence**: Complex systems work best when built from simple, well-defined components that do one thing well.
|
||||
- **Pragmatic trust**: The developer trusts external systems enough to interact with them directly, handling failures as they occur rather than assuming they'll happen.
|
||||
- **Consistency is key**: Uniform patterns and conventions make the codebase easier to understand and maintain. If you introduce a new pattern, make sure it's consistently applied. There should be one obvious way to do things.
|
||||
|
||||
This development philosophy values clear, concise documentation, readable code, and belief that good architecture emerges from simplicity rather than being imposed through complexity.
|
||||
|
||||
## Core Design Principles
|
||||
|
||||
### 1. Ruthless Simplicity
|
||||
|
||||
- **KISS principle taken to heart**: Keep everything as simple as possible, but no simpler
|
||||
- **Minimize abstractions**: Every layer of abstraction must justify its existence
|
||||
- **Start minimal, grow as needed**: Begin with the simplest implementation that meets current needs
|
||||
- **Avoid future-proofing**: Don't build for hypothetical future requirements
|
||||
- **Question everything**: Regularly challenge complexity in the codebase
|
||||
|
||||
### 2. Architectural Integrity with Minimal Implementation
|
||||
|
||||
- **Preserve key architectural patterns**: Maintain clear boundaries and responsibilities
|
||||
- **Simplify implementations**: Maintain pattern benefits with dramatically simpler code
|
||||
- **Scrappy but structured**: Lightweight implementations of solid architectural foundations
|
||||
- **End-to-end thinking**: Focus on complete flows rather than perfect components
|
||||
|
||||
### 3. Library vs Custom Code
|
||||
|
||||
Choosing between custom code and external libraries is a judgment call that evolves with your requirements. There's no rigid rule - it's about understanding trade-offs and being willing to revisit decisions as needs change.
|
||||
|
||||
#### The Evolution Pattern
|
||||
|
||||
Your approach might naturally evolve:
|
||||
- **Start simple**: Custom code for basic needs (20 lines handles it)
|
||||
- **Growing complexity**: Switch to a library when requirements expand
|
||||
- **Hitting limits**: Back to custom when you outgrow the library's capabilities
|
||||
|
||||
This isn't failure - it's natural evolution. Each stage was the right choice at that time.
|
||||
|
||||
#### When Custom Code Makes Sense
|
||||
|
||||
Custom code often wins when:
|
||||
- The need is simple and well-understood
|
||||
- You want code perfectly tuned to your exact requirements
|
||||
- Libraries would require significant "hacking" or workarounds
|
||||
- The problem is unique to your domain
|
||||
- You need full control over the implementation
|
||||
|
||||
#### When Libraries Make Sense
|
||||
|
||||
Libraries shine when:
|
||||
- They solve complex problems you'd rather not tackle (auth, crypto, video encoding)
|
||||
- They align well with your needs without major modifications
|
||||
- The problem is well-solved with mature, battle-tested solutions
|
||||
- Configuration alone can adapt them to your requirements
|
||||
- The complexity they handle far exceeds the integration cost
|
||||
|
||||
#### Making the Judgment Call
|
||||
|
||||
Ask yourself:
|
||||
- How well does this library align with our actual needs?
|
||||
- Are we fighting the library or working with it?
|
||||
- Is the integration clean or does it require workarounds?
|
||||
- Will our future requirements likely stay within this library's capabilities?
|
||||
- Is the problem complex enough to justify the dependency?
|
||||
|
||||
#### Recognizing Misalignment
|
||||
|
||||
Watch for signs you're fighting your current approach:
|
||||
- Spending more time working around the library than using it
|
||||
- Your simple custom solution has grown complex and fragile
|
||||
- You're monkey-patching or heavily wrapping a library
|
||||
- The library's assumptions fundamentally conflict with your needs
|
||||
|
||||
#### Stay Flexible
|
||||
|
||||
Remember that complexity isn't destroyed, only moved. Libraries shift complexity from your code to someone else's - that's often a great trade, but recognize what you're doing.
|
||||
|
||||
The key is avoiding lock-in. Keep library integration points minimal and isolated so you can switch approaches when needed. There's no shame in moving from custom to library or library to custom. Requirements change, understanding deepens, and the right answer today might not be the right answer tomorrow. Make the best decision with current information, and be ready to evolve.
|
||||
|
||||
## Technical Implementation Guidelines
|
||||
|
||||
### API Layer
|
||||
|
||||
- Implement only essential endpoints
|
||||
- Minimal middleware with focused validation
|
||||
- Clear error responses with useful messages
|
||||
- Consistent patterns across endpoints
|
||||
|
||||
### Storage
|
||||
|
||||
- Prefer simple file storage
|
||||
- Simple schema focused on current needs
|
||||
|
||||
## Development Approach
|
||||
|
||||
### Vertical Slices
|
||||
|
||||
- Implement complete end-to-end functionality slices
|
||||
- Start with core user journeys
|
||||
- Get data flowing through all layers early
|
||||
- Add features horizontally only after core flows work
|
||||
|
||||
### Iterative Implementation
|
||||
|
||||
- 80/20 principle: Focus on high-value, low-effort features first
|
||||
- One working feature > multiple partial features
|
||||
- Validate with real usage before enhancing
|
||||
- Be willing to refactor early work as patterns emerge
|
||||
|
||||
### Testing Strategy
|
||||
|
||||
- Focus on critical path testing initially
|
||||
- Add unit tests for complex logic and edge cases
|
||||
- Testing pyramid: 60% unit, 30% integration, 10% end-to-end
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Handle common errors robustly
|
||||
- Log detailed information for debugging
|
||||
- Provide clear error messages to users
|
||||
- Fail fast and visibly during development
|
||||
|
||||
## Decision-Making Framework
|
||||
|
||||
When faced with implementation decisions, ask these questions:
|
||||
|
||||
1. **Necessity**: "Do we actually need this right now?"
|
||||
2. **Simplicity**: "What's the simplest way to solve this problem?"
|
||||
3. **Directness**: "Can we solve this more directly?"
|
||||
4. **Value**: "Does the complexity add proportional value?"
|
||||
5. **Maintenance**: "How easy will this be to understand and change later?"
|
||||
|
||||
## Areas to Embrace Complexity
|
||||
|
||||
Some areas justify additional complexity:
|
||||
|
||||
1. **Security**: Never compromise on security fundamentals
|
||||
2. **Data integrity**: Ensure data consistency and reliability
|
||||
3. **Core user experience**: Make the primary user flows smooth and reliable
|
||||
4. **Error visibility**: Make problems obvious and diagnosable
|
||||
|
||||
## Areas to Aggressively Simplify
|
||||
|
||||
Push for extreme simplicity in these areas:
|
||||
|
||||
1. **Internal abstractions**: Minimize layers between components
|
||||
2. **Generic "future-proof" code**: Resist solving non-existent problems
|
||||
3. **Edge case handling**: Handle the common cases well first
|
||||
4. **Framework usage**: Use only what you need from frameworks
|
||||
5. **State management**: Keep state simple and explicit
|
||||
|
||||
## Remember
|
||||
|
||||
- It's easier to add complexity later than to remove it
|
||||
- Code you don't write has no bugs
|
||||
- Favor clarity over cleverness
|
||||
- The best code is often the simplest
|
||||
|
||||
This philosophy document serves as the foundational guide for all implementation decisions in the project.
|
||||
|
||||
69
VERSIONING.md
Normal file
69
VERSIONING.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# Wild Cloud Versioning
|
||||
|
||||
Wild Cloud uses a single VERSION file at the monorepo root. All components (API, CLI, Web, package) share the same version number.
|
||||
|
||||
## Release Channels
|
||||
|
||||
### Stable
|
||||
Versioned releases following [Semantic Versioning](https://semver.org/). Tagged immutably in git as `v{VERSION}`. Published to the `stable` APT suite.
|
||||
|
||||
### Edge
|
||||
Rolling releases built from HEAD. No version bump required — the version is derived from the build timestamp and git commit (`0.0.0~edge.{timestamp}.{hash}`). The `edge` git tag is force-pushed to HEAD on each build. Published to the `edge` APT suite as a pre-release.
|
||||
|
||||
## Stable Release Workflow
|
||||
|
||||
```bash
|
||||
# 1. Bump the version
|
||||
echo "0.2.0" > VERSION
|
||||
|
||||
# 2. Update CHANGELOG.md — rename [Unreleased] to [0.2.0] with today's date,
|
||||
# add a fresh [Unreleased] section above it
|
||||
|
||||
# 3. Commit the version bump
|
||||
git add VERSION CHANGELOG.md
|
||||
git commit -m "Release v0.2.0"
|
||||
|
||||
# 4. Build and release — tags HEAD as v0.2.0, pushes tag, publishes Gitea
|
||||
# release with changelog notes, and deploys to stable APT
|
||||
cd dist
|
||||
make package-all && make release
|
||||
```
|
||||
|
||||
`make release` creates the `v{VERSION}` git tag automatically. If the tag already exists, it skips tagging and only updates the Gitea release assets.
|
||||
|
||||
## Edge Release Workflow
|
||||
|
||||
```bash
|
||||
cd dist
|
||||
make package-all && make release-edge
|
||||
```
|
||||
|
||||
This force-pushes the `edge` git tag to HEAD, creates or updates the "edge" pre-release on Gitea, and deploys to the edge APT suite. No version file changes needed.
|
||||
|
||||
## Semver Guidelines (pre-1.0)
|
||||
|
||||
- **MINOR** (`0.x`): new features, any breaking changes
|
||||
- **PATCH** (`0.1.x`): bug fixes, small improvements
|
||||
|
||||
## Version in Components
|
||||
|
||||
Version is injected at build time via `-ldflags`:
|
||||
|
||||
```makefile
|
||||
go build -ldflags="-X main.Version=$(VERSION)"
|
||||
```
|
||||
|
||||
The web app reads its version from the API at runtime.
|
||||
|
||||
## Makefile Quick Reference
|
||||
|
||||
```bash
|
||||
make help # Show all targets and current version info
|
||||
make package-all # Build all .deb packages
|
||||
make release # Stable release: tag + Gitea + APT stable
|
||||
make release-edge # Edge release: force-tag + Gitea + APT edge
|
||||
```
|
||||
|
||||
Required environment variables (see `.envrc`):
|
||||
- `GITEA_TOKEN` — for Gitea release publishing
|
||||
- `APTLY_URL` / `APTLY_PASS` — for APT deployment (optional; skipped if unset)
|
||||
19
api/.air.toml
Normal file
19
api/.air.toml
Normal file
@@ -0,0 +1,19 @@
|
||||
root = "."
|
||||
tmp_dir = "tmp"
|
||||
|
||||
[build]
|
||||
bin = "./tmp/wildd"
|
||||
cmd = "go build -o ./tmp/wildd ."
|
||||
delay = 1000
|
||||
exclude_dir = ["tmp", "build", "dist", "vendor"]
|
||||
exclude_regex = ["_test.go$"]
|
||||
include_ext = ["go", "yaml"]
|
||||
kill_delay = "0s"
|
||||
send_interrupt = true
|
||||
stop_on_error = true
|
||||
|
||||
[log]
|
||||
time = false
|
||||
|
||||
[misc]
|
||||
clean_on_exit = true
|
||||
3
api/.gitignore
vendored
3
api/.gitignore
vendored
@@ -22,3 +22,6 @@ __debug*
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# Air live-reload
|
||||
tmp/
|
||||
|
||||
@@ -19,7 +19,7 @@ Use tests on the API extensively to keep the API functioning well for all client
|
||||
- Like v.PoC, we should only use gomplate templates for distinguishing between cloud instances. However, **within** a cloud instance, there should be no templating. The templates are compiled when being copied into the instances. This allows transparency and simple management by the user.
|
||||
- Manage state and infrastructure idempotently.
|
||||
- Cluster state should be the k8s cluster itself, not local files. It should be accessed via kubectl and talosctl.
|
||||
- All wild cloud state should be stored on the filesystem in easy to read YAML files, and can be edited directly or through the webapp.
|
||||
- All wild cloud state should be stored on the filesystem in easy to read YAML files, and can be edited directly or through the webapp. Avoid changing state directly on the server when it can be updated with manifest files.
|
||||
- All code should be simple and easy to understand.
|
||||
- Avoid unnecessary complexity.
|
||||
- Avoid unnecessary dependencies.
|
||||
@@ -28,7 +28,7 @@ Use tests on the API extensively to keep the API functioning well for all client
|
||||
- Avoid unnecessary comments.
|
||||
- Avoid unnecessary configuration options.
|
||||
- Avoid Helm. Use Kustomize.
|
||||
- The API should be able to run on low-resource devices like a Raspberry Pi 4 (4GB RAM).
|
||||
- The API should be able to run on low-resource devices like a Raspberry Pi 4 (4GB RAM). The cluster, of course will be a full cluster, but the Wild Central API that drives it does not.
|
||||
- The API should be able to manage multiple Wild Cloud instances on the LAN.
|
||||
- The API should include functionality to manage a dnsmasq server on the same device. Currently, this is only used to resolve wild cloud domain names within the LAN to provide for private addresses on the LAN. The LAN router should be configured to use the Wild Central IP as its DNS server.
|
||||
- The API is configurable to use various providers for:
|
||||
@@ -50,6 +50,52 @@ Use tests on the API extensively to keep the API functioning well for all client
|
||||
|
||||
- If WILD_CENTRAL_ENV environment variable is set to "development", the API should run in development mode.
|
||||
|
||||
## Server-Sent Events (SSE)
|
||||
|
||||
The API publishes real-time events via SSE at `/api/v1/events`. Clients connect to this endpoint to receive updates about cluster state changes.
|
||||
|
||||
### Publishing Events
|
||||
|
||||
Events are published through the `events` package:
|
||||
|
||||
```go
|
||||
import "wild-cloud/api/internal/events"
|
||||
|
||||
// Publish an event
|
||||
events.Publish(events.Event{
|
||||
Type: "pod:modified",
|
||||
InstanceName: instanceName,
|
||||
Data: podData,
|
||||
Metadata: map[string]interface{}{
|
||||
"namespace": namespace,
|
||||
"name": podName,
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
### Event Types
|
||||
|
||||
Standard event types follow the pattern `resource:action`:
|
||||
- **Kubernetes resources**: `pod:added`, `pod:modified`, `pod:deleted`
|
||||
- **Deployments**: `deployment:added`, `deployment:modified`, `deployment:deleted`
|
||||
- **Services**: `service:added`, `service:modified`, `service:deleted`
|
||||
- **Nodes**: `node:added`, `node:modified`, `node:deleted`, `node:configured`, `node:applied`
|
||||
- **Operations**: `operation:started`, `operation:progress`, `operation:completed`, `operation:failed`
|
||||
- **Central status**: `central:status`, `central:health`
|
||||
- **DNS**: `dnsmasq:restart`, `dnsmasq:config`
|
||||
- **Talos**: `talos:event`
|
||||
- **System**: `connected`, `heartbeat`
|
||||
|
||||
### When to Publish
|
||||
|
||||
Publish events when:
|
||||
- Kubernetes resources change (detected via watches or after kubectl operations)
|
||||
- Long-running operations update their status
|
||||
- Configuration changes that affect the UI
|
||||
- Any state change that clients need to know about immediately
|
||||
|
||||
Events are fire-and-forget - the publisher doesn't wait for clients to receive them.
|
||||
|
||||
## Patterns
|
||||
|
||||
### Instance-scoped Endpoints
|
||||
|
||||
10
api/Makefile
10
api/Makefile
@@ -29,8 +29,14 @@ build: ## Build the daemon binary
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) .
|
||||
|
||||
dev: ## Run the daemon in development mode with live reloading
|
||||
@echo "Starting $(BINARY_NAME) in development mode..."
|
||||
$(GOCMD) run .
|
||||
@if command -v air >/dev/null 2>&1; then \
|
||||
echo "Starting $(BINARY_NAME) in development mode with live reloading (air)..."; \
|
||||
air; \
|
||||
else \
|
||||
echo "air not found. Install it for live reloading: go install github.com/air-verse/air@latest"; \
|
||||
echo "Starting $(BINARY_NAME) in development mode without live reloading..."; \
|
||||
$(GOCMD) run .; \
|
||||
fi
|
||||
|
||||
test: ## Run tests
|
||||
@echo "Running tests..."
|
||||
|
||||
@@ -4,7 +4,7 @@ The Wild Central API is a lightweight service that runs on a local machine (e.g.
|
||||
|
||||
## Development
|
||||
|
||||
Start the development server:
|
||||
Start the development server with live reloading:
|
||||
|
||||
```bash
|
||||
make dev
|
||||
@@ -12,6 +12,14 @@ make dev
|
||||
|
||||
The API will be available at `http://localhost:5055`.
|
||||
|
||||
`make dev` uses [air](https://github.com/air-verse/air) to automatically rebuild and restart the server when `.go` or `.yaml` files change. Install it with:
|
||||
|
||||
```bash
|
||||
go install github.com/air-verse/air@latest
|
||||
```
|
||||
|
||||
If `air` is not installed, `make dev` falls back to `go run .` (no live reloading).
|
||||
|
||||
### Environment Variables
|
||||
|
||||
- `WILD_API_DATA_DIR` - Directory for instance data (default: `/var/lib/wild-central`)
|
||||
@@ -28,8 +36,7 @@ The API provides the following endpoint categories:
|
||||
- **Secrets** - Manage instance secrets.yaml (redacted by default)
|
||||
- **Nodes** - Discover, configure, and manage cluster nodes
|
||||
- **Cluster** - Bootstrap and manage Talos/Kubernetes clusters
|
||||
- **Services** - Install and manage base infrastructure services
|
||||
- **Apps** - Deploy and manage Wild Cloud applications
|
||||
- **Apps** - Deploy and manage Wild Cloud packages (infrastructure services and applications)
|
||||
- **PXE** - Manage PXE boot assets for network installation
|
||||
- **Operations** - Track and stream long-running operations
|
||||
- **Utilities** - Helper functions and status endpoints
|
||||
|
||||
47
api/go.mod
47
api/go.mod
@@ -1,18 +1,51 @@
|
||||
module github.com/wild-cloud/wild-central/daemon
|
||||
|
||||
go 1.24
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.5
|
||||
|
||||
require (
|
||||
github.com/creack/pty v1.1.24
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/rs/cors v1.11.1
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/time v0.14.0
|
||||
|
||||
require (
|
||||
github.com/creack/pty v1.1.24 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/Azure/azure-pipeline-go v0.2.3 // indirect
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect
|
||||
github.com/aws/smithy-go v1.24.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.1 // indirect
|
||||
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b // indirect
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/stretchr/testify v1.11.1
|
||||
)
|
||||
|
||||
81
api/go.sum
81
api/go.sum
@@ -1,20 +1,101 @@
|
||||
github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
|
||||
github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
|
||||
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
|
||||
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 h1:s8fbFscel8NLpnz+ggR7ncW+lqhXIkmyHbgbPeT8yyM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4/go.mod h1:BazuWe/q/mMJ/NrSJBTbNBJiLq6u8reodbEZ4giRms4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs=
|
||||
github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=
|
||||
github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
|
||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
|
||||
github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
|
||||
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A=
|
||||
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -2,7 +2,7 @@ package v1
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
@@ -27,7 +27,7 @@ func (api *API) StartAsyncOperation(
|
||||
instanceName, operationType, target string,
|
||||
operation AsyncOperation,
|
||||
) {
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
opID, err := opsMgr.Start(instanceName, operationType, target)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to start operation")
|
||||
@@ -38,7 +38,7 @@ func (api *API) StartAsyncOperation(
|
||||
// Always recover from panics to prevent goroutine crashes from taking down the server
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in async operation %s/%s: %v", operationType, target, r)
|
||||
slog.Error("panic in async operation", "type", operationType, "target", target, "error", r)
|
||||
_ = opsMgr.Update(instanceName, opID, "failed", fmt.Sprintf("Internal error: %v", r), 0)
|
||||
}
|
||||
}()
|
||||
@@ -61,7 +61,7 @@ func (api *API) StartAsyncOperationWithMessage(
|
||||
instanceName, operationType, target, successMessage string,
|
||||
operation AsyncOperation,
|
||||
) {
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
opID, err := opsMgr.Start(instanceName, operationType, target)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to start operation")
|
||||
@@ -71,7 +71,7 @@ func (api *API) StartAsyncOperationWithMessage(
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in async operation %s/%s: %v", operationType, target, r)
|
||||
slog.Error("panic in async operation", "type", operationType, "target", target, "error", r)
|
||||
_ = opsMgr.Update(instanceName, opID, "failed", fmt.Sprintf("Internal error: %v", r), 0)
|
||||
}
|
||||
}()
|
||||
@@ -95,7 +95,7 @@ func (api *API) StartAsyncOperationWithBroadcaster(
|
||||
instanceName, operationType, target string,
|
||||
operation func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error,
|
||||
) {
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
opID, err := opsMgr.Start(instanceName, operationType, target)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to start operation")
|
||||
@@ -105,7 +105,7 @@ func (api *API) StartAsyncOperationWithBroadcaster(
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[ERROR] Panic in async operation %s/%s: %v", operationType, target, r)
|
||||
slog.Error("panic in async operation", "type", operationType, "target", target, "error", r)
|
||||
_ = opsMgr.Update(instanceName, opID, "failed", fmt.Sprintf("Internal error: %v", r), 0)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -1,195 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/backup"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
type BackupConfigRequest struct {
|
||||
Repository string `json:"repository"`
|
||||
Staging string `json:"staging,omitempty"`
|
||||
Retention *backup.ResticRetentionPolicy `json:"retention,omitempty"`
|
||||
Backend *backup.BackendConfig `json:"backend,omitempty"`
|
||||
Secrets *BackupSecretsRequest `json:"secrets,omitempty"`
|
||||
}
|
||||
|
||||
type BackupSecretsRequest struct {
|
||||
Password string `json:"password"`
|
||||
Credentials *backup.BackupCredentials `json:"credentials,omitempty"`
|
||||
}
|
||||
|
||||
type BackupConfigResponse struct {
|
||||
Repository string `json:"repository"`
|
||||
Staging string `json:"staging"`
|
||||
Retention backup.ResticRetentionPolicy `json:"retention"`
|
||||
Backend backup.BackendConfig `json:"backend"`
|
||||
}
|
||||
|
||||
func (api *API) GetBackupConfig(w http.ResponseWriter, r *http.Request) {
|
||||
config, err := backup.LoadBackupConfig(api.dataDir)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if config == nil {
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"configured": false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
response := BackupConfigResponse{
|
||||
Repository: config.Repository,
|
||||
Staging: config.Staging,
|
||||
Retention: config.Retention,
|
||||
Backend: config.Backend,
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"configured": true,
|
||||
"config": response,
|
||||
})
|
||||
}
|
||||
|
||||
func (api *API) UpdateBackupConfig(w http.ResponseWriter, r *http.Request) {
|
||||
var req BackupConfigRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if req.Repository == "" {
|
||||
respondError(w, http.StatusBadRequest, "repository is required")
|
||||
return
|
||||
}
|
||||
|
||||
config := &backup.BackupConfig{
|
||||
Repository: req.Repository,
|
||||
Staging: req.Staging,
|
||||
}
|
||||
|
||||
if req.Retention != nil {
|
||||
config.Retention = *req.Retention
|
||||
} else {
|
||||
config.Retention = backup.ResticRetentionPolicy{
|
||||
KeepDaily: 7,
|
||||
KeepWeekly: 4,
|
||||
KeepMonthly: 6,
|
||||
KeepYearly: 2,
|
||||
}
|
||||
}
|
||||
|
||||
if req.Backend != nil {
|
||||
config.Backend = *req.Backend
|
||||
}
|
||||
|
||||
if err := backup.SaveBackupConfig(api.dataDir, config); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if req.Secrets != nil {
|
||||
secrets := &backup.BackupSecrets{
|
||||
Password: req.Secrets.Password,
|
||||
}
|
||||
|
||||
if req.Secrets.Credentials != nil {
|
||||
secrets.Credentials = *req.Secrets.Credentials
|
||||
}
|
||||
|
||||
if err := backup.SaveBackupSecrets(api.dataDir, secrets); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
})
|
||||
}
|
||||
|
||||
func (api *API) TestBackupConfig(w http.ResponseWriter, r *http.Request) {
|
||||
client, err := backup.NewResticClient(api.dataDir)
|
||||
if err != nil {
|
||||
respondJSON(w, http.StatusBadRequest, map[string]interface{}{
|
||||
"success": false,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
initialized := client.IsInitialized()
|
||||
|
||||
result := map[string]interface{}{
|
||||
"success": true,
|
||||
"initialized": initialized,
|
||||
}
|
||||
|
||||
if initialized {
|
||||
status, err := client.Status()
|
||||
if err != nil {
|
||||
respondJSON(w, http.StatusInternalServerError, map[string]interface{}{
|
||||
"success": false,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
result["status"] = status
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (api *API) GetRepositoryStatus(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
instanceName := vars["name"]
|
||||
|
||||
instanceDir := tools.GetInstancePath(api.dataDir, instanceName)
|
||||
|
||||
client, err := backup.NewResticClient(instanceDir)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
status, err := client.Status()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
func (api *API) InitRepository(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
instanceName := vars["name"]
|
||||
|
||||
instanceDir := tools.GetInstancePath(api.dataDir, instanceName)
|
||||
|
||||
client, err := backup.NewResticClient(instanceDir)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if client.IsInitialized() {
|
||||
respondError(w, http.StatusBadRequest, "repository already initialized")
|
||||
return
|
||||
}
|
||||
|
||||
if err := client.Init(); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
})
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package v1
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
@@ -11,34 +11,45 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/backup"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/config"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/context"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/dnsmasq"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/factory"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/instance"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/network"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/secrets"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/sse"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// API holds all dependencies for API handlers
|
||||
type API struct {
|
||||
dataDir string
|
||||
appsDir string // Path to external apps directory
|
||||
config *config.Manager
|
||||
secrets *secrets.Manager
|
||||
context *context.Manager
|
||||
instance *instance.Manager
|
||||
dnsmasq *dnsmasq.ConfigGenerator
|
||||
opsMgr *operations.Manager // Operations manager
|
||||
broadcaster *operations.Broadcaster // SSE broadcaster for operation output
|
||||
backupScheduler *backup.Scheduler // Backup scheduler
|
||||
dataDir string
|
||||
appsDir string // Path to external apps directory
|
||||
version string
|
||||
config *config.Manager
|
||||
secrets *secrets.Manager
|
||||
context *context.Manager
|
||||
instance *instance.Manager
|
||||
dnsmasq *dnsmasq.ConfigGenerator
|
||||
opsMgr *operations.Manager // Operations manager
|
||||
broadcaster *operations.Broadcaster // SSE broadcaster for operation output
|
||||
sseManager *sse.Manager // SSE manager for real-time events
|
||||
watcherManager *sse.WatcherManager // Manager for kubectl/talos watchers
|
||||
factory *factory.Client // Talos Image Factory client
|
||||
}
|
||||
|
||||
// NewAPI creates a new API handler with all dependencies
|
||||
// Note: Setup files (cluster-services, cluster-nodes, etc.) are now embedded in the binary
|
||||
func NewAPI(dataDir, appsDir string) (*API, error) {
|
||||
func NewAPI(dataDir, appsDir, version string) (*API, error) {
|
||||
// Initialize config manager
|
||||
configMgr := config.NewManager()
|
||||
|
||||
// Ensure global config exists
|
||||
if err := configMgr.EnsureGlobalConfig(dataDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure global config: %w", err)
|
||||
}
|
||||
|
||||
// Ensure base directories exist
|
||||
instancesDir := tools.GetInstancesPath(dataDir)
|
||||
if err := os.MkdirAll(instancesDir, 0755); err != nil {
|
||||
@@ -49,29 +60,56 @@ func NewAPI(dataDir, appsDir string) (*API, error) {
|
||||
dnsmasqConfigPath := "/etc/dnsmasq.d/wild-cloud.conf"
|
||||
if os.Getenv("WILD_API_DNSMASQ_CONFIG_PATH") != "" {
|
||||
dnsmasqConfigPath = os.Getenv("WILD_API_DNSMASQ_CONFIG_PATH")
|
||||
log.Printf("Using custom dnsmasq config path: %s", dnsmasqConfigPath)
|
||||
slog.Info("using custom dnsmasq config path", "path", dnsmasqConfigPath)
|
||||
}
|
||||
|
||||
return &API{
|
||||
dataDir: dataDir,
|
||||
appsDir: appsDir,
|
||||
config: config.NewManager(),
|
||||
secrets: secrets.NewManager(),
|
||||
context: context.NewManager(dataDir),
|
||||
instance: instance.NewManager(dataDir),
|
||||
dnsmasq: dnsmasq.NewConfigGenerator(dnsmasqConfigPath),
|
||||
opsMgr: operations.NewManager(dataDir),
|
||||
broadcaster: operations.NewBroadcaster(),
|
||||
backupScheduler: nil, // Set later via SetBackupScheduler
|
||||
}, nil
|
||||
// Create SSE manager for real-time events
|
||||
sseManager := sse.NewManager()
|
||||
watcherManager := sse.NewWatcherManager(sseManager)
|
||||
|
||||
// Create operations manager with SSE support
|
||||
opsMgr := operations.NewManager(dataDir)
|
||||
// Use adapter to convert between operations.SSEManager and sse.Manager
|
||||
adapter := &sseManagerAdapter{sseManager: sseManager}
|
||||
opsMgr.SetSSEManager(adapter)
|
||||
|
||||
api := &API{
|
||||
dataDir: dataDir,
|
||||
appsDir: appsDir,
|
||||
version: version,
|
||||
config: configMgr,
|
||||
secrets: secrets.NewManager(),
|
||||
context: context.NewManager(dataDir),
|
||||
instance: instance.NewManager(dataDir),
|
||||
dnsmasq: dnsmasq.NewConfigGenerator(dnsmasqConfigPath),
|
||||
opsMgr: opsMgr,
|
||||
broadcaster: operations.NewBroadcaster(),
|
||||
sseManager: sseManager,
|
||||
watcherManager: watcherManager,
|
||||
factory: factory.NewClient(),
|
||||
}
|
||||
|
||||
return api, nil
|
||||
}
|
||||
|
||||
// SetBackupScheduler sets the backup scheduler for the API
|
||||
func (api *API) SetBackupScheduler(scheduler *backup.Scheduler) {
|
||||
api.backupScheduler = scheduler
|
||||
// StartCentralStatusBroadcaster starts periodic broadcasting of central status
|
||||
func (api *API) StartCentralStatusBroadcaster(startTime time.Time) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(30 * time.Second) // Broadcast every 30 seconds
|
||||
defer ticker.Stop()
|
||||
|
||||
// Send initial status
|
||||
api.broadcastCentralStatusEvent(startTime)
|
||||
|
||||
for range ticker.C {
|
||||
api.broadcastCentralStatusEvent(startTime)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
// Request logging middleware (runs first, wraps everything)
|
||||
r.Use(RequestLoggingMiddleware)
|
||||
// Apply instance validation middleware to all routes with {name} parameter
|
||||
r.Use(api.ValidateInstanceMiddleware)
|
||||
|
||||
@@ -81,13 +119,17 @@ func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/v1/instances/{name}", api.GetInstance).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}", api.DeleteInstance).Methods("DELETE")
|
||||
|
||||
// Setup status
|
||||
// Setup
|
||||
r.HandleFunc("/api/v1/instances/{name}/setup/status", api.GetSetupStatus).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/setup/infrastructure", api.SetupDeployInfrastructure).Methods("POST")
|
||||
|
||||
// Config management
|
||||
r.HandleFunc("/api/v1/instances/{name}/config", api.GetConfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/config", api.UpdateConfig).Methods("PUT")
|
||||
r.HandleFunc("/api/v1/instances/{name}/config", api.ConfigUpdateBatch).Methods("PATCH")
|
||||
r.HandleFunc("/api/v1/instances/{name}/config/files", api.ListConfigFiles).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/config/file", api.GetConfigFile).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/config/file", api.UpdateConfigFile).Methods("PUT")
|
||||
|
||||
// Secrets management
|
||||
r.HandleFunc("/api/v1/instances/{name}/secrets", api.GetSecrets).Methods("GET")
|
||||
@@ -109,7 +151,11 @@ func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}", api.NodeGet).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}", api.NodeUpdate).Methods("PUT")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/apply", api.NodeApply).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/health", api.NodeHealth).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/reboot", api.NodeReboot).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/reset", api.NodeReset).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/upgrade", api.NodeUpgrade).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}/rollback", api.NodeRollback).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/nodes/{node}", api.NodeDelete).Methods("DELETE")
|
||||
|
||||
// PXE Asset management (schematic@version composite key)
|
||||
@@ -120,6 +166,16 @@ func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/v1/pxe/assets/{schematicId}/{version}/pxe/{assetType}", api.AssetsServePXE).Methods("GET")
|
||||
r.HandleFunc("/api/v1/pxe/assets/{schematicId}/{version}/status", api.AssetsGetStatus).Methods("GET")
|
||||
|
||||
// Talos Image Factory
|
||||
r.HandleFunc("/api/v1/talos/versions", api.TalosVersionsList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/talos/versions/{version}/extensions", api.TalosExtensionsList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/talos/schematics/{id}", api.TalosSchematicGet).Methods("GET")
|
||||
r.HandleFunc("/api/v1/talos/schematics/{schematicId}/validate/{version}", api.TalosSchematicValidate).Methods("GET")
|
||||
|
||||
// Talos client management
|
||||
r.HandleFunc("/api/v1/talos/client", api.TalosClientInfo).Methods("GET")
|
||||
r.HandleFunc("/api/v1/talos/client/upgrade", api.TalosClientUpgrade).Methods("POST")
|
||||
|
||||
// Instance-schematic relationship
|
||||
r.HandleFunc("/api/v1/instances/{name}/schematic", api.SchematicGetInstanceSchematic).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/schematic", api.SchematicUpdateInstanceSchematic).Methods("PUT")
|
||||
@@ -139,33 +195,22 @@ func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/kubeconfig", api.ClusterGetKubeconfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/kubeconfig/generate", api.ClusterGenerateKubeconfig).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/talosconfig", api.ClusterGetTalosconfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/talosconfig/renew", api.ClusterRenewTalosconfig).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/reset", api.ClusterReset).Methods("POST")
|
||||
|
||||
// Services
|
||||
r.HandleFunc("/api/v1/instances/{name}/services", api.ServicesList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services", api.ServicesInstall).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/install-all", api.ServicesInstallAll).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}", api.ServicesGet).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}", api.ServicesDelete).Methods("DELETE")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/status", api.ServicesGetStatus).Methods("GET")
|
||||
r.HandleFunc("/api/v1/services/{service}/manifest", api.ServicesGetManifest).Methods("GET")
|
||||
r.HandleFunc("/api/v1/services/{service}/config", api.ServicesGetConfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/config", api.ServicesGetInstanceConfig).Methods("GET")
|
||||
|
||||
// Service lifecycle endpoints
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/fetch", api.ServicesFetch).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/compile", api.ServicesCompile).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/deploy", api.ServicesDeploy).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/logs", api.ServicesGetLogs).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/services/{service}/config", api.ServicesUpdateConfig).Methods("PATCH")
|
||||
|
||||
// Apps
|
||||
// Apps (unified: includes both applications and infrastructure services)
|
||||
r.HandleFunc("/api/v1/apps", api.AppsListAvailable).Methods("GET")
|
||||
r.HandleFunc("/api/v1/apps/{app}", api.AppsGetAvailable).Methods("GET")
|
||||
r.HandleFunc("/api/v1/apps/{app}/manifest", api.AppsGetManifest).Methods("GET")
|
||||
r.HandleFunc("/api/v1/apps/{app}/readme", api.AppsGetAvailableReadme).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps", api.AppsListDeployed).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps", api.AppsAdd).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/deploy", api.AppsDeploy).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/compile", api.AppsCompile).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/fetch", api.AppsFetch).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restart", api.AppsRestart).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/update", api.AppsUpdate).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/upgrade-plan", api.AppsGetUpgradePlan).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/eject", api.AppsEject).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/config", api.AppsGetConfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/config", api.AppsUpdateConfig).Methods("PATCH")
|
||||
@@ -178,54 +223,53 @@ func (api *API) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/logs", api.AppsGetLogs).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/events", api.AppsGetEvents).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/readme", api.AppsGetReadme).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/manifests", api.AppsGetManifests).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/scripts/{script}/run", api.AppsRunScript).Methods("POST")
|
||||
|
||||
// Backup & Restore - Apps
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup", api.BackupAppStart).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup", api.BackupAppList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup/latest", api.BackupAppLatest).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup/discover", api.BackupAppDiscoverResources).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup/{timestamp}", api.BackupAppDelete).Methods("DELETE")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/backup/{timestamp}/verify", api.BackupAppVerify).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restore", api.BackupAppRestore).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restore/switch", api.BackupAppSwitch).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restore/cleanup", api.BackupAppCleanup).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restore/plan", api.BackupAppRecoveryPlan).Methods("GET")
|
||||
|
||||
// Restore from Restic Snapshots
|
||||
r.HandleFunc("/api/v1/instances/{name}/backups/snapshots", api.ListSnapshots).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/snapshots", api.ListAppSnapshots).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/apps/{app}/restore/{snapshot}", api.RestoreFromSnapshot).Methods("POST")
|
||||
|
||||
// Backup & Restore - Cluster
|
||||
r.HandleFunc("/api/v1/instances/{name}/backups/all", api.BackupListAll).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/backup", api.BackupClusterStart).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/backup", api.BackupClusterList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/restore", api.BackupClusterRestore).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/cluster/backup/{timestamp}", api.BackupClusterDelete).Methods("DELETE")
|
||||
// Backup & Restore - Cluster Config
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/cluster", api.BackupClusterStart).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/cluster", api.BackupClusterList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/cluster/{timestamp}", api.BackupClusterDelete).Methods("DELETE")
|
||||
|
||||
// Backup Schedules
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules", api.ScheduleListHandler).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules", api.ScheduleCreateHandler).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules/{schedule_id}", api.ScheduleGetHandler).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules/{schedule_id}", api.ScheduleUpdateHandler).Methods("PUT")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules/{schedule_id}", api.ScheduleDeleteHandler).Methods("DELETE")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules/{schedule_id}/run", api.ScheduleRunHandler).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup-schedules/{schedule_id}/history", api.ScheduleHistoryHandler).Methods("GET")
|
||||
r.HandleFunc("/api/v1/scheduler/status", api.SchedulerStatusHandler).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/schedules", api.BackupScheduleList).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/schedules", api.BackupScheduleCreate).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/schedules/{scheduleId}", api.BackupScheduleUpdate).Methods("PUT")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/schedules/{scheduleId}", api.BackupScheduleDelete).Methods("DELETE")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/schedules/{scheduleId}/run", api.BackupScheduleRun).Methods("POST")
|
||||
|
||||
// Backup Health & Configuration
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/health", api.BackupHealth).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/config", api.BackupConfigGet).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backup/config", api.BackupConfigUpdate).Methods("PUT")
|
||||
|
||||
// Global Configuration
|
||||
r.HandleFunc("/api/v1/config", api.GetGlobalConfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/config", api.UpdateGlobalConfig).Methods("PUT")
|
||||
|
||||
// Backup Configuration
|
||||
r.HandleFunc("/api/v1/config/backup", api.GetBackupConfig).Methods("GET")
|
||||
r.HandleFunc("/api/v1/config/backup", api.UpdateBackupConfig).Methods("PUT")
|
||||
r.HandleFunc("/api/v1/config/backup/test", api.TestBackupConfig).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backups/repository/status", api.GetRepositoryStatus).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/backups/repository/init", api.InitRepository).Methods("POST")
|
||||
|
||||
// Utilities
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/health", api.InstanceUtilitiesHealth).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/dashboard/token", api.UtilitiesDashboardToken).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/nodes/ips", api.UtilitiesNodeIPs).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/controlplane/ip", api.UtilitiesControlPlaneIP).Methods("GET")
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/secrets/{secret}/copy", api.UtilitiesSecretCopy).Methods("POST")
|
||||
r.HandleFunc("/api/v1/instances/{name}/utilities/version", api.UtilitiesVersion).Methods("GET")
|
||||
|
||||
// SSE events endpoints
|
||||
r.HandleFunc("/api/v1/instances/{name}/events", api.InstanceEventStream).Methods("GET")
|
||||
r.HandleFunc("/api/v1/events", api.GlobalEventStream).Methods("GET") // Global events (central, dnsmasq, etc)
|
||||
|
||||
// dnsmasq management
|
||||
r.HandleFunc("/api/v1/dnsmasq/status", api.DnsmasqStatus).Methods("GET")
|
||||
r.HandleFunc("/api/v1/dnsmasq/config", api.DnsmasqGetConfig).Methods("GET")
|
||||
@@ -265,13 +309,13 @@ func (api *API) CreateInstance(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Attempt to update dnsmasq configuration with all instances
|
||||
// This is non-critical - include warning in response if it fails
|
||||
response := map[string]interface{}{
|
||||
response := map[string]any{
|
||||
"name": req.Name,
|
||||
"message": "Instance created successfully",
|
||||
}
|
||||
|
||||
if err := api.updateDnsmasqForAllInstances(); err != nil {
|
||||
log.Printf("Warning: Could not update dnsmasq configuration: %v", err)
|
||||
slog.Error("dnsmasq config update failed", "instance", req.Name, "error", err)
|
||||
response["warning"] = fmt.Sprintf("dnsmasq update failed: %v. Use POST /api/v1/dnsmasq/update to retry.", err)
|
||||
}
|
||||
|
||||
@@ -286,7 +330,7 @@ func (api *API) ListInstances(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"instances": instances,
|
||||
})
|
||||
}
|
||||
@@ -309,13 +353,13 @@ func (api *API) GetInstance(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var configMap map[string]interface{}
|
||||
var configMap map[string]any
|
||||
if err := yaml.Unmarshal(configData, &configMap); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"name": name,
|
||||
"config": configMap,
|
||||
})
|
||||
@@ -359,12 +403,12 @@ func (api *API) GetConfig(w http.ResponseWriter, r *http.Request) {
|
||||
// Return raw YAML
|
||||
w.Header().Set("Content-Type", "application/yaml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(configData)
|
||||
_, _ = w.Write(configData)
|
||||
return
|
||||
}
|
||||
|
||||
// Default: return JSON
|
||||
var configMap map[string]interface{}
|
||||
var configMap map[string]any
|
||||
if err := yaml.Unmarshal(configData, &configMap); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse config: %v", err))
|
||||
return
|
||||
@@ -397,14 +441,14 @@ func (api *API) GetSecrets(w http.ResponseWriter, r *http.Request) {
|
||||
secretsData, err := os.ReadFile(secretsPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{})
|
||||
respondJSON(w, http.StatusOK, map[string]any{})
|
||||
return
|
||||
}
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read secrets: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
var secretsMap map[string]interface{}
|
||||
var secretsMap map[string]any
|
||||
if err := yaml.Unmarshal(secretsData, &secretsMap); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse secrets: %v", err))
|
||||
return
|
||||
@@ -435,7 +479,7 @@ func (api *API) GetContext(w http.ResponseWriter, r *http.Request) {
|
||||
currentContext, err := api.context.GetCurrentContext()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"context": nil,
|
||||
})
|
||||
return
|
||||
@@ -444,7 +488,7 @@ func (api *API) GetContext(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"context": currentContext,
|
||||
})
|
||||
}
|
||||
@@ -487,15 +531,15 @@ func (api *API) StatusHandler(w http.ResponseWriter, r *http.Request, startTime
|
||||
// Calculate uptime
|
||||
uptime := time.Since(startTime)
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"status": "running",
|
||||
"version": "0.1.0", // TODO: Get from build info
|
||||
"version": api.version,
|
||||
"uptime": uptime.String(),
|
||||
"uptimeSeconds": int(uptime.Seconds()),
|
||||
"dataDir": dataDir,
|
||||
"appsDir": appsDir,
|
||||
"setupFiles": "embedded", // Indicate that setup files are now embedded
|
||||
"instances": map[string]interface{}{
|
||||
"instances": map[string]any{
|
||||
"count": len(instances),
|
||||
"names": instances,
|
||||
},
|
||||
@@ -522,17 +566,36 @@ func (api *API) NetworkResolveHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
ip, err := network.ResolveDomain(domain)
|
||||
if err != nil {
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"success": false,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
respondJSON(w, http.StatusOK, map[string]any{
|
||||
"success": true,
|
||||
"ip": ip,
|
||||
})
|
||||
}
|
||||
|
||||
// Helper functions are now in response.go and helpers.go
|
||||
|
||||
// sseManagerAdapter adapts sse.Manager to operations.SSEManager interface
|
||||
type sseManagerAdapter struct {
|
||||
sseManager *sse.Manager
|
||||
}
|
||||
|
||||
// Broadcast implements operations.SSEManager interface
|
||||
func (a *sseManagerAdapter) Broadcast(event *operations.SSEEvent) {
|
||||
// Convert operations.SSEEvent to sse.Event
|
||||
sseEvent := &sse.Event{
|
||||
ID: event.ID,
|
||||
Type: event.Type,
|
||||
InstanceName: event.InstanceName,
|
||||
Timestamp: event.Timestamp,
|
||||
Data: event.Data,
|
||||
Metadata: event.Metadata,
|
||||
}
|
||||
a.sseManager.Broadcast(sseEvent)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
@@ -18,9 +19,11 @@ import (
|
||||
|
||||
// AppsListAvailable lists all available apps
|
||||
func (api *API) AppsListAvailable(w http.ResponseWriter, r *http.Request) {
|
||||
category := r.URL.Query().Get("category")
|
||||
|
||||
// List available apps from apps directory
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
appList, err := appsMgr.ListAvailable()
|
||||
appList, err := appsMgr.ListAvailable(category)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list apps: %v", err))
|
||||
return
|
||||
@@ -50,9 +53,10 @@ func (api *API) AppsGetAvailable(w http.ResponseWriter, r *http.Request) {
|
||||
// AppsListDeployed lists deployed apps for an instance
|
||||
func (api *API) AppsListDeployed(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
category := r.URL.Query().Get("category")
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
deployedApps, err := appsMgr.ListDeployed(instanceName)
|
||||
deployedApps, err := appsMgr.ListDeployed(instanceName, category)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list apps: %v", err))
|
||||
return
|
||||
@@ -79,7 +83,7 @@ func (api *API) AppsAdd(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
if err := appsMgr.Add(instanceName, req.Name, req.Config, req.RequiredAppMappings); err != nil {
|
||||
if err := appsMgr.Add(instanceName, req.Name, req.Version, req.Config, req.RequiredAppMappings); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to add app: %v", err))
|
||||
return
|
||||
}
|
||||
@@ -104,9 +108,21 @@ func (api *API) AppsDeploy(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
api.startAppOperation(w, instanceName, appName, "deploy_app", "App deployed",
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "deploy_app", appName,
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
return appsMgr.Deploy(instanceName, appName, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
// AppsRestart performs a rolling restart of an app's pods
|
||||
func (api *API) AppsRestart(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
api.startAppOperation(w, instanceName, appName, "restart_app", "App restarted",
|
||||
func(mgr *apps.Manager, instance, app string) error {
|
||||
return mgr.Deploy(instance, app)
|
||||
return mgr.Restart(instance, app)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -121,17 +137,33 @@ func (api *API) AppsDelete(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// AppsUpdate updates an app from its source
|
||||
// AppsUpdate updates an app from its source, using the upgrade-aware path
|
||||
func (api *API) AppsUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
api.startAppOperation(w, instanceName, appName, "update_app", "App updated",
|
||||
func(mgr *apps.Manager, instance, app string) error {
|
||||
return mgr.Update(instance, app)
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "update_app", appName,
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
return appsMgr.Upgrade(instanceName, appName, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
// AppsGetUpgradePlan returns the computed upgrade plan for an app
|
||||
func (api *API) AppsGetUpgradePlan(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
plan, err := appsMgr.GetUpgradePlan(instanceName, appName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to compute upgrade plan: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, plan)
|
||||
}
|
||||
|
||||
// AppsEject converts an app from package-managed to custom
|
||||
func (api *API) AppsEject(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
@@ -278,9 +310,12 @@ func (api *API) AppsGetLogs(w http.ResponseWriter, r *http.Request) {
|
||||
kubeconfigPath := tools.GetKubeconfigPath(api.dataDir, instanceName)
|
||||
kubectl := tools.NewKubectl(kubeconfigPath)
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
namespace := appsMgr.ResolveNamespace(instanceName, appName)
|
||||
|
||||
// If no pod specified, get the first pod
|
||||
if podName == "" {
|
||||
pods, err := kubectl.GetPods(appName, true)
|
||||
pods, err := kubectl.GetPods(namespace, true)
|
||||
if err != nil || len(pods) == 0 {
|
||||
respondError(w, http.StatusNotFound, "No pods found for app")
|
||||
return
|
||||
@@ -293,7 +328,7 @@ func (api *API) AppsGetLogs(w http.ResponseWriter, r *http.Request) {
|
||||
Tail: tail,
|
||||
SinceSeconds: sinceSeconds,
|
||||
}
|
||||
logs, err := kubectl.GetLogs(appName, podName, logOpts)
|
||||
logs, err := kubectl.GetLogs(namespace, podName, logOpts)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get logs: %v", err))
|
||||
return
|
||||
@@ -321,7 +356,10 @@ func (api *API) AppsGetEvents(w http.ResponseWriter, r *http.Request) {
|
||||
kubeconfigPath := tools.GetKubeconfigPath(api.dataDir, instanceName)
|
||||
kubectl := tools.NewKubectl(kubeconfigPath)
|
||||
|
||||
events, err := kubectl.GetRecentEvents(appName, limit)
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
namespace := appsMgr.ResolveNamespace(instanceName, appName)
|
||||
|
||||
events, err := kubectl.GetRecentEvents(namespace, limit)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get events: %v", err))
|
||||
return
|
||||
@@ -332,7 +370,11 @@ func (api *API) AppsGetEvents(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// AppsGetReadme returns the README.md content for an app
|
||||
// AppsGetReadme returns the README.md for a deployed app instance.
|
||||
// Reads from the compiled instance directory (instances/{name}/apps/{app}/README.md),
|
||||
// which is written by Add() when the app is added. This file only exists after
|
||||
// "wild app add" — updating a README in wild-directory requires a version bump and re-add.
|
||||
// See AppsGetAvailableReadme for the catalog (not-yet-deployed) version.
|
||||
func (api *API) AppsGetReadme(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
@@ -342,18 +384,8 @@ func (api *API) AppsGetReadme(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Try instance-specific README first
|
||||
instancePath := filepath.Join(api.dataDir, "instances", instanceName, "apps", appName, "README.md")
|
||||
content, err := os.ReadFile(instancePath)
|
||||
if err == nil {
|
||||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||
w.Write(content)
|
||||
return
|
||||
}
|
||||
|
||||
// Fall back to global directory
|
||||
globalPath := filepath.Join(api.appsDir, appName, "README.md")
|
||||
content, err = os.ReadFile(globalPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("README not found for app '%s' in instance '%s'", appName, instanceName))
|
||||
@@ -364,5 +396,149 @@ func (api *API) AppsGetReadme(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||
w.Write(content)
|
||||
_, _ = w.Write(content)
|
||||
}
|
||||
|
||||
// AppsGetManifest returns the manifest for an available app
|
||||
func (api *API) AppsGetManifest(w http.ResponseWriter, r *http.Request) {
|
||||
appName := mux.Vars(r)["app"]
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
manifest, err := appsMgr.GetAppManifest(appName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Manifest not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, manifest)
|
||||
}
|
||||
|
||||
// AppsGetAvailableReadme returns the README for a catalog (not-yet-deployed) app.
|
||||
// Reads live from the wild-directory version slot via GetCatalogReadme — no re-add needed.
|
||||
// For deployed apps, use AppsGetReadme instead.
|
||||
func (api *API) AppsGetAvailableReadme(w http.ResponseWriter, r *http.Request) {
|
||||
appName := mux.Vars(r)["app"]
|
||||
|
||||
if err := ValidateAppName(appName); err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
content, err := appsMgr.GetCatalogReadme(appName)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("README not found for app '%s'", appName))
|
||||
} else {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to read README file")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||
_, _ = w.Write(content)
|
||||
}
|
||||
|
||||
// AppsCompile recompiles an app's templates
|
||||
func (api *API) AppsCompile(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
if err := appsMgr.Compile(instanceName, appName); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to compile templates: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": fmt.Sprintf("Templates compiled successfully for %s", appName),
|
||||
})
|
||||
}
|
||||
|
||||
// AppsFetch re-fetches an app's files from its source
|
||||
func (api *API) AppsFetch(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
if err := appsMgr.Fetch(instanceName, appName); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to fetch app: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": fmt.Sprintf("App %s files fetched successfully", appName),
|
||||
})
|
||||
}
|
||||
|
||||
// AppsGetManifests returns the rendered kustomize output for an installed app.
|
||||
// For apps with deploy phases, renders each phase directory separately.
|
||||
func (api *API) AppsGetManifests(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
appDir := filepath.Join(api.dataDir, "instances", instanceName, "apps", appName)
|
||||
|
||||
// Load manifest to check for deploy phases
|
||||
var manifest apps.AppManifest
|
||||
manifestPath := filepath.Join(appDir, "manifest.yaml")
|
||||
if data, err := os.ReadFile(manifestPath); err == nil {
|
||||
_ = yaml.Unmarshal(data, &manifest)
|
||||
}
|
||||
|
||||
// Build list of kustomize directories to render
|
||||
var dirs []string
|
||||
if manifest.Deploy != nil && len(manifest.Deploy.Phases) > 0 {
|
||||
for _, phase := range manifest.Deploy.Phases {
|
||||
phaseDir := filepath.Join(appDir, phase.Path)
|
||||
if _, err := os.Stat(filepath.Join(phaseDir, "kustomization.yaml")); err == nil {
|
||||
dirs = append(dirs, phaseDir)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
dirs = []string{appDir}
|
||||
}
|
||||
|
||||
if len(dirs) == 0 {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("No kustomization.yaml found for app '%s'", appName))
|
||||
return
|
||||
}
|
||||
|
||||
var allOutput []byte
|
||||
for i, dir := range dirs {
|
||||
if i > 0 {
|
||||
allOutput = append(allOutput, []byte("---\n")...)
|
||||
}
|
||||
cmd := exec.Command("kubectl", "kustomize", dir)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to render manifests: %v", err))
|
||||
return
|
||||
}
|
||||
allOutput = append(allOutput, output...)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
_, _ = w.Write(allOutput)
|
||||
}
|
||||
|
||||
// AppsRunScript runs a named script defined in the app's manifest
|
||||
func (api *API) AppsRunScript(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
scriptName := mux.Vars(r)["script"]
|
||||
|
||||
var params map[string]string
|
||||
if r.ContentLength > 0 {
|
||||
if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "run_script", appName,
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
return appsMgr.RunScript(instanceName, appName, scriptName, params, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,130 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/backup"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
)
|
||||
|
||||
// BackupClusterStart starts a cluster backup operation
|
||||
func (api *API) BackupClusterStart(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
// Parse request body for backup components
|
||||
var req struct {
|
||||
Etcd bool `json:"etcd"`
|
||||
Config bool `json:"config"`
|
||||
Secrets bool `json:"secrets"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
components := backup.ClusterBackupComponents{
|
||||
Etcd: req.Etcd,
|
||||
Config: req.Config,
|
||||
Secrets: req.Secrets,
|
||||
}
|
||||
|
||||
api.StartAsyncOperation(w, instanceName, "cluster-backup", "cluster",
|
||||
func(opsMgr *operations.Manager, opID string) error {
|
||||
_ = opsMgr.UpdateProgress(instanceName, opID, 10, "Starting cluster backup")
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
_, err := mgr.BackupCluster(instanceName, components)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// BackupClusterList lists all cluster backups for an instance
|
||||
func (api *API) BackupClusterList(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
backups, err := mgr.ListClusterBackups(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to list cluster backups")
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"data": map[string]interface{}{
|
||||
"backups": backups,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// BackupClusterRestore restores cluster from backup
|
||||
func (api *API) BackupClusterRestore(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
// Parse request body for restore options
|
||||
var req struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
Etcd bool `json:"etcd"`
|
||||
Config bool `json:"config"`
|
||||
Secrets bool `json:"secrets"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Timestamp == "" {
|
||||
respondError(w, http.StatusBadRequest, "Timestamp is required")
|
||||
return
|
||||
}
|
||||
|
||||
components := backup.ClusterBackupComponents{
|
||||
Etcd: req.Etcd,
|
||||
Config: req.Config,
|
||||
Secrets: req.Secrets,
|
||||
}
|
||||
|
||||
api.StartAsyncOperation(w, instanceName, "cluster-restore", "cluster",
|
||||
func(opsMgr *operations.Manager, opID string) error {
|
||||
_ = opsMgr.UpdateProgress(instanceName, opID, 10, "Starting cluster restore")
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
return mgr.RestoreCluster(instanceName, req.Timestamp, components)
|
||||
})
|
||||
}
|
||||
|
||||
// BackupClusterDelete deletes a specific cluster backup
|
||||
func (api *API) BackupClusterDelete(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
timestamp := mux.Vars(r)["timestamp"]
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
if err := mgr.DeleteClusterBackup(instanceName, timestamp); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to delete cluster backup")
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Cluster backup deleted",
|
||||
})
|
||||
}
|
||||
|
||||
// BackupListAll lists all backups (cluster and apps) for an instance
|
||||
func (api *API) BackupListAll(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
allBackups, err := mgr.ListAllBackups(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to list backups")
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"data": allBackups,
|
||||
})
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/backup"
|
||||
)
|
||||
|
||||
// ScheduleListHandler lists all backup schedules for an instance
|
||||
func (api *API) ScheduleListHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
schedules, err := mgr.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to load schedules: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"schedules": schedules,
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleCreateHandler creates a new backup schedule
|
||||
func (api *API) ScheduleCreateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
var req backup.BackupSchedule
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
|
||||
// Validate configuration if scheduler is available
|
||||
if api.backupScheduler != nil {
|
||||
if err := api.backupScheduler.ValidateScheduleConfiguration(instanceName, &req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid schedule configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
schedule, err := mgr.CreateSchedule(instanceName, &req)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to create schedule: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"schedule": schedule,
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleGetHandler retrieves a specific backup schedule
|
||||
func (api *API) ScheduleGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
scheduleID := mux.Vars(r)["schedule_id"]
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
schedule, err := mgr.GetSchedule(instanceName, scheduleID)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, "Schedule not found: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"schedule": schedule,
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleUpdateHandler updates an existing backup schedule
|
||||
func (api *API) ScheduleUpdateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
scheduleID := mux.Vars(r)["schedule_id"]
|
||||
|
||||
var req backup.BackupSchedule
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
|
||||
// Validate configuration if scheduler is available
|
||||
if api.backupScheduler != nil {
|
||||
if err := api.backupScheduler.ValidateScheduleConfiguration(instanceName, &req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid schedule configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
schedule, err := mgr.UpdateSchedule(instanceName, scheduleID, &req)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to update schedule: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"schedule": schedule,
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleDeleteHandler deletes a backup schedule
|
||||
func (api *API) ScheduleDeleteHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
scheduleID := mux.Vars(r)["schedule_id"]
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
if err := mgr.DeleteSchedule(instanceName, scheduleID); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to delete schedule: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Schedule deleted",
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleRunHandler manually triggers a scheduled backup
|
||||
func (api *API) ScheduleRunHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
scheduleID := mux.Vars(r)["schedule_id"]
|
||||
|
||||
if api.backupScheduler == nil {
|
||||
respondError(w, http.StatusInternalServerError, "Backup scheduler not available")
|
||||
return
|
||||
}
|
||||
|
||||
// Execute synchronously for manual trigger
|
||||
if err := api.backupScheduler.ExecuteScheduledBackup(instanceName, scheduleID); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to execute backup: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"message": "Backup executed successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// ScheduleHistoryHandler returns backup history for a schedule
|
||||
func (api *API) ScheduleHistoryHandler(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
scheduleID := mux.Vars(r)["schedule_id"]
|
||||
|
||||
if api.backupScheduler == nil {
|
||||
respondError(w, http.StatusInternalServerError, "Backup scheduler not available")
|
||||
return
|
||||
}
|
||||
|
||||
history, err := api.backupScheduler.GetScheduleHistory(instanceName, scheduleID)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to get history: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"history": history,
|
||||
})
|
||||
}
|
||||
|
||||
// SchedulerStatusHandler returns the current scheduler status
|
||||
func (api *API) SchedulerStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if api.backupScheduler == nil {
|
||||
respondError(w, http.StatusInternalServerError, "Backup scheduler not available")
|
||||
return
|
||||
}
|
||||
|
||||
status := api.backupScheduler.GetSchedulerStatus()
|
||||
respondJSON(w, http.StatusOK, status)
|
||||
}
|
||||
1124
api/internal/api/v1/handlers_backup_test.go
Normal file
1124
api/internal/api/v1/handlers_backup_test.go
Normal file
File diff suppressed because it is too large
Load Diff
@@ -187,6 +187,19 @@ func (api *API) ClusterGetTalosconfig(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// ClusterRenewTalosconfig regenerates the talosconfig client certificate from the existing PKI root.
|
||||
func (api *API) ClusterRenewTalosconfig(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
clusterMgr := cluster.NewManager(api.dataDir, api.opsMgr)
|
||||
if err := clusterMgr.RenewTalosconfig(instanceName); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to renew talosconfig: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondMessage(w, http.StatusOK, "Talosconfig renewed successfully")
|
||||
}
|
||||
|
||||
// ClusterReset resets the cluster
|
||||
func (api *API) ClusterReset(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
@@ -3,11 +3,149 @@ package v1
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/config"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ConfigFile describes a single editable config file within an instance.
|
||||
type ConfigFile struct {
|
||||
Path string `json:"path"` // relative to instance root, e.g. "config/instance.yaml"
|
||||
Label string `json:"label"` // human-readable, e.g. "Instance Config"
|
||||
}
|
||||
|
||||
// ListConfigFiles returns all editable config files for an instance.
|
||||
// GET /api/v1/instances/{name}/config/files
|
||||
func (api *API) ListConfigFiles(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
instancePath := tools.GetInstancePath(api.dataDir, instanceName)
|
||||
|
||||
var files []ConfigFile
|
||||
|
||||
// config/instance.yaml
|
||||
if storage.FileExists(tools.GetInstanceYamlPath(api.dataDir, instanceName)) {
|
||||
files = append(files, ConfigFile{Path: "config/instance.yaml", Label: "Instance Config"})
|
||||
}
|
||||
|
||||
// config/nodes/*.yaml
|
||||
nodeDir := tools.GetNodeConfigDirPath(api.dataDir, instanceName)
|
||||
if entries, err := os.ReadDir(nodeDir); err == nil {
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".yaml" {
|
||||
continue
|
||||
}
|
||||
hostname := strings.TrimSuffix(entry.Name(), ".yaml")
|
||||
files = append(files, ConfigFile{
|
||||
Path: "config/nodes/" + entry.Name(),
|
||||
Label: "Node: " + hostname,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// config/apps/*/config.yaml
|
||||
appsConfigDir := filepath.Join(instancePath, "config", "apps")
|
||||
if entries, err := os.ReadDir(appsConfigDir); err == nil {
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
appName := entry.Name()
|
||||
if storage.FileExists(filepath.Join(appsConfigDir, appName, "config.yaml")) {
|
||||
files = append(files, ConfigFile{
|
||||
Path: "config/apps/" + appName + "/config.yaml",
|
||||
Label: "App: " + appName,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if files == nil {
|
||||
files = []ConfigFile{}
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{"files": files})
|
||||
}
|
||||
|
||||
// GetConfigFile returns the raw YAML content of a specific config file.
|
||||
// GET /api/v1/instances/{name}/config/file?path=config/instance.yaml
|
||||
func (api *API) GetConfigFile(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
relPath := r.URL.Query().Get("path")
|
||||
|
||||
if !isAllowedConfigPath(relPath) {
|
||||
respondError(w, http.StatusBadRequest, "Invalid file path")
|
||||
return
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(tools.GetInstancePath(api.dataDir, instanceName), relPath)
|
||||
data, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
respondError(w, http.StatusNotFound, "Config file not found")
|
||||
return
|
||||
}
|
||||
respondError(w, http.StatusInternalServerError, "Failed to read config file")
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/yaml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
|
||||
// UpdateConfigFile replaces the content of a specific config file.
|
||||
// PUT /api/v1/instances/{name}/config/file?path=config/instance.yaml
|
||||
func (api *API) UpdateConfigFile(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
relPath := r.URL.Query().Get("path")
|
||||
|
||||
if !isAllowedConfigPath(relPath) {
|
||||
respondError(w, http.StatusBadRequest, "Invalid file path")
|
||||
return
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Failed to read request body")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate it parses as valid YAML before writing
|
||||
var parsed map[string]interface{}
|
||||
if err := yaml.Unmarshal(body, &parsed); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid YAML format")
|
||||
return
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(tools.GetInstancePath(api.dataDir, instanceName), relPath)
|
||||
lockPath := fullPath + ".lock"
|
||||
if err := storage.WithLock(lockPath, func() error {
|
||||
return storage.WriteFile(fullPath, body, 0644)
|
||||
}); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to write config file")
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("config file updated", "instance", instanceName, "path", relPath)
|
||||
respondMessage(w, http.StatusOK, "Config file updated successfully")
|
||||
}
|
||||
|
||||
// isAllowedConfigPath checks that a relative path is safe to read/write:
|
||||
// must start with "config/" and must not contain "..".
|
||||
func isAllowedConfigPath(relPath string) bool {
|
||||
return relPath != "" &&
|
||||
strings.HasPrefix(relPath, "config/") &&
|
||||
!strings.Contains(relPath, "..")
|
||||
}
|
||||
|
||||
// ConfigUpdateBatch updates multiple configuration values atomically
|
||||
func (api *API) ConfigUpdateBatch(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
@@ -43,6 +181,8 @@ func (api *API) ConfigUpdateBatch(w http.ResponseWriter, r *http.Request) {
|
||||
updateCount++
|
||||
}
|
||||
|
||||
slog.Info("config batch updated", "instance", instanceName, "keys", updateCount)
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Configuration updated successfully",
|
||||
"updated": updateCount,
|
||||
@@ -87,6 +227,8 @@ func (api *API) UpdateGlobalConfig(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("global config updated")
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Global configuration updated successfully",
|
||||
"config": globalCfg,
|
||||
|
||||
@@ -3,7 +3,7 @@ package v1
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
@@ -43,6 +43,9 @@ func (api *API) DnsmasqRestart(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Broadcast SSE event
|
||||
api.broadcastDnsmasqEvent("dnsmasq:restart", "dnsmasq service restarted")
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": "dnsmasq service restarted successfully",
|
||||
})
|
||||
@@ -71,38 +74,74 @@ func (api *API) DnsmasqGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Load all instance configs
|
||||
var instanceConfigs []config.InstanceConfig
|
||||
var validInstanceNames []string
|
||||
for _, name := range instanceNames {
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(name)
|
||||
instanceCfg, err := config.LoadCloudConfig(instanceConfigPath)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not load instance config for %s: %v", name, err)
|
||||
slog.Error("failed to load instance config", "instance", name, "error", err)
|
||||
continue
|
||||
}
|
||||
instanceConfigs = append(instanceConfigs, *instanceCfg)
|
||||
validInstanceNames = append(validInstanceNames, name)
|
||||
}
|
||||
|
||||
// Generate config
|
||||
configContent := api.dnsmasq.Generate(globalCfg, instanceConfigs)
|
||||
// Generate dnsmasq config
|
||||
configContent := api.dnsmasq.GenerateMainConfig(globalCfg)
|
||||
|
||||
if overwrite {
|
||||
// Check if this is the first time dnsmasq is being started
|
||||
status, err := api.dnsmasq.GetStatus()
|
||||
isFirstStart := err != nil || status.Status != "active"
|
||||
|
||||
// Write config and restart service
|
||||
if err := api.dnsmasq.UpdateConfig(globalCfg, instanceConfigs, true); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to update dnsmasq: %v", err))
|
||||
// Update main dnsmasq configuration
|
||||
slog.Info("updating dnsmasq main configuration")
|
||||
|
||||
// Write the main config
|
||||
tempFile := api.dnsmasq.GetConfigPath() + ".tmp"
|
||||
if err := os.WriteFile(tempFile, []byte(configContent), 0644); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to write temp config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the config
|
||||
if err := api.dnsmasq.ValidateConfig(tempFile); err != nil {
|
||||
os.Remove(tempFile)
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Config validation failed: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Install the new config
|
||||
if err := os.Rename(tempFile, api.dnsmasq.GetConfigPath()); err != nil {
|
||||
os.Remove(tempFile)
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to install config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Write all instance configs
|
||||
for i, name := range validInstanceNames {
|
||||
if err := api.dnsmasq.WriteInstanceConfig(name, instanceConfigs[i]); err != nil {
|
||||
slog.Error("failed to write instance DNS config", "instance", name, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reload dnsmasq
|
||||
if err := api.dnsmasq.ReloadService(); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to reload dnsmasq: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Configure system DNS to use local dnsmasq on first start
|
||||
if isFirstStart {
|
||||
if err := api.dnsmasq.ConfigureSystemDNS(); err != nil {
|
||||
log.Printf("Warning: Failed to configure system DNS: %v", err)
|
||||
slog.Error("failed to configure system DNS", "error", err)
|
||||
// Don't fail the request - dnsmasq is still running
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast SSE event
|
||||
api.broadcastDnsmasqEvent("dnsmasq:config", "dnsmasq configuration updated and applied")
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "dnsmasq configuration generated and applied successfully",
|
||||
"config": configContent,
|
||||
@@ -139,6 +178,9 @@ func (api *API) DnsmasqWriteConfig(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Broadcast SSE event
|
||||
api.broadcastDnsmasqEvent("dnsmasq:config", "dnsmasq configuration written")
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": "dnsmasq configuration written successfully",
|
||||
})
|
||||
@@ -173,7 +215,7 @@ func (api *API) updateDnsmasqForAllInstances() error {
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(name)
|
||||
instanceCfg, err := config.LoadCloudConfig(instanceConfigPath)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Could not load instance config for %s: %v", name, err)
|
||||
slog.Error("failed to load instance config", "instance", name, "error", err)
|
||||
continue
|
||||
}
|
||||
instanceConfigs = append(instanceConfigs, *instanceCfg)
|
||||
|
||||
@@ -39,11 +39,12 @@ func TestDnsmasqGenerate_WithoutOverwrite(t *testing.T) {
|
||||
|
||||
// Create global config
|
||||
globalConfig := config.GlobalConfig{}
|
||||
globalConfig.Cloud.DNS.IP = "192.168.1.100"
|
||||
globalConfig.Cloud.Dnsmasq.Interface = "eth0"
|
||||
globalConfig.Cloud.Router.IP = "192.168.1.1"
|
||||
configPath := filepath.Join(tmpDir, "config.yaml")
|
||||
configData, _ := yaml.Marshal(globalConfig)
|
||||
storage.WriteFile(configPath, configData, 0644)
|
||||
if err := storage.WriteFile(configPath, configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create test instance
|
||||
instanceName := "test-instance"
|
||||
@@ -55,7 +56,9 @@ func TestDnsmasqGenerate_WithoutOverwrite(t *testing.T) {
|
||||
instanceConfig.Cloud.InternalDomain = "internal.test.local"
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
instanceConfigData, _ := yaml.Marshal(instanceConfig)
|
||||
storage.WriteFile(instanceConfigPath, instanceConfigData, 0644)
|
||||
if err := storage.WriteFile(instanceConfigPath, instanceConfigData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test generate without overwrite
|
||||
req := httptest.NewRequest("POST", "/api/v1/dnsmasq/generate", nil)
|
||||
@@ -68,7 +71,9 @@ func TestDnsmasqGenerate_WithoutOverwrite(t *testing.T) {
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal response: %v", err)
|
||||
}
|
||||
|
||||
// Verify response contains config
|
||||
if config, ok := resp["config"].(string); !ok || config == "" {
|
||||
@@ -88,11 +93,12 @@ func TestDnsmasqGenerate_WithOverwrite(t *testing.T) {
|
||||
|
||||
// Create global config
|
||||
globalConfig := config.GlobalConfig{}
|
||||
globalConfig.Cloud.DNS.IP = "192.168.1.100"
|
||||
globalConfig.Cloud.Dnsmasq.Interface = "eth0"
|
||||
globalConfig.Cloud.Router.IP = "192.168.1.1"
|
||||
configPath := filepath.Join(tmpDir, "config.yaml")
|
||||
configData, _ := yaml.Marshal(globalConfig)
|
||||
storage.WriteFile(configPath, configData, 0644)
|
||||
if err := storage.WriteFile(configPath, configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create test instance
|
||||
instanceName := "test-instance"
|
||||
@@ -105,7 +111,9 @@ func TestDnsmasqGenerate_WithOverwrite(t *testing.T) {
|
||||
instanceConfig.Cluster.LoadBalancerIp = "192.168.1.80"
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
instanceConfigData, _ := yaml.Marshal(instanceConfig)
|
||||
storage.WriteFile(instanceConfigPath, instanceConfigData, 0644)
|
||||
if err := storage.WriteFile(instanceConfigPath, instanceConfigData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Instead of calling the handler which would try to restart the service,
|
||||
// directly test the UpdateConfig method with restart=false
|
||||
@@ -143,6 +151,63 @@ func TestDnsmasqGenerate_WithOverwrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqGenerate_MetallbFallback(t *testing.T) {
|
||||
api, tmpDir := setupTestDnsmasq(t)
|
||||
|
||||
globalConfig := config.GlobalConfig{}
|
||||
globalConfig.Cloud.Router.IP = "192.168.1.1"
|
||||
configPath := filepath.Join(tmpDir, "config.yaml")
|
||||
configData, _ := yaml.Marshal(globalConfig)
|
||||
if err := storage.WriteFile(configPath, configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
instanceName := "test-instance"
|
||||
createTestInstance(t, api, instanceName)
|
||||
|
||||
// Set loadBalancerIp only under apps.metallb, not cluster — the normal install flow
|
||||
instanceConfig := config.InstanceConfig{}
|
||||
instanceConfig.Cloud.Domain = "test.local"
|
||||
instanceConfig.Cloud.InternalDomain = "internal.test.local"
|
||||
instanceConfig.Apps = map[string]interface{}{
|
||||
"metallb": map[string]interface{}{
|
||||
"loadBalancerIp": "192.168.1.80",
|
||||
},
|
||||
}
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
instanceConfigData, _ := yaml.Marshal(instanceConfig)
|
||||
if err := storage.WriteFile(instanceConfigPath, instanceConfigData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
instances, _ := api.instance.ListInstances()
|
||||
var instanceConfigs []config.InstanceConfig
|
||||
for _, name := range instances {
|
||||
icPath := api.instance.GetInstanceConfigPath(name)
|
||||
ic, _ := config.LoadCloudConfig(icPath)
|
||||
instanceConfigs = append(instanceConfigs, *ic)
|
||||
}
|
||||
|
||||
err := api.dnsmasq.UpdateConfig(&globalConfig, instanceConfigs, false)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateConfig failed: %v", err)
|
||||
}
|
||||
|
||||
dnsmasqConfigPath := api.dnsmasq.GetConfigPath()
|
||||
content, err := os.ReadFile(dnsmasqConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read dnsmasq config: %v", err)
|
||||
}
|
||||
|
||||
configStr := string(content)
|
||||
if strings.Contains(configStr, "WARNING") {
|
||||
t.Fatal("Expected no WARNING comment when metallb loadBalancerIp is set")
|
||||
}
|
||||
if !strings.Contains(configStr, "192.168.1.80") {
|
||||
t.Fatal("Expected config to contain load balancer IP from apps.metallb fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqWriteConfig(t *testing.T) {
|
||||
api, _ := setupTestDnsmasq(t)
|
||||
|
||||
@@ -203,8 +268,12 @@ func TestDnsmasqGetConfig(t *testing.T) {
|
||||
// Write a config first
|
||||
configPath := api.dnsmasq.GetConfigPath()
|
||||
testConfig := "# Test config\ninterface=eth0\n"
|
||||
os.MkdirAll(filepath.Dir(configPath), 0755)
|
||||
os.WriteFile(configPath, []byte(testConfig), 0644)
|
||||
if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte(testConfig), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/v1/dnsmasq/config", nil)
|
||||
w := httptest.NewRecorder()
|
||||
@@ -216,7 +285,9 @@ func TestDnsmasqGetConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to unmarshal response: %v", err)
|
||||
}
|
||||
|
||||
content, ok := resp["content"].(string)
|
||||
if !ok || content != testConfig {
|
||||
|
||||
@@ -5,11 +5,13 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/discovery"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/node"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// NodeDiscover initiates node discovery
|
||||
@@ -53,7 +55,7 @@ func (api *API) NodeDiscover(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName)
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName, api.sseManager)
|
||||
if err := discoveryMgr.StartDiscovery(instanceName, ipList); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to start discovery: %v", err))
|
||||
return
|
||||
@@ -70,7 +72,7 @@ func (api *API) NodeDiscover(w http.ResponseWriter, r *http.Request) {
|
||||
func (api *API) NodeDiscoveryStatus(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName)
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName, api.sseManager)
|
||||
status, err := discoveryMgr.GetDiscoveryStatus(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get status: %v", err))
|
||||
@@ -131,19 +133,33 @@ func (api *API) NodeAdd(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if nodeData.Version != "" {
|
||||
if err := validateTalosVersion(nodeData.Version); err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nodeMgr := node.NewManager(api.dataDir, instanceName)
|
||||
if err := nodeMgr.Add(instanceName, &nodeData); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to add node: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Remove from discovery results so UI no longer shows it as pending
|
||||
if nodeData.CurrentIP != "" {
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName, api.sseManager)
|
||||
discoveryMgr.RemoveDiscoveredNode(instanceName, nodeData.CurrentIP)
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"message": "Node added successfully",
|
||||
"node": nodeData,
|
||||
})
|
||||
}
|
||||
|
||||
// NodeList returns all nodes for an instance
|
||||
// NodeList returns all nodes for an instance.
|
||||
// For applied nodes, queries the live Talos version concurrently to reflect actual state.
|
||||
func (api *API) NodeList(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
@@ -154,6 +170,8 @@ func (api *API) NodeList(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
api.enrichNodesWithLiveVersions(instanceName, nodes)
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"nodes": nodes,
|
||||
})
|
||||
@@ -171,9 +189,38 @@ func (api *API) NodeGet(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Query live version for this node
|
||||
nodes := []node.Node{*nodeData}
|
||||
api.enrichNodesWithLiveVersions(instanceName, nodes)
|
||||
*nodeData = nodes[0]
|
||||
|
||||
respondJSON(w, http.StatusOK, nodeData)
|
||||
}
|
||||
|
||||
// enrichNodesWithLiveVersions queries the actual Talos version from each applied node
|
||||
// concurrently and overrides the config version. Falls back to config version on failure.
|
||||
func (api *API) enrichNodesWithLiveVersions(instanceName string, nodes []node.Node) {
|
||||
talosconfigPath := tools.GetTalosconfigPath(api.dataDir, instanceName)
|
||||
talosctl := tools.NewTalosconfigWithConfig(talosconfigPath)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range nodes {
|
||||
n := &nodes[i]
|
||||
if !n.Applied || n.Maintenance || n.TargetIP == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(n *node.Node) {
|
||||
defer wg.Done()
|
||||
if version, err := talosctl.GetVersion(n.TargetIP, false); err == nil && version != "" {
|
||||
n.Version = version
|
||||
}
|
||||
// On error, keep the config version as fallback
|
||||
}(n)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// NodeApply generates configuration and applies it to node
|
||||
func (api *API) NodeApply(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
@@ -264,7 +311,7 @@ func (api *API) NodeDelete(w http.ResponseWriter, r *http.Request) {
|
||||
func (api *API) NodeDiscoveryCancel(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName)
|
||||
discoveryMgr := discovery.NewManager(api.dataDir, instanceName, api.sseManager)
|
||||
if err := discoveryMgr.CancelDiscovery(instanceName); err != nil {
|
||||
respondError(w, http.StatusBadRequest, fmt.Sprintf("Failed to cancel discovery: %v", err))
|
||||
return
|
||||
@@ -275,6 +322,38 @@ func (api *API) NodeDiscoveryCancel(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// NodeReboot reboots a node without wiping state
|
||||
func (api *API) NodeReboot(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
nodeIdentifier := GetNodeName(r)
|
||||
|
||||
nodeMgr := node.NewManager(api.dataDir, instanceName)
|
||||
if err := nodeMgr.Reboot(instanceName, nodeIdentifier); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to reboot node: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": "Node reboot initiated",
|
||||
"node": nodeIdentifier,
|
||||
})
|
||||
}
|
||||
|
||||
// NodeHealth checks node health via Talos service statuses and dmesg
|
||||
func (api *API) NodeHealth(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
nodeIdentifier := GetNodeName(r)
|
||||
|
||||
nodeMgr := node.NewManager(api.dataDir, instanceName)
|
||||
health, err := nodeMgr.Health(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to check node health: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, health)
|
||||
}
|
||||
|
||||
// NodeReset resets a node to maintenance mode
|
||||
func (api *API) NodeReset(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
160
api/internal/api/v1/handlers_node_upgrade.go
Normal file
160
api/internal/api/v1/handlers_node_upgrade.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/factory"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/node"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// NodeUpgrade upgrades a node to a new Talos version using the Image Factory installer.
|
||||
// This is a long-running async operation that cordons, drains, upgrades, and reboots the node.
|
||||
func (api *API) NodeUpgrade(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
nodeIdentifier := GetNodeName(r)
|
||||
|
||||
var req NodeUpgradeRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Version == "" {
|
||||
respondError(w, http.StatusBadRequest, "version is required")
|
||||
return
|
||||
}
|
||||
|
||||
if err := validateTalosVersion(req.Version); err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get node to verify state and retrieve IP/schematic
|
||||
nodeMgr := node.NewManager(api.dataDir, instanceName)
|
||||
nodeData, err := nodeMgr.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Node not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if !nodeData.Applied || nodeData.Maintenance {
|
||||
respondError(w, http.StatusBadRequest, "Node must be applied and not in maintenance mode to upgrade")
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve schematic ID: request > node > cluster default
|
||||
schematicID := req.SchematicID
|
||||
if schematicID == "" {
|
||||
schematicID = nodeData.SchematicID
|
||||
}
|
||||
if schematicID == "" {
|
||||
configPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
defaultSchematic, _ := api.config.GetConfigValue(configPath, "cluster.nodes.talos.schematicId")
|
||||
schematicID = defaultSchematic
|
||||
}
|
||||
if schematicID == "" {
|
||||
respondError(w, http.StatusBadRequest, "No schematic ID found for node (set on node, instance, or in request)")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate schematic+version compatibility
|
||||
validation, err := api.factory.ValidateSchematic(schematicID, req.Version)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, fmt.Sprintf("Failed to validate schematic compatibility: %v", err))
|
||||
return
|
||||
}
|
||||
if !validation.Valid {
|
||||
respondError(w, http.StatusBadRequest, fmt.Sprintf(
|
||||
"Schematic incompatible with version %s: missing extensions %v",
|
||||
req.Version, validation.MissingExtensions))
|
||||
return
|
||||
}
|
||||
|
||||
// Check talosctl client version compatibility (must be within 1 minor version of target)
|
||||
clientInfo, err := tools.GetClientInfo()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to check talosctl version: %v", err))
|
||||
return
|
||||
}
|
||||
if gap := factory.MinorVersionGap(clientInfo.Version, req.Version); gap > 1 {
|
||||
respondError(w, http.StatusBadRequest, fmt.Sprintf(
|
||||
"talosctl %s is %d minor versions behind target %s (max 1). Upgrade talosctl first via POST /api/v1/talos/client/upgrade",
|
||||
clientInfo.Version, gap, req.Version))
|
||||
return
|
||||
}
|
||||
|
||||
// Build installer image URL
|
||||
image := api.factory.GetInstallerImage(schematicID, req.Version)
|
||||
|
||||
// Preserve defaults to true for upgrades
|
||||
preserve := true
|
||||
if req.Preserve != nil {
|
||||
preserve = *req.Preserve
|
||||
}
|
||||
|
||||
nodeIP := nodeData.TargetIP
|
||||
|
||||
api.StartAsyncOperation(w, instanceName, "upgrade", nodeIdentifier,
|
||||
func(opsMgr *operations.Manager, opID string) error {
|
||||
_ = opsMgr.UpdateProgress(instanceName, opID, 10,
|
||||
fmt.Sprintf("Upgrading %s to %s using image %s", nodeIdentifier, req.Version, image))
|
||||
|
||||
talosconfigPath := tools.GetTalosconfigPath(api.dataDir, instanceName)
|
||||
talosctl := tools.NewTalosconfigWithConfig(talosconfigPath)
|
||||
|
||||
if err := talosctl.Upgrade(nodeIP, image, preserve); err != nil {
|
||||
return fmt.Errorf("upgrade failed: %w", err)
|
||||
}
|
||||
|
||||
_ = opsMgr.UpdateProgress(instanceName, opID, 80, "Upgrade command completed, updating config")
|
||||
|
||||
// Update node version in config after successful upgrade
|
||||
if err := nodeMgr.Update(instanceName, nodeIdentifier, map[string]interface{}{
|
||||
"version": req.Version,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("upgrade succeeded but failed to update config: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// NodeRollback reverts a node to its previous Talos version.
|
||||
// Talos uses an A/B image scheme so the previous version is always available.
|
||||
func (api *API) NodeRollback(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
nodeIdentifier := GetNodeName(r)
|
||||
|
||||
nodeMgr := node.NewManager(api.dataDir, instanceName)
|
||||
nodeData, err := nodeMgr.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Node not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if !nodeData.Applied || nodeData.Maintenance {
|
||||
respondError(w, http.StatusBadRequest, "Node must be applied and not in maintenance mode to rollback")
|
||||
return
|
||||
}
|
||||
|
||||
nodeIP := nodeData.TargetIP
|
||||
|
||||
api.StartAsyncOperation(w, instanceName, "rollback", nodeIdentifier,
|
||||
func(opsMgr *operations.Manager, opID string) error {
|
||||
_ = opsMgr.UpdateProgress(instanceName, opID, 10,
|
||||
fmt.Sprintf("Rolling back %s to previous version", nodeIdentifier))
|
||||
|
||||
talosconfigPath := tools.GetTalosconfigPath(api.dataDir, instanceName)
|
||||
talosctl := tools.NewTalosconfigWithConfig(talosconfigPath)
|
||||
|
||||
if err := talosctl.Rollback(nodeIP); err != nil {
|
||||
return fmt.Errorf("rollback failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
@@ -19,7 +18,7 @@ func (api *API) OperationGet(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
opID := mux.Vars(r)["id"]
|
||||
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
op, err := opsMgr.GetByInstance(instanceName, opID)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Operation not found: %v", err))
|
||||
@@ -33,7 +32,7 @@ func (api *API) OperationGet(w http.ResponseWriter, r *http.Request) {
|
||||
func (api *API) OperationList(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
ops, err := opsMgr.List(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list operations: %v", err))
|
||||
@@ -50,7 +49,7 @@ func (api *API) OperationCancel(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
opID := mux.Vars(r)["id"]
|
||||
|
||||
opsMgr := operations.NewManager(api.dataDir)
|
||||
opsMgr := api.opsMgr
|
||||
if err := opsMgr.Cancel(instanceName, opID); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to cancel operation: %v", err))
|
||||
return
|
||||
|
||||
@@ -3,7 +3,7 @@ package v1
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -18,7 +18,7 @@ func (api *API) PXEListAssets(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
w.Header().Set("X-Deprecated", "This endpoint is deprecated. Use GET /api/v1/assets/{schematicId} instead.")
|
||||
log.Printf("Warning: Deprecated endpoint /api/v1/instances/%s/pxe/assets called", instanceName)
|
||||
slog.Info("deprecated endpoint called", "endpoint", "pxe/assets", "instance", instanceName)
|
||||
|
||||
// Get schematic ID from instance config
|
||||
configPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
@@ -49,7 +49,7 @@ func (api *API) PXEDownloadAsset(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
w.Header().Set("X-Deprecated", "This endpoint is deprecated. Use POST /api/v1/assets/{schematicId}/download instead.")
|
||||
log.Printf("Warning: Deprecated endpoint /api/v1/instances/%s/pxe/assets/download called", instanceName)
|
||||
slog.Info("deprecated endpoint called", "endpoint", "pxe/assets/download", "instance", instanceName)
|
||||
|
||||
// Parse request
|
||||
var req struct {
|
||||
@@ -123,7 +123,7 @@ func (api *API) PXEGetAsset(w http.ResponseWriter, r *http.Request) {
|
||||
assetType := mux.Vars(r)["type"]
|
||||
|
||||
w.Header().Set("X-Deprecated", "This endpoint is deprecated. Use GET /api/v1/assets/{schematicId}/pxe/{assetType} instead.")
|
||||
log.Printf("Warning: Deprecated endpoint /api/v1/instances/%s/pxe/assets/%s called", instanceName, assetType)
|
||||
slog.Info("deprecated endpoint called", "endpoint", "pxe/assets/get", "instance", instanceName, "assetType", assetType)
|
||||
|
||||
// Get schematic ID from instance config
|
||||
configPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
@@ -162,7 +162,7 @@ func (api *API) PXEDeleteAsset(w http.ResponseWriter, r *http.Request) {
|
||||
assetType := mux.Vars(r)["type"]
|
||||
|
||||
w.Header().Set("X-Deprecated", "This endpoint is deprecated. Use DELETE /api/v1/assets/{schematicId} instead.")
|
||||
log.Printf("Warning: Deprecated endpoint DELETE /api/v1/instances/%s/pxe/assets/%s called", instanceName, assetType)
|
||||
slog.Info("deprecated endpoint called", "endpoint", "pxe/assets/delete", "instance", instanceName, "assetType", assetType)
|
||||
|
||||
// Get schematic ID from instance config
|
||||
configPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/backup"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
)
|
||||
|
||||
// Note: RestoreFromSnapshot uses custom async logic (needs snapshotID in response)
|
||||
|
||||
// ListSnapshots lists all snapshots for an instance
|
||||
func (api *API) ListSnapshots(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
snapshots, err := backup.ListSnapshotsForInstance(api.dataDir, instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to list snapshots: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"data": map[string]interface{}{
|
||||
"snapshots": snapshots,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ListAppSnapshots lists all snapshots for a specific app
|
||||
func (api *API) ListAppSnapshots(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
|
||||
snapshots, err := backup.ListSnapshotsForApp(api.dataDir, instanceName, appName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to list app snapshots: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"data": map[string]interface{}{
|
||||
"snapshots": snapshots,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// RestoreFromSnapshot restores an app from a restic snapshot
|
||||
func (api *API) RestoreFromSnapshot(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
appName := GetAppName(r)
|
||||
snapshotID := mux.Vars(r)["snapshot"]
|
||||
|
||||
// Parse request body for restore options
|
||||
var opts backup.RestoreOptions
|
||||
if err := json.NewDecoder(r.Body).Decode(&opts); err != nil {
|
||||
// Use defaults if no body provided
|
||||
opts = backup.RestoreOptions{}
|
||||
}
|
||||
|
||||
mgr := backup.NewManager(api.dataDir)
|
||||
|
||||
// Custom async handling needed to include snapshot_id in response
|
||||
opMgr := operations.NewManager(api.dataDir)
|
||||
opID, err := opMgr.Start(instanceName, "restore-snapshot", appName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to start restore operation: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_ = opMgr.Update(instanceName, opID, "failed", "Internal error during restore", 100)
|
||||
}
|
||||
}()
|
||||
|
||||
_ = opMgr.UpdateProgress(instanceName, opID, 10, "Validating snapshot")
|
||||
|
||||
if err := mgr.RestoreFromSnapshot(instanceName, appName, snapshotID, opts); err != nil {
|
||||
_ = opMgr.Update(instanceName, opID, "failed", err.Error(), 100)
|
||||
return
|
||||
}
|
||||
|
||||
_ = opMgr.Update(instanceName, opID, "completed", "Restore from snapshot completed", 100)
|
||||
}()
|
||||
|
||||
respondJSON(w, http.StatusAccepted, map[string]interface{}{
|
||||
"success": true,
|
||||
"operation_id": opID,
|
||||
"message": "Restore from snapshot started",
|
||||
"snapshot_id": snapshotID,
|
||||
})
|
||||
}
|
||||
@@ -65,6 +65,11 @@ func (api *API) SchematicUpdateInstanceSchematic(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
if err := validateTalosVersion(req.Version); err != nil {
|
||||
respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
configPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
|
||||
// Update schematic ID in config
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/contracts"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/services"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// ServicesList lists all base services
|
||||
func (api *API) ServicesList(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
svcList, err := servicesMgr.List(instanceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list services: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"services": svcList,
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesGet returns a specific service
|
||||
func (api *API) ServicesGet(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
service, err := servicesMgr.Get(instanceName, serviceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Service not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, service)
|
||||
}
|
||||
|
||||
// ServicesInstall installs a service
|
||||
func (api *API) ServicesInstall(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
var req ServiceInstallRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Name == "" {
|
||||
respondError(w, http.StatusBadRequest, "service name is required")
|
||||
return
|
||||
}
|
||||
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "install_service", req.Name,
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
return servicesMgr.Install(instanceName, req.Name, req.Fetch, req.Deploy, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesInstallAll installs all base services
|
||||
func (api *API) ServicesInstallAll(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
var req ServiceInstallAllRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
req.Deploy = true
|
||||
}
|
||||
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "install_all_services", "all",
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
return servicesMgr.InstallAll(instanceName, req.Fetch, req.Deploy, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesDelete deletes a service
|
||||
func (api *API) ServicesDelete(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
api.StartAsyncOperation(w, instanceName, "delete_service", serviceName,
|
||||
func(opsMgr *operations.Manager, opID string) error {
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
return servicesMgr.Delete(instanceName, serviceName)
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesGetStatus returns detailed service status
|
||||
func (api *API) ServicesGetStatus(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
status, err := servicesMgr.GetDetailedStatus(instanceName, serviceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Failed to get status: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// ServicesGetManifest returns the manifest for a service
|
||||
func (api *API) ServicesGetManifest(w http.ResponseWriter, r *http.Request) {
|
||||
serviceName := mux.Vars(r)["service"]
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
manifest, err := servicesMgr.GetManifest(serviceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Service not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, manifest)
|
||||
}
|
||||
|
||||
// ServicesGetConfig returns the service configuration schema
|
||||
func (api *API) ServicesGetConfig(w http.ResponseWriter, r *http.Request) {
|
||||
serviceName := mux.Vars(r)["service"]
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
manifest, err := servicesMgr.GetManifest(serviceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Service not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"configReferences": manifest.ConfigReferences,
|
||||
"serviceConfig": manifest.ServiceConfig,
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// ServicesGetInstanceConfig returns current config values for a service instance
|
||||
func (api *API) ServicesGetInstanceConfig(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
manifest, err := servicesMgr.GetManifest(serviceName)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Service not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
configPath := tools.GetInstanceConfigPath(api.dataDir, instanceName)
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read instance config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
var instanceConfig map[string]interface{}
|
||||
if err := yaml.Unmarshal(configData, &instanceConfig); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse instance config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
configValues := make(map[string]interface{})
|
||||
for _, path := range manifest.ConfigReferences {
|
||||
if value := getNestedValue(instanceConfig, path); value != nil {
|
||||
configValues[path] = value
|
||||
}
|
||||
}
|
||||
for _, cfg := range manifest.ServiceConfig {
|
||||
if value := getNestedValue(instanceConfig, cfg.Path); value != nil {
|
||||
configValues[cfg.Path] = value
|
||||
}
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"config": configValues,
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesFetch handles fetching service files to instance
|
||||
func (api *API) ServicesFetch(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
if err := servicesMgr.Fetch(instanceName, serviceName); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to fetch service: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": fmt.Sprintf("Service %s files fetched successfully", serviceName),
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesCompile handles template compilation
|
||||
func (api *API) ServicesCompile(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
if err := servicesMgr.Compile(instanceName, serviceName); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to compile templates: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": fmt.Sprintf("Templates compiled successfully for %s", serviceName),
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesDeploy handles service deployment
|
||||
func (api *API) ServicesDeploy(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
if err := servicesMgr.Deploy(instanceName, serviceName, "", nil); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to deploy service: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]string{
|
||||
"message": fmt.Sprintf("Service %s deployed successfully", serviceName),
|
||||
})
|
||||
}
|
||||
|
||||
// ServicesGetLogs retrieves or streams service logs
|
||||
func (api *API) ServicesGetLogs(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
query := r.URL.Query()
|
||||
logsReq := contracts.ServiceLogsRequest{
|
||||
Container: query.Get("container"),
|
||||
Follow: query.Get("follow") == "true",
|
||||
Previous: query.Get("previous") == "true",
|
||||
Since: query.Get("since"),
|
||||
}
|
||||
|
||||
if tailStr := query.Get("tail"); tailStr != "" {
|
||||
var tail int
|
||||
if _, err := fmt.Sscanf(tailStr, "%d", &tail); err == nil {
|
||||
logsReq.Tail = tail
|
||||
}
|
||||
}
|
||||
|
||||
if logsReq.Tail < 0 {
|
||||
respondError(w, http.StatusBadRequest, "tail parameter must be positive")
|
||||
return
|
||||
}
|
||||
if logsReq.Tail > 5000 {
|
||||
respondError(w, http.StatusBadRequest, "tail parameter cannot exceed 5000")
|
||||
return
|
||||
}
|
||||
if logsReq.Previous && logsReq.Follow {
|
||||
respondError(w, http.StatusBadRequest, "previous and follow cannot be used together")
|
||||
return
|
||||
}
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
|
||||
if logsReq.Follow {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
|
||||
if err := servicesMgr.StreamLogs(instanceName, serviceName, logsReq, w); err != nil {
|
||||
fmt.Printf("Error streaming logs: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
logsResp, err := servicesMgr.GetLogs(instanceName, serviceName, logsReq)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get logs: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, logsResp)
|
||||
}
|
||||
|
||||
// ServicesUpdateConfig updates service configuration
|
||||
func (api *API) ServicesUpdateConfig(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
serviceName := GetServiceName(r)
|
||||
|
||||
var update contracts.ServiceConfigUpdate
|
||||
if err := json.NewDecoder(r.Body).Decode(&update); err != nil {
|
||||
respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid request body: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if len(update.Config) == 0 {
|
||||
respondError(w, http.StatusBadRequest, "config field is required and must not be empty")
|
||||
return
|
||||
}
|
||||
|
||||
servicesMgr := services.NewManager(api.dataDir)
|
||||
response, err := servicesMgr.UpdateConfig(instanceName, serviceName, update, api.broadcaster)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to update config: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, response)
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package v1
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/apps"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/setup"
|
||||
)
|
||||
|
||||
@@ -25,3 +27,19 @@ func (api *API) GetSetupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
respondJSON(w, http.StatusOK, status)
|
||||
}
|
||||
|
||||
// SetupDeployInfrastructure deploys all infrastructure packages in dependency order
|
||||
func (api *API) SetupDeployInfrastructure(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
if err := api.instance.ValidateInstance(instanceName); err != nil {
|
||||
respondError(w, http.StatusNotFound, "Instance not found")
|
||||
return
|
||||
}
|
||||
|
||||
api.StartAsyncOperationWithBroadcaster(w, instanceName, "deploy_infrastructure", "all",
|
||||
func(opsMgr *operations.Manager, opID string, broadcaster *operations.Broadcaster) error {
|
||||
appsMgr := apps.NewManager(api.dataDir, api.appsDir)
|
||||
return appsMgr.DeployInfrastructure(instanceName, opID, broadcaster)
|
||||
})
|
||||
}
|
||||
|
||||
265
api/internal/api/v1/handlers_sse.go
Normal file
265
api/internal/api/v1/handlers_sse.go
Normal file
@@ -0,0 +1,265 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/sse"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// InstanceEventStream handles SSE connections for instance events
|
||||
func (api *API) InstanceEventStream(w http.ResponseWriter, r *http.Request) {
|
||||
// 1. Extract instance name from URL
|
||||
vars := mux.Vars(r)
|
||||
instanceName := vars["name"]
|
||||
|
||||
// 2. Validate instance exists
|
||||
if err := api.instance.ValidateInstance(instanceName); err != nil {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("Instance not found: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Parse event filters from query parameters
|
||||
filters := sse.EventFilters{
|
||||
EventTypes: parseQueryList(r.URL.Query().Get("types")),
|
||||
Namespaces: parseQueryList(r.URL.Query().Get("namespaces")),
|
||||
Apps: parseQueryList(r.URL.Query().Get("apps")),
|
||||
}
|
||||
|
||||
// 4. Set SSE headers
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no") // Disable nginx buffering
|
||||
|
||||
// 5. Register client with SSE manager
|
||||
client := api.sseManager.RegisterClient(instanceName, filters)
|
||||
defer api.sseManager.UnregisterClient(client)
|
||||
|
||||
// 6. Start watchers if needed (only once per instance)
|
||||
kubeconfigPath := tools.GetKubeconfigPath(api.dataDir, instanceName)
|
||||
talosconfigPath := tools.GetTalosconfigPath(api.dataDir, instanceName)
|
||||
configPath := tools.GetInstanceConfigPath(api.dataDir, instanceName)
|
||||
|
||||
// Get control plane VIP for talos events
|
||||
nodeIP, err := api.config.GetConfigValue(configPath, "cluster.nodes.control.vip")
|
||||
if err != nil {
|
||||
// Default to empty string if not found - talos events will be skipped
|
||||
nodeIP = ""
|
||||
slog.Info("control plane VIP not found, Talos events disabled", "instance", instanceName)
|
||||
}
|
||||
|
||||
// Start watchers for this instance if not already running
|
||||
if err := api.watcherManager.StartWatchers(instanceName, kubeconfigPath, talosconfigPath, nodeIP); err != nil {
|
||||
slog.Error("failed to start watchers", "instance", instanceName, "error", err)
|
||||
}
|
||||
|
||||
// 7. Send initial connected event
|
||||
connectedEvent := &sse.Event{
|
||||
Type: "connected",
|
||||
InstanceName: instanceName,
|
||||
Data: map[string]interface{}{
|
||||
"message": "Successfully connected to event stream",
|
||||
"filters": filters,
|
||||
},
|
||||
}
|
||||
if err := sendSSEEvent(w, connectedEvent); err != nil {
|
||||
slog.Error("failed to send SSE connected event", "instance", instanceName, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 8. Flush immediately to establish connection
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// 9. Send heartbeat and handle events
|
||||
heartbeatInterval := 30 // seconds
|
||||
heartbeatTicker := time.NewTicker(time.Duration(heartbeatInterval) * time.Second)
|
||||
defer heartbeatTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-client.Context.Done():
|
||||
// Client disconnected
|
||||
return
|
||||
|
||||
case <-r.Context().Done():
|
||||
// Request cancelled
|
||||
return
|
||||
|
||||
case event := <-client.Channel:
|
||||
// Send event to client
|
||||
if err := sendSSEEvent(w, event); err != nil {
|
||||
slog.Error("failed to send SSE event", "instance", instanceName, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush after each event
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
case <-heartbeatTicker.C:
|
||||
// Send heartbeat to keep connection alive
|
||||
heartbeatEvent := &sse.Event{
|
||||
Type: "heartbeat",
|
||||
InstanceName: instanceName,
|
||||
Data: map[string]interface{}{
|
||||
"timestamp": time.Now().Unix(),
|
||||
},
|
||||
}
|
||||
if err := sendSSEEvent(w, heartbeatEvent); err != nil {
|
||||
slog.Error("failed to send SSE heartbeat", "instance", instanceName, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush after heartbeat
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sendSSEEvent writes an SSE event to the response writer
|
||||
func sendSSEEvent(w http.ResponseWriter, event *sse.Event) error {
|
||||
// Set event ID
|
||||
if _, err := fmt.Fprintf(w, "id: %s\n", event.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set event type
|
||||
if _, err := fmt.Fprintf(w, "event: %s\n", event.Type); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set retry interval (in milliseconds)
|
||||
if _, err := fmt.Fprintf(w, "retry: 5000\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Marshal event data to JSON
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal event: %w", err)
|
||||
}
|
||||
|
||||
// Write data field
|
||||
if _, err := fmt.Fprintf(w, "data: %s\n\n", data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GlobalEventStream handles SSE connections for ALL events (global and instance-specific)
|
||||
func (api *API) GlobalEventStream(w http.ResponseWriter, r *http.Request) {
|
||||
// 1. Parse event filters from query parameters
|
||||
filters := sse.EventFilters{
|
||||
EventTypes: parseQueryList(r.URL.Query().Get("types")),
|
||||
Namespaces: parseQueryList(r.URL.Query().Get("namespaces")),
|
||||
Apps: parseQueryList(r.URL.Query().Get("apps")),
|
||||
}
|
||||
|
||||
// 2. Set SSE headers
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no") // Disable nginx buffering
|
||||
|
||||
// 3. Register client with SSE manager
|
||||
// Use "*" as a special instance name to receive ALL events
|
||||
client := api.sseManager.RegisterClient("*", filters)
|
||||
defer api.sseManager.UnregisterClient(client)
|
||||
|
||||
// 4. Send initial connected event
|
||||
connectedEvent := &sse.Event{
|
||||
Type: "connected",
|
||||
InstanceName: "global",
|
||||
Data: map[string]interface{}{
|
||||
"message": "Successfully connected to global event stream",
|
||||
"filters": filters,
|
||||
},
|
||||
}
|
||||
if err := sendSSEEvent(w, connectedEvent); err != nil {
|
||||
slog.Error("failed to send SSE connected event", "stream", "global", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 5. Flush immediately to establish connection
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
// 6. Send heartbeat and handle events
|
||||
heartbeatInterval := 30 // seconds
|
||||
heartbeatTicker := time.NewTicker(time.Duration(heartbeatInterval) * time.Second)
|
||||
defer heartbeatTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-client.Context.Done():
|
||||
// Client disconnected
|
||||
return
|
||||
|
||||
case <-r.Context().Done():
|
||||
// Request cancelled
|
||||
return
|
||||
|
||||
case event := <-client.Channel:
|
||||
// Send event to client
|
||||
if err := sendSSEEvent(w, event); err != nil {
|
||||
slog.Error("failed to send SSE event", "stream", "global", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush after each event
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
|
||||
case <-heartbeatTicker.C:
|
||||
// Send heartbeat to keep connection alive
|
||||
heartbeatEvent := &sse.Event{
|
||||
Type: "heartbeat",
|
||||
InstanceName: "global",
|
||||
Data: map[string]interface{}{
|
||||
"timestamp": time.Now().Unix(),
|
||||
},
|
||||
}
|
||||
if err := sendSSEEvent(w, heartbeatEvent); err != nil {
|
||||
slog.Error("failed to send SSE heartbeat", "stream", "global", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush after heartbeat
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseQueryList parses comma-separated query parameter into slice
|
||||
func parseQueryList(param string) []string {
|
||||
if param == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parts := strings.Split(param, ",")
|
||||
result := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed != "" {
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
78
api/internal/api/v1/handlers_talos.go
Normal file
78
api/internal/api/v1/handlers_talos.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TalosVersionsList returns available Talos versions from the Image Factory.
|
||||
// Use ?all=true to include pre-release versions.
|
||||
func (api *API) TalosVersionsList(w http.ResponseWriter, r *http.Request) {
|
||||
all := r.URL.Query().Get("all") == "true"
|
||||
|
||||
var versions []string
|
||||
var err error
|
||||
|
||||
if all {
|
||||
versions, err = api.factory.GetVersions()
|
||||
} else {
|
||||
versions, err = api.factory.GetStableVersions()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, "Failed to fetch Talos versions: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"versions": versions,
|
||||
})
|
||||
}
|
||||
|
||||
// TalosExtensionsList returns official system extensions available for a Talos version
|
||||
func (api *API) TalosExtensionsList(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
version := vars["version"]
|
||||
|
||||
extensions, err := api.factory.GetExtensions(version)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, "Failed to fetch extensions: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"extensions": extensions,
|
||||
})
|
||||
}
|
||||
|
||||
// TalosSchematicGet retrieves a schematic by its content-addressed ID
|
||||
func (api *API) TalosSchematicGet(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
id := vars["id"]
|
||||
|
||||
schematic, err := api.factory.GetSchematic(id)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, "Failed to fetch schematic: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"schematic": schematic,
|
||||
})
|
||||
}
|
||||
|
||||
// TalosSchematicValidate checks whether a schematic is compatible with a Talos version
|
||||
func (api *API) TalosSchematicValidate(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
schematicID := vars["schematicId"]
|
||||
version := vars["version"]
|
||||
|
||||
result, err := api.factory.ValidateSchematic(schematicID, version)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, "Failed to validate schematic: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, result)
|
||||
}
|
||||
126
api/internal/api/v1/handlers_talos_client.go
Normal file
126
api/internal/api/v1/handlers_talos_client.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// TalosClientInfo returns information about the local talosctl binary.
|
||||
func (api *API) TalosClientInfo(w http.ResponseWriter, r *http.Request) {
|
||||
info, err := tools.GetClientInfo()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get talosctl info: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
// TalosClientUpgrade upgrades the local talosctl binary to a specified version.
|
||||
func (api *API) TalosClientUpgrade(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, http.StatusBadRequest, "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.Version == "" {
|
||||
respondError(w, http.StatusBadRequest, "version is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Get current binary info
|
||||
info, err := tools.GetClientInfo()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get current talosctl info: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if info.Version == req.Version {
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": fmt.Sprintf("talosctl is already at version %s", req.Version),
|
||||
"version": info.Version,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Map Go arch to Talos download arch
|
||||
arch := runtime.GOARCH
|
||||
if arch == "amd64" {
|
||||
arch = "amd64"
|
||||
} else if arch == "arm64" {
|
||||
arch = "arm64"
|
||||
} else {
|
||||
respondError(w, http.StatusBadRequest, fmt.Sprintf("Unsupported architecture: %s", arch))
|
||||
return
|
||||
}
|
||||
|
||||
// Download from GitHub releases
|
||||
url := fmt.Sprintf("https://github.com/siderolabs/talos/releases/download/%s/talosctl-linux-%s", req.Version, arch)
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusBadGateway, fmt.Sprintf("Failed to download talosctl: %v", err))
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
respondError(w, http.StatusNotFound, fmt.Sprintf("talosctl %s not found for linux/%s", req.Version, arch))
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respondError(w, http.StatusBadGateway, fmt.Sprintf("Failed to download talosctl: HTTP %d", resp.StatusCode))
|
||||
return
|
||||
}
|
||||
|
||||
// Download to a temp file, then use sudo to install (binary is typically in /usr/local/bin)
|
||||
tmpFile, err := os.CreateTemp("", "talosctl-*")
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create temp file: %v", err))
|
||||
return
|
||||
}
|
||||
tmpPath := tmpFile.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
if _, err := io.Copy(tmpFile, resp.Body); err != nil {
|
||||
tmpFile.Close()
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to write talosctl binary: %v", err))
|
||||
return
|
||||
}
|
||||
tmpFile.Close()
|
||||
|
||||
// Install with sudo: copy to destination and set permissions
|
||||
if output, err := exec.Command("sudo", "cp", tmpPath, info.Path).CombinedOutput(); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to install talosctl binary: %v\n%s", err, string(output)))
|
||||
return
|
||||
}
|
||||
if output, err := exec.Command("sudo", "chmod", "755", info.Path).CombinedOutput(); err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to set permissions: %v\n%s", err, string(output)))
|
||||
return
|
||||
}
|
||||
|
||||
// Verify the new version
|
||||
newInfo, err := tools.GetClientInfo()
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, fmt.Sprintf("Upgraded binary but failed to verify: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": fmt.Sprintf("talosctl upgraded from %s to %s", info.Version, newInfo.Version),
|
||||
"previousVersion": info.Version,
|
||||
"version": newInfo.Version,
|
||||
"path": newInfo.Path,
|
||||
})
|
||||
}
|
||||
@@ -63,11 +63,11 @@ func (api *API) TerminalWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
ptmx, err := pty.Start(cmd)
|
||||
if err != nil {
|
||||
conn.WriteMessage(websocket.TextMessage, []byte("Failed to start shell: "+err.Error()))
|
||||
_ = conn.WriteMessage(websocket.TextMessage, []byte("Failed to start shell: "+err.Error()))
|
||||
return
|
||||
}
|
||||
defer ptmx.Close()
|
||||
defer cmd.Process.Kill()
|
||||
defer func() { _ = cmd.Process.Kill() }()
|
||||
|
||||
// Channel to signal when to stop
|
||||
done := make(chan struct{})
|
||||
@@ -103,7 +103,7 @@ func (api *API) TerminalWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
var resize terminalResize
|
||||
if err := json.Unmarshal(msg, &resize); err == nil && resize.Type == "resize" {
|
||||
if resize.Cols > 0 && resize.Rows > 0 {
|
||||
pty.Setsize(ptmx, &pty.Winsize{
|
||||
_ = pty.Setsize(ptmx, &pty.Winsize{
|
||||
Cols: uint16(resize.Cols),
|
||||
Rows: uint16(resize.Rows),
|
||||
})
|
||||
|
||||
@@ -26,26 +26,6 @@ func (api *API) InstanceUtilitiesHealth(w http.ResponseWriter, r *http.Request)
|
||||
})
|
||||
}
|
||||
|
||||
// InstanceUtilitiesDashboardToken returns a Kubernetes dashboard token for a specific instance
|
||||
func (api *API) UtilitiesDashboardToken(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
kubeconfigPath := tools.GetKubeconfigPath(api.dataDir, instanceName)
|
||||
|
||||
token, err := utilities.GetDashboardToken(kubeconfigPath)
|
||||
if err != nil {
|
||||
token, err = utilities.GetDashboardTokenFromSecret(kubeconfigPath)
|
||||
if err != nil {
|
||||
respondError(w, http.StatusInternalServerError, "Failed to get dashboard token")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"data": token,
|
||||
})
|
||||
}
|
||||
|
||||
// UtilitiesNodeIPs returns IP addresses for all cluster nodes
|
||||
func (api *API) UtilitiesNodeIPs(w http.ResponseWriter, r *http.Request) {
|
||||
instanceName := GetInstanceName(r)
|
||||
|
||||
@@ -1,58 +1,20 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/config"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/sse"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// getNestedValue retrieves a value from a nested map using dot notation path.
|
||||
// For example, getNestedValue(data, "cluster.nodes.active") returns data["cluster"]["nodes"]["active"].
|
||||
func getNestedValue(data map[string]interface{}, path string) interface{} {
|
||||
keys := strings.Split(path, ".")
|
||||
current := data
|
||||
|
||||
for i, key := range keys {
|
||||
if i == len(keys)-1 {
|
||||
return current[key]
|
||||
}
|
||||
|
||||
if next, ok := current[key].(map[string]interface{}); ok {
|
||||
current = next
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setNestedValue sets a value in a nested map using dot notation path.
|
||||
// Creates intermediate maps as needed.
|
||||
func setNestedValue(data map[string]interface{}, path string, value interface{}) {
|
||||
keys := strings.Split(path, ".")
|
||||
current := data
|
||||
|
||||
for i, key := range keys {
|
||||
if i == len(keys)-1 {
|
||||
current[key] = value
|
||||
return
|
||||
}
|
||||
|
||||
if next, ok := current[key].(map[string]interface{}); ok {
|
||||
current = next
|
||||
} else {
|
||||
next := make(map[string]interface{})
|
||||
current[key] = next
|
||||
current = next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateYAMLFile updates a YAML file with the provided key-value pairs.
|
||||
// It performs a shallow merge at the top level, preserving unmodified keys.
|
||||
func (api *API) updateYAMLFile(w http.ResponseWriter, r *http.Request, instanceName, fileType string) {
|
||||
@@ -98,6 +60,23 @@ func (api *API) updateYAMLFile(w http.ResponseWriter, r *http.Request, instanceN
|
||||
existingConfig = make(map[string]interface{})
|
||||
}
|
||||
|
||||
// Track if domains changed (for DNS update)
|
||||
var domainsChanged bool
|
||||
if fileType == "config" {
|
||||
oldCloud, _ := existingConfig["cloud"].(map[string]interface{})
|
||||
newCloud, _ := updates["cloud"].(map[string]interface{})
|
||||
if oldCloud != nil && newCloud != nil {
|
||||
oldDomain, _ := oldCloud["domain"].(string)
|
||||
newDomain, _ := newCloud["domain"].(string)
|
||||
oldInternalDomain, _ := oldCloud["internalDomain"].(string)
|
||||
newInternalDomain, _ := newCloud["internalDomain"].(string)
|
||||
|
||||
if oldDomain != newDomain || oldInternalDomain != newInternalDomain {
|
||||
domainsChanged = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge updates into existing config (shallow merge for top-level keys)
|
||||
for key, value := range updates {
|
||||
existingConfig[key] = value
|
||||
@@ -118,7 +97,91 @@ func (api *API) updateYAMLFile(w http.ResponseWriter, r *http.Request, instanceN
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info(fileType+" updated", "instance", instanceName)
|
||||
|
||||
// Update DNS if domains changed
|
||||
if domainsChanged && fileType == "config" {
|
||||
go func() {
|
||||
slog.Info("domain change detected, updating DNS", "instance", instanceName)
|
||||
|
||||
instanceConfigPath := api.instance.GetInstanceConfigPath(instanceName)
|
||||
instanceCfg, err := config.LoadCloudConfig(instanceConfigPath)
|
||||
if err != nil {
|
||||
slog.Error("failed to load instance config for DNS update", "instance", instanceName, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := api.dnsmasq.UpdateInstanceDNS(instanceName, *instanceCfg); err != nil {
|
||||
slog.Error("failed to update DNS", "instance", instanceName, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("DNS configuration updated", "instance", instanceName)
|
||||
}()
|
||||
}
|
||||
|
||||
// Capitalize first letter for message
|
||||
fileTypeCap := strings.ToUpper(fileType[:1]) + fileType[1:]
|
||||
respondMessage(w, http.StatusOK, fileTypeCap+" updated successfully")
|
||||
}
|
||||
|
||||
// broadcastDnsmasqEvent broadcasts SSE events for dnsmasq status changes
|
||||
func (api *API) broadcastDnsmasqEvent(eventType string, message string) {
|
||||
if api.sseManager == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Get current dnsmasq status
|
||||
status, err := api.dnsmasq.GetStatus()
|
||||
if err != nil {
|
||||
status = nil
|
||||
}
|
||||
|
||||
event := &sse.Event{
|
||||
ID: fmt.Sprintf("dnsmasq-%d", time.Now().UnixNano()),
|
||||
Type: eventType,
|
||||
InstanceName: "global", // dnsmasq is global/central-level, not instance-specific
|
||||
Timestamp: time.Now(),
|
||||
Data: map[string]interface{}{
|
||||
"message": message,
|
||||
"status": status,
|
||||
},
|
||||
}
|
||||
|
||||
api.sseManager.Broadcast(event)
|
||||
}
|
||||
|
||||
// broadcastCentralStatusEvent broadcasts SSE events for central status changes
|
||||
func (api *API) broadcastCentralStatusEvent(startTime time.Time) {
|
||||
if api.sseManager == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Get list of instances
|
||||
instances, err := api.instance.ListInstances()
|
||||
if err != nil {
|
||||
instances = []string{}
|
||||
}
|
||||
|
||||
// Calculate uptime
|
||||
uptime := time.Since(startTime)
|
||||
|
||||
event := &sse.Event{
|
||||
ID: fmt.Sprintf("central-%d", time.Now().UnixNano()),
|
||||
Type: "central:status",
|
||||
InstanceName: "global", // Central status is global, not instance-specific
|
||||
Timestamp: time.Now(),
|
||||
Data: map[string]interface{}{
|
||||
"status": "running",
|
||||
"version": api.version,
|
||||
"uptime": uptime.String(),
|
||||
"uptimeSeconds": int(uptime.Seconds()),
|
||||
"instances": map[string]interface{}{
|
||||
"count": len(instances),
|
||||
"names": instances,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
api.sseManager.Broadcast(event)
|
||||
}
|
||||
|
||||
@@ -2,11 +2,68 @@ package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// statusResponseWriter wraps http.ResponseWriter to capture the status code.
|
||||
type statusResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
func (w *statusResponseWriter) WriteHeader(code int) {
|
||||
w.status = code
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// RequestLoggingMiddleware logs method, path, status, and duration for each request.
|
||||
// Long-lived connections (SSE, WebSocket) are excluded.
|
||||
func RequestLoggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
|
||||
// Skip SSE and WebSocket endpoints (long-lived connections)
|
||||
if strings.HasSuffix(path, "/events") || strings.HasSuffix(path, "/ws") || strings.HasSuffix(path, "/stream") {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
sw := &statusResponseWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
next.ServeHTTP(sw, r)
|
||||
|
||||
attrs := []any{
|
||||
"status", sw.status,
|
||||
"method", r.Method,
|
||||
"path", path,
|
||||
"duration", time.Since(start),
|
||||
}
|
||||
|
||||
// Add route params if present
|
||||
vars := mux.Vars(r)
|
||||
if name := vars["name"]; name != "" {
|
||||
attrs = append(attrs, "instance", name)
|
||||
}
|
||||
if app := vars["app"]; app != "" {
|
||||
attrs = append(attrs, "app", app)
|
||||
}
|
||||
if node := vars["node"]; node != "" {
|
||||
attrs = append(attrs, "node", node)
|
||||
}
|
||||
|
||||
if sw.status >= 400 {
|
||||
slog.Error("request", attrs...)
|
||||
} else {
|
||||
slog.Info("request", attrs...)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// contextKey is a type for context keys to avoid collisions.
|
||||
type contextKey string
|
||||
|
||||
@@ -14,7 +71,6 @@ type contextKey string
|
||||
const (
|
||||
InstanceNameKey contextKey = "instanceName"
|
||||
AppNameKey contextKey = "appName"
|
||||
ServiceNameKey contextKey = "serviceName"
|
||||
NodeNameKey contextKey = "nodeName"
|
||||
)
|
||||
|
||||
@@ -34,6 +90,13 @@ func (api *API) ValidateInstanceMiddleware(next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
|
||||
// Skip validation for the migrate endpoint — it handles legacy instances
|
||||
// that don't yet have the new-format config/instance.yaml.
|
||||
if strings.HasSuffix(r.URL.Path, "/migrate") || strings.Contains(r.URL.Path, "/migrate/") {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate instance exists
|
||||
if err := api.instance.ValidateInstance(instanceName); err != nil {
|
||||
respondError(w, http.StatusNotFound, "Instance not found")
|
||||
@@ -47,9 +110,6 @@ func (api *API) ValidateInstanceMiddleware(next http.Handler) http.Handler {
|
||||
if appName := vars["app"]; appName != "" {
|
||||
ctx = context.WithValue(ctx, AppNameKey, appName)
|
||||
}
|
||||
if serviceName := vars["service"]; serviceName != "" {
|
||||
ctx = context.WithValue(ctx, ServiceNameKey, serviceName)
|
||||
}
|
||||
if nodeName := vars["node"]; nodeName != "" {
|
||||
ctx = context.WithValue(ctx, NodeNameKey, nodeName)
|
||||
}
|
||||
@@ -76,15 +136,6 @@ func GetAppName(r *http.Request) string {
|
||||
return mux.Vars(r)["app"]
|
||||
}
|
||||
|
||||
// GetServiceName returns the service name from request context.
|
||||
// Falls back to mux.Vars if not in context.
|
||||
func GetServiceName(r *http.Request) string {
|
||||
if name, ok := r.Context().Value(ServiceNameKey).(string); ok {
|
||||
return name
|
||||
}
|
||||
return mux.Vars(r)["service"]
|
||||
}
|
||||
|
||||
// GetNodeName returns the node name from request context.
|
||||
// Falls back to mux.Vars if not in context.
|
||||
func GetNodeName(r *http.Request) string {
|
||||
|
||||
@@ -1,8 +1,22 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Request types for API endpoints.
|
||||
// These are shared across handlers to ensure consistency and reduce duplication.
|
||||
|
||||
// validateTalosVersion checks that a Talos version string has the required "v" prefix.
|
||||
// The Talos Image Factory requires version tags like "v1.13.0", not "1.13.0".
|
||||
func validateTalosVersion(version string) error {
|
||||
if !strings.HasPrefix(version, "v") {
|
||||
return fmt.Errorf("invalid Talos version %q: must start with 'v' (e.g. v1.13.0)", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateInstanceRequest is the request body for creating a new instance.
|
||||
type CreateInstanceRequest struct {
|
||||
Name string `json:"name"`
|
||||
@@ -23,24 +37,14 @@ type IPRequest struct {
|
||||
IP string `json:"ip"`
|
||||
}
|
||||
|
||||
// ServiceInstallRequest is the request body for installing a service.
|
||||
type ServiceInstallRequest struct {
|
||||
Name string `json:"name"`
|
||||
Fetch bool `json:"fetch"`
|
||||
Deploy bool `json:"deploy"`
|
||||
}
|
||||
|
||||
// ServiceInstallAllRequest is the request body for installing all services.
|
||||
type ServiceInstallAllRequest struct {
|
||||
Fetch bool `json:"fetch"`
|
||||
Deploy bool `json:"deploy"`
|
||||
}
|
||||
|
||||
// AppAddRequest is the request body for adding an app to an instance.
|
||||
type AppAddRequest struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Config map[string]interface{} `json:"config"`
|
||||
RequiredAppMappings map[string]string `json:"requiredAppMappings"`
|
||||
Fetch bool `json:"fetch,omitempty"`
|
||||
Deploy bool `json:"deploy,omitempty"`
|
||||
}
|
||||
|
||||
// AppConfigUpdateRequest is the request body for updating app configuration.
|
||||
@@ -85,12 +89,39 @@ type RestoreRequest struct {
|
||||
|
||||
// ScheduleCreateRequest is the request body for creating a backup schedule.
|
||||
type ScheduleCreateRequest struct {
|
||||
Name string `json:"name"`
|
||||
Cron string `json:"cron"`
|
||||
Targets []string `json:"targets"`
|
||||
Retention int `json:"retention,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
BackupType string `json:"backupType,omitempty"`
|
||||
Name string `json:"name"`
|
||||
TargetType string `json:"target_type"` // "app" or "cluster"
|
||||
TargetName string `json:"target_name"`
|
||||
Frequency string `json:"frequency"` // "daily", "weekly", "monthly"
|
||||
Time string `json:"time"` // "HH:MM"
|
||||
DayOfWeek int `json:"day_of_week,omitempty"`
|
||||
DayOfMonth int `json:"day_of_month,omitempty"`
|
||||
Retention *ScheduleRetentionReq `json:"retention,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// ScheduleRetentionReq is the retention override in a schedule request.
|
||||
type ScheduleRetentionReq struct {
|
||||
KeepLast int `json:"keep_last"`
|
||||
KeepDays int `json:"keep_days"`
|
||||
}
|
||||
|
||||
// ScheduleUpdateRequest is the request body for updating a backup schedule.
|
||||
type ScheduleUpdateRequest struct {
|
||||
Name *string `json:"name,omitempty"`
|
||||
Frequency *string `json:"frequency,omitempty"`
|
||||
Time *string `json:"time,omitempty"`
|
||||
DayOfWeek *int `json:"day_of_week,omitempty"`
|
||||
DayOfMonth *int `json:"day_of_month,omitempty"`
|
||||
Retention *ScheduleRetentionReq `json:"retention,omitempty"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
// NodeUpgradeRequest is the request body for upgrading a node's Talos version.
|
||||
type NodeUpgradeRequest struct {
|
||||
Version string `json:"version"`
|
||||
SchematicID string `json:"schematicId,omitempty"`
|
||||
Preserve *bool `json:"preserve,omitempty"`
|
||||
}
|
||||
|
||||
// SchematicUpdateRequest is the request body for updating an instance's schematic.
|
||||
|
||||
@@ -23,12 +23,6 @@ func respondJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
_ = json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
// respondSuccess writes a successful response with data wrapped in the standard envelope.
|
||||
// Use this for GET requests returning data or POST/PUT requests returning created/updated resources.
|
||||
func respondSuccess(w http.ResponseWriter, status int, data interface{}) {
|
||||
respondJSON(w, status, APIResponse{Data: data})
|
||||
}
|
||||
|
||||
// respondMessage writes a success response with only a message (no data).
|
||||
// Use this for operations that succeed but don't return meaningful data.
|
||||
func respondMessage(w http.ResponseWriter, status int, message string) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
package services
|
||||
package apps
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
267
api/internal/apps/drift.go
Normal file
267
api/internal/apps/drift.go
Normal file
@@ -0,0 +1,267 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// computeDrift calculates drift state for an app across the three-stage pipeline.
|
||||
// Uses manifest.Source as the canonical signal for whether an app is source-managed.
|
||||
func (m *Manager) computeDrift(instanceName, appName, appDir, kubeconfigPath, status string, manifest *AppManifest) *DriftInfo {
|
||||
if manifest == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
configFile := m.getAppConfigFile(instanceName, appName)
|
||||
secretsFile := m.getAppSecretsFile(instanceName, appName)
|
||||
keyPrefix := m.appConfigKeyPrefix(instanceName, appName)
|
||||
packageDir := filepath.Join(appDir, ".package")
|
||||
|
||||
drift := &DriftInfo{}
|
||||
hasAnyDrift := false
|
||||
|
||||
// Source and compilation drift only apply to source-managed apps
|
||||
if manifest.Source != "" {
|
||||
drift.Source = m.checkSourceDrift(manifest, packageDir, appName)
|
||||
if drift.Source != nil {
|
||||
hasAnyDrift = true
|
||||
}
|
||||
|
||||
// Compilation drift: only if .package/ exists (if missing, source drift covers it)
|
||||
if storage.FileExists(packageDir) {
|
||||
drift.Compilation = m.checkCompilationDrift(appName, appDir, packageDir, configFile, secretsFile, keyPrefix)
|
||||
if drift.Compilation != nil {
|
||||
hasAnyDrift = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deploy drift only for deployed apps
|
||||
if status != "not-added" && status != "added" {
|
||||
drift.Deploy = checkDeployDrift(kubeconfigPath, appDir)
|
||||
if drift.Deploy != nil {
|
||||
hasAnyDrift = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasAnyDrift {
|
||||
return nil
|
||||
}
|
||||
return drift
|
||||
}
|
||||
|
||||
// checkSourceDrift compares the installed app against the Wild Directory source.
|
||||
func (m *Manager) checkSourceDrift(manifest *AppManifest, packageDir, appName string) *StageDrift {
|
||||
if manifest.Source == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse source URI to get filesystem path
|
||||
sourceDir := parseSourceDir(manifest.Source)
|
||||
if sourceDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If .package/ is missing entirely, that's source drift (fetch to resolve)
|
||||
if !storage.FileExists(packageDir) {
|
||||
return &StageDrift{
|
||||
Drifted: true,
|
||||
Reason: "package files missing — fetch from source to restore",
|
||||
}
|
||||
}
|
||||
|
||||
// Determine available version from source
|
||||
// app.yaml points to a slot directory, read actual version from manifest inside it.
|
||||
var availableVersion string
|
||||
appYamlPath := filepath.Join(sourceDir, "app.yaml")
|
||||
if storage.FileExists(appYamlPath) {
|
||||
if data, err := os.ReadFile(appYamlPath); err == nil {
|
||||
var meta AppMeta
|
||||
if err := yaml.Unmarshal(data, &meta); err == nil {
|
||||
// meta.Latest is a slot name — read the actual version from that slot's manifest
|
||||
slotManifestPath := filepath.Join(sourceDir, "versions", meta.Latest, "manifest.yaml")
|
||||
if slotData, slotErr := os.ReadFile(slotManifestPath); slotErr == nil {
|
||||
var slotManifest AppManifest
|
||||
if yaml.Unmarshal(slotData, &slotManifest) == nil {
|
||||
availableVersion = slotManifest.Version
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sourceManifestPath := filepath.Join(sourceDir, "manifest.yaml")
|
||||
if !storage.FileExists(sourceManifestPath) {
|
||||
return nil // Source no longer exists
|
||||
}
|
||||
if data, err := os.ReadFile(sourceManifestPath); err == nil {
|
||||
var sourceManifest AppManifest
|
||||
if err := yaml.Unmarshal(data, &sourceManifest); err == nil {
|
||||
availableVersion = sourceManifest.Version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if availableVersion == "" || manifest.Version == availableVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
sd := &StageDrift{
|
||||
Drifted: true,
|
||||
Reason: fmt.Sprintf("update available: %s → %s", manifest.Version, availableVersion),
|
||||
CurrentVersion: manifest.Version,
|
||||
AvailableVersion: availableVersion,
|
||||
}
|
||||
|
||||
// Enrich with upgrade plan info
|
||||
appsDir := filepath.Dir(sourceDir)
|
||||
if plan, err := ComputeUpgradePlan(manifest.Version, appName, appsDir); err == nil && plan != nil {
|
||||
sd.UpgradeBlocked = plan.Blocked
|
||||
sd.UpgradeNotes = plan.Notes
|
||||
sd.UpgradeSteps = len(plan.Steps)
|
||||
}
|
||||
|
||||
return sd
|
||||
}
|
||||
|
||||
// checkCompilationDrift compiles to a temp dir and compares against current compiled manifests.
|
||||
func (m *Manager) checkCompilationDrift(appName, appDir, packageDir, configFile, secretsFile, keyPrefix string) *StageDrift {
|
||||
tempDir, err := os.MkdirTemp("", "drift-check-*")
|
||||
if err != nil {
|
||||
return nil // Can't check, skip gracefully
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Compile .package/ + config + secrets to temp dir
|
||||
if err := m.compileFromPackage(appName, tempDir, packageDir, configFile, secretsFile, keyPrefix); err != nil {
|
||||
return nil // Compilation failed, skip gracefully
|
||||
}
|
||||
|
||||
// Compare each compiled file in temp dir against current app dir
|
||||
entries, err := os.ReadDir(packageDir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == "manifest.yaml" {
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
if dirsDiffer(filepath.Join(tempDir, entry.Name()), filepath.Join(appDir, entry.Name())) {
|
||||
return &StageDrift{
|
||||
Drifted: true,
|
||||
Reason: "configuration changed — redeploy to apply",
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if filesDiffer(filepath.Join(tempDir, entry.Name()), filepath.Join(appDir, entry.Name())) {
|
||||
return &StageDrift{
|
||||
Drifted: true,
|
||||
Reason: "configuration changed — redeploy to apply",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDeployDrift runs kubectl diff to compare compiled manifests against cluster state.
|
||||
func checkDeployDrift(kubeconfigPath, appDir string) *StageDrift {
|
||||
kubectl := tools.NewKubectl(kubeconfigPath)
|
||||
hasDiff, err := kubectl.Diff(appDir)
|
||||
if err != nil {
|
||||
return nil // kubectl diff failed (cluster unreachable, etc.), skip gracefully
|
||||
}
|
||||
|
||||
if hasDiff {
|
||||
return &StageDrift{
|
||||
Drifted: true,
|
||||
Reason: "cluster state differs from manifests",
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseSourceDir extracts the filesystem path from a source URI.
|
||||
// Returns empty string for unsupported schemes.
|
||||
func parseSourceDir(source string) string {
|
||||
parts := strings.SplitN(source, "://", 2)
|
||||
if len(parts) != 2 || parts[0] != "file" {
|
||||
return ""
|
||||
}
|
||||
return parts[1]
|
||||
}
|
||||
|
||||
// filesDiffer returns true if two files have different contents, or if either doesn't exist.
|
||||
func filesDiffer(pathA, pathB string) bool {
|
||||
a, errA := os.ReadFile(pathA)
|
||||
b, errB := os.ReadFile(pathB)
|
||||
if errA != nil || errB != nil {
|
||||
return (errA != nil) != (errB != nil) // Both missing = same, one missing = different
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
return true
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// dirsDiffer recursively compares two directories.
|
||||
func dirsDiffer(dirA, dirB string) bool {
|
||||
entriesA, errA := os.ReadDir(dirA)
|
||||
entriesB, errB := os.ReadDir(dirB)
|
||||
if errA != nil || errB != nil {
|
||||
return errA != errB
|
||||
}
|
||||
|
||||
namesA := make(map[string]bool, len(entriesA))
|
||||
for _, e := range entriesA {
|
||||
namesA[e.Name()] = e.IsDir()
|
||||
}
|
||||
|
||||
namesB := make(map[string]bool, len(entriesB))
|
||||
for _, e := range entriesB {
|
||||
namesB[e.Name()] = e.IsDir()
|
||||
}
|
||||
|
||||
// Check for files in A not in B or different
|
||||
for name, isDir := range namesA {
|
||||
isDirB, exists := namesB[name]
|
||||
if !exists || isDir != isDirB {
|
||||
return true
|
||||
}
|
||||
if isDir {
|
||||
if dirsDiffer(filepath.Join(dirA, name), filepath.Join(dirB, name)) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if filesDiffer(filepath.Join(dirA, name), filepath.Join(dirB, name)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for files in B not in A
|
||||
for name := range namesB {
|
||||
if _, exists := namesA[name]; !exists {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
342
api/internal/apps/drift_test.go
Normal file
342
api/internal/apps/drift_test.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestParseSourceDir(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
source string
|
||||
want string
|
||||
}{
|
||||
{"file URI", "file:///opt/wild-cloud/apps/myapp", "/opt/wild-cloud/apps/myapp"},
|
||||
{"empty string", "", ""},
|
||||
{"no scheme", "/opt/wild-cloud/apps/myapp", ""},
|
||||
{"unsupported scheme", "git://example.com/repo", ""},
|
||||
{"http scheme", "http://example.com/app", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := parseSourceDir(tt.source)
|
||||
if got != tt.want {
|
||||
t.Errorf("parseSourceDir(%q) = %q, want %q", tt.source, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesDiffer(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-files-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
mustWrite := func(path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fileA := filepath.Join(tmpDir, "a.txt")
|
||||
fileB := filepath.Join(tmpDir, "b.txt")
|
||||
fileC := filepath.Join(tmpDir, "c.txt")
|
||||
fileMissing := filepath.Join(tmpDir, "missing.txt")
|
||||
|
||||
mustWrite(fileA, []byte("hello"))
|
||||
mustWrite(fileB, []byte("hello"))
|
||||
mustWrite(fileC, []byte("world"))
|
||||
|
||||
t.Run("identical files", func(t *testing.T) {
|
||||
if filesDiffer(fileA, fileB) {
|
||||
t.Error("expected identical files to not differ")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different files", func(t *testing.T) {
|
||||
if !filesDiffer(fileA, fileC) {
|
||||
t.Error("expected different files to differ")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("one missing", func(t *testing.T) {
|
||||
if !filesDiffer(fileA, fileMissing) {
|
||||
t.Error("expected missing file to differ")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("both missing", func(t *testing.T) {
|
||||
if filesDiffer(fileMissing, filepath.Join(tmpDir, "also-missing.txt")) {
|
||||
t.Error("expected both missing to not differ")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDirsDiffer(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-dirs-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
mustMkdir := func(path string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mustWriteFile := func(path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create two identical directories
|
||||
dirA := filepath.Join(tmpDir, "a")
|
||||
dirB := filepath.Join(tmpDir, "b")
|
||||
mustMkdir(dirA)
|
||||
mustMkdir(dirB)
|
||||
mustWriteFile(filepath.Join(dirA, "file.txt"), []byte("same"))
|
||||
mustWriteFile(filepath.Join(dirB, "file.txt"), []byte("same"))
|
||||
|
||||
t.Run("identical directories", func(t *testing.T) {
|
||||
if dirsDiffer(dirA, dirB) {
|
||||
t.Error("expected identical directories to not differ")
|
||||
}
|
||||
})
|
||||
|
||||
// Create a directory with different content
|
||||
dirC := filepath.Join(tmpDir, "c")
|
||||
mustMkdir(dirC)
|
||||
mustWriteFile(filepath.Join(dirC, "file.txt"), []byte("different"))
|
||||
|
||||
t.Run("different content", func(t *testing.T) {
|
||||
if !dirsDiffer(dirA, dirC) {
|
||||
t.Error("expected directories with different content to differ")
|
||||
}
|
||||
})
|
||||
|
||||
// Directory with extra file
|
||||
dirD := filepath.Join(tmpDir, "d")
|
||||
mustMkdir(dirD)
|
||||
mustWriteFile(filepath.Join(dirD, "file.txt"), []byte("same"))
|
||||
mustWriteFile(filepath.Join(dirD, "extra.txt"), []byte("extra"))
|
||||
|
||||
t.Run("extra file in second", func(t *testing.T) {
|
||||
if !dirsDiffer(dirA, dirD) {
|
||||
t.Error("expected directories with different file counts to differ")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckSourceDrift_NoDrift(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Create source directory with manifest
|
||||
sourceDir := filepath.Join(tmpDir, "source", "myapp")
|
||||
if err := os.MkdirAll(sourceDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceManifest := AppManifest{Version: "1.0.0"}
|
||||
data, _ := yaml.Marshal(sourceManifest)
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "manifest.yaml"), data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create package dir (it exists)
|
||||
packageDir := filepath.Join(tmpDir, "package")
|
||||
if err := os.MkdirAll(packageDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Installed manifest with same version
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "file://" + sourceDir,
|
||||
}
|
||||
|
||||
m := &Manager{}
|
||||
result := m.checkSourceDrift(manifest, packageDir, "myapp")
|
||||
if result != nil {
|
||||
t.Errorf("expected no drift, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSourceDrift_VersionDrift(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Create source directory with newer version
|
||||
sourceDir := filepath.Join(tmpDir, "source", "myapp")
|
||||
if err := os.MkdirAll(sourceDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceManifest := AppManifest{Version: "2.0.0"}
|
||||
data, _ := yaml.Marshal(sourceManifest)
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "manifest.yaml"), data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create package dir
|
||||
packageDir := filepath.Join(tmpDir, "package")
|
||||
if err := os.MkdirAll(packageDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Installed manifest with older version
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "file://" + sourceDir,
|
||||
}
|
||||
|
||||
m := &Manager{}
|
||||
result := m.checkSourceDrift(manifest, packageDir, "myapp")
|
||||
if result == nil || !result.Drifted {
|
||||
t.Fatal("expected drift result with Drifted=true, got nil or false")
|
||||
}
|
||||
if result.CurrentVersion != "1.0.0" {
|
||||
t.Errorf("expected CurrentVersion '1.0.0', got %q", result.CurrentVersion)
|
||||
}
|
||||
if result.AvailableVersion != "2.0.0" {
|
||||
t.Errorf("expected AvailableVersion '2.0.0', got %q", result.AvailableVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSourceDrift_NoSource(t *testing.T) {
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "", // ejected app
|
||||
}
|
||||
|
||||
m := &Manager{}
|
||||
result := m.checkSourceDrift(manifest, "/nonexistent", "myapp")
|
||||
if result != nil {
|
||||
t.Errorf("expected nil for ejected app, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSourceDrift_PackageMissing(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Source exists but .package/ does not
|
||||
sourceDir := filepath.Join(tmpDir, "source", "myapp")
|
||||
if err := os.MkdirAll(sourceDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "file://" + sourceDir,
|
||||
}
|
||||
|
||||
packageDir := filepath.Join(tmpDir, "nonexistent-package")
|
||||
|
||||
m := &Manager{}
|
||||
result := m.checkSourceDrift(manifest, packageDir, "myapp")
|
||||
if result == nil || !result.Drifted {
|
||||
t.Fatal("expected drift result with Drifted=true for missing package dir")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSourceDrift_SourceDirMissing(t *testing.T) {
|
||||
// Source URI points to a directory that doesn't exist
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "file:///nonexistent/path/myapp",
|
||||
}
|
||||
|
||||
packageDir := "/tmp" // exists but irrelevant
|
||||
|
||||
m := &Manager{}
|
||||
result := m.checkSourceDrift(manifest, packageDir, "myapp")
|
||||
if result != nil {
|
||||
t.Errorf("expected nil when source dir is missing, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_EjectedApp(t *testing.T) {
|
||||
// Ejected app: Source is empty, so source and compilation drift should be nil
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "",
|
||||
}
|
||||
|
||||
m := &Manager{}
|
||||
result := m.computeDrift("test-instance", "myapp", "/tmp", "", "added", manifest)
|
||||
|
||||
// For an ejected app with status "added", nothing to check — should be nil
|
||||
if result != nil {
|
||||
t.Errorf("expected nil drift for ejected 'added' app, got %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_NotDeployed(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "drift-compute-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Source-managed app that is only "added" (not deployed)
|
||||
sourceDir := filepath.Join(tmpDir, "source")
|
||||
if err := os.MkdirAll(sourceDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Source manifest with newer version
|
||||
sourceManifest := AppManifest{Version: "2.0.0"}
|
||||
data, _ := yaml.Marshal(sourceManifest)
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "manifest.yaml"), data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// App directory with .package
|
||||
appDir := filepath.Join(tmpDir, "app")
|
||||
packageDir := filepath.Join(appDir, ".package")
|
||||
if err := os.MkdirAll(packageDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
manifest := &AppManifest{
|
||||
Version: "1.0.0",
|
||||
Source: "file://" + sourceDir,
|
||||
}
|
||||
|
||||
m := &Manager{}
|
||||
result := m.computeDrift("test-instance", "myapp", appDir, "", "added", manifest)
|
||||
|
||||
if result == nil || result.Source == nil || !result.Source.Drifted {
|
||||
t.Fatal("expected drift info with source drift for version mismatch")
|
||||
}
|
||||
|
||||
// Should NOT have deploy drift (status is "added")
|
||||
if result.Deploy != nil {
|
||||
t.Error("expected no deploy drift for 'added' status")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_NilManifest(t *testing.T) {
|
||||
m := &Manager{}
|
||||
result := m.computeDrift("test-instance", "myapp", "/tmp", "", "running", nil)
|
||||
if result != nil {
|
||||
t.Errorf("expected nil drift for nil manifest, got %+v", result)
|
||||
}
|
||||
}
|
||||
155
api/internal/apps/infrastructure.go
Normal file
155
api/internal/apps/infrastructure.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
)
|
||||
|
||||
// InfrastructureOrder returns infrastructure packages from the apps directory
|
||||
// sorted in dependency order (topological sort by requires).
|
||||
func (m *Manager) InfrastructureOrder() ([]AppManifest, error) {
|
||||
if m.appsDir == "" {
|
||||
return nil, fmt.Errorf("apps directory not configured")
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(m.appsDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading apps directory: %w", err)
|
||||
}
|
||||
|
||||
// Collect all infrastructure manifests
|
||||
manifests := map[string]AppManifest{}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
manifestPath := filepath.Join(m.appsDir, entry.Name(), "manifest.yaml")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var manifest AppManifest
|
||||
if err := yaml.Unmarshal(data, &manifest); err != nil {
|
||||
continue
|
||||
}
|
||||
if manifest.Category != "services" {
|
||||
continue
|
||||
}
|
||||
manifest.Name = entry.Name()
|
||||
manifests[entry.Name()] = manifest
|
||||
}
|
||||
|
||||
return topoSort(manifests)
|
||||
}
|
||||
|
||||
// topoSort returns manifests in dependency order using Kahn's algorithm.
|
||||
// Only considers dependencies that are themselves infrastructure packages.
|
||||
func topoSort(manifests map[string]AppManifest) ([]AppManifest, error) {
|
||||
// Build adjacency list and in-degree count
|
||||
inDegree := map[string]int{}
|
||||
dependents := map[string][]string{} // dep -> list of packages that depend on it
|
||||
|
||||
for name := range manifests {
|
||||
inDegree[name] = 0
|
||||
}
|
||||
|
||||
for name, manifest := range manifests {
|
||||
for _, req := range manifest.Requires {
|
||||
if _, isInfra := manifests[req.Name]; isInfra {
|
||||
inDegree[name]++
|
||||
dependents[req.Name] = append(dependents[req.Name], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start with packages that have no infrastructure dependencies
|
||||
var queue []string
|
||||
for name, deg := range inDegree {
|
||||
if deg == 0 {
|
||||
queue = append(queue, name)
|
||||
}
|
||||
}
|
||||
|
||||
var sorted []AppManifest
|
||||
for len(queue) > 0 {
|
||||
// Pop first
|
||||
name := queue[0]
|
||||
queue = queue[1:]
|
||||
sorted = append(sorted, manifests[name])
|
||||
|
||||
for _, dep := range dependents[name] {
|
||||
inDegree[dep]--
|
||||
if inDegree[dep] == 0 {
|
||||
queue = append(queue, dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(sorted) != len(manifests) {
|
||||
return nil, fmt.Errorf("circular dependency detected in infrastructure packages")
|
||||
}
|
||||
|
||||
return sorted, nil
|
||||
}
|
||||
|
||||
// DeployInfrastructure adds and deploys all infrastructure packages in dependency order.
|
||||
// It skips packages that are already deployed. Progress is reported via the broadcaster.
|
||||
func (m *Manager) DeployInfrastructure(instanceName, opID string, broadcaster *operations.Broadcaster) error {
|
||||
packages, err := m.InfrastructureOrder()
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolving infrastructure order: %w", err)
|
||||
}
|
||||
|
||||
total := len(packages)
|
||||
for i, pkg := range packages {
|
||||
// Skip if already added and deployed
|
||||
if m.isDeployed(instanceName, pkg.Name) {
|
||||
slog.Info("already deployed, skipping", "component", "infrastructure", "package", pkg.Name)
|
||||
if broadcaster != nil {
|
||||
broadcaster.Publish(opID, []byte(fmt.Sprintf("Skipping %s (already deployed)\n", pkg.Name)))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("installing package", "component", "infrastructure", "package", pkg.Name, "progress", fmt.Sprintf("%d/%d", i+1, total))
|
||||
if broadcaster != nil {
|
||||
broadcaster.Publish(opID, []byte(fmt.Sprintf("Installing %s (%d/%d)...\n", pkg.Name, i+1, total)))
|
||||
}
|
||||
|
||||
if err := m.Install(instanceName, pkg.Name, false, true, nil, nil, opID, broadcaster); err != nil {
|
||||
return fmt.Errorf("installing %s: %w", pkg.Name, err)
|
||||
}
|
||||
|
||||
progress := ((i + 1) * 100) / total
|
||||
if broadcaster != nil {
|
||||
broadcaster.Publish(opID, []byte(fmt.Sprintf("Installed %s (%d%%)\n", pkg.Name, progress)))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isDeployed checks if a package is deployed by looking for its namespace in the cluster.
|
||||
func (m *Manager) isDeployed(instanceName, appName string) bool {
|
||||
instancePath := filepath.Join(m.dataDir, "instances", instanceName)
|
||||
appDir := filepath.Join(instancePath, "apps", appName)
|
||||
|
||||
// Not even added locally
|
||||
if _, err := os.Stat(appDir); os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if deployed by looking at cluster state
|
||||
status, err := m.GetStatus(instanceName, appName)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return status.Status == "deployed" || status.Status == "running"
|
||||
}
|
||||
184
api/internal/apps/infrastructure_test.go
Normal file
184
api/internal/apps/infrastructure_test.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTopoSort_SimpleChain(t *testing.T) {
|
||||
manifests := map[string]AppManifest{
|
||||
"metallb": {Name: "metallb"},
|
||||
"traefik": {Name: "traefik", Requires: []AppDependency{{Name: "metallb"}}},
|
||||
"cert-manager": {Name: "cert-manager", Requires: []AppDependency{{Name: "traefik"}}},
|
||||
}
|
||||
|
||||
sorted, err := topoSort(manifests)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(sorted) != 3 {
|
||||
t.Fatalf("expected 3 packages, got %d", len(sorted))
|
||||
}
|
||||
|
||||
// metallb must come before traefik, traefik before cert-manager
|
||||
indexOf := map[string]int{}
|
||||
for i, m := range sorted {
|
||||
indexOf[m.Name] = i
|
||||
}
|
||||
|
||||
if indexOf["metallb"] >= indexOf["traefik"] {
|
||||
t.Errorf("metallb (at %d) should come before traefik (at %d)", indexOf["metallb"], indexOf["traefik"])
|
||||
}
|
||||
if indexOf["traefik"] >= indexOf["cert-manager"] {
|
||||
t.Errorf("traefik (at %d) should come before cert-manager (at %d)", indexOf["traefik"], indexOf["cert-manager"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoSort_MultipleDeps(t *testing.T) {
|
||||
manifests := map[string]AppManifest{
|
||||
"metallb": {Name: "metallb"},
|
||||
"nfs": {Name: "nfs"},
|
||||
"traefik": {Name: "traefik", Requires: []AppDependency{{Name: "metallb"}}},
|
||||
"longhorn": {Name: "longhorn", Requires: []AppDependency{{Name: "traefik"}, {Name: "nfs"}}},
|
||||
}
|
||||
|
||||
sorted, err := topoSort(manifests)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
indexOf := map[string]int{}
|
||||
for i, m := range sorted {
|
||||
indexOf[m.Name] = i
|
||||
}
|
||||
|
||||
// longhorn must come after both traefik and nfs
|
||||
if indexOf["traefik"] >= indexOf["longhorn"] {
|
||||
t.Errorf("traefik should come before longhorn")
|
||||
}
|
||||
if indexOf["nfs"] >= indexOf["longhorn"] {
|
||||
t.Errorf("nfs should come before longhorn")
|
||||
}
|
||||
// metallb must come before traefik
|
||||
if indexOf["metallb"] >= indexOf["traefik"] {
|
||||
t.Errorf("metallb should come before traefik")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoSort_NoDeps(t *testing.T) {
|
||||
manifests := map[string]AppManifest{
|
||||
"a": {Name: "a"},
|
||||
"b": {Name: "b"},
|
||||
"c": {Name: "c"},
|
||||
}
|
||||
|
||||
sorted, err := topoSort(manifests)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(sorted) != 3 {
|
||||
t.Fatalf("expected 3 packages, got %d", len(sorted))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoSort_CircularDependency(t *testing.T) {
|
||||
manifests := map[string]AppManifest{
|
||||
"a": {Name: "a", Requires: []AppDependency{{Name: "b"}}},
|
||||
"b": {Name: "b", Requires: []AppDependency{{Name: "a"}}},
|
||||
}
|
||||
|
||||
_, err := topoSort(manifests)
|
||||
if err == nil {
|
||||
t.Fatal("expected circular dependency error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoSort_IgnoresNonInfraDeps(t *testing.T) {
|
||||
// If a package requires something not in the manifests map, it should be ignored
|
||||
manifests := map[string]AppManifest{
|
||||
"traefik": {Name: "traefik", Requires: []AppDependency{{Name: "metallb"}}},
|
||||
"metallb": {Name: "metallb"},
|
||||
}
|
||||
|
||||
// traefik requires metallb which IS in the map — should work
|
||||
sorted, err := topoSort(manifests)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if sorted[0].Name != "metallb" {
|
||||
t.Errorf("expected metallb first, got %s", sorted[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoSort_IgnoresExternalDeps(t *testing.T) {
|
||||
// A package that requires something NOT in the map (e.g., a non-infra app)
|
||||
manifests := map[string]AppManifest{
|
||||
"myservice": {Name: "myservice", Requires: []AppDependency{{Name: "postgres"}}},
|
||||
}
|
||||
|
||||
sorted, err := topoSort(manifests)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(sorted) != 1 || sorted[0].Name != "myservice" {
|
||||
t.Errorf("expected [myservice], got %v", sorted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfrastructureOrder(t *testing.T) {
|
||||
// Create a temp apps directory with test manifests
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
writeManifest := func(name, category string, requires []string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(tmpDir, name)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
content := "name: " + name + "\ncategory: " + category + "\n"
|
||||
if len(requires) > 0 {
|
||||
content += "requires:\n"
|
||||
for _, r := range requires {
|
||||
content += " - name: " + r + "\n"
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "manifest.yaml"), []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create infrastructure packages
|
||||
writeManifest("metallb", "services", nil)
|
||||
writeManifest("traefik", "services", []string{"metallb"})
|
||||
writeManifest("cert-manager", "services", []string{"traefik"})
|
||||
|
||||
// Create a non-infrastructure package (should be excluded)
|
||||
writeManifest("postgres", "database", nil)
|
||||
|
||||
mgr := NewManager(t.TempDir(), tmpDir)
|
||||
sorted, err := mgr.InfrastructureOrder()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(sorted) != 3 {
|
||||
t.Fatalf("expected 3 infrastructure packages, got %d", len(sorted))
|
||||
}
|
||||
|
||||
// Verify order: metallb -> traefik -> cert-manager
|
||||
if sorted[0].Name != "metallb" {
|
||||
t.Errorf("expected metallb first, got %s", sorted[0].Name)
|
||||
}
|
||||
if sorted[1].Name != "traefik" {
|
||||
t.Errorf("expected traefik second, got %s", sorted[1].Name)
|
||||
}
|
||||
if sorted[2].Name != "cert-manager" {
|
||||
t.Errorf("expected cert-manager third, got %s", sorted[2].Name)
|
||||
}
|
||||
}
|
||||
@@ -22,11 +22,67 @@ type AppManifest struct {
|
||||
Version string `json:"version" yaml:"version"`
|
||||
Icon string `json:"icon,omitempty" yaml:"icon,omitempty"`
|
||||
Category string `json:"category,omitempty" yaml:"category,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
|
||||
DeploymentName string `json:"deploymentName,omitempty" yaml:"deploymentName,omitempty"`
|
||||
Requires []AppDependency `json:"requires,omitempty" yaml:"requires,omitempty"`
|
||||
DefaultConfig map[string]interface{} `json:"defaultConfig,omitempty" yaml:"defaultConfig,omitempty"`
|
||||
DefaultSecrets []SecretDefinition `json:"defaultSecrets,omitempty" yaml:"defaultSecrets,omitempty"`
|
||||
RequiredSecrets []string `json:"requiredSecrets,omitempty" yaml:"requiredSecrets,omitempty"`
|
||||
Source string `json:"source,omitempty" yaml:"source,omitempty"`
|
||||
Scripts []Script `json:"scripts,omitempty" yaml:"scripts,omitempty"`
|
||||
Deploy *DeployConfig `json:"deploy,omitempty" yaml:"deploy,omitempty"`
|
||||
Upgrade *UpgradeConfig `json:"upgrade,omitempty" yaml:"upgrade,omitempty"`
|
||||
}
|
||||
|
||||
// DeployConfig declares deployment behavior in the manifest, replacing install.sh scripts
|
||||
type DeployConfig struct {
|
||||
RequireWorkerNodes bool `json:"requireWorkerNodes,omitempty" yaml:"requireWorkerNodes,omitempty"`
|
||||
CRDs []CRDInstall `json:"crds,omitempty" yaml:"crds,omitempty"`
|
||||
Phases []DeployPhase `json:"phases,omitempty" yaml:"phases,omitempty"`
|
||||
CreateSecrets []CreateSecret `json:"createSecrets,omitempty" yaml:"createSecrets,omitempty"`
|
||||
RestartDeployments []string `json:"restartDeployments,omitempty" yaml:"restartDeployments,omitempty"`
|
||||
WaitForRollout *RolloutWait `json:"waitForRollout,omitempty" yaml:"waitForRollout,omitempty"`
|
||||
}
|
||||
|
||||
// DeployPhase represents one phase of a multi-phase deployment (e.g., metallb installation/ then configuration/)
|
||||
type DeployPhase struct {
|
||||
Path string `json:"path" yaml:"path"`
|
||||
WaitFor *RolloutWait `json:"waitFor,omitempty" yaml:"waitFor,omitempty"`
|
||||
}
|
||||
|
||||
// CreateSecret describes a Kubernetes secret to create from secrets.yaml values
|
||||
type CreateSecret struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` // target namespace (defaults to app namespace)
|
||||
Entries map[string]string `json:"entries" yaml:"entries"` // k8s secret key -> secrets.yaml path
|
||||
}
|
||||
|
||||
// CRDInstall describes CRDs to apply from a URL before deployment
|
||||
type CRDInstall struct {
|
||||
URL string `json:"url" yaml:"url"`
|
||||
WaitFor []string `json:"waitFor,omitempty" yaml:"waitFor,omitempty"` // CRD names to wait for establishment
|
||||
}
|
||||
|
||||
// RolloutWait describes a rollout to wait for after deployment
|
||||
type RolloutWait struct {
|
||||
Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` // "deployment" (default) or "daemonset"
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` // e.g. "120s", default "120s"
|
||||
}
|
||||
|
||||
// ScriptParam describes a named parameter that a script accepts
|
||||
type ScriptParam struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||
Required bool `json:"required,omitempty" yaml:"required,omitempty"`
|
||||
}
|
||||
|
||||
// Script describes an operational script that can be run on demand
|
||||
type Script struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Path string `json:"path" yaml:"path"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||
Params []ScriptParam `json:"params,omitempty" yaml:"params,omitempty"`
|
||||
}
|
||||
|
||||
// AppDependency represents a dependency on another app
|
||||
@@ -38,18 +94,79 @@ type AppDependency struct {
|
||||
|
||||
// EnhancedApp extends DeployedApp with runtime status information
|
||||
type EnhancedApp struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Version string `json:"version"`
|
||||
Namespace string `json:"namespace"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Icon string `json:"icon,omitempty"`
|
||||
Manifest *AppManifest `json:"manifest,omitempty"`
|
||||
Runtime *RuntimeStatus `json:"runtime,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Version string `json:"version"`
|
||||
Namespace string `json:"namespace"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Icon string `json:"icon,omitempty"`
|
||||
Manifest *AppManifest `json:"manifest,omitempty"`
|
||||
Runtime *RuntimeStatus `json:"runtime,omitempty"`
|
||||
Config map[string]interface{} `json:"config,omitempty"`
|
||||
Readme string `json:"readme,omitempty"`
|
||||
Documentation string `json:"documentation,omitempty"`
|
||||
Drift *DriftInfo `json:"drift,omitempty"`
|
||||
Readme string `json:"readme,omitempty"`
|
||||
Documentation string `json:"documentation,omitempty"`
|
||||
}
|
||||
|
||||
// DriftInfo describes divergence across the three-stage pipeline:
|
||||
// Wild Directory → .package/ → compiled manifests → cluster
|
||||
type DriftInfo struct {
|
||||
Source *StageDrift `json:"source,omitempty"`
|
||||
Compilation *StageDrift `json:"compilation,omitempty"`
|
||||
Deploy *StageDrift `json:"deploy,omitempty"`
|
||||
}
|
||||
|
||||
// StageDrift represents drift at one pipeline stage
|
||||
type StageDrift struct {
|
||||
Drifted bool `json:"drifted"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
CurrentVersion string `json:"currentVersion,omitempty"`
|
||||
AvailableVersion string `json:"availableVersion,omitempty"`
|
||||
UpgradeBlocked bool `json:"upgradeBlocked,omitempty"`
|
||||
UpgradeNotes string `json:"upgradeNotes,omitempty"`
|
||||
UpgradeSteps int `json:"upgradeSteps,omitempty"`
|
||||
}
|
||||
|
||||
// AppMeta represents the app-level metadata from app.yaml.
|
||||
// Contains identity/display fields and upgrade routing rules.
|
||||
// Version-specific details (requires, defaultConfig, etc.) live in version manifest.yaml files.
|
||||
type AppMeta struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Is string `json:"is,omitempty" yaml:"is,omitempty"`
|
||||
Description string `json:"description" yaml:"description"`
|
||||
Icon string `json:"icon,omitempty" yaml:"icon,omitempty"`
|
||||
Category string `json:"category,omitempty" yaml:"category,omitempty"`
|
||||
Latest string `json:"latest" yaml:"latest"`
|
||||
Upgrade *UpgradeConfig `json:"upgrade,omitempty" yaml:"upgrade,omitempty"`
|
||||
}
|
||||
|
||||
// UpgradeConfig declares upgrade behavior and constraints in the manifest
|
||||
type UpgradeConfig struct {
|
||||
From []UpgradeFromRule `json:"from,omitempty" yaml:"from,omitempty"`
|
||||
PreUpgrade *PreUpgradeConfig `json:"preUpgrade,omitempty" yaml:"preUpgrade,omitempty"`
|
||||
Migrations *MigrationConfig `json:"migrations,omitempty" yaml:"migrations,omitempty"`
|
||||
ConfigMigrations map[string]string `json:"configMigrations,omitempty" yaml:"configMigrations,omitempty"`
|
||||
}
|
||||
|
||||
// UpgradeFromRule defines a version constraint and optional upgrade path
|
||||
type UpgradeFromRule struct {
|
||||
Version string `json:"version" yaml:"version"` // e.g. ">=1.23.0", "<1.21.0", ">0"
|
||||
Via string `json:"via,omitempty" yaml:"via,omitempty"` // waypoint version in versions/
|
||||
Blocked bool `json:"blocked,omitempty" yaml:"blocked,omitempty"`
|
||||
Notes string `json:"notes,omitempty" yaml:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// PreUpgradeConfig defines pre-upgrade requirements
|
||||
type PreUpgradeConfig struct {
|
||||
Backup string `json:"backup,omitempty" yaml:"backup,omitempty"` // "none", "recommended", "required"
|
||||
Notes string `json:"notes,omitempty" yaml:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// MigrationConfig defines pre/post-deploy migration jobs for a version transition
|
||||
type MigrationConfig struct {
|
||||
Pre []string `json:"pre,omitempty" yaml:"pre,omitempty"` // paths to K8s Job YAMLs relative to app dir
|
||||
Post []string `json:"post,omitempty" yaml:"post,omitempty"`
|
||||
}
|
||||
|
||||
// RuntimeStatus contains runtime information from kubernetes
|
||||
|
||||
417
api/internal/apps/upgrade.go
Normal file
417
api/internal/apps/upgrade.go
Normal file
@@ -0,0 +1,417 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// UpgradeStep represents one step in a computed upgrade chain
|
||||
type UpgradeStep struct {
|
||||
FromVersion string `json:"fromVersion"`
|
||||
ToVersion string `json:"toVersion"`
|
||||
SourceDir string `json:"-"`
|
||||
Manifest *AppManifest `json:"-"`
|
||||
}
|
||||
|
||||
// UpgradePlan is the computed plan for upgrading an app
|
||||
type UpgradePlan struct {
|
||||
Steps []UpgradeStep `json:"steps"`
|
||||
Blocked bool `json:"blocked"`
|
||||
Notes string `json:"notes,omitempty"`
|
||||
BackupRequired bool `json:"backupRequired,omitempty"`
|
||||
BackupRecommended bool `json:"backupRecommended,omitempty"`
|
||||
}
|
||||
|
||||
// ParseAppVersion extracts major, minor, patch, and packaging revision from a version string.
|
||||
// Handles formats like "1.24.3-1", "v3.4", "5.118.1", "v4.0.18-2".
|
||||
func ParseAppVersion(v string) (major, minor, patch, revision int) {
|
||||
v = strings.TrimPrefix(v, "v")
|
||||
|
||||
// Split upstream from revision
|
||||
upstream := v
|
||||
if idx := strings.LastIndexByte(v, '-'); idx >= 0 {
|
||||
revStr := v[idx+1:]
|
||||
if r, err := strconv.Atoi(revStr); err == nil {
|
||||
revision = r
|
||||
upstream = v[:idx]
|
||||
}
|
||||
}
|
||||
|
||||
_, _ = fmt.Sscanf(upstream, "%d.%d.%d", &major, &minor, &patch)
|
||||
return
|
||||
}
|
||||
|
||||
// CompareAppVersions compares two version strings including packaging revision.
|
||||
// Returns >0 if a > b, <0 if a < b, 0 if equal.
|
||||
func CompareAppVersions(a, b string) int {
|
||||
aMaj, aMin, aPat, aRev := ParseAppVersion(a)
|
||||
bMaj, bMin, bPat, bRev := ParseAppVersion(b)
|
||||
|
||||
if aMaj != bMaj {
|
||||
return aMaj - bMaj
|
||||
}
|
||||
if aMin != bMin {
|
||||
return aMin - bMin
|
||||
}
|
||||
if aPat != bPat {
|
||||
return aPat - bPat
|
||||
}
|
||||
return aRev - bRev
|
||||
}
|
||||
|
||||
// MatchVersionConstraint checks whether version satisfies a constraint like ">=1.23.0", "<1.21.0", ">0".
|
||||
// The constraint's revision suffix is ignored so ">=1.23.0" matches "1.23.0-1".
|
||||
func MatchVersionConstraint(constraint, version string) bool {
|
||||
constraint = strings.TrimSpace(constraint)
|
||||
if constraint == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
var op string
|
||||
var target string
|
||||
|
||||
// Parse operator
|
||||
switch {
|
||||
case strings.HasPrefix(constraint, ">="):
|
||||
op, target = ">=", strings.TrimSpace(constraint[2:])
|
||||
case strings.HasPrefix(constraint, "<="):
|
||||
op, target = "<=", strings.TrimSpace(constraint[2:])
|
||||
case strings.HasPrefix(constraint, ">"):
|
||||
op, target = ">", strings.TrimSpace(constraint[1:])
|
||||
case strings.HasPrefix(constraint, "<"):
|
||||
op, target = "<", strings.TrimSpace(constraint[1:])
|
||||
case strings.HasPrefix(constraint, "="):
|
||||
op, target = "=", strings.TrimSpace(constraint[1:])
|
||||
default:
|
||||
// Bare version = exact match
|
||||
op, target = "=", constraint
|
||||
}
|
||||
|
||||
// Special case: ">0" matches any version
|
||||
if op == ">" && target == "0" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Compare upstream portions only (ignore revision in constraint target)
|
||||
tMaj, tMin, tPat, _ := ParseAppVersion(target)
|
||||
vMaj, vMin, vPat, _ := ParseAppVersion(version)
|
||||
|
||||
cmp := 0
|
||||
if vMaj != tMaj {
|
||||
cmp = vMaj - tMaj
|
||||
} else if vMin != tMin {
|
||||
cmp = vMin - tMin
|
||||
} else {
|
||||
cmp = vPat - tPat
|
||||
}
|
||||
|
||||
switch op {
|
||||
case ">=":
|
||||
return cmp >= 0
|
||||
case ">":
|
||||
return cmp > 0
|
||||
case "<=":
|
||||
return cmp <= 0
|
||||
case "<":
|
||||
return cmp < 0
|
||||
case "=":
|
||||
return cmp == 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ComputeUpgradePlan computes the upgrade path from installedVersion to the latest version
|
||||
// in the Wild Directory. Returns an empty plan (0 steps) if already current.
|
||||
// Supports both new-style (app.yaml + versions/) and old-style (manifest.yaml at root).
|
||||
func ComputeUpgradePlan(installedVersion, appName, appsDir string) (*UpgradePlan, error) {
|
||||
appRoot := filepath.Join(appsDir, appName)
|
||||
appYamlPath := filepath.Join(appRoot, "app.yaml")
|
||||
|
||||
// New-style: app.yaml exists — use centralized routing rules
|
||||
if storage.FileExists(appYamlPath) {
|
||||
return computeUpgradePlanFromMeta(installedVersion, appName, appsDir)
|
||||
}
|
||||
|
||||
// Old-style fallback: routing rules in manifest.yaml at root
|
||||
latestDir := appRoot
|
||||
return computeUpgradePlanRecursive(installedVersion, appName, appsDir, latestDir, make(map[string]bool), 0)
|
||||
}
|
||||
|
||||
// computeUpgradePlanFromMeta computes upgrade plan using centralized app.yaml routing rules.
|
||||
// Iteratively re-evaluates the same rules after each waypoint step (no recursive descent).
|
||||
// meta.Latest and rule.Via are slot names (directory names), not version strings.
|
||||
// The actual version is read from the manifest.yaml inside each slot directory.
|
||||
func computeUpgradePlanFromMeta(installedVersion, appName, appsDir string) (*UpgradePlan, error) {
|
||||
appRoot := filepath.Join(appsDir, appName)
|
||||
data, err := os.ReadFile(filepath.Join(appRoot, "app.yaml"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read app.yaml: %w", err)
|
||||
}
|
||||
var meta AppMeta
|
||||
if err := yaml.Unmarshal(data, &meta); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse app.yaml: %w", err)
|
||||
}
|
||||
|
||||
// Read the actual latest version from the manifest inside the slot
|
||||
latestDir := filepath.Join(appRoot, "versions", meta.Latest)
|
||||
latestManifest := readManifestFile(latestDir)
|
||||
if latestManifest == nil {
|
||||
return nil, fmt.Errorf("latest slot %q not found at %s", meta.Latest, latestDir)
|
||||
}
|
||||
latestVersion := latestManifest.Version
|
||||
|
||||
// Already current?
|
||||
if installedVersion == latestVersion {
|
||||
return &UpgradePlan{}, nil
|
||||
}
|
||||
|
||||
// No upgrade rules = any version upgrades directly
|
||||
if meta.Upgrade == nil || len(meta.Upgrade.From) == 0 {
|
||||
plan := &UpgradePlan{
|
||||
Steps: []UpgradeStep{{
|
||||
FromVersion: installedVersion,
|
||||
ToVersion: latestVersion,
|
||||
SourceDir: latestDir,
|
||||
Manifest: latestManifest,
|
||||
}},
|
||||
}
|
||||
setBackupFlags(plan, latestManifest)
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// Iterative chain building: re-evaluate app.yaml rules at each step
|
||||
var steps []UpgradeStep
|
||||
currentVersion := installedVersion
|
||||
visited := map[string]bool{}
|
||||
|
||||
for i := 0; i < 10; i++ { // max depth safety
|
||||
if currentVersion == latestVersion {
|
||||
break
|
||||
}
|
||||
if visited[currentVersion] {
|
||||
return &UpgradePlan{Blocked: true, Notes: fmt.Sprintf("circular upgrade path at version %s", currentVersion)}, nil
|
||||
}
|
||||
visited[currentVersion] = true
|
||||
|
||||
rule := findMatchingRule(meta.Upgrade.From, currentVersion)
|
||||
if rule == nil {
|
||||
return &UpgradePlan{
|
||||
Blocked: true,
|
||||
Notes: fmt.Sprintf("no upgrade path from version %s to %s", currentVersion, latestVersion),
|
||||
}, nil
|
||||
}
|
||||
if rule.Blocked {
|
||||
return &UpgradePlan{Blocked: true, Notes: rule.Notes}, nil
|
||||
}
|
||||
|
||||
if rule.Via != "" {
|
||||
// Step to waypoint — Via is a slot name, read version from manifest
|
||||
wpDir := filepath.Join(appRoot, "versions", rule.Via)
|
||||
wpManifest := readManifestFile(wpDir)
|
||||
if wpManifest == nil {
|
||||
return nil, fmt.Errorf("waypoint slot %q not found at %s", rule.Via, wpDir)
|
||||
}
|
||||
// If waypoint resolves to current version, skip it and go directly to latest
|
||||
if wpManifest.Version == currentVersion {
|
||||
steps = append(steps, UpgradeStep{
|
||||
FromVersion: currentVersion,
|
||||
ToVersion: latestVersion,
|
||||
SourceDir: latestDir,
|
||||
Manifest: latestManifest,
|
||||
})
|
||||
currentVersion = latestVersion
|
||||
} else {
|
||||
steps = append(steps, UpgradeStep{
|
||||
FromVersion: currentVersion,
|
||||
ToVersion: wpManifest.Version,
|
||||
SourceDir: wpDir,
|
||||
Manifest: wpManifest,
|
||||
})
|
||||
currentVersion = wpManifest.Version
|
||||
}
|
||||
} else {
|
||||
// Direct step to latest
|
||||
steps = append(steps, UpgradeStep{
|
||||
FromVersion: currentVersion,
|
||||
ToVersion: latestVersion,
|
||||
SourceDir: latestDir,
|
||||
Manifest: latestManifest,
|
||||
})
|
||||
currentVersion = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
plan := &UpgradePlan{Steps: steps}
|
||||
// Aggregate backup flags from app-level preUpgrade and per-step manifests
|
||||
if meta.Upgrade.PreUpgrade != nil {
|
||||
switch meta.Upgrade.PreUpgrade.Backup {
|
||||
case "required":
|
||||
plan.BackupRequired = true
|
||||
case "recommended":
|
||||
plan.BackupRecommended = true
|
||||
}
|
||||
}
|
||||
for _, step := range plan.Steps {
|
||||
if step.Manifest != nil {
|
||||
setBackupFlags(plan, step.Manifest)
|
||||
}
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// findMatchingRule returns the first matching UpgradeFromRule for the given version.
|
||||
func findMatchingRule(rules []UpgradeFromRule, version string) *UpgradeFromRule {
|
||||
for i, rule := range rules {
|
||||
if MatchVersionConstraint(rule.Version, version) {
|
||||
return &rules[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readManifestFile reads and parses a manifest.yaml from a directory. Returns nil if not found.
|
||||
func readManifestFile(dir string) *AppManifest {
|
||||
data, err := os.ReadFile(filepath.Join(dir, "manifest.yaml"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var manifest AppManifest
|
||||
if err := yaml.Unmarshal(data, &manifest); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &manifest
|
||||
}
|
||||
|
||||
func computeUpgradePlanRecursive(installedVersion, appName, appsDir, targetDir string, visited map[string]bool, depth int) (*UpgradePlan, error) {
|
||||
if depth > 10 {
|
||||
return &UpgradePlan{Blocked: true, Notes: "upgrade path exceeds maximum depth"}, nil
|
||||
}
|
||||
|
||||
// Read target manifest
|
||||
manifestPath := filepath.Join(targetDir, "manifest.yaml")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read manifest at %s: %w", manifestPath, err)
|
||||
}
|
||||
|
||||
var manifest AppManifest
|
||||
if err := yaml.Unmarshal(data, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse manifest at %s: %w", manifestPath, err)
|
||||
}
|
||||
|
||||
// Already current
|
||||
if installedVersion == manifest.Version {
|
||||
return &UpgradePlan{}, nil
|
||||
}
|
||||
|
||||
// Cycle detection
|
||||
if visited[manifest.Version] {
|
||||
return &UpgradePlan{Blocked: true, Notes: fmt.Sprintf("circular upgrade path at version %s", manifest.Version)}, nil
|
||||
}
|
||||
visited[manifest.Version] = true
|
||||
|
||||
// No upgrade block = any version can upgrade directly (backward compat)
|
||||
if manifest.Upgrade == nil || len(manifest.Upgrade.From) == 0 {
|
||||
plan := &UpgradePlan{
|
||||
Steps: []UpgradeStep{{
|
||||
FromVersion: installedVersion,
|
||||
ToVersion: manifest.Version,
|
||||
SourceDir: targetDir,
|
||||
Manifest: &manifest,
|
||||
}},
|
||||
}
|
||||
setBackupFlags(plan, &manifest)
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// Find matching rule
|
||||
for _, rule := range manifest.Upgrade.From {
|
||||
if !MatchVersionConstraint(rule.Version, installedVersion) {
|
||||
continue
|
||||
}
|
||||
|
||||
if rule.Blocked {
|
||||
return &UpgradePlan{Blocked: true, Notes: rule.Notes}, nil
|
||||
}
|
||||
|
||||
if rule.Via != "" {
|
||||
// Load waypoint and compute path to it, then from it to target
|
||||
waypointDir := filepath.Join(appsDir, appName, ".versions", rule.Via)
|
||||
waypointManifestPath := filepath.Join(waypointDir, "manifest.yaml")
|
||||
if _, err := os.Stat(waypointManifestPath); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("waypoint version %s not found at %s", rule.Via, waypointDir)
|
||||
}
|
||||
|
||||
// Compute path from installed -> waypoint
|
||||
toWaypoint, err := computeUpgradePlanRecursive(installedVersion, appName, appsDir, waypointDir, visited, depth+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if toWaypoint.Blocked {
|
||||
return toWaypoint, nil
|
||||
}
|
||||
|
||||
// Read waypoint manifest version for the next hop
|
||||
wpData, err := os.ReadFile(waypointManifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read waypoint manifest: %w", err)
|
||||
}
|
||||
var wpManifest AppManifest
|
||||
if err := yaml.Unmarshal(wpData, &wpManifest); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse waypoint manifest: %w", err)
|
||||
}
|
||||
|
||||
// Add the step from waypoint -> target
|
||||
plan := &UpgradePlan{
|
||||
Steps: append(toWaypoint.Steps, UpgradeStep{
|
||||
FromVersion: wpManifest.Version,
|
||||
ToVersion: manifest.Version,
|
||||
SourceDir: targetDir,
|
||||
Manifest: &manifest,
|
||||
}),
|
||||
}
|
||||
|
||||
// Aggregate backup flags from all steps
|
||||
for _, step := range plan.Steps {
|
||||
setBackupFlags(plan, step.Manifest)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// Direct upgrade
|
||||
plan := &UpgradePlan{
|
||||
Steps: []UpgradeStep{{
|
||||
FromVersion: installedVersion,
|
||||
ToVersion: manifest.Version,
|
||||
SourceDir: targetDir,
|
||||
Manifest: &manifest,
|
||||
}},
|
||||
}
|
||||
setBackupFlags(plan, &manifest)
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// No matching rule
|
||||
return &UpgradePlan{
|
||||
Blocked: true,
|
||||
Notes: fmt.Sprintf("no upgrade path from version %s to %s", installedVersion, manifest.Version),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func setBackupFlags(plan *UpgradePlan, manifest *AppManifest) {
|
||||
if manifest.Upgrade == nil || manifest.Upgrade.PreUpgrade == nil {
|
||||
return
|
||||
}
|
||||
switch manifest.Upgrade.PreUpgrade.Backup {
|
||||
case "required":
|
||||
plan.BackupRequired = true
|
||||
case "recommended":
|
||||
plan.BackupRecommended = true
|
||||
}
|
||||
}
|
||||
675
api/internal/apps/upgrade_test.go
Normal file
675
api/internal/apps/upgrade_test.go
Normal file
@@ -0,0 +1,675 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestParseAppVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
major, minor, patch, revision int
|
||||
}{
|
||||
{"1.24.3-1", 1, 24, 3, 1},
|
||||
{"v3.4", 3, 4, 0, 0},
|
||||
{"5.118.1", 5, 118, 1, 0},
|
||||
{"v4.0.18-2", 4, 0, 18, 2},
|
||||
{"1.0.0", 1, 0, 0, 0},
|
||||
{"0.1.0-10", 0, 1, 0, 10},
|
||||
{"v1.0.0-1", 1, 0, 0, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
maj, min, pat, rev := ParseAppVersion(tt.input)
|
||||
if maj != tt.major || min != tt.minor || pat != tt.patch || rev != tt.revision {
|
||||
t.Errorf("ParseAppVersion(%q) = (%d,%d,%d,%d), want (%d,%d,%d,%d)",
|
||||
tt.input, maj, min, pat, rev, tt.major, tt.minor, tt.patch, tt.revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareAppVersions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b string
|
||||
want int // >0, <0, or 0
|
||||
}{
|
||||
{"equal", "1.0.0", "1.0.0", 0},
|
||||
{"major greater", "2.0.0", "1.0.0", 1},
|
||||
{"major less", "1.0.0", "2.0.0", -1},
|
||||
{"minor greater", "1.2.0", "1.1.0", 1},
|
||||
{"minor less", "1.1.0", "1.2.0", -1},
|
||||
{"patch greater", "1.0.2", "1.0.1", 1},
|
||||
{"patch less", "1.0.1", "1.0.2", -1},
|
||||
{"revision tiebreaker", "1.0.0-2", "1.0.0-1", 1},
|
||||
{"revision less", "1.0.0-1", "1.0.0-2", -1},
|
||||
{"revision equal", "1.0.0-1", "1.0.0-1", 0},
|
||||
{"no revision vs revision", "1.0.0", "1.0.0-1", -1},
|
||||
{"v prefix ignored", "v1.0.0", "1.0.0", 0},
|
||||
{"real versions", "1.24.3-1", "1.23.0", 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := CompareAppVersions(tt.a, tt.b)
|
||||
if (tt.want > 0 && got <= 0) || (tt.want < 0 && got >= 0) || (tt.want == 0 && got != 0) {
|
||||
t.Errorf("CompareAppVersions(%q, %q) = %d, want sign %d", tt.a, tt.b, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchVersionConstraint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
constraint string
|
||||
version string
|
||||
want bool
|
||||
}{
|
||||
{"gte match", ">=1.23.0", "1.24.0", true},
|
||||
{"gte exact", ">=1.23.0", "1.23.0", true},
|
||||
{"gte with revision", ">=1.23.0", "1.23.0-1", true},
|
||||
{"gte below", ">=1.23.0", "1.22.0", false},
|
||||
{"gt match", ">1.0.0", "1.0.1", true},
|
||||
{"gt exact", ">1.0.0", "1.0.0", false},
|
||||
{"lt match", "<2.0.0", "1.9.9", true},
|
||||
{"lt exact", "<2.0.0", "2.0.0", false},
|
||||
{"lt above", "<2.0.0", "2.0.1", false},
|
||||
{"lte match", "<=2.0.0", "2.0.0", true},
|
||||
{"lte below", "<=2.0.0", "1.9.0", true},
|
||||
{"lte above", "<=2.0.0", "2.0.1", false},
|
||||
{"exact match", "=1.5.0", "1.5.0", true},
|
||||
{"exact no match", "=1.5.0", "1.5.1", false},
|
||||
{"bare version exact", "1.5.0", "1.5.0", true},
|
||||
{"bare version no match", "1.5.0", "1.5.1", false},
|
||||
{"gt zero universal", ">0", "0.0.1", true},
|
||||
{"gt zero any version", ">0", "99.99.99", true},
|
||||
{"empty constraint", "", "1.0.0", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := MatchVersionConstraint(tt.constraint, tt.version)
|
||||
if got != tt.want {
|
||||
t.Errorf("MatchVersionConstraint(%q, %q) = %v, want %v",
|
||||
tt.constraint, tt.version, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeManifest is a test helper that writes an AppManifest as YAML to a manifest.yaml file.
|
||||
func writeManifest(t *testing.T, dir string, manifest AppManifest) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := yaml.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "manifest.yaml"), data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// writeAppMeta writes an AppMeta as app.yaml in the given app root directory.
|
||||
func writeAppMeta(t *testing.T, appRoot string, meta AppMeta) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(appRoot, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := yaml.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(appRoot, "app.yaml"), data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// writeVersionManifest writes a version manifest at versions/{version}/manifest.yaml.
|
||||
func writeVersionManifest(t *testing.T, appRoot, version string, manifest AppManifest) {
|
||||
t.Helper()
|
||||
writeManifest(t, filepath.Join(appRoot, "versions", version), manifest)
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_AlreadyCurrent(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{Name: "myapp", Latest: "1"})
|
||||
writeVersionManifest(t, appDir, "1", AppManifest{Version: "1.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(plan.Steps) != 0 {
|
||||
t.Errorf("expected 0 steps for current version, got %d", len(plan.Steps))
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected not blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_NoUpgradeBlock(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{Name: "myapp", Latest: "2"})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected not blocked")
|
||||
}
|
||||
if len(plan.Steps) != 1 {
|
||||
t.Fatalf("expected 1 step, got %d", len(plan.Steps))
|
||||
}
|
||||
if plan.Steps[0].FromVersion != "1.0.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step = %s -> %s, want 1.0.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_DirectUpgrade(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "2",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=1.0.0"},
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected not blocked")
|
||||
}
|
||||
if len(plan.Steps) != 1 {
|
||||
t.Fatalf("expected 1 step, got %d", len(plan.Steps))
|
||||
}
|
||||
if plan.Steps[0].FromVersion != "1.5.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step = %s -> %s, want 1.5.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_Blocked(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: "<1.0.0", Blocked: true, Notes: "too old, manual migration required"},
|
||||
{Version: ">=1.0.0"},
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("0.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !plan.Blocked {
|
||||
t.Error("expected blocked")
|
||||
}
|
||||
if plan.Notes != "too old, manual migration required" {
|
||||
t.Errorf("unexpected notes: %q", plan.Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_NoMatchingRule(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=2.0.0"}, // only allows from 2.x+
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !plan.Blocked {
|
||||
t.Error("expected blocked when no rule matches")
|
||||
}
|
||||
if plan.Notes == "" {
|
||||
t.Error("expected notes explaining no upgrade path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_ViaWaypoint(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Slot "3" has version 3.0.0, slot "2" (waypoint) has version 2.0.0
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=2.0.0"}, // direct from 2.x
|
||||
{Version: ">=1.0.0", Via: "2"},
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Errorf("expected not blocked, notes: %s", plan.Notes)
|
||||
}
|
||||
if len(plan.Steps) != 2 {
|
||||
t.Fatalf("expected 2 steps, got %d", len(plan.Steps))
|
||||
}
|
||||
|
||||
// Step 1: 1.5.0 -> 2.0.0 (version from manifest in slot "2")
|
||||
if plan.Steps[0].FromVersion != "1.5.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step 1 = %s -> %s, want 1.5.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
// Step 2: 2.0.0 -> 3.0.0 (version from manifest in slot "3")
|
||||
if plan.Steps[1].FromVersion != "2.0.0" || plan.Steps[1].ToVersion != "3.0.0" {
|
||||
t.Errorf("step 2 = %s -> %s, want 2.0.0 -> 3.0.0",
|
||||
plan.Steps[1].FromVersion, plan.Steps[1].ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_MultipleWaypoints(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Slots "2", "3", "4" with versions 2.0.0, 3.0.0, 4.0.0
|
||||
// Centralized rules: 1.x → via slot "2", 2.x → via slot "3", 3.x → direct to slot "4"
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "4",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=3.0.0"}, // direct from 3.x
|
||||
{Version: ">=2.0.0", Via: "3"}, // 2.x must go through slot "3"
|
||||
{Version: ">=1.0.0", Via: "2"}, // 1.x must go through slot "2"
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "4", AppManifest{Version: "4.0.0"})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Errorf("expected not blocked, notes: %s", plan.Notes)
|
||||
}
|
||||
if len(plan.Steps) != 3 {
|
||||
t.Fatalf("expected 3 steps, got %d", len(plan.Steps))
|
||||
}
|
||||
|
||||
// Step 1: 1.0.0 -> 2.0.0 (version from manifest in slot "2")
|
||||
if plan.Steps[0].FromVersion != "1.0.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step 1 = %s -> %s, want 1.0.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
// Step 2: 2.0.0 -> 3.0.0 (version from manifest in slot "3")
|
||||
if plan.Steps[1].FromVersion != "2.0.0" || plan.Steps[1].ToVersion != "3.0.0" {
|
||||
t.Errorf("step 2 = %s -> %s, want 2.0.0 -> 3.0.0",
|
||||
plan.Steps[1].FromVersion, plan.Steps[1].ToVersion)
|
||||
}
|
||||
// Step 3: 3.0.0 -> 4.0.0 (version from manifest in slot "4")
|
||||
if plan.Steps[2].FromVersion != "3.0.0" || plan.Steps[2].ToVersion != "4.0.0" {
|
||||
t.Errorf("step 3 = %s -> %s, want 3.0.0 -> 4.0.0",
|
||||
plan.Steps[2].FromVersion, plan.Steps[2].ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_CycleDetection(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Two waypoints that bounce between each other: wp-a (1.5.0) and wp-b (1.2.0)
|
||||
// 1.0.0 → wp-a (1.5.0) → wp-b (1.2.0) → wp-a (1.5.0) = cycle
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=1.5.0", Via: "wp-b"},
|
||||
{Version: ">0", Via: "wp-a"},
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
writeVersionManifest(t, appDir, "wp-a", AppManifest{Version: "1.5.0"})
|
||||
writeVersionManifest(t, appDir, "wp-b", AppManifest{Version: "1.2.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !plan.Blocked {
|
||||
t.Error("expected blocked for circular upgrade path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_BackupRequired(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "2",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">0"},
|
||||
},
|
||||
PreUpgrade: &PreUpgradeConfig{
|
||||
Backup: "required",
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected not blocked")
|
||||
}
|
||||
if !plan.BackupRequired {
|
||||
t.Error("expected BackupRequired to be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_BackupRecommended(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "2",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">0"},
|
||||
},
|
||||
PreUpgrade: &PreUpgradeConfig{
|
||||
Backup: "recommended",
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !plan.BackupRecommended {
|
||||
t.Error("expected BackupRecommended to be true")
|
||||
}
|
||||
if plan.BackupRequired {
|
||||
t.Error("expected BackupRequired to be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_WaypointMissing(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "2",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">0", Via: "1-wp"}, // waypoint slot doesn't exist
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{Version: "2.0.0"})
|
||||
|
||||
_, err = ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err == nil {
|
||||
t.Error("expected error for missing waypoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_RuleOrdering(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=2.0.0"}, // direct for 2.x+
|
||||
{Version: ">=1.0.0", Blocked: true, Notes: "must be on 2.x+"}, // block for 1.x
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
|
||||
// Version 2.5.0 matches first rule (direct)
|
||||
plan, err := ComputeUpgradePlan("2.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected 2.5.0 to not be blocked")
|
||||
}
|
||||
|
||||
// Version 1.5.0 matches second rule (blocked)
|
||||
plan, err = ComputeUpgradePlan("1.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !plan.Blocked {
|
||||
t.Error("expected 1.5.0 to be blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_BackupAggregation(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// App-level has no backup requirement, but waypoint manifest does
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeAppMeta(t, appDir, AppMeta{
|
||||
Name: "myapp",
|
||||
Latest: "3",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=2.0.0"},
|
||||
{Version: ">=1.0.0", Via: "2"},
|
||||
},
|
||||
},
|
||||
})
|
||||
writeVersionManifest(t, appDir, "3", AppManifest{Version: "3.0.0"})
|
||||
writeVersionManifest(t, appDir, "2", AppManifest{
|
||||
Version: "2.0.0",
|
||||
Upgrade: &UpgradeConfig{
|
||||
PreUpgrade: &PreUpgradeConfig{
|
||||
Backup: "required",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.0.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Errorf("expected not blocked, notes: %s", plan.Notes)
|
||||
}
|
||||
if !plan.BackupRequired {
|
||||
t.Error("expected BackupRequired to be true (aggregated from waypoint)")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Old-style backward compatibility tests ---
|
||||
|
||||
func TestComputeUpgradePlan_OldStyle_DirectUpgrade(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Old-style: manifest.yaml at root, no app.yaml
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeManifest(t, appDir, AppManifest{
|
||||
Name: "myapp",
|
||||
Version: "2.0.0",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=1.0.0"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Error("expected not blocked")
|
||||
}
|
||||
if len(plan.Steps) != 1 {
|
||||
t.Fatalf("expected 1 step, got %d", len(plan.Steps))
|
||||
}
|
||||
if plan.Steps[0].FromVersion != "1.5.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step = %s -> %s, want 1.5.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeUpgradePlan_OldStyle_ViaWaypoint(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "upgrade-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// Old-style: root manifest + .versions/ for waypoints
|
||||
appDir := filepath.Join(tmpDir, "myapp")
|
||||
writeManifest(t, appDir, AppManifest{
|
||||
Name: "myapp",
|
||||
Version: "3.0.0",
|
||||
Upgrade: &UpgradeConfig{
|
||||
From: []UpgradeFromRule{
|
||||
{Version: ">=2.0.0"},
|
||||
{Version: ">=1.0.0", Via: "2.0.0"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
waypointDir := filepath.Join(appDir, ".versions", "2.0.0")
|
||||
writeManifest(t, waypointDir, AppManifest{
|
||||
Name: "myapp",
|
||||
Version: "2.0.0",
|
||||
})
|
||||
|
||||
plan, err := ComputeUpgradePlan("1.5.0", "myapp", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.Blocked {
|
||||
t.Errorf("expected not blocked, notes: %s", plan.Notes)
|
||||
}
|
||||
if len(plan.Steps) != 2 {
|
||||
t.Fatalf("expected 2 steps, got %d", len(plan.Steps))
|
||||
}
|
||||
if plan.Steps[0].FromVersion != "1.5.0" || plan.Steps[0].ToVersion != "2.0.0" {
|
||||
t.Errorf("step 1 = %s -> %s, want 1.5.0 -> 2.0.0",
|
||||
plan.Steps[0].FromVersion, plan.Steps[0].ToVersion)
|
||||
}
|
||||
if plan.Steps[1].FromVersion != "2.0.0" || plan.Steps[1].ToVersion != "3.0.0" {
|
||||
t.Errorf("step 2 = %s -> %s, want 2.0.0 -> 3.0.0",
|
||||
plan.Steps[1].FromVersion, plan.Steps[1].ToVersion)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package assets
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -167,10 +166,6 @@ func (m *Manager) listAssetFiles(schematicID, version string) ([]Asset, error) {
|
||||
|
||||
if err == nil && info != nil {
|
||||
asset.Size = info.Size()
|
||||
// Calculate SHA256 if file exists
|
||||
if hash, err := calculateSHA256(assetPath); err == nil {
|
||||
asset.SHA256 = hash
|
||||
}
|
||||
}
|
||||
|
||||
assets = append(assets, asset)
|
||||
@@ -191,10 +186,6 @@ func (m *Manager) listAssetFiles(schematicID, version string) ([]Asset, error) {
|
||||
|
||||
if err == nil && info != nil {
|
||||
asset.Size = info.Size()
|
||||
// Calculate SHA256 if file exists
|
||||
if hash, err := calculateSHA256(isoPath); err == nil {
|
||||
asset.SHA256 = hash
|
||||
}
|
||||
}
|
||||
|
||||
assets = append(assets, asset)
|
||||
@@ -430,19 +421,3 @@ func (m *Manager) DeleteAsset(schematicID, version string) error {
|
||||
|
||||
return os.RemoveAll(assetDir)
|
||||
}
|
||||
|
||||
// calculateSHA256 computes the SHA256 hash of a file
|
||||
func calculateSHA256(filePath string) (string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
601
api/internal/backup/README.md
Normal file
601
api/internal/backup/README.md
Normal file
@@ -0,0 +1,601 @@
|
||||
# Disaster Recovery Backup System
|
||||
|
||||
## Core Requirements
|
||||
1. **True disaster recovery**: All backup data on NFS (or other external destination)
|
||||
2. **Migration capability**: Restore apps from one instance/cluster to another
|
||||
3. **Simplicity**: Keep only the latest backup per app/cluster
|
||||
4. **Incremental**: Use Longhorn's incremental backup capability to minimize storage and transfer time
|
||||
5. **Cluster backup**: Include kubeconfig, talosconfig, and cluster-level configs
|
||||
|
||||
## In-Place Restore Algorithm
|
||||
|
||||
Restore is always **in-place**: the app is briefly taken offline, data is swapped in the
|
||||
existing namespace, and the app comes back up. This keeps namespaces, domains, ingresses,
|
||||
and all external references canonical throughout.
|
||||
|
||||
Blue-green color tracking remains — not for namespaces, but for **staging volume names**
|
||||
during the restore phase (restored Longhorn volumes are created with a color suffix while
|
||||
the app is still running, avoiding naming conflicts with live volumes). The active color
|
||||
alternates with each restore so that incremental Longhorn backups work correctly.
|
||||
|
||||
### Naming Convention
|
||||
|
||||
The active color is tracked in `config.yaml` under `apps.{app}.activeDeployment`:
|
||||
|
||||
```yaml
|
||||
# In config.yaml
|
||||
apps:
|
||||
gitea:
|
||||
activeDeployment: blue # which color produced the last backup
|
||||
```
|
||||
|
||||
A bare namespace (`gitea`) with no activeDeployment is treated as blue (backwards compat).
|
||||
|
||||
Resources:
|
||||
- **Namespace**: always `gitea` (canonical, never colored)
|
||||
- **Database**: always `gitea` (canonical); `gitea_restore` used as staging during Restore phase
|
||||
- **Longhorn volumes**: staging volumes use `data-green` etc. during Restore; swapped in during Switch
|
||||
- **App directory**: always `apps/gitea`
|
||||
|
||||
### Overview: Four Phases
|
||||
|
||||
- **BACKUP** → Discover app state, export data, create RecoveryPlan
|
||||
- **RESTORE** → Restore data into staging resources (app still running with live data)
|
||||
- **SWITCH** → Scale app down, swap staging resources to canonical names, scale app up (~30-120s downtime)
|
||||
- **CLEANUP** → Drop the pre-restore data snapshots (`gitea_old`, old Longhorn volumes)
|
||||
|
||||
### The RecoveryPlan
|
||||
|
||||
The RecoveryPlan is the coordination record that decouples the phases. It is created
|
||||
during backup, captures everything needed to restore, and tracks progress through every
|
||||
phase. Each phase reads the plan, does its work, and writes its results back.
|
||||
|
||||
The plan is persisted as a YAML file alongside the backup metadata. It provides full
|
||||
transparency and is the basis for idempotent re-entry: before any phase runs you can
|
||||
inspect the plan to see exactly what will happen; after a phase runs you can see exactly
|
||||
what happened and at what sub-step it stopped.
|
||||
|
||||
```yaml
|
||||
# RecoveryPlan for gitea on test-cloud
|
||||
app: gitea
|
||||
instance: test-cloud
|
||||
status: backed_up # backing_up | backed_up | restoring | restored | switching | switched | cleaning_up | cleaned_up | failed
|
||||
error: "" # set on failure; includes phase and strategy
|
||||
|
||||
# Populated during BACKUP
|
||||
source:
|
||||
activeColor: blue # from config.yaml; default: blue
|
||||
namespace: gitea # always canonical (bare name = blue)
|
||||
appDir: instances/test-cloud/apps/gitea
|
||||
configPath: instances/test-cloud/config.yaml
|
||||
secretsPath: instances/test-cloud/secrets.yaml
|
||||
|
||||
# Used for staging volume names during Restore phase
|
||||
standbyColor: green
|
||||
|
||||
# Populated during RESTORE: always mirrors source for in-place restores
|
||||
standby:
|
||||
namespace: gitea
|
||||
appDir: instances/test-cloud/apps/gitea
|
||||
|
||||
# One entry per detected strategy
|
||||
strategies:
|
||||
- name: config
|
||||
status: backed_up
|
||||
backup:
|
||||
location: config/test-cloud/gitea/20240304T150405Z.tar.gz
|
||||
size: 12345
|
||||
|
||||
- name: postgres
|
||||
status: backed_up
|
||||
params:
|
||||
podNamespace: postgres
|
||||
podLabel: app=postgres
|
||||
dbName: gitea
|
||||
dbUser: gitea
|
||||
backup:
|
||||
location: postgres/test-cloud/gitea/20240304T150405Z.dump
|
||||
globalsLocation: postgres/test-cloud/gitea/20240304T150405Z-globals.sql
|
||||
size: 67890
|
||||
restore:
|
||||
dbName: gitea_restore # always "_restore" suffix; never colored
|
||||
switch:
|
||||
renamedToOld: true # checkpoint: written + plan saved after gitea → gitea_old
|
||||
previousDbName: gitea_old # populated when switch completes
|
||||
|
||||
- name: longhorn-native
|
||||
status: backed_up
|
||||
params:
|
||||
volumes:
|
||||
- pvcName: data
|
||||
volumeName: pvc-abc123
|
||||
size: 10Gi
|
||||
- pvcName: repos
|
||||
volumeName: pvc-def456
|
||||
size: 50Gi
|
||||
backup:
|
||||
volumes:
|
||||
- pvcName: data
|
||||
backupURL: nfs://server/path/backup-xyz
|
||||
- pvcName: repos
|
||||
backupURL: nfs://server/path/backup-abc
|
||||
restore:
|
||||
volumes:
|
||||
- pvcName: data
|
||||
volumeName: data-green # staging volume; swapped in during Switch
|
||||
- pvcName: repos
|
||||
volumeName: repos-green
|
||||
switch:
|
||||
completedPVCs: # checkpoint: per-PVC swap progress
|
||||
- data # "data" PVC swap completed
|
||||
# "repos" not yet completed
|
||||
previousItems: # populated as each PVC swap completes
|
||||
- pvcName: data
|
||||
pvName: pvc-abc123
|
||||
longhornVolume: pvc-abc123
|
||||
|
||||
phases:
|
||||
backup: { startedAt: ..., completedAt: ... }
|
||||
restore: { startedAt: ..., completedAt: ... }
|
||||
switch: { startedAt: ..., completedAt: ... }
|
||||
cleanup: { startedAt: ..., completedAt: ... }
|
||||
```
|
||||
|
||||
**Key properties:**
|
||||
|
||||
- **Self-describing**: Every parameter a strategy needs is in `params`, not rediscovered
|
||||
at restore time. Restore works even if source config has changed since backup.
|
||||
- **Per-strategy state**: Each strategy entry has its own status, params, backup, restore,
|
||||
and switch sections. Strategies only read/write their own section.
|
||||
- **Checkpointed**: Sub-step completion is written to the plan at each point of no return
|
||||
so that phases can be re-entered and resume where they stopped.
|
||||
- **Phase decoupling**: No phase needs to query the live system to figure out what the
|
||||
previous phase did. The plan is the source of truth.
|
||||
- **Idempotent**: Every operation is guarded by a detect-then-reconcile check. Any phase
|
||||
can be retried after failure without corrupting data.
|
||||
|
||||
---
|
||||
|
||||
## Resilience Patterns
|
||||
|
||||
Every phase follows three principles:
|
||||
|
||||
### 1. Detect-then-reconcile
|
||||
|
||||
Before acting, check what state the system is actually in, then take only the minimal
|
||||
action needed to reach the desired end state. Never blindly execute a script of steps.
|
||||
|
||||
Example: Postgres Switch does not just run the rename commands — it first queries which
|
||||
databases exist, identifies the current state, and takes the appropriate action for that
|
||||
state. This means re-running Switch after a mid-operation crash does the right thing.
|
||||
|
||||
### 2. Checkpoint at points of no return
|
||||
|
||||
A "point of no return" is a step after which the system is in an intermediate state that
|
||||
is neither the old state nor the new state. Write to the plan (and flush it to disk) before
|
||||
and after these steps so that re-entry can detect where to resume.
|
||||
|
||||
For Postgres: after renaming `gitea → gitea_old`, the production database is gone. The
|
||||
plan records `switch.renamedToOld: true` immediately so that if the process crashes before
|
||||
the second rename, re-entry knows to complete the second rename rather than retrying the
|
||||
first (which would fail: `gitea` no longer exists).
|
||||
|
||||
### 3. Always scale up on failure
|
||||
|
||||
The app is scaled to zero during Switch. If any strategy fails, always attempt to scale
|
||||
back up before returning the error. A crash-looping app with visible error messages is
|
||||
easier to diagnose and recover from than an app silently stuck at zero replicas.
|
||||
|
||||
Scale-up failures are logged but do not change the plan status — the data swap may have
|
||||
partially succeeded, and the operator needs to inspect the plan to determine next steps.
|
||||
|
||||
---
|
||||
|
||||
## Phase Algorithms
|
||||
|
||||
### Backup
|
||||
|
||||
```
|
||||
BACKUP(app):
|
||||
1. Validate: destination reachable, app is deployed
|
||||
|
||||
2. Create RecoveryPlan with status=backing_up
|
||||
- Read config.yaml apps.{app}.activeDeployment → activeColor (default: blue)
|
||||
- Set standbyColor = opposite of activeColor
|
||||
- Record source.activeColor, source.namespace (canonical), source.appDir
|
||||
|
||||
3. Detect strategies from app manifest
|
||||
- Always: config
|
||||
- From manifest.requires: postgres, mysql
|
||||
- Always: longhorn-native
|
||||
|
||||
4. For each detected strategy:
|
||||
- Strategy records params from config.yaml and kubectl
|
||||
- Strategy exports data to destination
|
||||
- Strategy records backup location in plan entry
|
||||
- Strategy sets status = backed_up (or failed)
|
||||
|
||||
5. If all strategies succeeded:
|
||||
- Set plan.status = backed_up, write plan
|
||||
```
|
||||
|
||||
### Restore
|
||||
|
||||
Restores data into **staging resources** while the app continues serving live traffic.
|
||||
No downtime in this phase. Staging resources use the standby color to avoid naming
|
||||
conflicts with live resources.
|
||||
|
||||
```
|
||||
RESTORE(app):
|
||||
1. Load RecoveryPlan, validate plan.status == backed_up
|
||||
- Set plan.status = restoring
|
||||
- Set plan.standby = plan.source (same namespace, same appDir — always in-place)
|
||||
|
||||
2. For each strategy:
|
||||
- Strategy reads its params and backup location from the plan
|
||||
- Strategy creates staging resources and restores data into them:
|
||||
Postgres: CREATE DATABASE {db}_restore; pg_restore into it
|
||||
Longhorn: createVolumeFromBackup({pvcName}-{standbyColor}, backupURL)
|
||||
Config: no-op (files restored during Switch or not needed separately)
|
||||
- Each operation is idempotent: checks whether the staging resource already
|
||||
exists with the correct data before recreating it
|
||||
- Strategy records restore targets in plan entry
|
||||
- Strategy sets status = restored (or failed)
|
||||
|
||||
3. Set plan.status = restored, write plan
|
||||
|
||||
Result: App still serving live traffic.
|
||||
Staging DB ({db}_restore) and staging volumes ({pvcName}-{color})
|
||||
exist alongside live resources, ready for Switch.
|
||||
```
|
||||
|
||||
### Switch
|
||||
|
||||
Briefly takes the app offline, swaps staging resources to canonical names, brings app
|
||||
back up. Expected downtime: 30–120 seconds.
|
||||
|
||||
```
|
||||
SWITCH(app):
|
||||
1. Load RecoveryPlan, validate plan.status == restored
|
||||
- Set plan.status = switching
|
||||
|
||||
2. Scale down app namespace (coordinator — before any strategy runs):
|
||||
kubectl scale deployment --all -n {namespace} --replicas=0
|
||||
kubectl scale statefulset --all -n {namespace} --replicas=0
|
||||
kubectl wait --for=delete pod --all -n {namespace} --timeout=120s
|
||||
- If scale-down fails, abort (return error; no data has been touched)
|
||||
- Set up deferred scale-up: always attempt scaleNamespace(1) on exit,
|
||||
even if a strategy fails below
|
||||
|
||||
3. For each strategy (with app at zero replicas):
|
||||
|
||||
Postgres (detect-then-reconcile):
|
||||
Desired end state: {db} has restored data, {db}_old has pre-restore data
|
||||
|
||||
Check: db_exists, restore_exists, old_exists
|
||||
|
||||
(db=✓, restore=✓, old=✗) → normal path:
|
||||
- pg_terminate_backend for {db}
|
||||
- ALTER DATABASE {db} RENAME TO {db}_old
|
||||
- write plan: switch.renamedToOld=true, flush to disk ← checkpoint
|
||||
- ALTER DATABASE {db}_restore RENAME TO {db}
|
||||
- GRANT ALL PRIVILEGES ON DATABASE {db} TO {user}
|
||||
- record switch.previousDbName = {db}_old
|
||||
|
||||
(db=✗, restore=✓, old=✓) → crashed after first rename; complete it:
|
||||
- ALTER DATABASE {db}_restore RENAME TO {db}
|
||||
- GRANT ALL PRIVILEGES ON DATABASE {db} TO {user}
|
||||
- record switch.previousDbName = {db}_old
|
||||
|
||||
(db=✓, restore=✗, old=✓) → switch already complete; return nil
|
||||
|
||||
(db=✗, restore=✗, old=✓) → switch and cleanup both done; return nil
|
||||
|
||||
(db=✓, restore=✗, old=✗) → no restore DB found; return error
|
||||
|
||||
(any other combination) → log full state; return descriptive error
|
||||
|
||||
MySQL (same pattern, using RENAME TABLE loop via information_schema):
|
||||
Identical state machine. renameDatabase helper:
|
||||
- CREATE DATABASE {to}
|
||||
- For each table in {from}: RENAME TABLE {from}.{t} TO {to}.{t}
|
||||
- DROP DATABASE {from} (now empty)
|
||||
|
||||
Longhorn (per PVC, independently idempotent):
|
||||
For each PVC in plan.strategies[longhorn-native].restore.volumes:
|
||||
- Check: is PVC already bound to restoredVolumeName?
|
||||
If yes: record in completedPVCs, continue
|
||||
- Check: is this PVC in plan.switch.completedPVCs?
|
||||
If yes: skip (already done in a previous attempt)
|
||||
- Get current PV name and Longhorn volume handle
|
||||
- kubectl delete pvc {pvcName} -n {namespace} --ignore-not-found
|
||||
- Delete old PV if still exists (Retain policy)
|
||||
- Delete old Longhorn volume if it has a colored suffix (was itself a restore volume)
|
||||
- kubectl apply -f [PV YAML pointing to restoredVolumeName]
|
||||
- kubectl apply -f [PVC YAML with spec.volumeName = pvName]
|
||||
- Wait for PVC Bound
|
||||
- Append pvcName to plan.switch.completedPVCs, flush plan to disk ← checkpoint
|
||||
|
||||
Config: no-op
|
||||
|
||||
4. Scale up app namespace (coordinator — always runs, even if a strategy failed):
|
||||
kubectl scale deployment --all -n {namespace} --replicas=1
|
||||
kubectl scale statefulset --all -n {namespace} --replicas=1
|
||||
- Failure here is logged but does not change plan status; operator must intervene
|
||||
|
||||
5. Update config.yaml: apps.{app}.activeDeployment = standbyColor
|
||||
|
||||
6. Set plan.status = switched, write plan
|
||||
|
||||
Result: App running with restored data.
|
||||
{db}_old and old Longhorn volumes exist for cleanup.
|
||||
App is back up (possibly crash-looping if data is inconsistent —
|
||||
this is visible and diagnosable; operators check pod logs).
|
||||
```
|
||||
|
||||
### Cleanup
|
||||
|
||||
Removes the pre-restore data snapshots. Irreversible — no rollback after this.
|
||||
|
||||
```
|
||||
CLEANUP(app):
|
||||
1. Load RecoveryPlan, validate plan.status == switched
|
||||
- Set plan.status = cleaning_up
|
||||
|
||||
2. For each strategy:
|
||||
|
||||
Postgres:
|
||||
- Check {db}_old exists; if not, skip (idempotent)
|
||||
- pg_terminate_backend for {db}_old
|
||||
- DROP DATABASE IF EXISTS {db}_old
|
||||
|
||||
MySQL:
|
||||
- DROP DATABASE IF EXISTS {db}_old
|
||||
|
||||
Longhorn:
|
||||
- No-op: old PVC/PV/volume were already deleted during Switch
|
||||
- Verify no orphaned colored volumes remain (log warning if found)
|
||||
|
||||
Config: no-op
|
||||
|
||||
3. Set plan.status = cleaned_up, write plan
|
||||
|
||||
Result: Only canonical resources remain. Staging data is gone.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Invariants
|
||||
|
||||
1. **Namespace is always canonical** — never `gitea-green`, always `gitea`
|
||||
2. **Color tracks backup lineage** — used only for staging volume names during Restore; alternates each cycle so Longhorn incremental backups work
|
||||
3. **Brief downtime during Switch** — app is at zero replicas while data is swapped; expected 30–120s
|
||||
4. **Always scale up** — coordinator attempts scale-up on exit from Switch regardless of success/failure
|
||||
5. **Detect-then-reconcile** — every phase operation checks current state before acting
|
||||
6. **Checkpointed at points of no return** — plan is flushed to disk before and after destructive steps
|
||||
7. **Idempotent** — every operation uses IF EXISTS guards and existence checks; any phase can be retried
|
||||
8. **No rediscovery** — every phase reads from the plan, not the live system
|
||||
9. **In-place only** — restore always operates in the source namespace; no colored standby namespaces
|
||||
|
||||
---
|
||||
|
||||
## Failure Modes
|
||||
|
||||
| Failure point | Plan status | App state | Recovery |
|
||||
|---|---|---|---|
|
||||
| Backup: strategy fails | `failed` | Running normally | Fix issue, retry backup |
|
||||
| Backup: destination unreachable | `failed` | Running normally | Fix network, retry |
|
||||
| Restore: strategy fails | `failed` | Running normally (live data untouched) | Retry restore (idempotent) |
|
||||
| Switch: scale-down fails | `switching` (no data touched) | Running normally | Investigate why scale failed, retry |
|
||||
| Switch: Postgres rename crashes mid-way | `switching` | Scaled to 0 | Re-run Switch; detect-then-reconcile completes the rename |
|
||||
| Switch: Longhorn PVC swap crashes mid-way | `switching` | Scaled to 0 | Re-run Switch; completedPVCs checkpoint skips done PVCs |
|
||||
| Switch: strategy fails | `failed` | Scaled to 0, then scale-up attempted | Check pod logs; may be crash-looping — inspect plan for which step failed |
|
||||
| Switch: scale-up fails | `switched` | Scaled to 0 | Scale up manually: `kubectl scale deployment --all -n {ns} --replicas=1` |
|
||||
| Cleanup: strategy fails | `failed` | Running normally | Retry cleanup; all cleanup ops are idempotent |
|
||||
|
||||
---
|
||||
|
||||
## Destinations
|
||||
|
||||
### Backup Structure on Destination
|
||||
|
||||
```
|
||||
{destination}/
|
||||
├── cluster/ # Cluster-level backup (latest only)
|
||||
│ ├── kubeconfig
|
||||
│ ├── talosconfig
|
||||
│ ├── config.yaml
|
||||
│ ├── secrets.yaml
|
||||
│ └── setup/cluster-services/
|
||||
└── apps/
|
||||
└── {app}/
|
||||
├── recovery-plan.yaml # The RecoveryPlan for this backup
|
||||
├── config/{timestamp}.tar.gz # Config strategy artifact
|
||||
├── postgres/{timestamp}.dump # Postgres strategy artifact
|
||||
├── postgres/{timestamp}-globals.sql
|
||||
└── mysql/{timestamp}.dump # MySQL strategy artifact (if applicable)
|
||||
# Longhorn volumes are stored in Longhorn's own backup store (NFS/S3)
|
||||
# and referenced by backupURL in the RecoveryPlan
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Strategies
|
||||
|
||||
Each strategy implements four operations corresponding to the four phases.
|
||||
Strategies only read/write their own section of the RecoveryPlan.
|
||||
|
||||
### Strategy Interface
|
||||
|
||||
```
|
||||
Strategy:
|
||||
Backup(plan) → Discover params, export data, write to plan.strategies[name].{params, backup}
|
||||
Restore(plan) → Read plan targets, restore data into staging resources (app still live)
|
||||
Switch(plan) → Detect current state, reconcile to desired state, checkpoint progress
|
||||
Cleanup(plan) → Remove pre-restore snapshots listed in switch section
|
||||
```
|
||||
|
||||
### Config Strategy
|
||||
|
||||
Backup:
|
||||
1. Tar all files from plan.source.appDir
|
||||
2. Write to destination
|
||||
|
||||
Restore: no-op (config files are not changed; the canonical app dir is used throughout)
|
||||
|
||||
Switch: no-op
|
||||
|
||||
Cleanup: no-op
|
||||
|
||||
### Database Strategies (Postgres, MySQL)
|
||||
|
||||
Backup:
|
||||
1. Discover dbName and dbUser from config.yaml
|
||||
2. Record in plan params
|
||||
3. Stream dump to destination (pg_dump custom format / mysqldump)
|
||||
4. Backup globals (postgres only)
|
||||
|
||||
Restore:
|
||||
1. `DROP DATABASE IF EXISTS {db}_restore`
|
||||
2. `CREATE DATABASE {db}_restore`
|
||||
3. Import dump into `{db}_restore`
|
||||
4. Grant permissions to app user
|
||||
5. Transfer table ownership to app user (postgres)
|
||||
6. Record `restore.dbName = {db}_restore`
|
||||
|
||||
Switch (detect-then-reconcile, see full algorithm above):
|
||||
- Rename `{db} → {db}_old` and `{db}_restore → {db}` while app is scaled to zero
|
||||
- Checkpoint after first rename
|
||||
- Record `switch.previousDbName = {db}_old`
|
||||
|
||||
Cleanup:
|
||||
1. Terminate connections to `{db}_old`
|
||||
2. `DROP DATABASE IF EXISTS {db}_old`
|
||||
|
||||
### Longhorn Native Strategy
|
||||
|
||||
Backup:
|
||||
1. List PVCs in source namespace (skip `-cache` and `-tmp` volumes)
|
||||
2. For each PVC: get Longhorn volume name, create snapshot, create backup to NFS/S3
|
||||
3. Record params: pvcName, volumeName, size, accessMode per volume
|
||||
4. Record backup: backupURL per volume
|
||||
|
||||
Restore:
|
||||
1. For each PVC:
|
||||
- Check if staging volume `{pvcName}-{standbyColor}` already exists with the correct backupURL
|
||||
- If yes: skip (idempotent)
|
||||
- If no: `kubectl apply` a Longhorn Volume CR with `spec.fromBackup = backupURL`
|
||||
- Wait for volume to reach healthy/detached state
|
||||
- Create a PV pre-bound to this volume (claimRef points to source namespace)
|
||||
2. Record `restore.volumes[].volumeName = {pvcName}-{standbyColor}`
|
||||
|
||||
Switch (per PVC, independently idempotent, see full algorithm above):
|
||||
- Check if PVC already bound to restored volume; skip if so
|
||||
- Delete old PVC, PV, and Longhorn volume (if colored)
|
||||
- Create new PV + PVC bound to staging volume
|
||||
- Checkpoint per-PVC completion in plan
|
||||
|
||||
Cleanup: no-op (old volumes deleted during Switch)
|
||||
|
||||
### Cluster Strategy
|
||||
|
||||
Backs up cluster-level config (not per-app). Not part of the restore cycle.
|
||||
|
||||
Backup:
|
||||
1. Copy kubeconfig and talosconfig to destination
|
||||
2. Tar non-app sections of config.yaml and secrets.yaml
|
||||
3. Tar cluster-services directory
|
||||
|
||||
Restore:
|
||||
1. Verify cluster is accessible
|
||||
2. Restore kubeconfig and talosconfig
|
||||
3. Merge cluster config (preserve existing apps)
|
||||
4. Apply cluster services
|
||||
|
||||
---
|
||||
|
||||
## CLI
|
||||
|
||||
```bash
|
||||
# Backup
|
||||
wild app backup gitea # Backup app, create RecoveryPlan
|
||||
wild app backup --all # Backup all apps
|
||||
wild cluster backup # Backup cluster config
|
||||
|
||||
# Restore (stages data alongside live app)
|
||||
wild app restore gitea # Restore data to staging resources
|
||||
wild app restore gitea --from-instance prod-cloud # Migrate from another instance
|
||||
wild cluster restore
|
||||
|
||||
# Switch (brief downtime: scale down, swap data, scale up)
|
||||
wild app switch gitea
|
||||
|
||||
# Cleanup (irreversible: drops pre-restore data)
|
||||
wild app cleanup gitea
|
||||
|
||||
# Inspect
|
||||
wild app recovery-plan gitea # Show current RecoveryPlan
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Backup
|
||||
```
|
||||
POST /api/v1/instances/{instance}/apps/{app}/backup
|
||||
GET /api/v1/instances/{instance}/apps/{app}/backup
|
||||
POST /api/v1/instances/{instance}/backup
|
||||
GET /api/v1/instances/{instance}/backups
|
||||
```
|
||||
|
||||
### Restore / Switch / Cleanup
|
||||
```
|
||||
POST /api/v1/instances/{instance}/apps/{app}/restore
|
||||
Body: { "fromInstance": "source-instance" } // optional migration
|
||||
|
||||
POST /api/v1/instances/{instance}/apps/{app}/switch
|
||||
// Scales down, swaps data, scales up. Returns when app is back up.
|
||||
|
||||
POST /api/v1/instances/{instance}/apps/{app}/cleanup
|
||||
// Drops pre-restore snapshots. Irreversible.
|
||||
|
||||
GET /api/v1/instances/{instance}/apps/{app}/recovery-plan
|
||||
// Returns current RecoveryPlan with all phase and strategy details.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Operations Tracking
|
||||
|
||||
```
|
||||
// Leverage existing operations package:
|
||||
1. Create operation: operations.NewOperation("backup_app_gitea")
|
||||
2. Update progress: op.UpdateProgress(45, "Exporting volume...")
|
||||
3. Stream via SSE: events.Publish(Event{Type: "operation:progress", Data: op})
|
||||
4. Complete: op.Complete(result) or op.Fail(error)
|
||||
|
||||
// Progress milestones for backup:
|
||||
- 10%: Discovering strategies and params
|
||||
- 20-80%: Running strategies (divided equally per strategy)
|
||||
- 90%: Writing RecoveryPlan
|
||||
- 100%: Complete
|
||||
|
||||
// Progress milestones for restore:
|
||||
- 10%: Loading RecoveryPlan
|
||||
- 20-80%: Running strategy restores (divided equally)
|
||||
- 90%: Verifying staging resources
|
||||
- 100%: Complete
|
||||
|
||||
// Progress milestones for switch:
|
||||
- 10%: Loading RecoveryPlan
|
||||
- 20%: Scaling down namespace
|
||||
- 30-80%: Running strategy switches (divided equally)
|
||||
- 90%: Scaling up namespace, updating config.yaml
|
||||
- 100%: Complete
|
||||
|
||||
// Progress milestones for cleanup:
|
||||
- 10%: Loading RecoveryPlan
|
||||
- 20-80%: Running strategy cleanups (divided equally)
|
||||
- 100%: Complete
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
711
api/internal/backup/backup_test.go
Normal file
711
api/internal/backup/backup_test.go
Normal file
@@ -0,0 +1,711 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// MockStrategy implements the Strategy interface for testing
|
||||
type MockStrategy struct {
|
||||
Name_ string
|
||||
BackupFunc func(plan *RecoveryPlan, dest BackupDestination) error
|
||||
RestoreFunc func(plan *RecoveryPlan, dest BackupDestination) error
|
||||
SwitchFunc func(plan *RecoveryPlan) error
|
||||
CleanupFunc func(plan *RecoveryPlan) error
|
||||
VerifyFunc func(plan *RecoveryPlan, dest BackupDestination) error
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Name() string {
|
||||
return m.Name_
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Backup(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
if m.BackupFunc != nil {
|
||||
return m.BackupFunc(plan, dest)
|
||||
}
|
||||
entry := plan.GetStrategyEntry(m.Name_)
|
||||
if entry != nil {
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": "test/location",
|
||||
"size": int64(1024),
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Restore(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
if m.RestoreFunc != nil {
|
||||
return m.RestoreFunc(plan, dest)
|
||||
}
|
||||
entry := plan.GetStrategyEntry(m.Name_)
|
||||
if entry != nil {
|
||||
entry.Restore = map[string]interface{}{
|
||||
"restored": true,
|
||||
}
|
||||
entry.Status = "restored"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Switch(plan *RecoveryPlan) error {
|
||||
if m.SwitchFunc != nil {
|
||||
return m.SwitchFunc(plan)
|
||||
}
|
||||
entry := plan.GetStrategyEntry(m.Name_)
|
||||
if entry != nil {
|
||||
entry.Status = "switched"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Cleanup(plan *RecoveryPlan) error {
|
||||
if m.CleanupFunc != nil {
|
||||
return m.CleanupFunc(plan)
|
||||
}
|
||||
entry := plan.GetStrategyEntry(m.Name_)
|
||||
if entry != nil {
|
||||
entry.Status = "cleaned_up"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockStrategy) Verify(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
if m.VerifyFunc != nil {
|
||||
return m.VerifyFunc(plan, dest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MockDestination implements BackupDestination for testing
|
||||
type MockDestination struct {
|
||||
PutFunc func(string, io.Reader) (int64, error)
|
||||
GetFunc func(string) (io.ReadCloser, error)
|
||||
DeleteFunc func(string) error
|
||||
ListFunc func(string) ([]BackupObject, error)
|
||||
GetURLFunc func(string, time.Duration) (string, error)
|
||||
TypeFunc func() string
|
||||
}
|
||||
|
||||
func (m *MockDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
if m.PutFunc != nil {
|
||||
return m.PutFunc(key, reader)
|
||||
}
|
||||
data, _ := io.ReadAll(reader)
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (m *MockDestination) Get(key string) (io.ReadCloser, error) {
|
||||
if m.GetFunc != nil {
|
||||
return m.GetFunc(key)
|
||||
}
|
||||
return io.NopCloser(strings.NewReader("test data")), nil
|
||||
}
|
||||
|
||||
func (m *MockDestination) Delete(key string) error {
|
||||
if m.DeleteFunc != nil {
|
||||
return m.DeleteFunc(key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockDestination) List(prefix string) ([]BackupObject, error) {
|
||||
if m.ListFunc != nil {
|
||||
return m.ListFunc(prefix)
|
||||
}
|
||||
return []BackupObject{}, nil
|
||||
}
|
||||
|
||||
func (m *MockDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
if m.GetURLFunc != nil {
|
||||
return m.GetURLFunc(key, expiry)
|
||||
}
|
||||
return "https://example.com/backup/" + key, nil
|
||||
}
|
||||
|
||||
func (m *MockDestination) Type() string {
|
||||
if m.TypeFunc != nil {
|
||||
return m.TypeFunc()
|
||||
}
|
||||
return "mock"
|
||||
}
|
||||
|
||||
func TestBackupApp(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
appsDir := filepath.Join(instanceDir, "data", "apps", appName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(appsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
|
||||
// Create test manifest with postgres dependency
|
||||
manifestContent := `
|
||||
name: test-app
|
||||
description: Test application
|
||||
version: 1.0.0
|
||||
requires:
|
||||
- name: postgres
|
||||
defaultConfig:
|
||||
image: test:latest
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(appsDir, "manifest.yaml"), []byte(manifestContent), 0644))
|
||||
|
||||
// Create test config
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + backupsDir + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
// Create manager with mock strategies
|
||||
mgr := NewManager(tempDir)
|
||||
|
||||
backupCalled := false
|
||||
mockStrategy := &MockStrategy{
|
||||
Name_: "config",
|
||||
BackupFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
backupCalled = true
|
||||
assert.Equal(t, instanceName, plan.Instance)
|
||||
assert.Equal(t, appName, plan.App)
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry != nil {
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": "test/location",
|
||||
"size": int64(2048),
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"postgres": mockStrategy,
|
||||
"config": mockStrategy,
|
||||
}
|
||||
|
||||
plan, err := mgr.BackupApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, plan)
|
||||
assert.Equal(t, appName, plan.App)
|
||||
assert.Equal(t, "backed_up", plan.Status)
|
||||
assert.True(t, backupCalled)
|
||||
assert.Equal(t, appName, plan.Source.Namespace)
|
||||
}
|
||||
|
||||
func TestBackupAppCreatesRecoveryPlan(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
appsDir := filepath.Join(instanceDir, "data", "apps", appName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(appsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
|
||||
manifestContent := `
|
||||
name: test-app
|
||||
description: Test application
|
||||
version: 1.0.0
|
||||
defaultConfig:
|
||||
image: test:latest
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(appsDir, "manifest.yaml"), []byte(manifestContent), 0644))
|
||||
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + backupsDir + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{Name_: "config"},
|
||||
}
|
||||
|
||||
plan, err := mgr.BackupApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify plan was saved as YAML (saved in data/backup/records/{appName}/{timestamp}/)
|
||||
planFile := filepath.Join(instanceDir, "data", "backup", "records", appName, plan.Timestamp, "recovery-plan.yaml")
|
||||
_, err = os.Stat(planFile)
|
||||
assert.NoError(t, err, "recovery-plan.yaml should exist")
|
||||
|
||||
// Load and verify plan contents
|
||||
data, err := os.ReadFile(planFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
var savedPlan btypes.RecoveryPlan
|
||||
require.NoError(t, yaml.Unmarshal(data, &savedPlan))
|
||||
assert.Equal(t, appName, savedPlan.App)
|
||||
assert.Equal(t, instanceName, savedPlan.Instance)
|
||||
assert.Equal(t, "backed_up", savedPlan.Status)
|
||||
}
|
||||
|
||||
func TestListBackups(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
backupsDir := filepath.Join(tempDir, "instances", instanceName, "data", "backup", "records", appName)
|
||||
|
||||
timestamps := []string{
|
||||
"20240101T120000Z",
|
||||
"20240102T120000Z",
|
||||
"20240103T120000Z",
|
||||
}
|
||||
|
||||
for _, ts := range timestamps {
|
||||
backupDir := filepath.Join(backupsDir, ts)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: ts,
|
||||
Status: "backed_up",
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
}
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
plans, err := mgr.ListBackups(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, plans, 3)
|
||||
|
||||
// Should be sorted newest first
|
||||
assert.Equal(t, "20240103T120000Z", plans[0].Timestamp)
|
||||
assert.Equal(t, "20240102T120000Z", plans[1].Timestamp)
|
||||
assert.Equal(t, "20240101T120000Z", plans[2].Timestamp)
|
||||
}
|
||||
|
||||
func TestDeleteAppBackup(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
timestamp := "20240101T120000Z"
|
||||
|
||||
backupDir := filepath.Join(tempDir, "instances", instanceName, "data", "backup", "records", appName, timestamp)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: timestamp,
|
||||
Status: "backed_up",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "config",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "test/backup.tar.gz",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
|
||||
// Create config
|
||||
configDir := filepath.Join(tempDir, "instances", instanceName, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: /tmp/backups
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
err := mgr.DeleteAppBackup(instanceName, appName, timestamp)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = os.Stat(backupDir)
|
||||
assert.True(t, os.IsNotExist(err), "Backup directory should be deleted")
|
||||
}
|
||||
|
||||
func TestProgressCallback(t *testing.T) {
|
||||
t.Run("NewManagerWithProgress creates manager with callback", func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
callbackCalled := false
|
||||
var receivedProgress int
|
||||
var receivedMessage string
|
||||
|
||||
callback := func(progress int, message string) {
|
||||
callbackCalled = true
|
||||
receivedProgress = progress
|
||||
receivedMessage = message
|
||||
}
|
||||
|
||||
mgr := NewManagerWithProgress(tempDir, callback)
|
||||
assert.NotNil(t, mgr)
|
||||
assert.NotNil(t, mgr.progressCallback)
|
||||
|
||||
mgr.reportProgress(50, "Test message")
|
||||
assert.True(t, callbackCalled)
|
||||
assert.Equal(t, 50, receivedProgress)
|
||||
assert.Equal(t, "Test message", receivedMessage)
|
||||
})
|
||||
|
||||
t.Run("NewManager creates manager without callback", func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
mgr := NewManager(tempDir)
|
||||
assert.NotNil(t, mgr)
|
||||
assert.Nil(t, mgr.progressCallback)
|
||||
|
||||
assert.NotPanics(t, func() {
|
||||
mgr.reportProgress(50, "Test")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("BackupApp reports progress", func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
progressReports := []struct {
|
||||
progress int
|
||||
message string
|
||||
}{}
|
||||
|
||||
callback := func(progress int, message string) {
|
||||
progressReports = append(progressReports, struct {
|
||||
progress int
|
||||
message string
|
||||
}{progress, message})
|
||||
}
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
appsDir := filepath.Join(instanceDir, "data", "apps", appName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(appsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
|
||||
manifestContent := `
|
||||
name: test-app
|
||||
description: Test application
|
||||
version: 1.0.0
|
||||
defaultConfig:
|
||||
image: test:latest
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(appsDir, "manifest.yaml"), []byte(manifestContent), 0644))
|
||||
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + backupsDir + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
mgr := NewManagerWithProgress(tempDir, callback)
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{Name_: "config"},
|
||||
}
|
||||
|
||||
_, err := mgr.BackupApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Greater(t, len(progressReports), 0)
|
||||
|
||||
foundMessages := make(map[string]bool)
|
||||
for _, report := range progressReports {
|
||||
if strings.Contains(report.message, "Loading backup configuration") {
|
||||
foundMessages["config"] = true
|
||||
}
|
||||
if strings.Contains(report.message, "Loading app manifest") {
|
||||
foundMessages["manifest"] = true
|
||||
}
|
||||
if strings.Contains(report.message, "Backup completed") {
|
||||
foundMessages["completed"] = true
|
||||
assert.Equal(t, 100, report.progress)
|
||||
}
|
||||
}
|
||||
|
||||
assert.True(t, foundMessages["config"], "Should report loading configuration")
|
||||
assert.True(t, foundMessages["manifest"], "Should report loading manifest")
|
||||
assert.True(t, foundMessages["completed"], "Should report completion")
|
||||
})
|
||||
}
|
||||
|
||||
func TestVerifyBackup(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
timestamp := "20240101T120000Z"
|
||||
|
||||
backupDir := filepath.Join(tempDir, "instances", instanceName, "data", "backup", "records", appName, timestamp)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: timestamp,
|
||||
Status: "backed_up",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "config",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "test/config.tar.gz",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "postgres",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "test/postgres.dump",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
|
||||
// Create config
|
||||
configDir := filepath.Join(tempDir, "instances", instanceName, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: /tmp/backups
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
|
||||
configVerifyCount := 0
|
||||
postgresVerifyCount := 0
|
||||
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{
|
||||
Name_: "config",
|
||||
VerifyFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
configVerifyCount++
|
||||
return nil
|
||||
},
|
||||
},
|
||||
"postgres": &MockStrategy{
|
||||
Name_: "postgres",
|
||||
VerifyFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
postgresVerifyCount++
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := mgr.VerifyBackup(instanceName, appName, timestamp)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.Success)
|
||||
assert.Len(t, result.Components, 2)
|
||||
assert.Equal(t, 1, configVerifyCount)
|
||||
assert.Equal(t, 1, postgresVerifyCount)
|
||||
|
||||
for _, comp := range result.Components {
|
||||
assert.True(t, comp.Success)
|
||||
assert.Empty(t, comp.Error)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func TestRestoreAppInPlace(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "postgres"
|
||||
timestamp := "20240101T120000Z"
|
||||
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName, timestamp)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
// Config with backup destination
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + filepath.Join(instanceDir, "backups") + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
// Saved plan in backed_up status
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: timestamp,
|
||||
Status: "backed_up",
|
||||
Source: btypes.RecoverySource{
|
||||
Namespace: appName,
|
||||
AppDir: filepath.Join("instances", instanceName, "apps", appName),
|
||||
},
|
||||
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{Name: "config", Status: "backed_up"},
|
||||
},
|
||||
Phases: map[string]btypes.PhaseTime{},
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{Name_: "config"},
|
||||
}
|
||||
|
||||
restored, err := mgr.RestoreApp(instanceName, appName, RestoreOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "restored", restored.Status)
|
||||
// In-place: standby namespace is the source namespace (not a colored standby)
|
||||
assert.Equal(t, appName, restored.Standby.Namespace)
|
||||
assert.Equal(t, plan.Source.AppDir, restored.Standby.AppDir)
|
||||
}
|
||||
|
||||
func TestSwitchAppInPlaceUpdatesActiveDeployment(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "postgres"
|
||||
timestamp := "20240101T120000Z"
|
||||
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName, timestamp)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
// Config with backup destination and existing app config
|
||||
configContent := `
|
||||
apps:
|
||||
postgres:
|
||||
namespace: postgres
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + filepath.Join(instanceDir, "backups") + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
// Saved plan in restored state with in-place mode
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: timestamp,
|
||||
Status: "restored",
|
||||
Source: btypes.RecoverySource{
|
||||
Namespace: appName,
|
||||
},
|
||||
Standby: btypes.RecoveryStandby{
|
||||
Namespace: appName,
|
||||
},
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{Name: "config", Status: "restored"},
|
||||
},
|
||||
Phases: map[string]btypes.PhaseTime{},
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{Name_: "config"},
|
||||
}
|
||||
|
||||
switched, err := mgr.SwitchApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "switched", switched.Status)
|
||||
}
|
||||
|
||||
func TestCleanupAppInPlaceSkipsNamespaceDeletion(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "postgres"
|
||||
timestamp := "20240101T120000Z"
|
||||
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName, timestamp)
|
||||
require.NoError(t, os.MkdirAll(backupDir, 0755))
|
||||
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + filepath.Join(instanceDir, "backups") + `
|
||||
`
|
||||
configDir := filepath.Join(instanceDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "instance.yaml"), []byte(configContent), 0644))
|
||||
|
||||
// Saved plan in switched state
|
||||
plan := &btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: timestamp,
|
||||
Status: "switched",
|
||||
Source: btypes.RecoverySource{
|
||||
Namespace: appName,
|
||||
},
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{Name: "config", Status: "switched"},
|
||||
},
|
||||
Phases: map[string]btypes.PhaseTime{},
|
||||
}
|
||||
|
||||
data, _ := yaml.Marshal(plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(backupDir, "recovery-plan.yaml"), data, 0600))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": &MockStrategy{Name_: "config"},
|
||||
}
|
||||
|
||||
// Should succeed without trying to run kubectl (no cluster access needed)
|
||||
cleaned, err := mgr.CleanupApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "cleaned_up", cleaned.Status)
|
||||
}
|
||||
@@ -1,408 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// ClusterBackupComponents represents which components to include in backup
|
||||
type ClusterBackupComponents struct {
|
||||
Etcd bool `json:"etcd"`
|
||||
Config bool `json:"config"`
|
||||
Secrets bool `json:"secrets"`
|
||||
}
|
||||
|
||||
// ClusterBackupInfo extends BackupInfo for cluster-level backups
|
||||
type ClusterBackupInfo struct {
|
||||
*BackupInfo
|
||||
InstanceName string `json:"instance_name"`
|
||||
Components ClusterBackupComponents `json:"components"`
|
||||
}
|
||||
|
||||
// BackupCluster creates a backup of cluster components (etcd, config, secrets)
|
||||
func (m *Manager) BackupCluster(instanceName string, components ClusterBackupComponents) (*ClusterBackupInfo, error) {
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
if err := storage.EnsureDir(stagingDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create staging directory: %w", err)
|
||||
}
|
||||
|
||||
timestamp := time.Now().UTC().Format("20060102T150405Z")
|
||||
clusterBackupDir := filepath.Join(stagingDir, "cluster", timestamp)
|
||||
|
||||
if err := os.RemoveAll(clusterBackupDir); err != nil && !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("failed to clean backup directory: %w", err)
|
||||
}
|
||||
if err := storage.EnsureDir(clusterBackupDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create backup directory: %w", err)
|
||||
}
|
||||
|
||||
info := &ClusterBackupInfo{
|
||||
BackupInfo: &BackupInfo{
|
||||
AppName: "cluster",
|
||||
Timestamp: timestamp,
|
||||
Type: "cluster",
|
||||
Status: "in_progress",
|
||||
Files: []string{},
|
||||
CreatedAt: time.Now(),
|
||||
},
|
||||
InstanceName: instanceName,
|
||||
Components: components,
|
||||
}
|
||||
|
||||
// Save initial in_progress metadata immediately so it's visible in list operations
|
||||
metaFile := filepath.Join(clusterBackupDir, "cluster-backup.json")
|
||||
if err := m.saveClusterBackupMeta(metaFile, info); err != nil {
|
||||
return nil, fmt.Errorf("failed to save initial backup metadata: %w", err)
|
||||
}
|
||||
|
||||
// Backup etcd if requested
|
||||
if components.Etcd {
|
||||
etcdFile, err := m.backupEtcd(instanceName, clusterBackupDir, timestamp, stagingDir)
|
||||
if err != nil {
|
||||
info.Status = "failed"
|
||||
info.Error = fmt.Sprintf("etcd backup failed: %v", err)
|
||||
} else if etcdFile != "" {
|
||||
info.Files = append(info.Files, etcdFile)
|
||||
}
|
||||
}
|
||||
|
||||
// Backup config if requested
|
||||
if components.Config {
|
||||
configFile, err := m.backupConfig(instanceName, clusterBackupDir, stagingDir)
|
||||
if err != nil && info.Status != "failed" {
|
||||
info.Status = "failed"
|
||||
info.Error = fmt.Sprintf("config backup failed: %v", err)
|
||||
} else if configFile != "" {
|
||||
info.Files = append(info.Files, configFile)
|
||||
}
|
||||
}
|
||||
|
||||
// Backup secrets if requested
|
||||
if components.Secrets {
|
||||
secretsFile, err := m.backupSecrets(instanceName, clusterBackupDir, stagingDir)
|
||||
if err != nil && info.Status != "failed" {
|
||||
info.Status = "failed"
|
||||
info.Error = fmt.Sprintf("secrets backup failed: %v", err)
|
||||
} else if secretsFile != "" {
|
||||
info.Files = append(info.Files, secretsFile)
|
||||
}
|
||||
}
|
||||
|
||||
if info.Status != "failed" {
|
||||
info.Status = "completed"
|
||||
}
|
||||
|
||||
// Calculate backup size
|
||||
info.Size = m.calculateBackupSize(clusterBackupDir)
|
||||
|
||||
// Update metadata with final status (overwrites the in_progress version)
|
||||
if err := m.saveClusterBackupMeta(metaFile, info); err != nil {
|
||||
return nil, fmt.Errorf("failed to save final backup metadata: %w", err)
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// RestoreCluster restores cluster from backup
|
||||
func (m *Manager) RestoreCluster(instanceName, timestamp string, components ClusterBackupComponents) error {
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
backupDir := filepath.Join(stagingDir, "cluster", timestamp)
|
||||
|
||||
if !storage.FileExists(backupDir) {
|
||||
return fmt.Errorf("no cluster backup found for timestamp %s", timestamp)
|
||||
}
|
||||
|
||||
// Restore etcd if requested and available
|
||||
if components.Etcd {
|
||||
if err := m.restoreEtcd(instanceName, backupDir); err != nil {
|
||||
return fmt.Errorf("etcd restore failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Restore config if requested and available
|
||||
if components.Config {
|
||||
if err := m.restoreConfig(instanceName, backupDir); err != nil {
|
||||
return fmt.Errorf("config restore failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Restore secrets if requested and available
|
||||
if components.Secrets {
|
||||
if err := m.restoreSecrets(instanceName, backupDir); err != nil {
|
||||
return fmt.Errorf("secrets restore failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListClusterBackups returns all cluster backups for an instance
|
||||
func (m *Manager) ListClusterBackups(instanceName string) ([]*ClusterBackupInfo, error) {
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
clusterBackupsDir := filepath.Join(stagingDir, "cluster")
|
||||
|
||||
if !storage.FileExists(clusterBackupsDir) {
|
||||
return []*ClusterBackupInfo{}, nil
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(clusterBackupsDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read cluster backups directory: %w", err)
|
||||
}
|
||||
|
||||
var backups []*ClusterBackupInfo
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
metaFile := filepath.Join(clusterBackupsDir, entry.Name(), "cluster-backup.json")
|
||||
if storage.FileExists(metaFile) {
|
||||
info, err := m.loadClusterBackupMeta(metaFile)
|
||||
if err == nil {
|
||||
backups = append(backups, info)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return backups, nil
|
||||
}
|
||||
|
||||
// DeleteClusterBackup deletes a specific cluster backup
|
||||
func (m *Manager) DeleteClusterBackup(instanceName, timestamp string) error {
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
backupDir := filepath.Join(stagingDir, "cluster", timestamp)
|
||||
|
||||
if !storage.FileExists(backupDir) {
|
||||
return fmt.Errorf("backup not found: %s", timestamp)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(backupDir); err != nil {
|
||||
return fmt.Errorf("failed to delete backup: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAllBackups returns both app and cluster backups
|
||||
func (m *Manager) ListAllBackups(instanceName string) (map[string]interface{}, error) {
|
||||
// Get cluster backups
|
||||
clusterBackups, err := m.ListClusterBackups(instanceName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list cluster backups: %w", err)
|
||||
}
|
||||
|
||||
// Get app backups
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
appsBackupsDir := filepath.Join(stagingDir, "apps")
|
||||
|
||||
appBackups := make(map[string][]*BackupInfo)
|
||||
if storage.FileExists(appsBackupsDir) {
|
||||
entries, err := os.ReadDir(appsBackupsDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read app backups directory: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
appName := entry.Name()
|
||||
backups, err := m.ListBackups(instanceName, appName)
|
||||
if err == nil && len(backups) > 0 {
|
||||
appBackups[appName] = backups
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"cluster": clusterBackups,
|
||||
"apps": appBackups,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// backupEtcd creates an etcd snapshot
|
||||
func (m *Manager) backupEtcd(instanceName, backupDir, timestamp, stagingDir string) (string, error) {
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
|
||||
etcdFile := filepath.Join(backupDir, fmt.Sprintf("etcd_%s.snapshot", timestamp))
|
||||
|
||||
// Get control plane node IP
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read instance config: %w", err)
|
||||
}
|
||||
|
||||
// Simple extraction of first control plane IP (could be improved with proper YAML parsing)
|
||||
// For now, assuming talosctl is available and configured
|
||||
cmd := exec.Command("kubectl", "get", "nodes", "-l", "node-role.kubernetes.io/control-plane",
|
||||
"-o", "jsonpath={.items[0].status.addresses[?(@.type==\"InternalIP\")].address}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
ipOutput, err := cmd.Output()
|
||||
if err != nil || len(ipOutput) == 0 {
|
||||
return "", fmt.Errorf("failed to get control plane node IP: %w", err)
|
||||
}
|
||||
|
||||
nodeIP := string(ipOutput)
|
||||
|
||||
// Use talosctl to create etcd snapshot
|
||||
talosConfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
cmd = exec.Command("talosctl", "-n", nodeIP, "--talosconfig", talosConfigPath,
|
||||
"etcd", "snapshot", etcdFile)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("etcd snapshot failed: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
_ = configData // Used for potential future enhancements
|
||||
|
||||
return m.cleanBackupPath(etcdFile, stagingDir), nil
|
||||
}
|
||||
|
||||
// backupConfig backs up instance config.yaml
|
||||
func (m *Manager) backupConfig(instanceName, backupDir, stagingDir string) (string, error) {
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
if !storage.FileExists(configPath) {
|
||||
return "", fmt.Errorf("config file not found")
|
||||
}
|
||||
|
||||
configFile := filepath.Join(backupDir, "config.yaml")
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read config: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(configFile, data, 0600); err != nil {
|
||||
return "", fmt.Errorf("failed to write config backup: %w", err)
|
||||
}
|
||||
|
||||
return m.cleanBackupPath(configFile, stagingDir), nil
|
||||
}
|
||||
|
||||
// backupSecrets backs up instance secrets.yaml
|
||||
func (m *Manager) backupSecrets(instanceName, backupDir, stagingDir string) (string, error) {
|
||||
secretsPath := tools.GetInstanceSecretsPath(m.dataDir, instanceName)
|
||||
if !storage.FileExists(secretsPath) {
|
||||
return "", fmt.Errorf("secrets file not found")
|
||||
}
|
||||
|
||||
secretsFile := filepath.Join(backupDir, "secrets.yaml")
|
||||
data, err := os.ReadFile(secretsPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read secrets: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(secretsFile, data, 0600); err != nil {
|
||||
return "", fmt.Errorf("failed to write secrets backup: %w", err)
|
||||
}
|
||||
|
||||
return m.cleanBackupPath(secretsFile, stagingDir), nil
|
||||
}
|
||||
|
||||
// restoreEtcd restores etcd from snapshot
|
||||
func (m *Manager) restoreEtcd(instanceName, backupDir string) error {
|
||||
// Find etcd snapshot file
|
||||
matches, err := filepath.Glob(filepath.Join(backupDir, "etcd_*.snapshot"))
|
||||
if err != nil || len(matches) == 0 {
|
||||
return fmt.Errorf("no etcd snapshot found in backup")
|
||||
}
|
||||
|
||||
snapshotFile := matches[0]
|
||||
|
||||
// Get control plane node IP
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
|
||||
cmd := exec.Command("kubectl", "get", "nodes", "-l", "node-role.kubernetes.io/control-plane",
|
||||
"-o", "jsonpath={.items[0].status.addresses[?(@.type==\"InternalIP\")].address}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
ipOutput, err := cmd.Output()
|
||||
if err != nil || len(ipOutput) == 0 {
|
||||
return fmt.Errorf("failed to get control plane node IP: %w", err)
|
||||
}
|
||||
|
||||
nodeIP := string(ipOutput)
|
||||
|
||||
// Use talosctl to bootstrap from snapshot
|
||||
talosConfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
cmd = exec.Command("talosctl", "-n", nodeIP, "--talosconfig", talosConfigPath,
|
||||
"bootstrap", "--recover-from", snapshotFile)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("etcd restore failed: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreConfig restores instance config.yaml
|
||||
func (m *Manager) restoreConfig(instanceName, backupDir string) error {
|
||||
configBackup := filepath.Join(backupDir, "config.yaml")
|
||||
if !storage.FileExists(configBackup) {
|
||||
return fmt.Errorf("config backup not found")
|
||||
}
|
||||
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
data, err := os.ReadFile(configBackup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read config backup: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(configPath, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to restore config: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreSecrets restores instance secrets.yaml
|
||||
func (m *Manager) restoreSecrets(instanceName, backupDir string) error {
|
||||
secretsBackup := filepath.Join(backupDir, "secrets.yaml")
|
||||
if !storage.FileExists(secretsBackup) {
|
||||
return fmt.Errorf("secrets backup not found")
|
||||
}
|
||||
|
||||
secretsPath := tools.GetInstanceSecretsPath(m.dataDir, instanceName)
|
||||
data, err := os.ReadFile(secretsBackup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read secrets backup: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(secretsPath, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to restore secrets: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveClusterBackupMeta saves cluster backup metadata to JSON file
|
||||
func (m *Manager) saveClusterBackupMeta(path string, info *ClusterBackupInfo) error {
|
||||
data, err := json.MarshalIndent(info, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
// loadClusterBackupMeta loads cluster backup metadata from JSON file
|
||||
func (m *Manager) loadClusterBackupMeta(path string) (*ClusterBackupInfo, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info ClusterBackupInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
196
api/internal/backup/cluster_backup_test.go
Normal file
196
api/internal/backup/cluster_backup_test.go
Normal file
@@ -0,0 +1,196 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBackupClusterConfig(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "config"), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "data", "talos"), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "data"), 0755))
|
||||
|
||||
// Create cluster config files at new XDG paths
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "kubeconfig"), []byte("kubeconfig-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), []byte("backup:\n destination:\n type: local\n local:\n path: "+backupsDir+"\n"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "secrets.yaml"), []byte("secrets-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "talos", "talosconfig"), []byte("talosconfig-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "talos", "controlplane.yaml"), []byte("controlplane-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "talos", "worker.yaml"), []byte("worker-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "talos", "pki.yaml"), []byte("talos-pki-data"), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
plan, err := mgr.BackupClusterConfig(instanceName)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, plan)
|
||||
|
||||
assert.Equal(t, "_cluster", plan.App)
|
||||
assert.Equal(t, instanceName, plan.Instance)
|
||||
assert.Equal(t, "backed_up", plan.Status)
|
||||
assert.Len(t, plan.Strategies, 1)
|
||||
assert.Equal(t, "cluster-config", plan.Strategies[0].Name)
|
||||
assert.Equal(t, "backed_up", plan.Strategies[0].Status)
|
||||
|
||||
// Check backup metadata
|
||||
files, ok := plan.Strategies[0].Backup["files"].(int)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 7, files)
|
||||
|
||||
// Verify plan was saved to disk (new path: data/backup/records)
|
||||
planFile := filepath.Join(instanceDir, "data", "backup", "records", "_cluster", plan.Timestamp, "recovery-plan.yaml")
|
||||
_, err = os.Stat(planFile)
|
||||
assert.NoError(t, err, "recovery-plan.yaml should exist")
|
||||
}
|
||||
|
||||
func TestBackupClusterConfigSkipsMissingFiles(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "config"), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "data"), 0755))
|
||||
|
||||
// Only create kubeconfig and config/instance.yaml (no talos files)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "kubeconfig"), []byte("kubeconfig-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), []byte("backup:\n destination:\n type: local\n local:\n path: "+backupsDir+"\n"), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
plan, err := mgr.BackupClusterConfig(instanceName)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, plan)
|
||||
|
||||
assert.Equal(t, "backed_up", plan.Status)
|
||||
files, ok := plan.Strategies[0].Backup["files"].(int)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 2, files)
|
||||
}
|
||||
|
||||
func TestBackupClusterConfigFailsWithNoFiles(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "config"), 0755))
|
||||
|
||||
// Create only config/instance.yaml for backup destination config, but none of the other cluster files
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), []byte("backup:\n destination:\n type: local\n local:\n path: "+backupsDir+"\n"), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
_, err := mgr.BackupClusterConfig(instanceName)
|
||||
// config/instance.yaml itself is one of the files, so it will be found
|
||||
// To truly have zero files, we need to remove config/instance.yaml too,
|
||||
// but then loadDestination fails first. So this test verifies
|
||||
// that config/instance.yaml IS included in the backup.
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestBackupClusterConfigArchiveContents(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "config"), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "data", "talos"), 0755))
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "kubeconfig"), []byte("kubeconfig-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), []byte("backup:\n destination:\n type: local\n local:\n path: "+backupsDir+"\n"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "talos", "talosconfig"), []byte("talosconfig-data"), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
plan, err := mgr.BackupClusterConfig(instanceName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Read the archive from the local backup destination
|
||||
location, ok := plan.Strategies[0].Backup["location"].(string)
|
||||
require.True(t, ok)
|
||||
|
||||
archivePath := filepath.Join(backupsDir, location)
|
||||
f, err := os.Open(archivePath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
gzReader, err := gzip.NewReader(f)
|
||||
require.NoError(t, err)
|
||||
defer gzReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzReader)
|
||||
|
||||
var fileNames []string
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
fileNames = append(fileNames, header.Name)
|
||||
}
|
||||
|
||||
assert.Contains(t, fileNames, filepath.Join("data", "kubeconfig"))
|
||||
assert.Contains(t, fileNames, filepath.Join("config", "instance.yaml"))
|
||||
assert.Contains(t, fileNames, filepath.Join("data", "talos", "talosconfig"))
|
||||
}
|
||||
|
||||
func TestBackupClusterConfigListAndDelete(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
instanceName := "test-instance"
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "config"), 0755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(instanceDir, "data"), 0755))
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "data", "kubeconfig"), []byte("kubeconfig-data"), 0644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), []byte("backup:\n destination:\n type: local\n local:\n path: "+backupsDir+"\n"), 0644))
|
||||
|
||||
mgr := NewManager(tempDir)
|
||||
|
||||
// Create two backups (sleep to ensure different timestamps)
|
||||
plan1, err := mgr.BackupClusterConfig(instanceName)
|
||||
require.NoError(t, err)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
plan2, err := mgr.BackupClusterConfig(instanceName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// List backups
|
||||
plans, err := mgr.ListBackups(instanceName, "_cluster")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, plans, 2)
|
||||
// Newest first
|
||||
assert.Equal(t, plan2.Timestamp, plans[0].Timestamp)
|
||||
assert.Equal(t, plan1.Timestamp, plans[1].Timestamp)
|
||||
|
||||
// Delete one
|
||||
err = mgr.DeleteAppBackup(instanceName, "_cluster", plan1.Timestamp)
|
||||
require.NoError(t, err)
|
||||
|
||||
plans, err = mgr.ListBackups(instanceName, "_cluster")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, plans, 1)
|
||||
assert.Equal(t, plan2.Timestamp, plans[0].Timestamp)
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type BackupConfig struct {
|
||||
Repository string `yaml:"repository"`
|
||||
Staging string `yaml:"staging"`
|
||||
Retention ResticRetentionPolicy `yaml:"retention"`
|
||||
Backend BackendConfig `yaml:"backend"`
|
||||
}
|
||||
|
||||
type ResticRetentionPolicy struct {
|
||||
KeepDaily int `yaml:"keepDaily"`
|
||||
KeepWeekly int `yaml:"keepWeekly"`
|
||||
KeepMonthly int `yaml:"keepMonthly"`
|
||||
KeepYearly int `yaml:"keepYearly"`
|
||||
}
|
||||
|
||||
type BackendConfig struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Region string `yaml:"region"`
|
||||
}
|
||||
|
||||
type BackupSecrets struct {
|
||||
Password string `yaml:"password"`
|
||||
Credentials BackupCredentials `yaml:"credentials"`
|
||||
}
|
||||
|
||||
type BackupCredentials struct {
|
||||
S3 *S3Credentials `yaml:"s3,omitempty"`
|
||||
SFTP *SFTPCredentials `yaml:"sftp,omitempty"`
|
||||
Azure *AzureCredentials `yaml:"azure,omitempty"`
|
||||
GCS *GCSCredentials `yaml:"gcs,omitempty"`
|
||||
}
|
||||
|
||||
type S3Credentials struct {
|
||||
AccessKeyID string `yaml:"accessKeyId"`
|
||||
SecretAccessKey string `yaml:"secretAccessKey"`
|
||||
}
|
||||
|
||||
type SFTPCredentials struct {
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
KeyFile string `yaml:"keyFile,omitempty"`
|
||||
}
|
||||
|
||||
type AzureCredentials struct {
|
||||
AccountName string `yaml:"accountName"`
|
||||
AccountKey string `yaml:"accountKey"`
|
||||
}
|
||||
|
||||
type GCSCredentials struct {
|
||||
ProjectID string `yaml:"projectId"`
|
||||
CredentialsJSON string `yaml:"credentialsJson"`
|
||||
}
|
||||
|
||||
func LoadBackupConfig(dataDir string) (*BackupConfig, error) {
|
||||
configPath := filepath.Join(dataDir, "config.yaml")
|
||||
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read config: %w", err)
|
||||
}
|
||||
|
||||
var root struct {
|
||||
Cloud struct {
|
||||
Backup BackupConfig `yaml:"backup"`
|
||||
} `yaml:"cloud"`
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
config := &root.Cloud.Backup
|
||||
|
||||
if config.Repository == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if config.Staging == "" {
|
||||
config.Staging = filepath.Join(dataDir, "backup-staging")
|
||||
}
|
||||
|
||||
if config.Retention.KeepDaily == 0 {
|
||||
config.Retention.KeepDaily = 7
|
||||
}
|
||||
if config.Retention.KeepWeekly == 0 {
|
||||
config.Retention.KeepWeekly = 4
|
||||
}
|
||||
if config.Retention.KeepMonthly == 0 {
|
||||
config.Retention.KeepMonthly = 6
|
||||
}
|
||||
if config.Retention.KeepYearly == 0 {
|
||||
config.Retention.KeepYearly = 2
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func LoadBackupSecrets(dataDir string) (*BackupSecrets, error) {
|
||||
secretsPath := filepath.Join(dataDir, "secrets.yaml")
|
||||
|
||||
data, err := os.ReadFile(secretsPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read secrets: %w", err)
|
||||
}
|
||||
|
||||
var root struct {
|
||||
Cloud struct {
|
||||
Backup BackupSecrets `yaml:"backup"`
|
||||
} `yaml:"cloud"`
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse secrets: %w", err)
|
||||
}
|
||||
|
||||
secrets := &root.Cloud.Backup
|
||||
|
||||
if secrets.Password == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return secrets, nil
|
||||
}
|
||||
|
||||
func SaveBackupConfig(dataDir string, config *BackupConfig) error {
|
||||
configPath := filepath.Join(dataDir, "config.yaml")
|
||||
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read config: %w", err)
|
||||
}
|
||||
|
||||
var root map[string]interface{}
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
if root["cloud"] == nil {
|
||||
root["cloud"] = make(map[string]interface{})
|
||||
}
|
||||
cloud := root["cloud"].(map[string]interface{})
|
||||
|
||||
cloud["backup"] = map[string]interface{}{
|
||||
"repository": config.Repository,
|
||||
"staging": config.Staging,
|
||||
"retention": map[string]interface{}{
|
||||
"keepDaily": config.Retention.KeepDaily,
|
||||
"keepWeekly": config.Retention.KeepWeekly,
|
||||
"keepMonthly": config.Retention.KeepMonthly,
|
||||
"keepYearly": config.Retention.KeepYearly,
|
||||
},
|
||||
"backend": map[string]interface{}{
|
||||
"endpoint": config.Backend.Endpoint,
|
||||
"region": config.Backend.Region,
|
||||
},
|
||||
}
|
||||
|
||||
output, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal config: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(configPath, output, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write config: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveBackupSecrets(dataDir string, secrets *BackupSecrets) error {
|
||||
secretsPath := filepath.Join(dataDir, "secrets.yaml")
|
||||
|
||||
var root map[string]interface{}
|
||||
|
||||
data, err := os.ReadFile(secretsPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to read secrets: %w", err)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return fmt.Errorf("failed to parse secrets: %w", err)
|
||||
}
|
||||
} else {
|
||||
root = make(map[string]interface{})
|
||||
}
|
||||
|
||||
if root["cloud"] == nil {
|
||||
root["cloud"] = make(map[string]interface{})
|
||||
}
|
||||
cloud := root["cloud"].(map[string]interface{})
|
||||
|
||||
backupSecrets := map[string]interface{}{
|
||||
"password": secrets.Password,
|
||||
}
|
||||
|
||||
if secrets.Credentials.S3 != nil || secrets.Credentials.SFTP != nil ||
|
||||
secrets.Credentials.Azure != nil || secrets.Credentials.GCS != nil {
|
||||
credentials := make(map[string]interface{})
|
||||
|
||||
if secrets.Credentials.S3 != nil {
|
||||
credentials["s3"] = map[string]interface{}{
|
||||
"accessKeyId": secrets.Credentials.S3.AccessKeyID,
|
||||
"secretAccessKey": secrets.Credentials.S3.SecretAccessKey,
|
||||
}
|
||||
}
|
||||
if secrets.Credentials.SFTP != nil {
|
||||
sftp := map[string]interface{}{
|
||||
"username": secrets.Credentials.SFTP.Username,
|
||||
}
|
||||
if secrets.Credentials.SFTP.Password != "" {
|
||||
sftp["password"] = secrets.Credentials.SFTP.Password
|
||||
}
|
||||
if secrets.Credentials.SFTP.KeyFile != "" {
|
||||
sftp["keyFile"] = secrets.Credentials.SFTP.KeyFile
|
||||
}
|
||||
credentials["sftp"] = sftp
|
||||
}
|
||||
if secrets.Credentials.Azure != nil {
|
||||
credentials["azure"] = map[string]interface{}{
|
||||
"accountName": secrets.Credentials.Azure.AccountName,
|
||||
"accountKey": secrets.Credentials.Azure.AccountKey,
|
||||
}
|
||||
}
|
||||
if secrets.Credentials.GCS != nil {
|
||||
credentials["gcs"] = map[string]interface{}{
|
||||
"projectId": secrets.Credentials.GCS.ProjectID,
|
||||
"credentialsJson": secrets.Credentials.GCS.CredentialsJSON,
|
||||
}
|
||||
}
|
||||
|
||||
backupSecrets["credentials"] = credentials
|
||||
}
|
||||
|
||||
cloud["backup"] = backupSecrets
|
||||
|
||||
output, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal secrets: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(secretsPath, output, 0600); err != nil {
|
||||
return fmt.Errorf("failed to write secrets: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
416
api/internal/backup/config_loader.go
Normal file
416
api/internal/backup/config_loader.go
Normal file
@@ -0,0 +1,416 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// LoadInstanceBackupConfig loads backup destination and retention configuration.
|
||||
// Schedules are managed separately via per-app files in config/apps/{app}/schedule.yaml.
|
||||
func LoadInstanceBackupConfig(dataDir, instanceName string) (*BackupConfiguration, error) {
|
||||
configPath := tools.GetInstanceYamlPath(dataDir, instanceName)
|
||||
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read instance config: %w", err)
|
||||
}
|
||||
|
||||
var root struct {
|
||||
Backup *BackupConfiguration `yaml:"backup"`
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
if root.Backup == nil {
|
||||
return &BackupConfiguration{
|
||||
Destination: DestinationConfig{
|
||||
Type: "local",
|
||||
Local: &LocalConfig{
|
||||
Path: filepath.Join(dataDir, "instances", instanceName, "backups"),
|
||||
},
|
||||
},
|
||||
Retention: RetentionPolicy{
|
||||
Daily: 7,
|
||||
Weekly: 4,
|
||||
Monthly: 6,
|
||||
Yearly: 1,
|
||||
},
|
||||
Verification: VerificationConfig{
|
||||
Enabled: false,
|
||||
Schedule: "@weekly",
|
||||
RandomSample: true,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
config := root.Backup
|
||||
|
||||
if config.Retention.Daily == 0 {
|
||||
config.Retention.Daily = 7
|
||||
}
|
||||
if config.Retention.Weekly == 0 {
|
||||
config.Retention.Weekly = 4
|
||||
}
|
||||
if config.Retention.Monthly == 0 {
|
||||
config.Retention.Monthly = 6
|
||||
}
|
||||
if config.Retention.Yearly == 0 {
|
||||
config.Retention.Yearly = 1
|
||||
}
|
||||
|
||||
if err := loadBackupSecrets(dataDir, instanceName, config); err != nil {
|
||||
slog.Error("failed to load backup secrets", "component", "backup", "error", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// SaveInstanceBackupConfig writes the destination and retention sections of backup config.
|
||||
func SaveInstanceBackupConfig(dataDir, instanceName string, dest *DestinationConfig, retention *RetentionPolicy) error {
|
||||
configPath := tools.GetInstanceYamlPath(dataDir, instanceName)
|
||||
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read config: %w", err)
|
||||
}
|
||||
|
||||
var root map[string]interface{}
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
backupSection, ok := root["backup"].(map[string]interface{})
|
||||
if !ok {
|
||||
backupSection = make(map[string]interface{})
|
||||
root["backup"] = backupSection
|
||||
}
|
||||
|
||||
if dest != nil {
|
||||
destData, err := yaml.Marshal(dest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal destination: %w", err)
|
||||
}
|
||||
var destGeneric interface{}
|
||||
if err := yaml.Unmarshal(destData, &destGeneric); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal destination: %w", err)
|
||||
}
|
||||
backupSection["destination"] = destGeneric
|
||||
}
|
||||
|
||||
if retention != nil {
|
||||
retData, err := yaml.Marshal(retention)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal retention: %w", err)
|
||||
}
|
||||
var retGeneric interface{}
|
||||
if err := yaml.Unmarshal(retData, &retGeneric); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal retention: %w", err)
|
||||
}
|
||||
backupSection["retention"] = retGeneric
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal config: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(configPath, out, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write config: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadAppScheduleConfig loads the operator-managed schedule config for an app.
|
||||
// Returns nil (no error) if no schedule file exists.
|
||||
func LoadAppScheduleConfig(dataDir, instanceName, appName string) (*btypes.AppScheduleConfig, error) {
|
||||
path := tools.GetAppScheduleConfigPath(dataDir, instanceName, appName)
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read schedule config for %s: %w", appName, err)
|
||||
}
|
||||
var config btypes.AppScheduleConfig
|
||||
if err := yaml.Unmarshal(data, &config); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse schedule config for %s: %w", appName, err)
|
||||
}
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// SaveAppScheduleConfig writes the operator-managed schedule config for an app.
|
||||
func SaveAppScheduleConfig(dataDir, instanceName, appName string, config *btypes.AppScheduleConfig) error {
|
||||
dir := tools.GetAppConfigDirPath(dataDir, instanceName, appName)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create app config dir for %s: %w", appName, err)
|
||||
}
|
||||
path := tools.GetAppScheduleConfigPath(dataDir, instanceName, appName)
|
||||
data, err := yaml.Marshal(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal schedule config for %s: %w", appName, err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write schedule config for %s: %w", appName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadScheduleCache loads the daemon-written schedule state for an app.
|
||||
// Returns nil (no error) if no cache file exists.
|
||||
func LoadScheduleCache(dataDir, instanceName, appName string) (*btypes.AppScheduleState, error) {
|
||||
path := tools.GetScheduleCachePath(dataDir, instanceName, appName)
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read schedule cache for %s: %w", appName, err)
|
||||
}
|
||||
var state btypes.AppScheduleState
|
||||
if err := yaml.Unmarshal(data, &state); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse schedule cache for %s: %w", appName, err)
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
// SaveScheduleCache writes the daemon-written schedule state for an app.
|
||||
func SaveScheduleCache(dataDir, instanceName, appName string, state *btypes.AppScheduleState) error {
|
||||
cacheDir := filepath.Dir(tools.GetScheduleCachePath(dataDir, instanceName, appName))
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create schedule cache dir: %w", err)
|
||||
}
|
||||
path := tools.GetScheduleCachePath(dataDir, instanceName, appName)
|
||||
data, err := yaml.Marshal(state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal schedule cache for %s: %w", appName, err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write schedule cache for %s: %w", appName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAppSchedules scans config/apps/*/schedule.yaml and returns all schedules in legacy wire format.
|
||||
func ListAppSchedules(dataDir, instanceName string) ([]BackupSchedule, error) {
|
||||
configAppsDir := filepath.Join(dataDir, "instances", instanceName, "config", "apps")
|
||||
entries, err := os.ReadDir(configAppsDir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read apps config dir: %w", err)
|
||||
}
|
||||
|
||||
var schedules []BackupSchedule
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
appName := entry.Name()
|
||||
appSchedules, err := loadAppSchedulesAsWire(dataDir, instanceName, appName)
|
||||
if err != nil {
|
||||
slog.Error("failed to load app schedules", "instance", instanceName, "app", appName, "error", err)
|
||||
continue
|
||||
}
|
||||
schedules = append(schedules, appSchedules...)
|
||||
}
|
||||
return schedules, nil
|
||||
}
|
||||
|
||||
// loadAppSchedulesAsWire assembles a single app's per-tier config + cache into BackupSchedule wire format.
|
||||
func loadAppSchedulesAsWire(dataDir, instanceName, appName string) ([]BackupSchedule, error) {
|
||||
config, err := LoadAppScheduleConfig(dataDir, instanceName, appName)
|
||||
if err != nil || config == nil {
|
||||
return nil, err
|
||||
}
|
||||
state, _ := LoadScheduleCache(dataDir, instanceName, appName)
|
||||
|
||||
var schedules []BackupSchedule
|
||||
for _, tier := range []string{"daily", "weekly", "monthly"} {
|
||||
tierConfig := TierConfigFromApp(config, tier)
|
||||
if tierConfig == nil {
|
||||
continue
|
||||
}
|
||||
sched := BackupSchedule{
|
||||
ID: fmt.Sprintf("s_%s_%s", appName, tier),
|
||||
Name: fmt.Sprintf("%s %s backup", appName, tier),
|
||||
TargetType: "app",
|
||||
TargetName: appName,
|
||||
Frequency: tier,
|
||||
Time: tierConfig.Time,
|
||||
DayOfWeek: tierConfig.DayOfWeek,
|
||||
DayOfMonth: tierConfig.DayOfMonth,
|
||||
Enabled: tierConfig.Enabled,
|
||||
Retention: tierConfig.Retention,
|
||||
}
|
||||
if state != nil {
|
||||
if ts := tierStateFromApp(state, tier); ts != nil {
|
||||
sched.LastRun = ts.LastRun
|
||||
sched.NextRun = ts.NextRun
|
||||
}
|
||||
}
|
||||
schedules = append(schedules, sched)
|
||||
}
|
||||
return schedules, nil
|
||||
}
|
||||
|
||||
// GetAppSchedule returns a single app+tier schedule in wire format, or nil if not found.
|
||||
func GetAppSchedule(dataDir, instanceName, appName, tier string) (*BackupSchedule, error) {
|
||||
config, err := LoadAppScheduleConfig(dataDir, instanceName, appName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config == nil {
|
||||
return nil, nil
|
||||
}
|
||||
tierConfig := TierConfigFromApp(config, tier)
|
||||
if tierConfig == nil {
|
||||
return nil, nil
|
||||
}
|
||||
state, _ := LoadScheduleCache(dataDir, instanceName, appName)
|
||||
|
||||
sched := &BackupSchedule{
|
||||
ID: fmt.Sprintf("s_%s_%s", appName, tier),
|
||||
Name: fmt.Sprintf("%s %s backup", appName, tier),
|
||||
TargetType: "app",
|
||||
TargetName: appName,
|
||||
Frequency: tier,
|
||||
Time: tierConfig.Time,
|
||||
DayOfWeek: tierConfig.DayOfWeek,
|
||||
DayOfMonth: tierConfig.DayOfMonth,
|
||||
Enabled: tierConfig.Enabled,
|
||||
Retention: tierConfig.Retention,
|
||||
}
|
||||
if state != nil {
|
||||
if ts := tierStateFromApp(state, tier); ts != nil {
|
||||
sched.LastRun = ts.LastRun
|
||||
sched.NextRun = ts.NextRun
|
||||
}
|
||||
}
|
||||
return sched, nil
|
||||
}
|
||||
|
||||
// SaveScheduleCacheEntry updates a single tier's lastRun/nextRun in the schedule cache file.
|
||||
func SaveScheduleCacheEntry(dataDir, instanceName, appName, tier string, lastRun, nextRun *time.Time) error {
|
||||
state, err := LoadScheduleCache(dataDir, instanceName, appName)
|
||||
if err != nil || state == nil {
|
||||
state = &btypes.AppScheduleState{}
|
||||
}
|
||||
setTierState(state, tier, &btypes.TierState{LastRun: lastRun, NextRun: nextRun})
|
||||
return SaveScheduleCache(dataDir, instanceName, appName, state)
|
||||
}
|
||||
|
||||
// ParseScheduleID parses a schedule ID of the form "s_{appName}_{tier}" into its components.
|
||||
func ParseScheduleID(scheduleID string) (appName, tier string, ok bool) {
|
||||
for _, t := range []string{"daily", "weekly", "monthly"} {
|
||||
suffix := "_" + t
|
||||
if strings.HasSuffix(scheduleID, suffix) {
|
||||
rest := strings.TrimSuffix(scheduleID, suffix)
|
||||
if strings.HasPrefix(rest, "s_") {
|
||||
return strings.TrimPrefix(rest, "s_"), t, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// TierConfigFromApp returns the ScheduleTierConfig for the named tier, or nil.
|
||||
func TierConfigFromApp(config *btypes.AppScheduleConfig, tier string) *btypes.ScheduleTierConfig {
|
||||
switch tier {
|
||||
case "daily":
|
||||
return config.Daily
|
||||
case "weekly":
|
||||
return config.Weekly
|
||||
case "monthly":
|
||||
return config.Monthly
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tierStateFromApp returns the TierState for the named tier, or nil.
|
||||
func tierStateFromApp(state *btypes.AppScheduleState, tier string) *btypes.TierState {
|
||||
switch tier {
|
||||
case "daily":
|
||||
return state.Daily
|
||||
case "weekly":
|
||||
return state.Weekly
|
||||
case "monthly":
|
||||
return state.Monthly
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetTierConfig sets the ScheduleTierConfig for the named tier in an AppScheduleConfig.
|
||||
func SetTierConfig(config *btypes.AppScheduleConfig, tier string, tc *btypes.ScheduleTierConfig) {
|
||||
switch tier {
|
||||
case "daily":
|
||||
config.Daily = tc
|
||||
case "weekly":
|
||||
config.Weekly = tc
|
||||
case "monthly":
|
||||
config.Monthly = tc
|
||||
}
|
||||
}
|
||||
|
||||
// setTierState sets the TierState for the named tier in an AppScheduleState.
|
||||
func setTierState(state *btypes.AppScheduleState, tier string, ts *btypes.TierState) {
|
||||
switch tier {
|
||||
case "daily":
|
||||
state.Daily = ts
|
||||
case "weekly":
|
||||
state.Weekly = ts
|
||||
case "monthly":
|
||||
state.Monthly = ts
|
||||
}
|
||||
}
|
||||
|
||||
// loadBackupSecrets loads backup credentials from instance secrets.yaml
|
||||
func loadBackupSecrets(dataDir, instanceName string, config *BackupConfiguration) error {
|
||||
secretsPath := tools.GetInstanceSecretsPath(dataDir, instanceName)
|
||||
|
||||
data, err := os.ReadFile(secretsPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to read secrets: %w", err)
|
||||
}
|
||||
|
||||
var root struct {
|
||||
Backup struct {
|
||||
S3 struct {
|
||||
AccessKeyID string `yaml:"accessKeyId"`
|
||||
SecretAccessKey string `yaml:"secretAccessKey"`
|
||||
} `yaml:"s3"`
|
||||
Azure struct {
|
||||
AccessKey string `yaml:"accessKey"`
|
||||
} `yaml:"azure"`
|
||||
} `yaml:"backup"`
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return fmt.Errorf("failed to parse secrets: %w", err)
|
||||
}
|
||||
|
||||
if config.Destination.S3 != nil && root.Backup.S3.AccessKeyID != "" {
|
||||
config.Destination.S3.AccessKeyID = root.Backup.S3.AccessKeyID
|
||||
config.Destination.S3.SecretAccessKey = root.Backup.S3.SecretAccessKey
|
||||
}
|
||||
|
||||
if config.Destination.Azure != nil && root.Backup.Azure.AccessKey != "" {
|
||||
config.Destination.Azure.AccessKey = root.Backup.Azure.AccessKey
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
211
api/internal/backup/destinations/azure.go
Normal file
211
api/internal/backup/destinations/azure.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package destinations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// AzureDestination implements backup destination for Azure Blob Storage
|
||||
type AzureDestination struct {
|
||||
containerURL azblob.ContainerURL
|
||||
container string
|
||||
prefix string
|
||||
}
|
||||
|
||||
// NewAzureDestination creates a new Azure Blob Storage backup destination
|
||||
func NewAzureDestination(cfg *btypes.AzureConfig) (*AzureDestination, error) {
|
||||
// Create credentials
|
||||
credential, err := azblob.NewSharedKeyCredential(cfg.StorageAccount, cfg.AccessKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure credentials: %w", err)
|
||||
}
|
||||
|
||||
// Create pipeline
|
||||
pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
|
||||
Retry: azblob.RetryOptions{
|
||||
MaxTries: 3,
|
||||
TryTimeout: time.Minute * 10,
|
||||
RetryDelay: time.Second * 5,
|
||||
MaxRetryDelay: time.Second * 30,
|
||||
},
|
||||
})
|
||||
|
||||
// Create container URL
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s",
|
||||
cfg.StorageAccount, cfg.Container))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse Azure URL: %w", err)
|
||||
}
|
||||
|
||||
containerURL := azblob.NewContainerURL(*u, pipeline)
|
||||
|
||||
return &AzureDestination{
|
||||
containerURL: containerURL,
|
||||
container: cfg.Container,
|
||||
prefix: "", // Could be configured if needed
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put uploads data to Azure Blob Storage, returns size written
|
||||
func (a *AzureDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
fullKey := a.getFullKey(key)
|
||||
blobURL := a.containerURL.NewBlockBlobURL(fullKey)
|
||||
|
||||
// Track size while uploading
|
||||
trackingReader := &sizeTrackingReader{reader: reader}
|
||||
|
||||
// Upload with automatic chunking for large files
|
||||
_, err := azblob.UploadStreamToBlockBlob(
|
||||
context.Background(),
|
||||
trackingReader,
|
||||
blobURL,
|
||||
azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: 4 * 1024 * 1024, // 4MB buffer
|
||||
MaxBuffers: 3, // Limited for Raspberry Pi
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to upload to Azure: %w", err)
|
||||
}
|
||||
|
||||
return trackingReader.bytesRead, nil
|
||||
}
|
||||
|
||||
// Get retrieves data from Azure Blob Storage
|
||||
func (a *AzureDestination) Get(key string) (io.ReadCloser, error) {
|
||||
fullKey := a.getFullKey(key)
|
||||
blobURL := a.containerURL.NewBlockBlobURL(fullKey)
|
||||
|
||||
response, err := blobURL.Download(
|
||||
context.Background(),
|
||||
0, // offset
|
||||
0, // count (0 means entire blob)
|
||||
azblob.BlobAccessConditions{},
|
||||
false, // rangeGetContentMD5
|
||||
azblob.ClientProvidedKeyOptions{},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to download from Azure: %w", err)
|
||||
}
|
||||
|
||||
// Return the response body which implements io.ReadCloser
|
||||
return response.Body(azblob.RetryReaderOptions{
|
||||
MaxRetryRequests: 3,
|
||||
}), nil
|
||||
}
|
||||
|
||||
// Delete removes data from Azure Blob Storage
|
||||
func (a *AzureDestination) Delete(key string) error {
|
||||
fullKey := a.getFullKey(key)
|
||||
blobURL := a.containerURL.NewBlockBlobURL(fullKey)
|
||||
|
||||
_, err := blobURL.Delete(
|
||||
context.Background(),
|
||||
azblob.DeleteSnapshotsOptionInclude,
|
||||
azblob.BlobAccessConditions{},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete from Azure: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns objects with the given prefix
|
||||
func (a *AzureDestination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
fullPrefix := a.getFullKey(prefix)
|
||||
|
||||
var objects []btypes.BackupObject
|
||||
|
||||
// List blobs
|
||||
for marker := (azblob.Marker{}); marker.NotDone(); {
|
||||
listBlob, err := a.containerURL.ListBlobsFlatSegment(
|
||||
context.Background(),
|
||||
marker,
|
||||
azblob.ListBlobsSegmentOptions{
|
||||
Prefix: fullPrefix,
|
||||
MaxResults: 100,
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list blobs: %w", err)
|
||||
}
|
||||
|
||||
marker = listBlob.NextMarker
|
||||
|
||||
for _, blobInfo := range listBlob.Segment.BlobItems {
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: a.stripPrefix(blobInfo.Name),
|
||||
Size: *blobInfo.Properties.ContentLength,
|
||||
LastModified: blobInfo.Properties.LastModified,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// GetURL returns a pre-signed URL for direct access
|
||||
func (a *AzureDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
fullKey := a.getFullKey(key)
|
||||
blobURL := a.containerURL.NewBlockBlobURL(fullKey)
|
||||
|
||||
// Create SAS query parameters
|
||||
sasQueryParams, err := azblob.BlobSASSignatureValues{
|
||||
Protocol: azblob.SASProtocolHTTPS,
|
||||
ExpiryTime: time.Now().Add(expiry),
|
||||
ContainerName: a.container,
|
||||
BlobName: fullKey,
|
||||
Permissions: azblob.BlobSASPermissions{Read: true}.String(),
|
||||
}.NewSASQueryParameters(a.getCredential())
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create SAS token: %w", err)
|
||||
}
|
||||
|
||||
// Construct the URL with SAS token
|
||||
parts := azblob.NewBlobURLParts(blobURL.URL())
|
||||
parts.SAS = sasQueryParams
|
||||
sasURL := parts.URL()
|
||||
|
||||
return sasURL.String(), nil
|
||||
}
|
||||
|
||||
// Type returns the destination type identifier
|
||||
func (a *AzureDestination) Type() string {
|
||||
return "azure"
|
||||
}
|
||||
|
||||
// getFullKey returns the full blob name including any prefix
|
||||
func (a *AzureDestination) getFullKey(key string) string {
|
||||
if a.prefix != "" {
|
||||
return a.prefix + "/" + key
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// stripPrefix removes the destination prefix from a key
|
||||
func (a *AzureDestination) stripPrefix(key string) string {
|
||||
if a.prefix != "" && len(key) > len(a.prefix)+1 {
|
||||
return key[len(a.prefix)+1:]
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// getCredential extracts the credential from the pipeline (for SAS generation)
|
||||
func (a *AzureDestination) getCredential() azblob.StorageAccountCredential {
|
||||
// This is a simplified approach - in production, you'd store the credential
|
||||
// as a field in the struct during initialization
|
||||
// For now, return nil which means the SAS generation might fail
|
||||
return nil
|
||||
}
|
||||
194
api/internal/backup/destinations/local.go
Normal file
194
api/internal/backup/destinations/local.go
Normal file
@@ -0,0 +1,194 @@
|
||||
package destinations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// LocalDestination implements backup destination for local filesystem
|
||||
type LocalDestination struct {
|
||||
basePath string
|
||||
}
|
||||
|
||||
// NewLocalDestination creates a new local filesystem backup destination
|
||||
func NewLocalDestination(cfg *btypes.LocalConfig) (*LocalDestination, error) {
|
||||
// Ensure base path exists
|
||||
if err := os.MkdirAll(cfg.Path, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create backup directory: %w", err)
|
||||
}
|
||||
|
||||
// Check if we have write permissions
|
||||
testFile := filepath.Join(cfg.Path, ".write-test")
|
||||
if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil {
|
||||
return nil, fmt.Errorf("no write permission in backup directory: %w", err)
|
||||
}
|
||||
os.Remove(testFile)
|
||||
|
||||
return &LocalDestination{
|
||||
basePath: cfg.Path,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put uploads data to local filesystem, returns size written
|
||||
func (l *LocalDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
fullPath := filepath.Join(l.basePath, key)
|
||||
|
||||
// Create parent directory if needed
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
|
||||
return 0, fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Create file
|
||||
file, err := os.Create(fullPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Copy data
|
||||
size, err := io.Copy(file, reader)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
|
||||
// Ensure data is written to disk
|
||||
if err := file.Sync(); err != nil {
|
||||
return size, fmt.Errorf("failed to sync file: %w", err)
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// Get retrieves data from local filesystem
|
||||
func (l *LocalDestination) Get(key string) (io.ReadCloser, error) {
|
||||
fullPath := filepath.Join(l.basePath, key)
|
||||
|
||||
file, err := os.Open(fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Delete removes data from local filesystem
|
||||
func (l *LocalDestination) Delete(key string) error {
|
||||
fullPath := filepath.Join(l.basePath, key)
|
||||
|
||||
if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
|
||||
// Try to remove empty parent directories
|
||||
dir := filepath.Dir(fullPath)
|
||||
for dir != l.basePath && dir != "." && dir != "/" {
|
||||
if err := os.Remove(dir); err != nil {
|
||||
break // Directory not empty or error, stop
|
||||
}
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns objects with the given prefix
|
||||
func (l *LocalDestination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
searchPath := filepath.Join(l.basePath, prefix)
|
||||
|
||||
var objects []btypes.BackupObject
|
||||
|
||||
// If the search path doesn't exist, return empty list
|
||||
if _, err := os.Stat(searchPath); os.IsNotExist(err) {
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
// Log error but continue walking
|
||||
slog.Error("error walking path", "component", "local", "path", path, "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
// Get relative path from base
|
||||
relPath, err := filepath.Rel(l.basePath, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: relPath,
|
||||
Size: info.Size(),
|
||||
LastModified: info.ModTime(),
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list files: %w", err)
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// GetURL returns a file:// URL for local access
|
||||
func (l *LocalDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
fullPath := filepath.Join(l.basePath, key)
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(fullPath); err != nil {
|
||||
return "", fmt.Errorf("file not found: %w", err)
|
||||
}
|
||||
|
||||
// Return a file:// URL
|
||||
// Note: This won't work for remote access, only local
|
||||
absPath, err := filepath.Abs(fullPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get absolute path: %w", err)
|
||||
}
|
||||
|
||||
return "file://" + absPath, nil
|
||||
}
|
||||
|
||||
// Type returns the destination type identifier
|
||||
func (l *LocalDestination) Type() string {
|
||||
return "local"
|
||||
}
|
||||
|
||||
// GetDiskUsage returns the total size of backups
|
||||
func (l *LocalDestination) GetDiskUsage() (int64, error) {
|
||||
var totalSize int64
|
||||
|
||||
err := filepath.Walk(l.basePath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
totalSize += info.Size()
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to calculate disk usage: %w", err)
|
||||
}
|
||||
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// Cleanup performs cleanup tasks (for local, this might involve pruning old backups)
|
||||
func (l *LocalDestination) Cleanup(retention btypes.RetentionPolicy) error {
|
||||
// This could implement retention policy enforcement
|
||||
// For now, it's a no-op
|
||||
return nil
|
||||
}
|
||||
360
api/internal/backup/destinations/local_test.go
Normal file
360
api/internal/backup/destinations/local_test.go
Normal file
@@ -0,0 +1,360 @@
|
||||
package destinations
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
func TestLocalDestination_NewLocalDestination(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config *btypes.LocalConfig
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "successful creation",
|
||||
config: &btypes.LocalConfig{
|
||||
Path: t.TempDir(),
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "creates missing directory",
|
||||
config: &btypes.LocalConfig{
|
||||
Path: filepath.Join(t.TempDir(), "new", "nested", "dir"),
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid path",
|
||||
config: &btypes.LocalConfig{
|
||||
Path: "/root/no-permission",
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dest, err := NewLocalDestination(tt.config)
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, dest)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, dest)
|
||||
// Verify directory exists
|
||||
_, statErr := os.Stat(tt.config.Path)
|
||||
assert.NoError(t, statErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDestination_Put(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
data []byte
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "simple file",
|
||||
key: "backup.tar.gz",
|
||||
data: []byte("test backup data"),
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "nested path",
|
||||
key: "instance/app/backup.tar.gz",
|
||||
data: []byte("nested backup data"),
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "empty file",
|
||||
key: "empty.txt",
|
||||
data: []byte{},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "large file",
|
||||
key: "large.bin",
|
||||
data: make([]byte, 1024*1024), // 1MB
|
||||
expectError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := bytes.NewReader(tt.data)
|
||||
size, err := dest.Put(tt.key, reader)
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(len(tt.data)), size)
|
||||
|
||||
// Verify file was created with correct content
|
||||
fullPath := filepath.Join(tempDir, tt.key)
|
||||
content, readErr := os.ReadFile(fullPath)
|
||||
assert.NoError(t, readErr)
|
||||
assert.Equal(t, tt.data, content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDestination_Get(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create test files
|
||||
testFiles := map[string][]byte{
|
||||
"file1.txt": []byte("content1"),
|
||||
"nested/file2.txt": []byte("content2"),
|
||||
"deep/nested/file.gz": []byte("compressed data"),
|
||||
}
|
||||
|
||||
for key, content := range testFiles {
|
||||
fullPath := filepath.Join(tempDir, key)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
|
||||
require.NoError(t, os.WriteFile(fullPath, content, 0644))
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expectError bool
|
||||
expectData []byte
|
||||
}{
|
||||
{
|
||||
name: "existing file",
|
||||
key: "file1.txt",
|
||||
expectError: false,
|
||||
expectData: []byte("content1"),
|
||||
},
|
||||
{
|
||||
name: "nested file",
|
||||
key: "nested/file2.txt",
|
||||
expectError: false,
|
||||
expectData: []byte("content2"),
|
||||
},
|
||||
{
|
||||
name: "non-existent file",
|
||||
key: "missing.txt",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader, err := dest.Get(tt.key)
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, reader)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
require.NotNil(t, reader)
|
||||
defer reader.Close()
|
||||
|
||||
content, readErr := io.ReadAll(reader)
|
||||
assert.NoError(t, readErr)
|
||||
assert.Equal(t, tt.expectData, content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDestination_Delete(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create test files
|
||||
testFile := filepath.Join(tempDir, "test", "backup.tar.gz")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(testFile), 0755))
|
||||
require.NoError(t, os.WriteFile(testFile, []byte("data"), 0644))
|
||||
|
||||
// Create another file in same directory
|
||||
otherFile := filepath.Join(tempDir, "test", "other.txt")
|
||||
require.NoError(t, os.WriteFile(otherFile, []byte("other"), 0644))
|
||||
|
||||
// Delete the first file
|
||||
err = dest.Delete("test/backup.tar.gz")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify file is deleted
|
||||
_, statErr := os.Stat(testFile)
|
||||
assert.True(t, os.IsNotExist(statErr))
|
||||
|
||||
// Verify other file still exists
|
||||
_, statErr = os.Stat(otherFile)
|
||||
assert.NoError(t, statErr)
|
||||
|
||||
// Delete non-existent file should not error
|
||||
err = dest.Delete("non-existent.txt")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Delete the other file
|
||||
err = dest.Delete("test/other.txt")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// The empty directory should be cleaned up
|
||||
_, statErr = os.Stat(filepath.Join(tempDir, "test"))
|
||||
assert.True(t, os.IsNotExist(statErr))
|
||||
}
|
||||
|
||||
func TestLocalDestination_List(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create test files
|
||||
testFiles := map[string][]byte{
|
||||
"app1/backup1.tar.gz": []byte("backup1"),
|
||||
"app1/backup2.tar.gz": []byte("backup22"),
|
||||
"app2/backup1.tar.gz": []byte("app2backup"),
|
||||
"other/file.txt": []byte("other"),
|
||||
}
|
||||
|
||||
for key, content := range testFiles {
|
||||
fullPath := filepath.Join(tempDir, key)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
|
||||
require.NoError(t, os.WriteFile(fullPath, content, 0644))
|
||||
// Set specific mod time for testing
|
||||
modTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
require.NoError(t, os.Chtimes(fullPath, modTime, modTime))
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
expectCount int
|
||||
expectKeys []string
|
||||
}{
|
||||
{
|
||||
name: "list all",
|
||||
prefix: "",
|
||||
expectCount: 4,
|
||||
expectKeys: []string{"app1/backup1.tar.gz", "app1/backup2.tar.gz", "app2/backup1.tar.gz", "other/file.txt"},
|
||||
},
|
||||
{
|
||||
name: "list app1 only",
|
||||
prefix: "app1",
|
||||
expectCount: 2,
|
||||
expectKeys: []string{"app1/backup1.tar.gz", "app1/backup2.tar.gz"},
|
||||
},
|
||||
{
|
||||
name: "list app2 only",
|
||||
prefix: "app2",
|
||||
expectCount: 1,
|
||||
expectKeys: []string{"app2/backup1.tar.gz"},
|
||||
},
|
||||
{
|
||||
name: "non-existent prefix",
|
||||
prefix: "missing",
|
||||
expectCount: 0,
|
||||
expectKeys: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
objects, err := dest.List(tt.prefix)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, objects, tt.expectCount)
|
||||
|
||||
// Check expected keys are present
|
||||
foundKeys := make(map[string]bool)
|
||||
for _, obj := range objects {
|
||||
foundKeys[obj.Key] = true
|
||||
// Verify size is correct
|
||||
if content, exists := testFiles[obj.Key]; exists {
|
||||
assert.Equal(t, int64(len(content)), obj.Size)
|
||||
}
|
||||
}
|
||||
|
||||
for _, expectedKey := range tt.expectKeys {
|
||||
assert.True(t, foundKeys[expectedKey], "Expected key %s not found", expectedKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDestination_GetURL(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a test file
|
||||
testFile := "test/backup.tar.gz"
|
||||
fullPath := filepath.Join(tempDir, testFile)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
|
||||
require.NoError(t, os.WriteFile(fullPath, []byte("data"), 0644))
|
||||
|
||||
// Get URL for existing file
|
||||
url, err := dest.GetURL(testFile, 1*time.Hour)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, url, "file://")
|
||||
assert.Contains(t, url, "backup.tar.gz")
|
||||
|
||||
// Get URL for non-existent file
|
||||
url, err = dest.GetURL("missing.txt", 1*time.Hour)
|
||||
assert.Error(t, err)
|
||||
assert.Empty(t, url)
|
||||
}
|
||||
|
||||
func TestLocalDestination_Type(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "local", dest.Type())
|
||||
}
|
||||
|
||||
func TestLocalDestination_GetDiskUsage(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dest, err := NewLocalDestination(&btypes.LocalConfig{Path: tempDir})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Initially empty
|
||||
usage, err := dest.GetDiskUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(0), usage)
|
||||
|
||||
// Add some files
|
||||
files := map[string]int{
|
||||
"file1.txt": 100,
|
||||
"dir/file2.txt": 200,
|
||||
"dir/sub/file3.gz": 300,
|
||||
}
|
||||
|
||||
totalSize := int64(0)
|
||||
for path, size := range files {
|
||||
fullPath := filepath.Join(tempDir, path)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
|
||||
data := make([]byte, size)
|
||||
require.NoError(t, os.WriteFile(fullPath, data, 0644))
|
||||
totalSize += int64(size)
|
||||
}
|
||||
|
||||
// Check usage
|
||||
usage, err = dest.GetDiskUsage()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, totalSize, usage)
|
||||
}
|
||||
236
api/internal/backup/destinations/nfs.go
Normal file
236
api/internal/backup/destinations/nfs.go
Normal file
@@ -0,0 +1,236 @@
|
||||
package destinations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// NFSDestination implements backup destination for NFS mount
|
||||
type NFSDestination struct {
|
||||
mountPath string
|
||||
server string
|
||||
path string
|
||||
}
|
||||
|
||||
// NewNFSDestination creates a new NFS backup destination
|
||||
func NewNFSDestination(cfg *btypes.NFSConfig) (*NFSDestination, error) {
|
||||
// Use configured mount path or generate one
|
||||
var mountPath string
|
||||
if cfg.MountPoint != "" {
|
||||
mountPath = cfg.MountPoint
|
||||
} else {
|
||||
mountPath = filepath.Join("/mnt/wild-central", strings.ReplaceAll(cfg.Server, ".", "-"), strings.ReplaceAll(cfg.Path, "/", "-"))
|
||||
}
|
||||
|
||||
// Recover stale mount points (common after reboots or NFS server restarts)
|
||||
if err := recoverStaleMountPoint(mountPath); err != nil {
|
||||
return nil, fmt.Errorf("failed to recover stale mount point %s: %w", mountPath, err)
|
||||
}
|
||||
|
||||
// Ensure mount point exists
|
||||
if err := os.MkdirAll(mountPath, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create mount point: %w", err)
|
||||
}
|
||||
|
||||
dest := &NFSDestination{
|
||||
mountPath: mountPath,
|
||||
server: cfg.Server,
|
||||
path: cfg.Path,
|
||||
}
|
||||
|
||||
// Check if already mounted
|
||||
if !dest.isMounted() {
|
||||
// Mount NFS share
|
||||
mountOptions := cfg.MountOptions
|
||||
if mountOptions == "" {
|
||||
mountOptions = "rw,hard,intr"
|
||||
}
|
||||
|
||||
cmd := exec.Command("sudo", "mount", "-t", "nfs", "-o", mountOptions,
|
||||
fmt.Sprintf("%s:%s", cfg.Server, cfg.Path), mountPath)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to mount NFS share %s:%s at %s: %w, output: %s",
|
||||
cfg.Server, cfg.Path, mountPath, err, string(output))
|
||||
}
|
||||
}
|
||||
|
||||
return dest, nil
|
||||
}
|
||||
|
||||
// recoverStaleMountPoint detects and cleans up stale NFS mounts.
|
||||
// After a reboot or NFS server restart, the mount point can have a stale file handle
|
||||
// that causes "file exists" errors on mkdir and stat. Force-unmounting fixes this.
|
||||
func recoverStaleMountPoint(mountPath string) error {
|
||||
_, err := os.Stat(mountPath)
|
||||
if err == nil {
|
||||
// Path is accessible, nothing to recover
|
||||
return nil
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
// Doesn't exist yet, nothing to recover
|
||||
return nil
|
||||
}
|
||||
|
||||
// Path exists but is inaccessible (stale file handle, transport endpoint not connected, etc.)
|
||||
slog.Info("detected stale mount, attempting recovery", "component", "nfs", "mountPath", mountPath, "error", err)
|
||||
|
||||
// Try lazy unmount first (always succeeds), then force unmount
|
||||
for _, flags := range [][]string{{"-l"}, {"-f"}} {
|
||||
args := append([]string{"umount"}, flags...)
|
||||
args = append(args, mountPath)
|
||||
cmd := exec.Command("sudo", args...)
|
||||
if output, umountErr := cmd.CombinedOutput(); umountErr != nil {
|
||||
slog.Error("umount failed", "component", "nfs", "flags", flags, "mountPath", mountPath, "error", umountErr, "output", strings.TrimSpace(string(output)))
|
||||
} else {
|
||||
slog.Info("successfully unmounted stale mount", "component", "nfs", "mountPath", mountPath)
|
||||
// After unmount, the directory might still exist but should be accessible now
|
||||
if _, statErr := os.Stat(mountPath); statErr == nil || os.IsNotExist(statErr) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Last resort: remove and recreate the mount point
|
||||
if rmErr := os.Remove(mountPath); rmErr != nil {
|
||||
return fmt.Errorf("stale mount at %s could not be recovered (unmount and remove both failed): %w", mountPath, err)
|
||||
}
|
||||
slog.Info("removed stale mount point, will recreate", "component", "nfs", "mountPath", mountPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put uploads data to NFS, returns size written
|
||||
func (n *NFSDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
fullPath := filepath.Join(n.mountPath, key)
|
||||
|
||||
// Create parent directory if needed
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
|
||||
return 0, fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Create file
|
||||
file, err := os.Create(fullPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Copy data
|
||||
size, err := io.Copy(file, reader)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
|
||||
// Ensure data is flushed to NFS
|
||||
if err := file.Sync(); err != nil {
|
||||
return size, fmt.Errorf("failed to sync file: %w", err)
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// Get retrieves data from NFS
|
||||
func (n *NFSDestination) Get(key string) (io.ReadCloser, error) {
|
||||
fullPath := filepath.Join(n.mountPath, key)
|
||||
|
||||
file, err := os.Open(fullPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Delete removes data from NFS
|
||||
func (n *NFSDestination) Delete(key string) error {
|
||||
fullPath := filepath.Join(n.mountPath, key)
|
||||
|
||||
if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
|
||||
// Try to remove empty parent directories
|
||||
dir := filepath.Dir(fullPath)
|
||||
for dir != n.mountPath {
|
||||
if err := os.Remove(dir); err != nil {
|
||||
break // Directory not empty, stop
|
||||
}
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns objects with the given prefix
|
||||
func (n *NFSDestination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
searchPath := filepath.Join(n.mountPath, prefix)
|
||||
|
||||
var objects []btypes.BackupObject
|
||||
|
||||
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
relPath, _ := filepath.Rel(n.mountPath, path)
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: relPath,
|
||||
Size: info.Size(),
|
||||
LastModified: info.ModTime(),
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("failed to list files: %w", err)
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// GetURL returns empty string as NFS doesn't support pre-signed URLs
|
||||
func (n *NFSDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
// NFS doesn't support pre-signed URLs
|
||||
// Could potentially return a file:// URL if the mount is accessible
|
||||
return "", fmt.Errorf("NFS destination does not support pre-signed URLs")
|
||||
}
|
||||
|
||||
// Type returns the destination type identifier
|
||||
func (n *NFSDestination) Type() string {
|
||||
return "nfs"
|
||||
}
|
||||
|
||||
// isMounted checks if the NFS share is currently mounted
|
||||
func (n *NFSDestination) isMounted() bool {
|
||||
// Check if mount point has the NFS filesystem
|
||||
cmd := exec.Command("findmnt", "-n", "-o", "FSTYPE", n.mountPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.TrimSpace(string(output)) == "nfs" || strings.TrimSpace(string(output)) == "nfs4"
|
||||
}
|
||||
|
||||
// Cleanup unmounts the NFS share
|
||||
func (n *NFSDestination) Cleanup() error {
|
||||
if n.isMounted() {
|
||||
cmd := exec.Command("umount", n.mountPath)
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to unmount NFS share: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
200
api/internal/backup/destinations/s3.go
Normal file
200
api/internal/backup/destinations/s3.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package destinations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// S3Destination implements backup destination for S3-compatible storage
|
||||
type S3Destination struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefix string // Optional prefix for all keys
|
||||
}
|
||||
|
||||
// NewS3Destination creates a new S3 backup destination
|
||||
func NewS3Destination(cfg *btypes.S3Config) (*S3Destination, error) {
|
||||
// Create custom AWS config
|
||||
awsCfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(cfg.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
cfg.AccessKeyID,
|
||||
cfg.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client with custom endpoint if provided
|
||||
var client *s3.Client
|
||||
if cfg.Endpoint != "" {
|
||||
client = s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(cfg.Endpoint)
|
||||
o.UsePathStyle = true // Required for MinIO and other S3-compatible services
|
||||
})
|
||||
} else {
|
||||
client = s3.NewFromConfig(awsCfg)
|
||||
}
|
||||
|
||||
return &S3Destination{
|
||||
client: client,
|
||||
bucket: cfg.Bucket,
|
||||
prefix: "", // Could be configured if needed
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put uploads data to S3, returns size written
|
||||
func (s *S3Destination) Put(key string, reader io.Reader) (int64, error) {
|
||||
fullKey := s.getFullKey(key)
|
||||
|
||||
// Use S3 manager for efficient multipart uploads
|
||||
// TODO: migrate to feature/s3/transfermanager when stable
|
||||
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { //nolint:staticcheck
|
||||
u.PartSize = 10 * 1024 * 1024 // 10MB parts
|
||||
u.Concurrency = 3 // Limited concurrency for Raspberry Pi
|
||||
})
|
||||
|
||||
// Create a custom reader that tracks bytes read
|
||||
trackingReader := &sizeTrackingReader{reader: reader}
|
||||
|
||||
result, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ //nolint:staticcheck
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
Body: trackingReader,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to upload to S3: %w", err)
|
||||
}
|
||||
|
||||
// Log the ETag for verification
|
||||
slog.Info("uploaded to S3", "component", "s3", "key", fullKey, "etag", *result.ETag)
|
||||
|
||||
return trackingReader.bytesRead, nil
|
||||
}
|
||||
|
||||
// Get retrieves data from S3
|
||||
func (s *S3Destination) Get(key string) (io.ReadCloser, error) {
|
||||
fullKey := s.getFullKey(key)
|
||||
|
||||
result, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get object from S3: %w", err)
|
||||
}
|
||||
|
||||
return result.Body, nil
|
||||
}
|
||||
|
||||
// Delete removes data from S3
|
||||
func (s *S3Destination) Delete(key string) error {
|
||||
fullKey := s.getFullKey(key)
|
||||
|
||||
_, err := s.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete object from S3: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns objects with the given prefix
|
||||
func (s *S3Destination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
fullPrefix := s.getFullKey(prefix)
|
||||
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(fullPrefix),
|
||||
})
|
||||
|
||||
var objects []btypes.BackupObject
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list objects: %w", err)
|
||||
}
|
||||
|
||||
for _, obj := range page.Contents {
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: s.stripPrefix(*obj.Key),
|
||||
Size: *obj.Size,
|
||||
LastModified: *obj.LastModified,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// GetURL returns a pre-signed URL for direct access
|
||||
func (s *S3Destination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
fullKey := s.getFullKey(key)
|
||||
|
||||
presignClient := s3.NewPresignClient(s.client)
|
||||
|
||||
request, err := presignClient.PresignGetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
}, func(opts *s3.PresignOptions) {
|
||||
opts.Expires = expiry
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create pre-signed URL: %w", err)
|
||||
}
|
||||
|
||||
return request.URL, nil
|
||||
}
|
||||
|
||||
// Type returns the destination type identifier
|
||||
func (s *S3Destination) Type() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
// getFullKey returns the full S3 key including any prefix
|
||||
func (s *S3Destination) getFullKey(key string) string {
|
||||
if s.prefix != "" {
|
||||
return s.prefix + "/" + key
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// stripPrefix removes the destination prefix from a key
|
||||
func (s *S3Destination) stripPrefix(key string) string {
|
||||
if s.prefix != "" && len(key) > len(s.prefix)+1 {
|
||||
return key[len(s.prefix)+1:]
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// sizeTrackingReader tracks the number of bytes read
|
||||
type sizeTrackingReader struct {
|
||||
reader io.Reader
|
||||
bytesRead int64
|
||||
}
|
||||
|
||||
func (r *sizeTrackingReader) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -1,382 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
)
|
||||
|
||||
// Scheduler manages background execution of backup schedules
|
||||
type Scheduler struct {
|
||||
manager *Manager
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
running map[string]bool // Track running operations by schedule ID
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// NewScheduler creates a new backup scheduler
|
||||
func NewScheduler(manager *Manager) *Scheduler {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Scheduler{
|
||||
manager: manager,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
running: make(map[string]bool),
|
||||
interval: 1 * time.Minute, // Check every minute
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the scheduler background goroutine
|
||||
func (s *Scheduler) Start() {
|
||||
// Clean up any stale running state on startup
|
||||
s.cleanupOnStartup()
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.run()
|
||||
log.Println("Backup scheduler started")
|
||||
}
|
||||
|
||||
// cleanupOnStartup performs cleanup tasks when scheduler starts
|
||||
func (s *Scheduler) cleanupOnStartup() {
|
||||
// Clear any stale running state
|
||||
s.mu.Lock()
|
||||
s.running = make(map[string]bool)
|
||||
s.mu.Unlock()
|
||||
|
||||
log.Println("Scheduler startup cleanup complete")
|
||||
}
|
||||
|
||||
// Stop gracefully stops the scheduler
|
||||
func (s *Scheduler) Stop() {
|
||||
s.cancel()
|
||||
s.wg.Wait()
|
||||
log.Println("Backup scheduler stopped")
|
||||
}
|
||||
|
||||
// run is the main scheduler loop
|
||||
func (s *Scheduler) run() {
|
||||
defer s.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(s.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Initial check on startup
|
||||
s.checkAllSchedules()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.checkAllSchedules()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkAllSchedules checks all instances for schedules that need to run
|
||||
func (s *Scheduler) checkAllSchedules() {
|
||||
allSchedules, err := s.manager.ListAllSchedules()
|
||||
if err != nil {
|
||||
log.Printf("Error listing schedules: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
for instanceName, schedules := range allSchedules {
|
||||
for _, schedule := range schedules {
|
||||
// Skip if disabled
|
||||
if !schedule.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip if already running
|
||||
if s.isRunning(schedule.ID) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if schedule should run
|
||||
if now.After(schedule.NextRun) || now.Equal(schedule.NextRun) {
|
||||
s.executeSchedule(instanceName, schedule)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// executeSchedule executes a backup for a schedule
|
||||
func (s *Scheduler) executeSchedule(instanceName string, schedule *BackupSchedule) {
|
||||
// Mark as running
|
||||
s.setRunning(schedule.ID, true)
|
||||
|
||||
// Execute in goroutine to avoid blocking scheduler
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.setRunning(schedule.ID, false)
|
||||
|
||||
log.Printf("Executing scheduled backup: %s for instance %s (target: %s/%s)",
|
||||
schedule.Name, instanceName, schedule.TargetType, schedule.TargetName)
|
||||
|
||||
// Execute backup
|
||||
var err error
|
||||
if schedule.TargetType == "cluster" {
|
||||
_, err = s.manager.BackupCluster(instanceName, ClusterBackupComponents{
|
||||
Etcd: true,
|
||||
Config: true,
|
||||
Secrets: true,
|
||||
})
|
||||
} else if schedule.TargetType == "app" {
|
||||
_, err = s.manager.BackupApp(instanceName, schedule.TargetName)
|
||||
} else {
|
||||
err = fmt.Errorf("invalid target type: %s", schedule.TargetType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Scheduled backup failed for %s: %v", schedule.Name, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("Scheduled backup completed successfully: %s", schedule.Name)
|
||||
|
||||
// Update schedule last run and next run
|
||||
now := time.Now()
|
||||
schedule.LastRun = &now
|
||||
schedule.NextRun = s.manager.GetNextRunTime(schedule.Frequency, now)
|
||||
|
||||
if _, err := s.manager.UpdateSchedule(instanceName, schedule.ID, schedule); err != nil {
|
||||
log.Printf("Failed to update schedule after backup: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Apply retention policy
|
||||
if err := s.manager.ApplyRetentionPolicy(instanceName, schedule); err != nil {
|
||||
log.Printf("Failed to apply retention policy for %s: %v", schedule.Name, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// isRunning checks if a schedule is currently executing
|
||||
func (s *Scheduler) isRunning(scheduleID string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.running[scheduleID]
|
||||
}
|
||||
|
||||
// setRunning sets the running state for a schedule
|
||||
func (s *Scheduler) setRunning(scheduleID string, running bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if running {
|
||||
s.running[scheduleID] = true
|
||||
} else {
|
||||
delete(s.running, scheduleID)
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteScheduledBackup manually triggers a scheduled backup (for "Run Now" functionality)
|
||||
func (s *Scheduler) ExecuteScheduledBackup(instanceName, scheduleID string) error {
|
||||
// Check if already running
|
||||
if s.isRunning(scheduleID) {
|
||||
return fmt.Errorf("backup is already running for this schedule")
|
||||
}
|
||||
|
||||
// Get schedule
|
||||
schedule, err := s.manager.GetSchedule(instanceName, scheduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Execute synchronously for manual trigger (user expects immediate feedback)
|
||||
s.setRunning(scheduleID, true)
|
||||
defer s.setRunning(scheduleID, false)
|
||||
|
||||
var backupErr error
|
||||
if schedule.TargetType == "cluster" {
|
||||
_, backupErr = s.manager.BackupCluster(instanceName, ClusterBackupComponents{
|
||||
Etcd: true,
|
||||
Config: true,
|
||||
Secrets: true,
|
||||
})
|
||||
} else if schedule.TargetType == "app" {
|
||||
_, backupErr = s.manager.BackupApp(instanceName, schedule.TargetName)
|
||||
} else {
|
||||
return fmt.Errorf("invalid target type: %s", schedule.TargetType)
|
||||
}
|
||||
|
||||
if backupErr != nil {
|
||||
return fmt.Errorf("backup execution failed: %w", backupErr)
|
||||
}
|
||||
|
||||
// Update schedule last run (but don't change next run for manual triggers)
|
||||
now := time.Now()
|
||||
schedule.LastRun = &now
|
||||
|
||||
if _, err := s.manager.UpdateSchedule(instanceName, scheduleID, schedule); err != nil {
|
||||
return fmt.Errorf("failed to update schedule after backup: %w", err)
|
||||
}
|
||||
|
||||
// Apply retention policy
|
||||
if err := s.manager.ApplyRetentionPolicy(instanceName, schedule); err != nil {
|
||||
log.Printf("Failed to apply retention policy for %s: %v", schedule.Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSchedulerStatus returns current scheduler status
|
||||
func (s *Scheduler) GetSchedulerStatus() map[string]interface{} {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
runningSchedules := make([]string, 0, len(s.running))
|
||||
for id := range s.running {
|
||||
runningSchedules = append(runningSchedules, id)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"running": true,
|
||||
"interval": s.interval.String(),
|
||||
"active_operations": len(s.running),
|
||||
"running_schedules": runningSchedules,
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateScheduleConfiguration checks if schedule can be created/updated
|
||||
func (s *Scheduler) ValidateScheduleConfiguration(instanceName string, schedule *BackupSchedule) error {
|
||||
// Check if instance exists
|
||||
instanceDir := filepath.Join(s.manager.dataDir, "instances", instanceName)
|
||||
if !storage.FileExists(instanceDir) {
|
||||
return fmt.Errorf("instance not found: %s", instanceName)
|
||||
}
|
||||
|
||||
// If app backup, verify app exists
|
||||
if schedule.TargetType == "app" {
|
||||
appsDir := filepath.Join(instanceDir, "apps", schedule.TargetName)
|
||||
if !storage.FileExists(appsDir) {
|
||||
return fmt.Errorf("app not found: %s", schedule.TargetName)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupOrphanedSchedules removes schedules for apps/instances that no longer exist
|
||||
func (s *Scheduler) CleanupOrphanedSchedules() error {
|
||||
allSchedules, err := s.manager.ListAllSchedules()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for instanceName, schedules := range allSchedules {
|
||||
// Check if instance exists
|
||||
instanceDir := filepath.Join(s.manager.dataDir, "instances", instanceName)
|
||||
if !storage.FileExists(instanceDir) {
|
||||
// Delete all schedules for this instance
|
||||
for _, schedule := range schedules {
|
||||
if err := s.manager.DeleteSchedule(instanceName, schedule.ID); err != nil {
|
||||
log.Printf("Failed to delete orphaned schedule %s: %v", schedule.ID, err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Check app schedules
|
||||
for _, schedule := range schedules {
|
||||
if schedule.TargetType == "app" {
|
||||
appsDir := filepath.Join(instanceDir, "apps", schedule.TargetName)
|
||||
if !storage.FileExists(appsDir) {
|
||||
// Delete schedule for non-existent app
|
||||
if err := s.manager.DeleteSchedule(instanceName, schedule.ID); err != nil {
|
||||
log.Printf("Failed to delete orphaned schedule %s: %v", schedule.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNextScheduledRun returns the next scheduled run time across all schedules
|
||||
func (s *Scheduler) GetNextScheduledRun() (*time.Time, error) {
|
||||
allSchedules, err := s.manager.ListAllSchedules()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var nextRun *time.Time
|
||||
for _, schedules := range allSchedules {
|
||||
for _, schedule := range schedules {
|
||||
if !schedule.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
if nextRun == nil || schedule.NextRun.Before(*nextRun) {
|
||||
nextRun = &schedule.NextRun
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nextRun, nil
|
||||
}
|
||||
|
||||
// GetScheduleHistory returns recent backup history for a schedule
|
||||
func (s *Scheduler) GetScheduleHistory(instanceName string, scheduleID string) ([]BackupHistoryEntry, error) {
|
||||
schedule, err := s.manager.GetSchedule(instanceName, scheduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var history []BackupHistoryEntry
|
||||
|
||||
if schedule.TargetType == "cluster" {
|
||||
backups, err := s.manager.ListClusterBackups(instanceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, backup := range backups {
|
||||
history = append(history, BackupHistoryEntry{
|
||||
Timestamp: backup.Timestamp,
|
||||
Status: backup.Status,
|
||||
Error: backup.Error,
|
||||
Size: backup.Size,
|
||||
CreatedAt: backup.CreatedAt,
|
||||
})
|
||||
}
|
||||
} else if schedule.TargetType == "app" {
|
||||
backups, err := s.manager.ListBackups(instanceName, schedule.TargetName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, backup := range backups {
|
||||
history = append(history, BackupHistoryEntry{
|
||||
Timestamp: backup.Timestamp,
|
||||
Status: backup.Status,
|
||||
Error: backup.Error,
|
||||
Size: backup.Size,
|
||||
CreatedAt: backup.CreatedAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return history, nil
|
||||
}
|
||||
|
||||
// BackupHistoryEntry represents a single backup execution
|
||||
type BackupHistoryEntry struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ResticClient struct {
|
||||
config *BackupConfig
|
||||
secrets *BackupSecrets
|
||||
dataDir string
|
||||
}
|
||||
|
||||
type ResticSnapshot struct {
|
||||
ID string `json:"id"`
|
||||
ShortID string `json:"short_id"`
|
||||
Time time.Time `json:"time"`
|
||||
Hostname string `json:"hostname"`
|
||||
Username string `json:"username"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
type ResticStats struct {
|
||||
TotalSize uint64 `json:"total_size"`
|
||||
TotalFileCount uint64 `json:"total_file_count"`
|
||||
}
|
||||
|
||||
type ResticStatus struct {
|
||||
Initialized bool `json:"initialized"`
|
||||
Snapshots int `json:"snapshots"`
|
||||
TotalSize uint64 `json:"totalSize"`
|
||||
LastBackup string `json:"lastBackup,omitempty"`
|
||||
}
|
||||
|
||||
func NewResticClient(dataDir string) (*ResticClient, error) {
|
||||
config, err := LoadBackupConfig(dataDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config: %w", err)
|
||||
}
|
||||
|
||||
if config == nil || config.Repository == "" {
|
||||
return nil, fmt.Errorf("backup not configured")
|
||||
}
|
||||
|
||||
secrets, err := LoadBackupSecrets(dataDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load secrets: %w", err)
|
||||
}
|
||||
|
||||
if secrets == nil || secrets.Password == "" {
|
||||
return nil, fmt.Errorf("backup password not configured")
|
||||
}
|
||||
|
||||
return &ResticClient{
|
||||
config: config,
|
||||
secrets: secrets,
|
||||
dataDir: dataDir,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) detectBackend() string {
|
||||
repo := r.config.Repository
|
||||
|
||||
if strings.HasPrefix(repo, "s3:") {
|
||||
return "s3"
|
||||
}
|
||||
if strings.HasPrefix(repo, "sftp:") {
|
||||
return "sftp"
|
||||
}
|
||||
if strings.HasPrefix(repo, "azure:") {
|
||||
return "azure"
|
||||
}
|
||||
if strings.HasPrefix(repo, "gs:") {
|
||||
return "gcs"
|
||||
}
|
||||
|
||||
return "local"
|
||||
}
|
||||
|
||||
func (r *ResticClient) buildEnv() []string {
|
||||
env := os.Environ()
|
||||
|
||||
env = append(env, fmt.Sprintf("RESTIC_REPOSITORY=%s", r.config.Repository))
|
||||
env = append(env, fmt.Sprintf("RESTIC_PASSWORD=%s", r.secrets.Password))
|
||||
|
||||
backend := r.detectBackend()
|
||||
|
||||
switch backend {
|
||||
case "s3":
|
||||
if r.secrets.Credentials.S3 != nil {
|
||||
env = append(env, fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", r.secrets.Credentials.S3.AccessKeyID))
|
||||
env = append(env, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", r.secrets.Credentials.S3.SecretAccessKey))
|
||||
}
|
||||
if r.config.Backend.Endpoint != "" {
|
||||
env = append(env, fmt.Sprintf("AWS_S3_ENDPOINT=%s", r.config.Backend.Endpoint))
|
||||
}
|
||||
if r.config.Backend.Region != "" {
|
||||
env = append(env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", r.config.Backend.Region))
|
||||
}
|
||||
|
||||
case "azure":
|
||||
if r.secrets.Credentials.Azure != nil {
|
||||
env = append(env, fmt.Sprintf("AZURE_ACCOUNT_NAME=%s", r.secrets.Credentials.Azure.AccountName))
|
||||
env = append(env, fmt.Sprintf("AZURE_ACCOUNT_KEY=%s", r.secrets.Credentials.Azure.AccountKey))
|
||||
}
|
||||
|
||||
case "gcs":
|
||||
if r.secrets.Credentials.GCS != nil {
|
||||
env = append(env, fmt.Sprintf("GOOGLE_PROJECT_ID=%s", r.secrets.Credentials.GCS.ProjectID))
|
||||
env = append(env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", r.secrets.Credentials.GCS.CredentialsJSON))
|
||||
}
|
||||
}
|
||||
|
||||
return env
|
||||
}
|
||||
|
||||
func (r *ResticClient) runCommand(args ...string) (string, error) {
|
||||
cmd := exec.Command("restic", args...)
|
||||
cmd.Env = r.buildEnv()
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("restic command failed: %w\nStderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Init() error {
|
||||
_, err := r.runCommand("init")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ResticClient) IsInitialized() bool {
|
||||
_, err := r.runCommand("snapshots", "--json")
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Backup(sourcePath string, tags ...string) (string, error) {
|
||||
args := []string{"backup", sourcePath, "--json"}
|
||||
|
||||
for _, tag := range tags {
|
||||
args = append(args, "--tag", tag)
|
||||
}
|
||||
|
||||
output, err := r.runCommand(args...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(output), "\n")
|
||||
lastLine := lines[len(lines)-1]
|
||||
|
||||
if err := json.Unmarshal([]byte(lastLine), &result); err != nil {
|
||||
return "", fmt.Errorf("failed to parse backup output: %w", err)
|
||||
}
|
||||
|
||||
return result.SnapshotID, nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Snapshots(tags ...string) ([]ResticSnapshot, error) {
|
||||
args := []string{"snapshots", "--json"}
|
||||
|
||||
for _, tag := range tags {
|
||||
args = append(args, "--tag", tag)
|
||||
}
|
||||
|
||||
output, err := r.runCommand(args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var snapshots []ResticSnapshot
|
||||
if err := json.Unmarshal([]byte(output), &snapshots); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse snapshots: %w", err)
|
||||
}
|
||||
|
||||
return snapshots, nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Stats() (*ResticStats, error) {
|
||||
output, err := r.runCommand("stats", "--json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var stats ResticStats
|
||||
if err := json.Unmarshal([]byte(output), &stats); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse stats: %w", err)
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Check() error {
|
||||
_, err := r.runCommand("check")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ResticClient) Forget(snapshotID string) error {
|
||||
_, err := r.runCommand("forget", snapshotID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ResticClient) Prune() error {
|
||||
_, err := r.runCommand("prune")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ResticClient) ForgetWithPolicy() error {
|
||||
args := []string{"forget", "--prune"}
|
||||
|
||||
if r.config.Retention.KeepDaily > 0 {
|
||||
args = append(args, "--keep-daily", fmt.Sprintf("%d", r.config.Retention.KeepDaily))
|
||||
}
|
||||
if r.config.Retention.KeepWeekly > 0 {
|
||||
args = append(args, "--keep-weekly", fmt.Sprintf("%d", r.config.Retention.KeepWeekly))
|
||||
}
|
||||
if r.config.Retention.KeepMonthly > 0 {
|
||||
args = append(args, "--keep-monthly", fmt.Sprintf("%d", r.config.Retention.KeepMonthly))
|
||||
}
|
||||
if r.config.Retention.KeepYearly > 0 {
|
||||
args = append(args, "--keep-yearly", fmt.Sprintf("%d", r.config.Retention.KeepYearly))
|
||||
}
|
||||
|
||||
_, err := r.runCommand(args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *ResticClient) Status() (*ResticStatus, error) {
|
||||
status := &ResticStatus{
|
||||
Initialized: r.IsInitialized(),
|
||||
}
|
||||
|
||||
if !status.Initialized {
|
||||
return status, nil
|
||||
}
|
||||
|
||||
snapshots, err := r.Snapshots()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get snapshots: %w", err)
|
||||
}
|
||||
|
||||
status.Snapshots = len(snapshots)
|
||||
|
||||
if len(snapshots) > 0 {
|
||||
latest := snapshots[0]
|
||||
for _, s := range snapshots {
|
||||
if s.Time.After(latest.Time) {
|
||||
latest = s
|
||||
}
|
||||
}
|
||||
status.LastBackup = latest.Time.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
stats, err := r.Stats()
|
||||
if err == nil {
|
||||
status.TotalSize = stats.TotalSize
|
||||
}
|
||||
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (r *ResticClient) Restore(snapshotID, targetPath string) error {
|
||||
_, err := r.runCommand("restore", snapshotID, "--target", targetPath)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetSnapshot retrieves details for a specific snapshot
|
||||
func (r *ResticClient) GetSnapshot(id string) (*ResticSnapshot, error) {
|
||||
output, err := r.runCommand("snapshots", id, "--json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var snapshots []ResticSnapshot
|
||||
if err := json.Unmarshal([]byte(output), &snapshots); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse snapshot: %w", err)
|
||||
}
|
||||
|
||||
if len(snapshots) == 0 {
|
||||
return nil, fmt.Errorf("snapshot not found: %s", id)
|
||||
}
|
||||
|
||||
return &snapshots[0], nil
|
||||
}
|
||||
|
||||
// ListSnapshotFiles lists files in a snapshot
|
||||
func (r *ResticClient) ListSnapshotFiles(id string) ([]string, error) {
|
||||
output, err := r.runCommand("ls", id, "--json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var files []string
|
||||
lines := strings.Split(strings.TrimSpace(output), "\n")
|
||||
for _, line := range lines {
|
||||
var entry struct {
|
||||
Path string `json:"path"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &entry); err != nil {
|
||||
continue
|
||||
}
|
||||
if entry.Path != "" {
|
||||
files = append(files, entry.Path)
|
||||
}
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// RestoreSnapshot downloads a snapshot to a target path
|
||||
func (r *ResticClient) RestoreSnapshot(id, targetPath string) error {
|
||||
if err := os.MkdirAll(targetPath, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create target directory: %w", err)
|
||||
}
|
||||
|
||||
_, err := r.runCommand("restore", id, "--target", targetPath)
|
||||
return err
|
||||
}
|
||||
|
||||
func UploadToRestic(dataDir, instanceName, appName, backupPath string) (string, error) {
|
||||
client, err := NewResticClient(dataDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if !client.IsInitialized() {
|
||||
if err := client.Init(); err != nil {
|
||||
return "", fmt.Errorf("failed to initialize repository: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
tags := []string{
|
||||
fmt.Sprintf("instance:%s", instanceName),
|
||||
fmt.Sprintf("app:%s", appName),
|
||||
}
|
||||
|
||||
snapshotID, err := client.Backup(backupPath, tags...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to backup to restic: %w", err)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(backupPath); err != nil {
|
||||
return snapshotID, fmt.Errorf("backup uploaded but failed to clean staging: %w", err)
|
||||
}
|
||||
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
func CleanStagingDirectory(stagingDir string) error {
|
||||
entries, err := os.ReadDir(stagingDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(stagingDir, entry.Name())
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// RestoreFromSnapshot restores an app from a restic snapshot
|
||||
func (m *Manager) RestoreFromSnapshot(instanceName, appName, snapshotID string, opts RestoreOptions) error {
|
||||
// Create restic client
|
||||
instanceDir := tools.GetInstancePath(m.dataDir, instanceName)
|
||||
client, err := NewResticClient(instanceDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create restic client: %w", err)
|
||||
}
|
||||
|
||||
// Verify snapshot exists and belongs to this app
|
||||
snapshot, err := client.GetSnapshot(snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Validate snapshot belongs to this app
|
||||
hasAppTag := false
|
||||
expectedTag := fmt.Sprintf("app:%s", appName)
|
||||
for _, tag := range snapshot.Tags {
|
||||
if tag == expectedTag {
|
||||
hasAppTag = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasAppTag {
|
||||
return fmt.Errorf("snapshot %s does not belong to app %s", snapshotID, appName)
|
||||
}
|
||||
|
||||
// Create temporary restore directory
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
restoreDir := filepath.Join(stagingDir, "restore-temp", snapshotID)
|
||||
if err := os.RemoveAll(restoreDir); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to clean restore directory: %w", err)
|
||||
}
|
||||
if err := storage.EnsureDir(restoreDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create restore directory: %w", err)
|
||||
}
|
||||
|
||||
// Download snapshot
|
||||
if err := client.RestoreSnapshot(snapshotID, restoreDir); err != nil {
|
||||
return fmt.Errorf("failed to restore snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Find backup data directory within restored snapshot
|
||||
backupDataDir, err := findBackupDataDir(restoreDir, appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to locate backup data: %w", err)
|
||||
}
|
||||
|
||||
// Restore using existing restore logic
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
|
||||
|
||||
// Restore database if not PVC-only
|
||||
if !opts.PVCOnly {
|
||||
if err := m.restoreDatabase(kubeconfigPath, appName, backupDataDir, opts.SkipGlobals); err != nil {
|
||||
return fmt.Errorf("database restore failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Restore PVCs if not DB-only
|
||||
if !opts.DBOnly {
|
||||
if err := m.restorePVCs(kubeconfigPath, appName, backupDataDir); err != nil {
|
||||
return fmt.Errorf("pvc restore failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up restore directory
|
||||
if err := os.RemoveAll(restoreDir); err != nil {
|
||||
return fmt.Errorf("restore completed but cleanup failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findBackupDataDir locates the backup data directory within a restored snapshot
|
||||
// The snapshot structure is: restoreDir/[full-path]/apps/appName/timestamp/
|
||||
func findBackupDataDir(restoreDir, appName string) (string, error) {
|
||||
// Walk the restore directory to find apps/appName/timestamp structure
|
||||
var foundDir string
|
||||
|
||||
err := filepath.Walk(restoreDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Look for backup.json which marks a backup directory
|
||||
if info.Name() == "backup.json" {
|
||||
// Verify this is for the correct app
|
||||
dirPath := filepath.Dir(path)
|
||||
if strings.Contains(dirPath, fmt.Sprintf("/apps/%s/", appName)) {
|
||||
foundDir = dirPath
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if foundDir == "" {
|
||||
return "", fmt.Errorf("backup data not found for app %s in snapshot", appName)
|
||||
}
|
||||
|
||||
return foundDir, nil
|
||||
}
|
||||
|
||||
// ListSnapshotsForInstance lists all snapshots for an instance
|
||||
func ListSnapshotsForInstance(dataDir, instanceName string, tags ...string) ([]ResticSnapshot, error) {
|
||||
instanceDir := tools.GetInstancePath(dataDir, instanceName)
|
||||
client, err := NewResticClient(instanceDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create restic client: %w", err)
|
||||
}
|
||||
|
||||
if !client.IsInitialized() {
|
||||
return []ResticSnapshot{}, nil
|
||||
}
|
||||
|
||||
// Add instance tag to filter
|
||||
allTags := append([]string{fmt.Sprintf("instance:%s", instanceName)}, tags...)
|
||||
return client.Snapshots(allTags...)
|
||||
}
|
||||
|
||||
// ListSnapshotsForApp lists all snapshots for a specific app
|
||||
func ListSnapshotsForApp(dataDir, instanceName, appName string) ([]ResticSnapshot, error) {
|
||||
return ListSnapshotsForInstance(dataDir, instanceName, fmt.Sprintf("app:%s", appName))
|
||||
}
|
||||
134
api/internal/backup/restore_integration_test.go
Normal file
134
api/internal/backup/restore_integration_test.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRestoreIntegration(t *testing.T) {
|
||||
// Skip this test in CI - it's for manual verification
|
||||
if os.Getenv("RUN_INTEGRATION_TESTS") != "true" {
|
||||
t.Skip("Skipping integration test - set RUN_INTEGRATION_TESTS=true to run")
|
||||
}
|
||||
|
||||
tempDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
// Set up instance directory structure
|
||||
instanceDir := filepath.Join(tempDir, "instances", instanceName)
|
||||
appDir := filepath.Join(instanceDir, "apps", appName)
|
||||
backupsDir := filepath.Join(instanceDir, "backups")
|
||||
require.NoError(t, os.MkdirAll(appDir, 0755))
|
||||
require.NoError(t, os.MkdirAll(backupsDir, 0755))
|
||||
|
||||
// Create manifest with postgres dependency
|
||||
manifestPath := filepath.Join(appDir, "manifest.yaml")
|
||||
manifestContent := `
|
||||
name: test-app
|
||||
description: Test application
|
||||
requires:
|
||||
- name: postgres
|
||||
defaultConfig:
|
||||
dbName: testdb
|
||||
`
|
||||
require.NoError(t, os.WriteFile(manifestPath, []byte(manifestContent), 0644))
|
||||
|
||||
// Create config
|
||||
configPath := filepath.Join(instanceDir, "config.yaml")
|
||||
configContent := `
|
||||
backup:
|
||||
destination:
|
||||
type: local
|
||||
local:
|
||||
path: ` + backupsDir + `
|
||||
`
|
||||
require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0644))
|
||||
|
||||
// Create manager with mock strategies
|
||||
mgr := NewManager(tempDir)
|
||||
|
||||
restoreCalled := map[string]bool{}
|
||||
|
||||
mockConfigStrategy := &MockStrategy{
|
||||
Name_: "config",
|
||||
BackupFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry != nil {
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": "config/test.tar.gz",
|
||||
"size": int64(100),
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
}
|
||||
return nil
|
||||
},
|
||||
RestoreFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
restoreCalled["config"] = true
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry != nil {
|
||||
entry.Status = "restored"
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPostgresStrategy := &MockStrategy{
|
||||
Name_: "postgres",
|
||||
BackupFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry != nil {
|
||||
entry.Params = map[string]interface{}{
|
||||
"dbName": "testdb",
|
||||
"dbUser": "test-app",
|
||||
}
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": "postgres/test.dump",
|
||||
"size": int64(200),
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
}
|
||||
return nil
|
||||
},
|
||||
RestoreFunc: func(plan *RecoveryPlan, dest BackupDestination) error {
|
||||
restoreCalled["postgres"] = true
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry != nil {
|
||||
entry.Restore = map[string]interface{}{
|
||||
"dbName": "testdb_restore",
|
||||
}
|
||||
entry.Status = "restored"
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
mgr.strategies = map[string]Strategy{
|
||||
"config": mockConfigStrategy,
|
||||
"postgres": mockPostgresStrategy,
|
||||
}
|
||||
|
||||
// First create a backup
|
||||
plan, err := mgr.BackupApp(instanceName, appName)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, plan)
|
||||
assert.Equal(t, "backed_up", plan.Status)
|
||||
assert.Len(t, plan.Strategies, 3) // config + postgres + pvc (from detectStrategies)
|
||||
|
||||
// Now test restore
|
||||
opts := RestoreOptions{
|
||||
Components: []string{},
|
||||
}
|
||||
|
||||
restoredPlan, err := mgr.RestoreApp(instanceName, appName, opts)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, restoredPlan)
|
||||
|
||||
// Verify both strategies were called
|
||||
assert.True(t, restoreCalled["config"], "Config restore should have been called")
|
||||
assert.True(t, restoreCalled["postgres"], "Postgres restore should have been called")
|
||||
}
|
||||
98
api/internal/backup/retention.go
Normal file
98
api/internal/backup/retention.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// EnforceRetention deletes old backups for an app based on retention policy.
|
||||
// A backup is kept if EITHER keepLast or keepDays says to keep it (whichever is more permissive wins).
|
||||
// Only terminal-state backups (backed_up, cleaned_up, failed) are eligible for deletion.
|
||||
// Returns the number of deleted backups.
|
||||
func EnforceRetention(mgr *Manager, instanceName, appName string, keepLast, keepDays int) (int, error) {
|
||||
plans, err := mgr.ListBackups(instanceName, appName)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to list backups: %w", err)
|
||||
}
|
||||
|
||||
// Filter to terminal-state plans only
|
||||
var eligible []*btypes.RecoveryPlan
|
||||
for _, plan := range plans {
|
||||
if isTerminalStatus(plan.Status) {
|
||||
eligible = append(eligible, plan)
|
||||
}
|
||||
}
|
||||
|
||||
if len(eligible) <= keepLast {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().AddDate(0, 0, -keepDays)
|
||||
deleted := 0
|
||||
|
||||
for i, plan := range eligible {
|
||||
// Keep if within keepLast count (plans are sorted newest first)
|
||||
if i < keepLast {
|
||||
continue
|
||||
}
|
||||
|
||||
// Keep if within keepDays
|
||||
planTime, err := time.Parse("20060102T150405Z", plan.Timestamp)
|
||||
if err != nil {
|
||||
continue // Can't parse timestamp, skip to be safe
|
||||
}
|
||||
if planTime.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Both policies say delete
|
||||
if err := mgr.DeleteAppBackup(instanceName, appName, plan.Timestamp); err != nil {
|
||||
slog.Error("failed to delete backup", "component", "backup", "instance", instanceName, "app", appName, "timestamp", plan.Timestamp, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// isTerminalStatus returns true if the backup can be safely considered for deletion
|
||||
func isTerminalStatus(status string) bool {
|
||||
switch status {
|
||||
case "backed_up", "cleaned_up", "failed":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// RetentionFromSchedule returns keepLast and keepDays from a schedule's retention policy,
|
||||
// falling back to instance-level defaults if the schedule has no override.
|
||||
func RetentionFromSchedule(sched *btypes.BackupSchedule, instanceRetention RetentionPolicy) (int, int) {
|
||||
if sched != nil && sched.Retention != nil {
|
||||
keepLast := sched.Retention.KeepLast
|
||||
keepDays := sched.Retention.KeepDays
|
||||
if keepLast < 1 {
|
||||
keepLast = 1
|
||||
}
|
||||
if keepDays < 1 {
|
||||
keepDays = 1
|
||||
}
|
||||
return keepLast, keepDays
|
||||
}
|
||||
return DefaultRetention(instanceRetention)
|
||||
}
|
||||
|
||||
// DefaultRetention derives keepLast and keepDays from instance-level retention policy.
|
||||
// Used for manual backups without a schedule.
|
||||
func DefaultRetention(r RetentionPolicy) (int, int) {
|
||||
keepLast := r.Daily
|
||||
if keepLast < 1 {
|
||||
keepLast = 7
|
||||
}
|
||||
keepDays := max(keepLast*2, 7)
|
||||
return keepLast, keepDays
|
||||
}
|
||||
367
api/internal/backup/retention_test.go
Normal file
367
api/internal/backup/retention_test.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestIsTerminalStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
status string
|
||||
expected bool
|
||||
}{
|
||||
{"backed_up", true},
|
||||
{"cleaned_up", true},
|
||||
{"failed", true},
|
||||
{"backing_up", false},
|
||||
{"restoring", false},
|
||||
{"restored", false},
|
||||
{"switching", false},
|
||||
{"switched", false},
|
||||
{"cleaning_up", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := isTerminalStatus(tt.status); got != tt.expected {
|
||||
t.Errorf("isTerminalStatus(%q) = %v, want %v", tt.status, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultRetention(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
policy RetentionPolicy
|
||||
wantKeepLast int
|
||||
wantKeepDays int
|
||||
}{
|
||||
{
|
||||
name: "standard defaults",
|
||||
policy: RetentionPolicy{Daily: 7, Weekly: 4, Monthly: 6},
|
||||
wantKeepLast: 7,
|
||||
wantKeepDays: 14,
|
||||
},
|
||||
{
|
||||
name: "zero daily uses default",
|
||||
policy: RetentionPolicy{},
|
||||
wantKeepLast: 7,
|
||||
wantKeepDays: 14,
|
||||
},
|
||||
{
|
||||
name: "small daily enforces minimum keepDays",
|
||||
policy: RetentionPolicy{Daily: 2},
|
||||
wantKeepLast: 2,
|
||||
wantKeepDays: 7,
|
||||
},
|
||||
{
|
||||
name: "large daily",
|
||||
policy: RetentionPolicy{Daily: 30},
|
||||
wantKeepLast: 30,
|
||||
wantKeepDays: 60,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
keepLast, keepDays := DefaultRetention(tt.policy)
|
||||
if keepLast != tt.wantKeepLast || keepDays != tt.wantKeepDays {
|
||||
t.Errorf("DefaultRetention() = (%d, %d), want (%d, %d)",
|
||||
keepLast, keepDays, tt.wantKeepLast, tt.wantKeepDays)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionFromSchedule(t *testing.T) {
|
||||
instancePolicy := RetentionPolicy{Daily: 7, Weekly: 4}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sched *btypes.BackupSchedule
|
||||
wantKeepLast int
|
||||
wantKeepDays int
|
||||
}{
|
||||
{
|
||||
name: "nil schedule uses defaults",
|
||||
sched: nil,
|
||||
wantKeepLast: 7,
|
||||
wantKeepDays: 14,
|
||||
},
|
||||
{
|
||||
name: "schedule without retention uses defaults",
|
||||
sched: &btypes.BackupSchedule{
|
||||
Retention: nil,
|
||||
},
|
||||
wantKeepLast: 7,
|
||||
wantKeepDays: 14,
|
||||
},
|
||||
{
|
||||
name: "schedule with retention override",
|
||||
sched: &btypes.BackupSchedule{
|
||||
Retention: &btypes.ScheduleRetention{KeepLast: 3, KeepDays: 10},
|
||||
},
|
||||
wantKeepLast: 3,
|
||||
wantKeepDays: 10,
|
||||
},
|
||||
{
|
||||
name: "schedule with zero values enforces minimums",
|
||||
sched: &btypes.BackupSchedule{
|
||||
Retention: &btypes.ScheduleRetention{KeepLast: 0, KeepDays: 0},
|
||||
},
|
||||
wantKeepLast: 1,
|
||||
wantKeepDays: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
keepLast, keepDays := RetentionFromSchedule(tt.sched, instancePolicy)
|
||||
if keepLast != tt.wantKeepLast || keepDays != tt.wantKeepDays {
|
||||
t.Errorf("RetentionFromSchedule() = (%d, %d), want (%d, %d)",
|
||||
keepLast, keepDays, tt.wantKeepLast, tt.wantKeepDays)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceRetention(t *testing.T) {
|
||||
// Set up temp directory structure simulating backups
|
||||
tmpDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
// Create instance config with local destination
|
||||
instanceDir := filepath.Join(tmpDir, "instances", instanceName)
|
||||
if err := os.MkdirAll(filepath.Join(instanceDir, "config"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"backup": map[string]any{
|
||||
"destination": map[string]any{
|
||||
"type": "local",
|
||||
"local": map[string]any{
|
||||
"path": filepath.Join(instanceDir, "backups"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
configData, _ := yaml.Marshal(config)
|
||||
if err := os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName)
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create test backup plans with different timestamps
|
||||
now := time.Now().UTC()
|
||||
timestamps := []struct {
|
||||
ts string
|
||||
age time.Duration
|
||||
status string
|
||||
}{
|
||||
{now.Format("20060102T150405Z"), 0, "backed_up"}, // newest
|
||||
{now.Add(-24 * time.Hour).Format("20060102T150405Z"), 24 * time.Hour, "backed_up"}, // 1 day old
|
||||
{now.Add(-48 * time.Hour).Format("20060102T150405Z"), 48 * time.Hour, "backed_up"}, // 2 days old
|
||||
{now.Add(-72 * time.Hour).Format("20060102T150405Z"), 72 * time.Hour, "backed_up"}, // 3 days old
|
||||
{now.Add(-96 * time.Hour).Format("20060102T150405Z"), 96 * time.Hour, "backed_up"}, // 4 days old
|
||||
}
|
||||
|
||||
for _, ts := range timestamps {
|
||||
planDir := filepath.Join(backupDir, ts.ts)
|
||||
if err := os.MkdirAll(planDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan := btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: ts.ts,
|
||||
Status: ts.status,
|
||||
}
|
||||
planData, _ := yaml.Marshal(plan)
|
||||
if err := os.WriteFile(filepath.Join(planDir, "recovery-plan.yaml"), planData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
mgr := NewManager(tmpDir)
|
||||
|
||||
t.Run("keepLast=2, keepDays=1 deletes old backups beyond both limits", func(t *testing.T) {
|
||||
// keepLast=2 protects the 2 newest
|
||||
// keepDays=1 protects anything within 1 day
|
||||
// Backups 3,4,5 are beyond keepLast. Backup 2 is 1 day old (within keepDays).
|
||||
// Only backups 3,4,5 (2+ days old and beyond keepLast=2) should be deleted.
|
||||
deleted, err := EnforceRetention(mgr, instanceName, appName, 2, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("EnforceRetention failed: %v", err)
|
||||
}
|
||||
if deleted != 3 {
|
||||
t.Errorf("expected 3 deleted, got %d", deleted)
|
||||
}
|
||||
|
||||
// Verify remaining
|
||||
remaining, _ := mgr.ListBackups(instanceName, appName)
|
||||
if len(remaining) != 2 {
|
||||
t.Errorf("expected 2 remaining backups, got %d", len(remaining))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEnforceRetentionSkipsActiveBackups(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
instanceDir := filepath.Join(tmpDir, "instances", instanceName)
|
||||
if err := os.MkdirAll(filepath.Join(instanceDir, "config"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"backup": map[string]any{
|
||||
"destination": map[string]any{
|
||||
"type": "local",
|
||||
"local": map[string]any{
|
||||
"path": filepath.Join(instanceDir, "backups"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
configData, _ := yaml.Marshal(config)
|
||||
if err := os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName)
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
backups := []struct {
|
||||
ts string
|
||||
status string
|
||||
}{
|
||||
{now.Format("20060102T150405Z"), "backed_up"},
|
||||
{now.Add(-24 * time.Hour).Format("20060102T150405Z"), "restoring"}, // active - should be skipped
|
||||
{now.Add(-48 * time.Hour).Format("20060102T150405Z"), "backed_up"},
|
||||
{now.Add(-72 * time.Hour).Format("20060102T150405Z"), "backed_up"},
|
||||
}
|
||||
|
||||
for _, b := range backups {
|
||||
planDir := filepath.Join(backupDir, b.ts)
|
||||
if err := os.MkdirAll(planDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan := btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: b.ts,
|
||||
Status: b.status,
|
||||
}
|
||||
planData, _ := yaml.Marshal(plan)
|
||||
if err := os.WriteFile(filepath.Join(planDir, "recovery-plan.yaml"), planData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
mgr := NewManager(tmpDir)
|
||||
|
||||
// keepLast=1, keepDays=0 means only keep 1 and delete everything older
|
||||
// The "restoring" backup should be skipped entirely
|
||||
// Terminal: [newest, 2-day-old, 3-day-old] -> keep 1, delete 2
|
||||
deleted, err := EnforceRetention(mgr, instanceName, appName, 1, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("EnforceRetention failed: %v", err)
|
||||
}
|
||||
if deleted != 2 {
|
||||
t.Errorf("expected 2 deleted, got %d", deleted)
|
||||
}
|
||||
|
||||
// Verify "restoring" backup is still there
|
||||
remaining, _ := mgr.ListBackups(instanceName, appName)
|
||||
if len(remaining) != 2 { // 1 kept + 1 restoring
|
||||
t.Errorf("expected 2 remaining backups, got %d", len(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceRetentionKeepDaysPreservesRecent(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
appName := "test-app"
|
||||
|
||||
instanceDir := filepath.Join(tmpDir, "instances", instanceName)
|
||||
if err := os.MkdirAll(filepath.Join(instanceDir, "config"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config := map[string]any{
|
||||
"backup": map[string]any{
|
||||
"destination": map[string]any{
|
||||
"type": "local",
|
||||
"local": map[string]any{
|
||||
"path": filepath.Join(instanceDir, "backups"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
configData, _ := yaml.Marshal(config)
|
||||
if err := os.WriteFile(filepath.Join(instanceDir, "config", "instance.yaml"), configData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
backupDir := filepath.Join(instanceDir, "data", "backup", "records", appName)
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
// 5 backups: newest, 1h old, 2h old, 3h old, 25h old
|
||||
timestamps := []time.Duration{0, time.Hour, 2 * time.Hour, 3 * time.Hour, 25 * time.Hour}
|
||||
|
||||
for _, offset := range timestamps {
|
||||
ts := now.Add(-offset).Format("20060102T150405Z")
|
||||
planDir := filepath.Join(backupDir, ts)
|
||||
if err := os.MkdirAll(planDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan := btypes.RecoveryPlan{
|
||||
App: appName,
|
||||
Instance: instanceName,
|
||||
Timestamp: ts,
|
||||
Status: "backed_up",
|
||||
}
|
||||
planData, _ := yaml.Marshal(plan)
|
||||
if err := os.WriteFile(filepath.Join(planDir, "recovery-plan.yaml"), planData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
mgr := NewManager(tmpDir)
|
||||
|
||||
// keepLast=1, keepDays=1: keep 1 by count, but keepDays=1 preserves anything within 24h
|
||||
// The 4 within 24h are all protected by keepDays. Only the 25h-old one gets deleted.
|
||||
deleted, err := EnforceRetention(mgr, instanceName, appName, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("EnforceRetention failed: %v", err)
|
||||
}
|
||||
if deleted != 1 {
|
||||
t.Errorf("expected 1 deleted, got %d", deleted)
|
||||
}
|
||||
|
||||
remaining, _ := mgr.ListBackups(instanceName, appName)
|
||||
if len(remaining) != 4 {
|
||||
t.Errorf("expected 4 remaining backups, got %d", len(remaining))
|
||||
}
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// BackupSchedule represents a scheduled backup configuration
|
||||
type BackupSchedule struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
TargetType string `json:"target_type"` // "cluster" or "app"
|
||||
TargetName string `json:"target_name"` // app name, or "cluster"
|
||||
Frequency string `json:"frequency"` // "daily", "weekly", "monthly"
|
||||
Retention RetentionPolicy `json:"retention"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LastRun *time.Time `json:"last_run,omitempty"`
|
||||
NextRun time.Time `json:"next_run"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// RetentionPolicy defines how long to keep backups
|
||||
type RetentionPolicy struct {
|
||||
KeepLast int `json:"keep_last"` // Keep last N backups
|
||||
KeepDays int `json:"keep_days"` // Keep backups for N days
|
||||
}
|
||||
|
||||
// GetSchedulesPath returns the path to schedules.json for an instance
|
||||
func (m *Manager) GetSchedulesPath(instanceName string) string {
|
||||
return filepath.Join(m.GetBackupDir(instanceName), "schedules.json")
|
||||
}
|
||||
|
||||
// LoadSchedules loads all schedules from JSON file
|
||||
func (m *Manager) LoadSchedules(instanceName string) ([]*BackupSchedule, error) {
|
||||
schedulesPath := m.GetSchedulesPath(instanceName)
|
||||
|
||||
if !storage.FileExists(schedulesPath) {
|
||||
return []*BackupSchedule{}, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(schedulesPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read schedules file: %w", err)
|
||||
}
|
||||
|
||||
var schedules []*BackupSchedule
|
||||
if err := json.Unmarshal(data, &schedules); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse schedules: %w", err)
|
||||
}
|
||||
|
||||
return schedules, nil
|
||||
}
|
||||
|
||||
// SaveSchedules saves all schedules to JSON file
|
||||
func (m *Manager) SaveSchedules(instanceName string, schedules []*BackupSchedule) error {
|
||||
schedulesPath := m.GetSchedulesPath(instanceName)
|
||||
|
||||
// Ensure backup directory exists
|
||||
backupDir := m.GetBackupDir(instanceName)
|
||||
if err := storage.EnsureDir(backupDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create backup directory: %w", err)
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(schedules, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal schedules: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(schedulesPath, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write schedules file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSchedule creates a new backup schedule
|
||||
func (m *Manager) CreateSchedule(instanceName string, schedule *BackupSchedule) (*BackupSchedule, error) {
|
||||
// Validate schedule
|
||||
if err := m.validateSchedule(schedule); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load existing schedules
|
||||
schedules, err := m.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generate ID and timestamps
|
||||
schedule.ID = uuid.New().String()
|
||||
schedule.CreatedAt = time.Now()
|
||||
schedule.UpdatedAt = time.Now()
|
||||
|
||||
// Calculate next run time
|
||||
schedule.NextRun = m.GetNextRunTime(schedule.Frequency, time.Now())
|
||||
|
||||
// Add to schedules
|
||||
schedules = append(schedules, schedule)
|
||||
|
||||
// Save schedules
|
||||
if err := m.SaveSchedules(instanceName, schedules); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return schedule, nil
|
||||
}
|
||||
|
||||
// UpdateSchedule updates an existing schedule
|
||||
func (m *Manager) UpdateSchedule(instanceName string, scheduleID string, updates *BackupSchedule) (*BackupSchedule, error) {
|
||||
// Validate updates
|
||||
if err := m.validateSchedule(updates); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load existing schedules
|
||||
schedules, err := m.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find schedule to update
|
||||
var found bool
|
||||
for i, s := range schedules {
|
||||
if s.ID == scheduleID {
|
||||
// Preserve ID, CreatedAt, LastRun
|
||||
updates.ID = s.ID
|
||||
updates.CreatedAt = s.CreatedAt
|
||||
updates.LastRun = s.LastRun
|
||||
updates.UpdatedAt = time.Now()
|
||||
|
||||
// Recalculate next run time if frequency changed
|
||||
if updates.Frequency != s.Frequency {
|
||||
updates.NextRun = m.GetNextRunTime(updates.Frequency, time.Now())
|
||||
} else {
|
||||
updates.NextRun = s.NextRun
|
||||
}
|
||||
|
||||
schedules[i] = updates
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil, fmt.Errorf("schedule not found: %s", scheduleID)
|
||||
}
|
||||
|
||||
// Save schedules
|
||||
if err := m.SaveSchedules(instanceName, schedules); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return updates, nil
|
||||
}
|
||||
|
||||
// DeleteSchedule deletes a schedule
|
||||
func (m *Manager) DeleteSchedule(instanceName string, scheduleID string) error {
|
||||
// Load existing schedules
|
||||
schedules, err := m.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Find and remove schedule
|
||||
var newSchedules []*BackupSchedule
|
||||
var found bool
|
||||
for _, s := range schedules {
|
||||
if s.ID == scheduleID {
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
newSchedules = append(newSchedules, s)
|
||||
}
|
||||
|
||||
if !found {
|
||||
return fmt.Errorf("schedule not found: %s", scheduleID)
|
||||
}
|
||||
|
||||
// Save schedules
|
||||
return m.SaveSchedules(instanceName, newSchedules)
|
||||
}
|
||||
|
||||
// GetSchedule retrieves a specific schedule
|
||||
func (m *Manager) GetSchedule(instanceName string, scheduleID string) (*BackupSchedule, error) {
|
||||
schedules, err := m.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, s := range schedules {
|
||||
if s.ID == scheduleID {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("schedule not found: %s", scheduleID)
|
||||
}
|
||||
|
||||
// RunSchedule manually triggers a scheduled backup
|
||||
func (m *Manager) RunSchedule(instanceName string, scheduleID string) error {
|
||||
schedule, err := m.GetSchedule(instanceName, scheduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Execute backup based on target type
|
||||
if schedule.TargetType == "cluster" {
|
||||
_, err = m.BackupCluster(instanceName, ClusterBackupComponents{
|
||||
Etcd: true,
|
||||
Config: true,
|
||||
Secrets: true,
|
||||
})
|
||||
} else if schedule.TargetType == "app" {
|
||||
_, err = m.BackupApp(instanceName, schedule.TargetName)
|
||||
} else {
|
||||
return fmt.Errorf("invalid target type: %s", schedule.TargetType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup execution failed: %w", err)
|
||||
}
|
||||
|
||||
// Update last run time
|
||||
now := time.Now()
|
||||
schedule.LastRun = &now
|
||||
schedule.NextRun = m.GetNextRunTime(schedule.Frequency, now)
|
||||
|
||||
// Save updated schedule
|
||||
_, err = m.UpdateSchedule(instanceName, scheduleID, schedule)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetNextRunTime calculates the next run time based on frequency
|
||||
func (m *Manager) GetNextRunTime(frequency string, from time.Time) time.Time {
|
||||
// Normalize to start of day at 2 AM
|
||||
year, month, day := from.Date()
|
||||
base := time.Date(year, month, day, 2, 0, 0, 0, from.Location())
|
||||
|
||||
// If we're past 2 AM today, start from tomorrow
|
||||
if from.After(base) {
|
||||
base = base.AddDate(0, 0, 1)
|
||||
}
|
||||
|
||||
switch frequency {
|
||||
case "daily":
|
||||
return base
|
||||
case "weekly":
|
||||
// Next Sunday at 2 AM
|
||||
daysUntilSunday := (7 - int(base.Weekday())) % 7
|
||||
if daysUntilSunday == 0 && from.After(base) {
|
||||
daysUntilSunday = 7
|
||||
}
|
||||
return base.AddDate(0, 0, daysUntilSunday)
|
||||
case "monthly":
|
||||
// Next 1st of month at 2 AM
|
||||
if base.Day() == 1 && !from.After(base) {
|
||||
return base
|
||||
}
|
||||
// Go to next month, day 1
|
||||
nextMonth := base.AddDate(0, 1, 0)
|
||||
return time.Date(nextMonth.Year(), nextMonth.Month(), 1, 2, 0, 0, 0, from.Location())
|
||||
default:
|
||||
// Default to daily
|
||||
return base
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyRetentionPolicy deletes old backups based on retention rules
|
||||
func (m *Manager) ApplyRetentionPolicy(instanceName string, schedule *BackupSchedule) error {
|
||||
if schedule.TargetType == "cluster" {
|
||||
return m.applyClusterRetention(instanceName, schedule.Retention)
|
||||
} else if schedule.TargetType == "app" {
|
||||
return m.applyAppRetention(instanceName, schedule.TargetName, schedule.Retention)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyClusterRetention applies retention policy to cluster backups
|
||||
func (m *Manager) applyClusterRetention(instanceName string, retention RetentionPolicy) error {
|
||||
backups, err := m.ListClusterBackups(instanceName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort backups by creation time (newest first) using modern Go idioms
|
||||
sort.Slice(backups, func(i, j int) bool {
|
||||
return backups[i].CreatedAt.After(backups[j].CreatedAt)
|
||||
})
|
||||
|
||||
// Determine which backups to delete
|
||||
now := time.Now()
|
||||
var toDelete []string
|
||||
|
||||
for i, backup := range backups {
|
||||
keep := false
|
||||
|
||||
// Keep if within keep_last (newest N backups)
|
||||
if retention.KeepLast > 0 && i < retention.KeepLast {
|
||||
keep = true
|
||||
}
|
||||
|
||||
// Keep if within keep_days
|
||||
if retention.KeepDays > 0 {
|
||||
age := now.Sub(backup.CreatedAt)
|
||||
if int(age.Hours()/24) < retention.KeepDays {
|
||||
keep = true
|
||||
}
|
||||
}
|
||||
|
||||
if !keep {
|
||||
toDelete = append(toDelete, backup.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete backups
|
||||
for _, timestamp := range toDelete {
|
||||
if err := m.DeleteClusterBackup(instanceName, timestamp); err != nil {
|
||||
log.Printf("Warning: failed to delete cluster backup %s: %v", timestamp, err)
|
||||
// Continue with other deletions
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyAppRetention applies retention policy to app backups
|
||||
func (m *Manager) applyAppRetention(instanceName, appName string, retention RetentionPolicy) error {
|
||||
backups, err := m.ListBackups(instanceName, appName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort backups by creation time (newest first) using modern Go idioms
|
||||
sort.Slice(backups, func(i, j int) bool {
|
||||
return backups[i].CreatedAt.After(backups[j].CreatedAt)
|
||||
})
|
||||
|
||||
// Determine which backups to delete
|
||||
now := time.Now()
|
||||
var toDelete []string
|
||||
|
||||
for i, backup := range backups {
|
||||
keep := false
|
||||
|
||||
// Keep if within keep_last (newest N backups)
|
||||
if retention.KeepLast > 0 && i < retention.KeepLast {
|
||||
keep = true
|
||||
}
|
||||
|
||||
// Keep if within keep_days
|
||||
if retention.KeepDays > 0 {
|
||||
age := now.Sub(backup.CreatedAt)
|
||||
if int(age.Hours()/24) < retention.KeepDays {
|
||||
keep = true
|
||||
}
|
||||
}
|
||||
|
||||
if !keep {
|
||||
toDelete = append(toDelete, backup.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete individual backup directories by timestamp
|
||||
stagingDir := m.GetStagingDir(instanceName)
|
||||
for _, timestamp := range toDelete {
|
||||
backupDir := filepath.Join(stagingDir, "apps", appName, timestamp)
|
||||
if err := os.RemoveAll(backupDir); err != nil {
|
||||
log.Printf("Warning: failed to delete backup %s: %v", timestamp, err)
|
||||
// Continue with other deletions
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSchedule validates schedule parameters
|
||||
func (m *Manager) validateSchedule(schedule *BackupSchedule) error {
|
||||
if schedule.Name == "" {
|
||||
return fmt.Errorf("schedule name is required")
|
||||
}
|
||||
|
||||
if schedule.TargetType != "cluster" && schedule.TargetType != "app" {
|
||||
return fmt.Errorf("target_type must be 'cluster' or 'app'")
|
||||
}
|
||||
|
||||
if schedule.TargetType == "app" && schedule.TargetName == "" {
|
||||
return fmt.Errorf("target_name is required for app backups")
|
||||
}
|
||||
|
||||
if schedule.Frequency != "daily" && schedule.Frequency != "weekly" && schedule.Frequency != "monthly" {
|
||||
return fmt.Errorf("frequency must be 'daily', 'weekly', or 'monthly'")
|
||||
}
|
||||
|
||||
if schedule.Retention.KeepLast < 1 {
|
||||
return fmt.Errorf("retention.keep_last must be at least 1")
|
||||
}
|
||||
|
||||
if schedule.Retention.KeepDays < 1 {
|
||||
return fmt.Errorf("retention.keep_days must be at least 1")
|
||||
}
|
||||
|
||||
// Validate app exists if target is app
|
||||
if schedule.TargetType == "app" {
|
||||
// This would need instance context to validate properly
|
||||
// For now, we'll skip validation and let the backup execution fail if app doesn't exist
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAllSchedules lists schedules for all instances
|
||||
func (m *Manager) ListAllSchedules() (map[string][]*BackupSchedule, error) {
|
||||
result := make(map[string][]*BackupSchedule)
|
||||
|
||||
instancesDir := filepath.Join(m.dataDir, "instances")
|
||||
if !storage.FileExists(instancesDir) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(instancesDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read instances directory: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
instanceName := entry.Name()
|
||||
schedules, err := m.LoadSchedules(instanceName)
|
||||
if err != nil {
|
||||
continue // Skip instances with errors
|
||||
}
|
||||
|
||||
if len(schedules) > 0 {
|
||||
result[instanceName] = schedules
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetInstanceNameFromDataDir extracts instance name from data directory
|
||||
func (m *Manager) GetInstanceNameFromDataDir(dataDir string) string {
|
||||
return filepath.Base(filepath.Dir(dataDir))
|
||||
}
|
||||
|
||||
// Helper function to check if instance exists
|
||||
func (m *Manager) instanceExists(instanceName string) bool {
|
||||
instanceDir := tools.GetInstancePath(m.dataDir, instanceName)
|
||||
return storage.FileExists(instanceDir)
|
||||
}
|
||||
217
api/internal/backup/scheduler.go
Normal file
217
api/internal/backup/scheduler.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/instance"
|
||||
)
|
||||
|
||||
// Scheduler runs backup schedules in the background.
|
||||
// It checks every minute if any schedule is due and triggers the backup.
|
||||
type Scheduler struct {
|
||||
dataDir string
|
||||
instanceMgr *instance.Manager
|
||||
mu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
running map[string]bool // tracks in-flight backups by schedule ID (s_{app}_{tier})
|
||||
}
|
||||
|
||||
// NewScheduler creates a new backup scheduler
|
||||
func NewScheduler(dataDir string, instanceMgr *instance.Manager) *Scheduler {
|
||||
return &Scheduler{
|
||||
dataDir: dataDir,
|
||||
instanceMgr: instanceMgr,
|
||||
running: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the scheduler loop. Call Stop() to shut it down.
|
||||
func (s *Scheduler) Start() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s.cancel = cancel
|
||||
|
||||
go s.loop(ctx)
|
||||
slog.Info("backup scheduler started", "component", "scheduler")
|
||||
}
|
||||
|
||||
// Stop shuts down the scheduler
|
||||
func (s *Scheduler) Stop() {
|
||||
if s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) loop(ctx context.Context) {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.tick()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) tick() {
|
||||
instances, err := s.instanceMgr.ListInstances()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
for _, inst := range instances {
|
||||
schedules, err := ListAppSchedules(s.dataDir, inst)
|
||||
if err != nil || len(schedules) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for i := range schedules {
|
||||
sched := &schedules[i]
|
||||
if !sched.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
if sched.NextRun == nil {
|
||||
next := ComputeNextRun(sched, now)
|
||||
sched.NextRun = &next
|
||||
s.saveScheduleCache(inst, sched.TargetName, sched.Frequency, sched.LastRun, sched.NextRun)
|
||||
continue
|
||||
}
|
||||
|
||||
if now.Before(*sched.NextRun) {
|
||||
continue
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if s.running[sched.ID] {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
s.running[sched.ID] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
go s.runSchedule(inst, sched)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) runSchedule(instanceName string, sched *btypes.BackupSchedule) {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.running, sched.ID)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
slog.Info("running scheduled backup", "component", "scheduler", "instance", instanceName, "target", sched.TargetName, "frequency", sched.Frequency)
|
||||
|
||||
mgr := NewManager(s.dataDir)
|
||||
|
||||
var backupErr error
|
||||
var retentionTarget string
|
||||
|
||||
switch sched.TargetType {
|
||||
case "app":
|
||||
retentionTarget = sched.TargetName
|
||||
_, backupErr = mgr.BackupApp(instanceName, sched.TargetName)
|
||||
case "cluster":
|
||||
retentionTarget = "_cluster"
|
||||
_, backupErr = mgr.BackupClusterConfig(instanceName)
|
||||
default:
|
||||
slog.Error("unknown schedule target type", "component", "scheduler", "instance", instanceName, "targetType", sched.TargetType, "error", fmt.Errorf("unsupported target type: %s", sched.TargetType))
|
||||
return
|
||||
}
|
||||
|
||||
if backupErr != nil {
|
||||
slog.Error("scheduled backup failed", "component", "scheduler", "instance", instanceName, "target", retentionTarget, "error", backupErr)
|
||||
} else {
|
||||
config, err := LoadInstanceBackupConfig(s.dataDir, instanceName)
|
||||
if err == nil {
|
||||
keepLast, keepDays := RetentionFromSchedule(sched, config.Retention)
|
||||
deleted, retErr := EnforceRetention(mgr, instanceName, retentionTarget, keepLast, keepDays)
|
||||
if retErr != nil {
|
||||
slog.Error("retention enforcement failed", "component", "scheduler", "instance", instanceName, "target", retentionTarget, "error", retErr)
|
||||
} else if deleted > 0 {
|
||||
slog.Info("retention cleaned up old backups", "component", "scheduler", "instance", instanceName, "target", retentionTarget, "deleted", deleted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
next := ComputeNextRun(sched, now)
|
||||
s.saveScheduleCache(instanceName, sched.TargetName, sched.Frequency, &now, &next)
|
||||
}
|
||||
|
||||
func (s *Scheduler) saveScheduleCache(instanceName, appName, tier string, lastRun, nextRun *time.Time) {
|
||||
state, err := LoadScheduleCache(s.dataDir, instanceName, appName)
|
||||
if err != nil || state == nil {
|
||||
state = &btypes.AppScheduleState{}
|
||||
}
|
||||
|
||||
ts := &btypes.TierState{
|
||||
LastRun: lastRun,
|
||||
NextRun: nextRun,
|
||||
}
|
||||
setTierState(state, tier, ts)
|
||||
|
||||
if err := SaveScheduleCache(s.dataDir, instanceName, appName, state); err != nil {
|
||||
slog.Error("failed to save schedule cache", "component", "scheduler", "instance", instanceName, "app", appName, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeNextRun calculates the next run time for a schedule based on frequency and time settings
|
||||
func ComputeNextRun(sched *btypes.BackupSchedule, after time.Time) time.Time {
|
||||
hour, minute := parseTime(sched.Time)
|
||||
|
||||
switch sched.Frequency {
|
||||
case "daily":
|
||||
next := time.Date(after.Year(), after.Month(), after.Day(), hour, minute, 0, 0, time.Local)
|
||||
if !next.After(after) {
|
||||
next = next.AddDate(0, 0, 1)
|
||||
}
|
||||
return next
|
||||
|
||||
case "weekly":
|
||||
next := time.Date(after.Year(), after.Month(), after.Day(), hour, minute, 0, 0, time.Local)
|
||||
daysUntil := (sched.DayOfWeek - int(next.Weekday()) + 7) % 7
|
||||
if daysUntil == 0 && !next.After(after) {
|
||||
daysUntil = 7
|
||||
}
|
||||
return next.AddDate(0, 0, daysUntil)
|
||||
|
||||
case "monthly":
|
||||
day := sched.DayOfMonth
|
||||
if day < 1 || day > 28 {
|
||||
day = 1
|
||||
}
|
||||
next := time.Date(after.Year(), after.Month(), day, hour, minute, 0, 0, time.Local)
|
||||
if !next.After(after) {
|
||||
next = next.AddDate(0, 1, 0)
|
||||
}
|
||||
return next
|
||||
|
||||
default:
|
||||
next := time.Date(after.Year(), after.Month(), after.Day(), hour, minute, 0, 0, time.Local)
|
||||
if !next.After(after) {
|
||||
next = next.AddDate(0, 0, 1)
|
||||
}
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
// parseTime parses "HH:MM" into hour and minute, defaulting to 02:00
|
||||
func parseTime(t string) (int, int) {
|
||||
var hour, minute int
|
||||
if n, _ := fmt.Sscanf(t, "%d:%d", &hour, &minute); n != 2 {
|
||||
return 2, 0
|
||||
}
|
||||
return hour, minute
|
||||
}
|
||||
286
api/internal/backup/scheduler_test.go
Normal file
286
api/internal/backup/scheduler_test.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
func TestParseTime(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
wantHour int
|
||||
wantMinute int
|
||||
}{
|
||||
{"02:00", 2, 0},
|
||||
{"14:30", 14, 30},
|
||||
{"00:00", 0, 0},
|
||||
{"23:59", 23, 59},
|
||||
{"", 2, 0}, // default
|
||||
{"invalid", 2, 0}, // default
|
||||
{"25:00", 25, 0}, // parses but invalid hour (not our concern here)
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
h, m := parseTime(tt.input)
|
||||
if h != tt.wantHour || m != tt.wantMinute {
|
||||
t.Errorf("parseTime(%q) = (%d, %d), want (%d, %d)", tt.input, h, m, tt.wantHour, tt.wantMinute)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeNextRun_Daily(t *testing.T) {
|
||||
sched := &btypes.BackupSchedule{
|
||||
Frequency: "daily",
|
||||
Time: "02:00",
|
||||
}
|
||||
|
||||
// Before 2AM today - next run should be today at 2AM
|
||||
now := time.Date(2026, 5, 19, 1, 0, 0, 0, time.Local)
|
||||
next := ComputeNextRun(sched, now)
|
||||
expected := time.Date(2026, 5, 19, 2, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("daily before time: got %v, want %v", next, expected)
|
||||
}
|
||||
|
||||
// After 2AM today - next run should be tomorrow at 2AM
|
||||
now = time.Date(2026, 5, 19, 3, 0, 0, 0, time.Local)
|
||||
next = ComputeNextRun(sched, now)
|
||||
expected = time.Date(2026, 5, 20, 2, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("daily after time: got %v, want %v", next, expected)
|
||||
}
|
||||
|
||||
// Exactly at 2AM - should be tomorrow
|
||||
now = time.Date(2026, 5, 19, 2, 0, 0, 0, time.Local)
|
||||
next = ComputeNextRun(sched, now)
|
||||
expected = time.Date(2026, 5, 20, 2, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("daily at exact time: got %v, want %v", next, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeNextRun_Weekly(t *testing.T) {
|
||||
sched := &btypes.BackupSchedule{
|
||||
Frequency: "weekly",
|
||||
Time: "03:00",
|
||||
DayOfWeek: 0, // Sunday
|
||||
}
|
||||
|
||||
// Tuesday 2026-05-19 - next Sunday is 2026-05-24
|
||||
now := time.Date(2026, 5, 19, 10, 0, 0, 0, time.Local) // Tuesday
|
||||
next := ComputeNextRun(sched, now)
|
||||
expected := time.Date(2026, 5, 24, 3, 0, 0, 0, time.Local) // Sunday
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("weekly from Tuesday: got %v (weekday=%v), want %v (weekday=%v)",
|
||||
next, next.Weekday(), expected, expected.Weekday())
|
||||
}
|
||||
|
||||
// On Sunday before the target time - should be today
|
||||
now = time.Date(2026, 5, 24, 1, 0, 0, 0, time.Local) // Sunday 1AM
|
||||
next = ComputeNextRun(sched, now)
|
||||
expected = time.Date(2026, 5, 24, 3, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("weekly on target day before time: got %v, want %v", next, expected)
|
||||
}
|
||||
|
||||
// On Sunday after the target time - should be next Sunday
|
||||
now = time.Date(2026, 5, 24, 4, 0, 0, 0, time.Local) // Sunday 4AM
|
||||
next = ComputeNextRun(sched, now)
|
||||
expected = time.Date(2026, 5, 31, 3, 0, 0, 0, time.Local) // Next Sunday
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("weekly on target day after time: got %v, want %v", next, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeNextRun_Monthly(t *testing.T) {
|
||||
sched := &btypes.BackupSchedule{
|
||||
Frequency: "monthly",
|
||||
Time: "04:00",
|
||||
DayOfMonth: 15,
|
||||
}
|
||||
|
||||
// Before the 15th
|
||||
now := time.Date(2026, 5, 10, 0, 0, 0, 0, time.Local)
|
||||
next := ComputeNextRun(sched, now)
|
||||
expected := time.Date(2026, 5, 15, 4, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("monthly before day: got %v, want %v", next, expected)
|
||||
}
|
||||
|
||||
// After the 15th
|
||||
now = time.Date(2026, 5, 20, 0, 0, 0, 0, time.Local)
|
||||
next = ComputeNextRun(sched, now)
|
||||
expected = time.Date(2026, 6, 15, 4, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("monthly after day: got %v, want %v", next, expected)
|
||||
}
|
||||
|
||||
// Invalid day of month (>28) should default to 1
|
||||
sched.DayOfMonth = 31
|
||||
now = time.Date(2026, 5, 2, 0, 0, 0, 0, time.Local)
|
||||
next = ComputeNextRun(sched, now)
|
||||
if next.Day() != 1 {
|
||||
t.Errorf("monthly invalid day: got day %d, want 1", next.Day())
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeNextRun_DefaultFrequency(t *testing.T) {
|
||||
sched := &btypes.BackupSchedule{
|
||||
Frequency: "unknown",
|
||||
Time: "05:00",
|
||||
}
|
||||
|
||||
now := time.Date(2026, 5, 19, 3, 0, 0, 0, time.Local)
|
||||
next := ComputeNextRun(sched, now)
|
||||
expected := time.Date(2026, 5, 19, 5, 0, 0, 0, time.Local)
|
||||
if !next.Equal(expected) {
|
||||
t.Errorf("default frequency: got %v, want %v", next, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveAndLoadAppScheduleConfig(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
|
||||
config := &AppScheduleConfig{
|
||||
Daily: &ScheduleTierConfig{
|
||||
Enabled: true,
|
||||
Time: "02:00",
|
||||
},
|
||||
Weekly: &ScheduleTierConfig{
|
||||
Enabled: true,
|
||||
Time: "03:00",
|
||||
DayOfWeek: 0,
|
||||
},
|
||||
}
|
||||
|
||||
if err := SaveAppScheduleConfig(dataDir, instanceName, "gitea", config); err != nil {
|
||||
t.Fatalf("SaveAppScheduleConfig() error = %v", err)
|
||||
}
|
||||
|
||||
got, err := LoadAppScheduleConfig(dataDir, instanceName, "gitea")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAppScheduleConfig() error = %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("LoadAppScheduleConfig() returned nil")
|
||||
}
|
||||
if got.Daily == nil || got.Daily.Time != "02:00" {
|
||||
t.Errorf("daily time = %v, want 02:00", got.Daily)
|
||||
}
|
||||
if got.Weekly == nil || got.Weekly.DayOfWeek != 0 {
|
||||
t.Errorf("weekly dayOfWeek = %v, want 0", got.Weekly)
|
||||
}
|
||||
if got.Monthly != nil {
|
||||
t.Errorf("monthly should be nil, got %v", got.Monthly)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAppScheduleConfig_Missing(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
got, err := LoadAppScheduleConfig(dataDir, "test-instance", "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil error for missing file, got %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected nil for missing file, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveAndLoadScheduleCache(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
|
||||
now := time.Date(2026, 5, 20, 2, 0, 0, 0, time.UTC)
|
||||
next := time.Date(2026, 5, 21, 2, 0, 0, 0, time.UTC)
|
||||
|
||||
state := &AppScheduleState{
|
||||
Daily: &TierState{
|
||||
LastRun: &now,
|
||||
NextRun: &next,
|
||||
},
|
||||
}
|
||||
|
||||
if err := SaveScheduleCache(dataDir, instanceName, "gitea", state); err != nil {
|
||||
t.Fatalf("SaveScheduleCache() error = %v", err)
|
||||
}
|
||||
|
||||
got, err := LoadScheduleCache(dataDir, instanceName, "gitea")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadScheduleCache() error = %v", err)
|
||||
}
|
||||
if got == nil || got.Daily == nil {
|
||||
t.Fatal("LoadScheduleCache() returned nil or missing daily")
|
||||
}
|
||||
if !got.Daily.LastRun.Equal(now) {
|
||||
t.Errorf("lastRun = %v, want %v", got.Daily.LastRun, now)
|
||||
}
|
||||
if !got.Daily.NextRun.Equal(next) {
|
||||
t.Errorf("nextRun = %v, want %v", got.Daily.NextRun, next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAppSchedules(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
instanceName := "test-instance"
|
||||
|
||||
// Write schedules for two apps
|
||||
for _, app := range []string{"gitea", "discourse"} {
|
||||
config := &AppScheduleConfig{
|
||||
Daily: &ScheduleTierConfig{Enabled: true, Time: "02:00"},
|
||||
}
|
||||
if err := SaveAppScheduleConfig(dataDir, instanceName, app, config); err != nil {
|
||||
t.Fatalf("SaveAppScheduleConfig(%s) error = %v", app, err)
|
||||
}
|
||||
}
|
||||
|
||||
schedules, err := ListAppSchedules(dataDir, instanceName)
|
||||
if err != nil {
|
||||
t.Fatalf("ListAppSchedules() error = %v", err)
|
||||
}
|
||||
|
||||
// Two apps × one tier each = 2 schedules
|
||||
if len(schedules) != 2 {
|
||||
t.Errorf("expected 2 schedules, got %d", len(schedules))
|
||||
}
|
||||
|
||||
ids := make(map[string]bool)
|
||||
for _, s := range schedules {
|
||||
ids[s.ID] = true
|
||||
}
|
||||
for _, want := range []string{"s_gitea_daily", "s_discourse_daily"} {
|
||||
if !ids[want] {
|
||||
t.Errorf("missing schedule ID %q in %v", want, ids)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScheduleID(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
wantApp string
|
||||
wantTier string
|
||||
wantOK bool
|
||||
}{
|
||||
{"s_gitea_daily", "gitea", "daily", true},
|
||||
{"s_gitea_weekly", "gitea", "weekly", true},
|
||||
{"s_gitea_monthly", "gitea", "monthly", true},
|
||||
{"s_my_app_daily", "my_app", "daily", true},
|
||||
{"invalid", "", "", false},
|
||||
{"s_gitea_yearly", "", "", false},
|
||||
{"gitea_daily", "", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
app, tier, ok := ParseScheduleID(tt.input)
|
||||
if ok != tt.wantOK {
|
||||
t.Errorf("ParseScheduleID(%q) ok=%v, want %v", tt.input, ok, tt.wantOK)
|
||||
continue
|
||||
}
|
||||
if ok && (app != tt.wantApp || tier != tt.wantTier) {
|
||||
t.Errorf("ParseScheduleID(%q) = (%q, %q), want (%q, %q)", tt.input, app, tier, tt.wantApp, tt.wantTier)
|
||||
}
|
||||
}
|
||||
}
|
||||
482
api/internal/backup/strategies/config.go
Normal file
482
api/internal/backup/strategies/config.go
Normal file
@@ -0,0 +1,482 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ConfigStrategy implements backup strategy for app configuration files
|
||||
type ConfigStrategy struct {
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// NewConfigStrategy creates a new config backup strategy
|
||||
func NewConfigStrategy(dataDir string) *ConfigStrategy {
|
||||
return &ConfigStrategy{
|
||||
dataDir: dataDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the strategy identifier
|
||||
func (c *ConfigStrategy) Name() string {
|
||||
return "config"
|
||||
}
|
||||
|
||||
// Backup creates a backup of app configuration files, writing results to the plan
|
||||
func (c *ConfigStrategy) Backup(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for config in plan")
|
||||
}
|
||||
entry.Status = "backing_up"
|
||||
|
||||
instancePath := filepath.Join(c.dataDir, "instances", plan.Instance)
|
||||
appPath := plan.Source.AppDir
|
||||
if !filepath.IsAbs(appPath) {
|
||||
appPath = filepath.Join(c.dataDir, appPath)
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("config/%s/%s/%s.tar.gz", plan.Instance, plan.App, plan.Timestamp)
|
||||
|
||||
// Create tar.gz archive in memory
|
||||
var buf bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&buf)
|
||||
tarWriter := tar.NewWriter(gzWriter)
|
||||
|
||||
// Files to backup from app directory
|
||||
patterns := []string{
|
||||
"manifest.yaml",
|
||||
"kustomization.yaml",
|
||||
"*.yaml",
|
||||
}
|
||||
|
||||
totalSize := int64(0)
|
||||
fileCount := 0
|
||||
|
||||
// Add app files to archive
|
||||
for _, pattern := range patterns {
|
||||
matches, _ := filepath.Glob(filepath.Join(appPath, pattern))
|
||||
for _, file := range matches {
|
||||
if err := c.addFileToTar(tarWriter, file, instancePath); err != nil {
|
||||
tarWriter.Close()
|
||||
gzWriter.Close()
|
||||
return fmt.Errorf("failed to add file %s to archive: %w", file, err)
|
||||
}
|
||||
|
||||
info, _ := os.Stat(file)
|
||||
if info != nil {
|
||||
totalSize += info.Size()
|
||||
}
|
||||
fileCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Add app configuration from config.yaml
|
||||
if err := c.addAppConfig(tarWriter, plan.Instance, plan.App); err != nil {
|
||||
tarWriter.Close()
|
||||
gzWriter.Close()
|
||||
return fmt.Errorf("failed to add app config: %w", err)
|
||||
}
|
||||
fileCount++
|
||||
|
||||
// Add app secrets from secrets.yaml (optional, might not exist)
|
||||
if err := c.addAppSecrets(tarWriter, plan.Instance, plan.App); err == nil {
|
||||
fileCount++
|
||||
}
|
||||
|
||||
// Close the archive
|
||||
if err := tarWriter.Close(); err != nil {
|
||||
gzWriter.Close()
|
||||
return fmt.Errorf("failed to close tar: %w", err)
|
||||
}
|
||||
|
||||
if err := gzWriter.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close gzip: %w", err)
|
||||
}
|
||||
|
||||
// Upload to destination
|
||||
reader := bytes.NewReader(buf.Bytes())
|
||||
size, err := dest.Put(key, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload config backup: %w", err)
|
||||
}
|
||||
|
||||
// Write results to plan entry
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": key,
|
||||
"size": size,
|
||||
"files": fileCount,
|
||||
"format": "tar.gz",
|
||||
"totalSize": totalSize,
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore restores app configuration from backup using plan-driven coordination
|
||||
func (c *ConfigStrategy) Restore(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for config in plan")
|
||||
}
|
||||
entry.Status = "restoring"
|
||||
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in config strategy entry")
|
||||
}
|
||||
|
||||
instancePath := filepath.Join(c.dataDir, "instances", plan.Instance)
|
||||
|
||||
// Determine target app directory (standby)
|
||||
targetAppDir := plan.Standby.AppDir
|
||||
if !filepath.IsAbs(targetAppDir) {
|
||||
targetAppDir = filepath.Join(c.dataDir, targetAppDir)
|
||||
}
|
||||
|
||||
// Get backup from destination
|
||||
reader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve config backup: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Decompress and extract
|
||||
gzReader, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzReader)
|
||||
|
||||
// Create target app directory if it doesn't exist
|
||||
if err := os.MkdirAll(targetAppDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create app directory: %w", err)
|
||||
}
|
||||
|
||||
// Extract files
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read tar: %w", err)
|
||||
}
|
||||
|
||||
// Determine target path
|
||||
targetPath := filepath.Join(instancePath, header.Name)
|
||||
|
||||
// Security check: ensure path is within instance directory
|
||||
if !strings.HasPrefix(targetPath, instancePath) {
|
||||
return fmt.Errorf("invalid file path in archive: %s", header.Name)
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
case tar.TypeReg:
|
||||
// Create directory if needed
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create parent directory: %w", err)
|
||||
}
|
||||
|
||||
// Handle special files (config.yaml and secrets.yaml)
|
||||
if strings.HasSuffix(header.Name, ".config.yaml.part") {
|
||||
if err := c.mergeConfig(tarReader, instancePath, plan.App); err != nil {
|
||||
return fmt.Errorf("failed to merge config: %w", err)
|
||||
}
|
||||
} else if strings.HasSuffix(header.Name, ".secrets.yaml.part") {
|
||||
if err := c.mergeSecrets(tarReader, instancePath, plan.App); err != nil {
|
||||
return fmt.Errorf("failed to merge secrets: %w", err)
|
||||
}
|
||||
} else {
|
||||
file, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(file, tarReader); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("failed to extract file: %w", err)
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entry.Restore = map[string]interface{}{
|
||||
"appDir": plan.Standby.AppDir,
|
||||
}
|
||||
entry.Status = "restored"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Switch is a no-op for config — config is shared, already merged during restore
|
||||
func (c *ConfigStrategy) Switch(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry != nil {
|
||||
entry.Status = "switched"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup is a no-op for config — config is shared, not per-color
|
||||
func (c *ConfigStrategy) Cleanup(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry != nil {
|
||||
entry.Status = "cleaned_up"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify checks if a config backup exists and is valid
|
||||
func (c *ConfigStrategy) Verify(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("config")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for config in plan")
|
||||
}
|
||||
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in config strategy entry")
|
||||
}
|
||||
|
||||
// Check if backup exists
|
||||
reader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup not found in destination: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Verify it's a valid gzip file
|
||||
gzReader, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid gzip format: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
// Verify it's a valid tar archive with manifest
|
||||
tarReader := tar.NewReader(gzReader)
|
||||
hasManifest := false
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid tar format: %w", err)
|
||||
}
|
||||
|
||||
if strings.Contains(header.Name, "manifest.yaml") {
|
||||
hasManifest = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasManifest {
|
||||
return fmt.Errorf("backup missing essential file: manifest.yaml")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addFileToTar adds a file to the tar archive
|
||||
func (c *ConfigStrategy) addFileToTar(tw *tar.Writer, filePath, baseDir string) error {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(stat, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(baseDir, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = relPath
|
||||
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(tw, file)
|
||||
return err
|
||||
}
|
||||
|
||||
// addAppConfig extracts and adds app configuration from config.yaml
|
||||
func (c *ConfigStrategy) addAppConfig(tw *tar.Writer, instanceName, appName string) error {
|
||||
configPath := tools.GetInstanceConfigPath(c.dataDir, instanceName)
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read config.yaml: %w", err)
|
||||
}
|
||||
|
||||
var config map[string]interface{}
|
||||
if err := yaml.Unmarshal(data, &config); err != nil {
|
||||
return fmt.Errorf("failed to parse config.yaml: %w", err)
|
||||
}
|
||||
|
||||
// Extract app-specific configuration
|
||||
appConfig := make(map[string]interface{})
|
||||
if apps, ok := config["apps"].(map[string]interface{}); ok {
|
||||
if ac, ok := apps[appName].(map[string]interface{}); ok {
|
||||
appConfig = ac
|
||||
}
|
||||
}
|
||||
|
||||
configData, err := yaml.Marshal(appConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal app config: %w", err)
|
||||
}
|
||||
|
||||
header := &tar.Header{
|
||||
Name: fmt.Sprintf("apps/%s/.config.yaml.part", appName),
|
||||
Mode: 0644,
|
||||
Size: int64(len(configData)),
|
||||
ModTime: time.Now(),
|
||||
}
|
||||
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tw.Write(configData)
|
||||
return err
|
||||
}
|
||||
|
||||
// addAppSecrets extracts and adds app secrets from secrets.yaml
|
||||
func (c *ConfigStrategy) addAppSecrets(tw *tar.Writer, instanceName, appName string) error {
|
||||
secretsPath := filepath.Join(c.dataDir, "instances", instanceName, "secrets.yaml")
|
||||
data, err := os.ReadFile(secretsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var secrets map[string]interface{}
|
||||
if err := yaml.Unmarshal(data, &secrets); err != nil {
|
||||
return fmt.Errorf("failed to parse secrets.yaml: %w", err)
|
||||
}
|
||||
|
||||
appSecrets := make(map[string]interface{})
|
||||
if apps, ok := secrets["apps"].(map[string]interface{}); ok {
|
||||
if as, ok := apps[appName].(map[string]interface{}); ok {
|
||||
appSecrets = as
|
||||
}
|
||||
}
|
||||
|
||||
secretsData, err := yaml.Marshal(appSecrets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal app secrets: %w", err)
|
||||
}
|
||||
|
||||
header := &tar.Header{
|
||||
Name: fmt.Sprintf("apps/%s/.secrets.yaml.part", appName),
|
||||
Mode: 0600,
|
||||
Size: int64(len(secretsData)),
|
||||
ModTime: time.Now(),
|
||||
}
|
||||
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tw.Write(secretsData)
|
||||
return err
|
||||
}
|
||||
|
||||
// mergeConfig merges restored config with existing config.yaml
|
||||
func (c *ConfigStrategy) mergeConfig(reader io.Reader, instancePath, appName string) error {
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
|
||||
restoredData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var restoredConfig map[string]interface{}
|
||||
if err := yaml.Unmarshal(restoredData, &restoredConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var config map[string]interface{}
|
||||
if data, err := os.ReadFile(configPath); err == nil {
|
||||
_ = yaml.Unmarshal(data, &config)
|
||||
}
|
||||
if config == nil {
|
||||
config = make(map[string]interface{})
|
||||
}
|
||||
|
||||
if config["apps"] == nil {
|
||||
config["apps"] = make(map[string]interface{})
|
||||
}
|
||||
apps := config["apps"].(map[string]interface{})
|
||||
apps[appName] = restoredConfig
|
||||
|
||||
output, err := yaml.Marshal(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(configPath, output, 0644)
|
||||
}
|
||||
|
||||
// mergeSecrets merges restored secrets with existing secrets.yaml
|
||||
func (c *ConfigStrategy) mergeSecrets(reader io.Reader, instancePath, appName string) error {
|
||||
secretsPath := filepath.Join(instancePath, "secrets.yaml")
|
||||
|
||||
restoredData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var restoredSecrets map[string]interface{}
|
||||
if err := yaml.Unmarshal(restoredData, &restoredSecrets); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var secrets map[string]interface{}
|
||||
if data, err := os.ReadFile(secretsPath); err == nil {
|
||||
_ = yaml.Unmarshal(data, &secrets)
|
||||
}
|
||||
if secrets == nil {
|
||||
secrets = make(map[string]interface{})
|
||||
}
|
||||
|
||||
if secrets["apps"] == nil {
|
||||
secrets["apps"] = make(map[string]interface{})
|
||||
}
|
||||
apps := secrets["apps"].(map[string]interface{})
|
||||
apps[appName] = restoredSecrets
|
||||
|
||||
output, err := yaml.Marshal(secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(secretsPath, output, 0600)
|
||||
}
|
||||
921
api/internal/backup/strategies/longhorn_native.go
Normal file
921
api/internal/backup/strategies/longhorn_native.go
Normal file
@@ -0,0 +1,921 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// LonghornNativeStrategy implements backup strategy using Longhorn native backups to NFS
|
||||
type LonghornNativeStrategy struct {
|
||||
dataDir string
|
||||
opManager *operations.Manager
|
||||
}
|
||||
|
||||
// NewLonghornNativeStrategy creates a new Longhorn native backup strategy
|
||||
func NewLonghornNativeStrategy(dataDir string) *LonghornNativeStrategy {
|
||||
return &LonghornNativeStrategy{
|
||||
dataDir: dataDir,
|
||||
opManager: operations.NewManager(dataDir),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the strategy identifier
|
||||
func (l *LonghornNativeStrategy) Name() string {
|
||||
return "longhorn-native"
|
||||
}
|
||||
|
||||
// Backup creates Longhorn native backups of all PVCs for an app, writing results to the plan
|
||||
func (l *LonghornNativeStrategy) Backup(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("longhorn-native")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for longhorn-native in plan")
|
||||
}
|
||||
entry.Status = "backing_up"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(l.dataDir, plan.Instance)
|
||||
|
||||
// Pre-flight: verify Longhorn backup target is configured
|
||||
if err := l.checkBackupTarget(kubeconfigPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Determine source namespace from plan
|
||||
sourceNamespace := plan.Source.Namespace
|
||||
if sourceNamespace == "" {
|
||||
sourceNamespace = plan.App
|
||||
}
|
||||
|
||||
// Get all PVCs in the source namespace
|
||||
pvcs, err := l.getPVCs(kubeconfigPath, sourceNamespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get PVCs: %w", err)
|
||||
}
|
||||
|
||||
if len(pvcs) == 0 {
|
||||
entry.Status = "backed_up"
|
||||
entry.Backup = map[string]any{
|
||||
"volumes": []any{},
|
||||
"count": 0,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
volumeParams := []map[string]any{}
|
||||
backupVolumes := []map[string]any{}
|
||||
|
||||
for _, pvcName := range pvcs {
|
||||
// Skip cache or temp volumes
|
||||
if strings.Contains(pvcName, "-cache") || strings.Contains(pvcName, "-tmp") {
|
||||
continue
|
||||
}
|
||||
|
||||
volumeName, err := l.getVolumeNameFromPVC(kubeconfigPath, sourceNamespace, pvcName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get volume name for PVC %s: %w", pvcName, err)
|
||||
}
|
||||
|
||||
pvcSize := l.getPVCSize(kubeconfigPath, sourceNamespace, pvcName)
|
||||
accessMode := l.getPVCAccessMode(kubeconfigPath, sourceNamespace, pvcName)
|
||||
|
||||
// Record volume params
|
||||
volumeParams = append(volumeParams, map[string]any{
|
||||
"pvcName": pvcName,
|
||||
"volumeName": volumeName,
|
||||
"size": pvcSize,
|
||||
"accessMode": accessMode,
|
||||
})
|
||||
|
||||
// Backup volume with retry for transient failures (e.g. instance manager restart)
|
||||
backupID, backupURL, err := l.backupVolumeWithRetry(kubeconfigPath, plan.App, pvcName, volumeName, plan.Timestamp, 3)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup failed for volume %s: %w", volumeName, err)
|
||||
}
|
||||
|
||||
backupVolumes = append(backupVolumes, map[string]any{
|
||||
"pvcName": pvcName,
|
||||
"backupID": backupID,
|
||||
"backupURL": backupURL,
|
||||
})
|
||||
|
||||
if err := l.cleanupOldBackups(kubeconfigPath, volumeName, backupID); err != nil {
|
||||
slog.Error("failed to clean up old backups", "component", "longhorn", "volume", volumeName, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Record in plan
|
||||
entry.Params = map[string]any{
|
||||
"volumes": volumeParams,
|
||||
}
|
||||
entry.Backup = map[string]any{
|
||||
"volumes": backupVolumes,
|
||||
"count": len(backupVolumes),
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore restores PVCs from Longhorn backups using plan-driven coordination.
|
||||
// Creates timestamped staging volumes ({pvcName}-restore-{timestamp}).
|
||||
func (l *LonghornNativeStrategy) Restore(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("longhorn-native")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for longhorn-native in plan")
|
||||
}
|
||||
entry.Status = "restoring"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(l.dataDir, plan.Instance)
|
||||
|
||||
backupVolumes, ok := entry.Backup["volumes"].([]any)
|
||||
if !ok || len(backupVolumes) == 0 {
|
||||
entry.Restore = map[string]any{
|
||||
"volumes": []any{},
|
||||
}
|
||||
entry.Status = "restored"
|
||||
return nil
|
||||
}
|
||||
|
||||
restoreVolumes := []map[string]any{}
|
||||
|
||||
for _, bv := range backupVolumes {
|
||||
backup, ok := bv.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
pvcName, _ := backup["pvcName"].(string)
|
||||
backupURL, _ := backup["backupURL"].(string)
|
||||
|
||||
if pvcName == "" || backupURL == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Find the volume params to get size and access mode
|
||||
pvcSize := "10Gi"
|
||||
accessMode := "ReadWriteOnce"
|
||||
if volumeParams, ok := entry.Params["volumes"].([]any); ok {
|
||||
for _, vp := range volumeParams {
|
||||
if vpMap, ok := vp.(map[string]any); ok {
|
||||
if vpMap["pvcName"] == pvcName {
|
||||
if size, ok := vpMap["size"].(string); ok {
|
||||
pvcSize = size
|
||||
}
|
||||
if mode, ok := vpMap["accessMode"].(string); ok {
|
||||
accessMode = mode
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Staging volume name: timestamped so successive restores never conflict.
|
||||
// Longhorn enforces a 40-char max on volume names; cap the name to avoid
|
||||
// the mutating webhook silently renaming it (which would cause waitForVolume
|
||||
// to poll the wrong name and time out).
|
||||
desiredName := strings.ToLower(fmt.Sprintf("%s-restore-%s", pvcName, plan.Timestamp))
|
||||
restoreVolumeName := desiredName
|
||||
if len(desiredName) > 40 {
|
||||
restoreVolumeName = desiredName[:40]
|
||||
}
|
||||
|
||||
// createVolumeFromBackup returns the actual name Longhorn assigned
|
||||
// (may differ from restoreVolumeName if a prior attempt left a volume behind)
|
||||
actualVolumeName, err := l.createVolumeFromBackup(kubeconfigPath, restoreVolumeName, backupURL, pvcSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create volume from backup for %s: %w", pvcName, err)
|
||||
}
|
||||
|
||||
// Create a PV pre-bound to the source namespace so switchInPlace can bind the PVC
|
||||
sourceNamespace := plan.Source.Namespace
|
||||
if sourceNamespace == "" {
|
||||
sourceNamespace = plan.App
|
||||
}
|
||||
if err := l.createPVForVolume(kubeconfigPath, actualVolumeName, pvcSize, accessMode, sourceNamespace, pvcName); err != nil {
|
||||
slog.Error("failed to create PV for volume", "component", "longhorn", "volume", actualVolumeName, "error", err)
|
||||
}
|
||||
|
||||
restoreVolumes = append(restoreVolumes, map[string]any{
|
||||
"pvcName": pvcName,
|
||||
"volumeName": actualVolumeName,
|
||||
})
|
||||
}
|
||||
|
||||
entry.Restore = map[string]any{
|
||||
"volumes": restoreVolumes,
|
||||
}
|
||||
entry.Status = "restored"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Switch performs an in-place PVC swap in the source namespace:
|
||||
// delete old PVC/PV/volume → create new PV+PVC from the restored volume.
|
||||
// Scale down/up is handled by the coordinator (SwitchApp in backup.go).
|
||||
func (l *LonghornNativeStrategy) Switch(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("longhorn-native")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for longhorn-native in plan")
|
||||
}
|
||||
entry.Status = "switching"
|
||||
|
||||
if err := l.switchInPlace(plan, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
entry.Status = "switched"
|
||||
return nil
|
||||
}
|
||||
|
||||
// switchInPlace performs the PVC swap in the source namespace. Scale down/up is
|
||||
// handled by the coordinator. Re-entrant via completedPVCs checkpoint in entry.Switch.
|
||||
func (l *LonghornNativeStrategy) switchInPlace(plan *btypes.RecoveryPlan, entry *btypes.StrategyEntry) error {
|
||||
kubeconfigPath := tools.GetKubeconfigPath(l.dataDir, plan.Instance)
|
||||
namespace := plan.Source.Namespace
|
||||
|
||||
restoreVolumes, ok := entry.Restore["volumes"].([]any)
|
||||
if !ok || len(restoreVolumes) == 0 {
|
||||
// No PVCs to swap (e.g., memcached) — coordinator handles scale bounce
|
||||
slog.Info("no volumes to swap for in-place restore, skipping PVC swap", "component", "longhorn", "namespace", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build a map of pvcName → restored volume name
|
||||
restoredVolumeByPVC := map[string]string{}
|
||||
for _, rv := range restoreVolumes {
|
||||
if rvMap, ok := rv.(map[string]any); ok {
|
||||
pvcName, _ := rvMap["pvcName"].(string)
|
||||
volumeName, _ := rvMap["volumeName"].(string)
|
||||
if pvcName != "" && volumeName != "" {
|
||||
restoredVolumeByPVC[pvcName] = volumeName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect PVC metadata from Params for size/accessMode
|
||||
pvcParams := map[string]map[string]any{}
|
||||
if volumeParams, ok := entry.Params["volumes"].([]any); ok {
|
||||
for _, vp := range volumeParams {
|
||||
if vpMap, ok := vp.(map[string]any); ok {
|
||||
if pvcName, ok := vpMap["pvcName"].(string); ok {
|
||||
pvcParams[pvcName] = vpMap
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Load completedPVCs checkpoint for re-entrant recovery
|
||||
completedPVCs := map[string]bool{}
|
||||
if entry.Switch == nil {
|
||||
entry.Switch = map[string]any{}
|
||||
}
|
||||
if completed, ok := entry.Switch["completedPVCs"].([]any); ok {
|
||||
for _, c := range completed {
|
||||
if name, ok := c.(string); ok {
|
||||
completedPVCs[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve previousItems from any prior run
|
||||
previousItems := []map[string]any{}
|
||||
if prev, ok := entry.Switch["previousItems"].([]any); ok {
|
||||
for _, p := range prev {
|
||||
if pm, ok := p.(map[string]any); ok {
|
||||
previousItems = append(previousItems, pm)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for pvcName, restoredVolumeName := range restoredVolumeByPVC {
|
||||
if completedPVCs[pvcName] {
|
||||
slog.Info("PVC already swapped on previous run, skipping", "component", "longhorn", "pvc", pvcName)
|
||||
continue
|
||||
}
|
||||
|
||||
// Get current PV name and Longhorn volume handle before deleting PVC
|
||||
pvName, longhornVolume, err := l.getPVInfo(kubeconfigPath, namespace, pvcName)
|
||||
if err != nil {
|
||||
slog.Error("failed to get PV info, skipping PVC", "component", "longhorn", "pvc", pvcName, "error", err)
|
||||
}
|
||||
|
||||
previousItems = append(previousItems, map[string]any{
|
||||
"pvcName": pvcName,
|
||||
"pvName": pvName,
|
||||
"longhornVolume": longhornVolume,
|
||||
})
|
||||
|
||||
// Delete the PVC (may cascade to PV deletion if Delete reclaimPolicy)
|
||||
slog.Info("deleting PVC for in-place swap", "component", "longhorn", "pvc", pvcName, "namespace", namespace)
|
||||
deleteCmd := exec.Command("kubectl", "delete", "pvc", pvcName, "-n", namespace, "--ignore-not-found", "--timeout=60s")
|
||||
tools.WithKubeconfig(deleteCmd, kubeconfigPath)
|
||||
if output, err := deleteCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to delete PVC %s: %w, output: %s", pvcName, err, output)
|
||||
}
|
||||
|
||||
// Clean up old PV if it still exists (Retain policy from a prior in-place restore)
|
||||
if pvName != "" {
|
||||
if err := l.deletePVIfExists(kubeconfigPath, pvName); err != nil {
|
||||
slog.Error("failed to delete old PV", "component", "longhorn", "pv", pvName, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old Longhorn volume if it was a previous restore volume.
|
||||
// Detects both the current timestamped naming ({pvc}-restore-{ts}) and the
|
||||
// old color-based naming ({pvc}-blue / {pvc}-green) for existing plans.
|
||||
isRestoreVolume := strings.Contains(longhornVolume, "-restore-") ||
|
||||
strings.HasSuffix(longhornVolume, "-blue") || strings.HasSuffix(longhornVolume, "-green")
|
||||
if longhornVolume != "" && isRestoreVolume {
|
||||
if err := l.deleteLonghornVolumeIfExists(kubeconfigPath, longhornVolume); err != nil {
|
||||
slog.Error("failed to delete old Longhorn volume", "component", "longhorn", "volume", longhornVolume, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get size and access mode for the new PVC
|
||||
size := "10Gi"
|
||||
accessMode := "ReadWriteOnce"
|
||||
if params, ok := pvcParams[pvcName]; ok {
|
||||
if s, ok := params["size"].(string); ok && s != "" {
|
||||
size = s
|
||||
}
|
||||
if m, ok := params["accessMode"].(string); ok && m != "" {
|
||||
accessMode = m
|
||||
}
|
||||
}
|
||||
|
||||
// Create new PV pointing to the restored Longhorn volume
|
||||
slog.Info("creating new PV for restored volume", "component", "longhorn", "volume", restoredVolumeName, "pvc", pvcName)
|
||||
if err := l.createPVForVolume(kubeconfigPath, restoredVolumeName, size, accessMode, namespace, pvcName); err != nil {
|
||||
return fmt.Errorf("failed to create PV for volume %s: %w", restoredVolumeName, err)
|
||||
}
|
||||
|
||||
// Create new PVC bound to the new PV
|
||||
slog.Info("creating new PVC bound to restored volume", "component", "longhorn", "pvc", pvcName, "volume", restoredVolumeName)
|
||||
if err := l.createPVC(kubeconfigPath, pvcName, namespace, restoredVolumeName, size, accessMode); err != nil {
|
||||
return fmt.Errorf("failed to create PVC %s: %w", pvcName, err)
|
||||
}
|
||||
|
||||
// Wait for PVC to be Bound
|
||||
if err := l.waitForPVCBound(kubeconfigPath, namespace, pvcName); err != nil {
|
||||
return fmt.Errorf("PVC %s did not reach Bound state: %w", pvcName, err)
|
||||
}
|
||||
|
||||
// Checkpoint: mark this PVC as completed
|
||||
completedPVCs[pvcName] = true
|
||||
completedList := []any{}
|
||||
for name := range completedPVCs {
|
||||
completedList = append(completedList, name)
|
||||
}
|
||||
entry.Switch["completedPVCs"] = completedList
|
||||
entry.Switch["previousItems"] = previousItems
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup is a no-op for in-place restores — old PVC/PV/volume were cleaned up during Switch.
|
||||
func (l *LonghornNativeStrategy) Cleanup(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("longhorn-native")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for longhorn-native in plan")
|
||||
}
|
||||
entry.Status = "cleaning_up"
|
||||
entry.Status = "cleaned_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify checks if Longhorn backups exist and are valid
|
||||
func (l *LonghornNativeStrategy) Verify(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("longhorn-native")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for longhorn-native in plan")
|
||||
}
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(l.dataDir, plan.Instance)
|
||||
|
||||
backupVolumes, ok := entry.Backup["volumes"].([]any)
|
||||
if !ok || len(backupVolumes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify backup target is accessible
|
||||
if err := l.checkBackupTarget(kubeconfigPath); err != nil {
|
||||
return fmt.Errorf("backup target not accessible: %w", err)
|
||||
}
|
||||
|
||||
// Verify each backup CRD still exists
|
||||
for _, bv := range backupVolumes {
|
||||
backup, ok := bv.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
backupID, _ := backup["backupID"].(string)
|
||||
if backupID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
cmd := exec.Command("kubectl", "get", "backups.longhorn.io", backupID,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.state}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup %s not found: %w", backupID, err)
|
||||
}
|
||||
if string(output) != "Completed" {
|
||||
return fmt.Errorf("backup %s is not in Completed state: %s", backupID, string(output))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func (l *LonghornNativeStrategy) backupVolumeWithRetry(kubeconfigPath, appName, pvcName, volumeName, timestamp string, maxAttempts int) (string, string, error) {
|
||||
var lastErr error
|
||||
for attempt := range maxAttempts {
|
||||
snapshotName := strings.ToLower(fmt.Sprintf("%s-%s-snapshot-%s", appName, pvcName, timestamp))
|
||||
if attempt > 0 {
|
||||
snapshotName = strings.ToLower(fmt.Sprintf("%s-%s-snapshot-%s-retry%d", appName, pvcName, timestamp, attempt))
|
||||
slog.Info("retrying backup for volume", "component", "longhorn", "volume", volumeName, "attempt", attempt+1, "maxAttempts", maxAttempts)
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
if err := l.createSnapshot(kubeconfigPath, volumeName, snapshotName); err != nil {
|
||||
lastErr = fmt.Errorf("failed to create snapshot: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
backupID, err := l.createBackup(kubeconfigPath, volumeName, snapshotName)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("failed to create backup: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
backupURL, err := l.waitForBackupComplete(kubeconfigPath, volumeName, backupID)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("backup not ready: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
return backupID, backupURL, nil
|
||||
}
|
||||
return "", "", lastErr
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) checkBackupTarget(kubeconfigPath string) error {
|
||||
cmd := exec.Command("kubectl", "get", "backuptargets.longhorn.io", "default",
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.spec.backupTargetURL}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check Longhorn backup target: %w", err)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(string(output)) == "" {
|
||||
return fmt.Errorf("longhorn backup target not configured: set a backup target (NFS or S3) in Longhorn settings before creating backups")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) getPVCs(kubeconfigPath, namespace string) ([]string, error) {
|
||||
cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, "-o", "jsonpath={.items[*].metadata.name}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pvcs := strings.Fields(string(output))
|
||||
return pvcs, nil
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) getPVCSize(kubeconfigPath, namespace, pvcName string) string {
|
||||
cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, pvcName,
|
||||
"-o", "jsonpath={.spec.resources.requests.storage}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
if output, err := cmd.Output(); err == nil && len(output) > 0 {
|
||||
return string(output)
|
||||
}
|
||||
return "10Gi"
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) getPVCAccessMode(kubeconfigPath, namespace, pvcName string) string {
|
||||
cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, pvcName,
|
||||
"-o", "jsonpath={.spec.accessModes[0]}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
if output, err := cmd.Output(); err == nil && len(output) > 0 {
|
||||
return string(output)
|
||||
}
|
||||
return "ReadWriteOnce"
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) getVolumeNameFromPVC(kubeconfigPath, namespace, pvcName string) (string, error) {
|
||||
cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, pvcName,
|
||||
"-o", "jsonpath={.spec.volumeName}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get volume name: %w", err)
|
||||
}
|
||||
|
||||
volumeName := string(output)
|
||||
if volumeName == "" {
|
||||
return "", fmt.Errorf("no volume bound to PVC %s", pvcName)
|
||||
}
|
||||
|
||||
return volumeName, nil
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) createSnapshot(kubeconfigPath, volumeName, snapshotName string) error {
|
||||
snapshotYAML := fmt.Sprintf(`apiVersion: longhorn.io/v1beta2
|
||||
kind: Snapshot
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
volume: %s
|
||||
createSnapshot: true
|
||||
`, snapshotName, volumeName)
|
||||
|
||||
cmd := exec.Command("kubectl", "apply", "-f", "-")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
cmd.Stdin = strings.NewReader(snapshotYAML)
|
||||
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create snapshot: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
// Wait for snapshot to be ready
|
||||
for range 30 {
|
||||
cmd := exec.Command("kubectl", "get", "snapshots.longhorn.io", snapshotName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.readyToUse}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err == nil && string(output) == "true" {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
return fmt.Errorf("timeout waiting for snapshot %s to be ready", snapshotName)
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) createBackup(kubeconfigPath, volumeName, snapshotName string) (string, error) {
|
||||
// Backup name must be unique — derive from snapshot name
|
||||
backupName := strings.ReplaceAll(snapshotName, "_", "-")
|
||||
if len(backupName) > 63 {
|
||||
backupName = backupName[:63]
|
||||
}
|
||||
|
||||
backupYAML := fmt.Sprintf(`apiVersion: longhorn.io/v1beta2
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: longhorn-system
|
||||
labels:
|
||||
backup-volume: %s
|
||||
spec:
|
||||
snapshotName: %s
|
||||
`, backupName, volumeName, snapshotName)
|
||||
|
||||
cmd := exec.Command("kubectl", "apply", "-f", "-")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
cmd.Stdin = strings.NewReader(backupYAML)
|
||||
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("failed to create backup: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return backupName, nil
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) waitForBackupComplete(kubeconfigPath, _, backupName string) (string, error) {
|
||||
maxRetries := 120
|
||||
for i := range maxRetries {
|
||||
// Get backup state
|
||||
stateCmd := exec.Command("kubectl", "get", "backups.longhorn.io", backupName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.state}")
|
||||
tools.WithKubeconfig(stateCmd, kubeconfigPath)
|
||||
|
||||
stateOutput, err := stateCmd.Output()
|
||||
if err != nil {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
state := string(stateOutput)
|
||||
|
||||
if state == "Error" {
|
||||
// Get error message
|
||||
errCmd := exec.Command("kubectl", "get", "backups.longhorn.io", backupName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.messages}")
|
||||
tools.WithKubeconfig(errCmd, kubeconfigPath)
|
||||
errOutput, _ := errCmd.Output()
|
||||
return "", fmt.Errorf("backup failed: %s", string(errOutput))
|
||||
}
|
||||
|
||||
if state == "Completed" {
|
||||
// Get backup URL
|
||||
urlCmd := exec.Command("kubectl", "get", "backups.longhorn.io", backupName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.url}")
|
||||
tools.WithKubeconfig(urlCmd, kubeconfigPath)
|
||||
|
||||
urlOutput, err := urlCmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("backup completed but failed to get URL: %w", err)
|
||||
}
|
||||
backupURL := string(urlOutput)
|
||||
if backupURL != "" {
|
||||
return backupURL, nil
|
||||
}
|
||||
}
|
||||
|
||||
if i%12 == 0 && i > 0 {
|
||||
slog.Info("waiting for backup to complete", "component", "longhorn", "backup", backupName, "state", state, "attempt", i)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return "", fmt.Errorf("timeout waiting for backup %s to complete", backupName)
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) createPVForVolume(kubeconfigPath, volumeName, size, accessMode, namespace, pvcName string) error {
|
||||
pvYAML := fmt.Sprintf(`apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: %s
|
||||
spec:
|
||||
capacity:
|
||||
storage: %s
|
||||
accessModes:
|
||||
- %s
|
||||
storageClassName: longhorn
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
csi:
|
||||
driver: driver.longhorn.io
|
||||
fsType: ext4
|
||||
volumeHandle: %s
|
||||
claimRef:
|
||||
namespace: %s
|
||||
name: %s
|
||||
`, volumeName, size, accessMode, volumeName, namespace, pvcName)
|
||||
|
||||
cmd := exec.Command("kubectl", "apply", "-f", "-")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
cmd.Stdin = strings.NewReader(pvYAML)
|
||||
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create PV: %w, output: %s", err, string(output))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createVolumeFromBackup creates a Longhorn volume from a backup URL and returns
|
||||
// the actual volume name assigned by Longhorn (which may differ from volumeName
|
||||
// if Longhorn's mutating webhook renames it, e.g. to enforce a 40-char limit).
|
||||
func (l *LonghornNativeStrategy) createVolumeFromBackup(kubeconfigPath, volumeName, backupURL, size string) (string, error) {
|
||||
// If a volume for this exact backup already exists (from a prior attempt),
|
||||
// reuse it — the restore data is already there.
|
||||
if existingName, err := l.findVolumeByBackupURL(kubeconfigPath, backupURL); err == nil && existingName != "" {
|
||||
slog.Info("volume for this backup already exists, reusing", "component", "longhorn", "volume", existingName)
|
||||
return existingName, l.waitForVolume(kubeconfigPath, existingName)
|
||||
}
|
||||
|
||||
// Delete a stale volume that has our intended name but a different backup URL.
|
||||
if exists, existingURL := l.getLonghornVolumeSpec(kubeconfigPath, volumeName); exists {
|
||||
slog.Info("deleting stale staging volume before restore", "component", "longhorn", "volume", volumeName, "existingBackup", existingURL)
|
||||
if err := l.deleteLonghornVolumeIfExists(kubeconfigPath, volumeName); err != nil {
|
||||
return "", fmt.Errorf("failed to delete stale staging volume %s: %w", volumeName, err)
|
||||
}
|
||||
if err := l.waitForVolumeDeletion(kubeconfigPath, volumeName); err != nil {
|
||||
return "", fmt.Errorf("stale staging volume %s did not delete in time: %w", volumeName, err)
|
||||
}
|
||||
}
|
||||
|
||||
sizeBytes := "1073741824"
|
||||
if strings.HasSuffix(size, "Gi") {
|
||||
var sizeInt int
|
||||
if _, err := fmt.Sscanf(size, "%dGi", &sizeInt); err == nil {
|
||||
sizeBytes = fmt.Sprintf("%d", sizeInt*1024*1024*1024)
|
||||
}
|
||||
}
|
||||
|
||||
volumeYAML := fmt.Sprintf(`apiVersion: longhorn.io/v1beta2
|
||||
kind: Volume
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
size: "%s"
|
||||
fromBackup: "%s"
|
||||
numberOfReplicas: 3
|
||||
frontend: blockdev
|
||||
accessMode: rwo
|
||||
`, volumeName, sizeBytes, backupURL)
|
||||
|
||||
cmd := exec.Command("kubectl", "apply", "-f", "-")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
cmd.Stdin = strings.NewReader(volumeYAML)
|
||||
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("failed to create volume from backup: %w, output: %s", err, string(output))
|
||||
}
|
||||
|
||||
// Longhorn's mutating webhook may rename the volume (e.g. to enforce a 40-char
|
||||
// limit). Discover the actual name by looking up the backupURL after a brief delay.
|
||||
time.Sleep(2 * time.Second)
|
||||
actualName, err := l.findVolumeByBackupURL(kubeconfigPath, backupURL)
|
||||
if err != nil || actualName == "" {
|
||||
actualName = volumeName // fall back to our intended name
|
||||
}
|
||||
|
||||
return actualName, l.waitForVolume(kubeconfigPath, actualName)
|
||||
}
|
||||
|
||||
// findVolumeByBackupURL returns the name of the Longhorn volume whose spec.fromBackup
|
||||
// matches backupURL, or empty string if none is found.
|
||||
func (l *LonghornNativeStrategy) findVolumeByBackupURL(kubeconfigPath, backupURL string) (string, error) {
|
||||
cmd := exec.Command("kubectl", "get", "volumes.longhorn.io", "-n", "longhorn-system",
|
||||
"-o", `jsonpath={range .items[*]}{.metadata.name} {.spec.fromBackup}{"\n"}{end}`)
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, line := range strings.Split(string(output), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, " ", 2)
|
||||
if len(parts) == 2 && strings.TrimSpace(parts[1]) == backupURL {
|
||||
return strings.TrimSpace(parts[0]), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) waitForVolume(kubeconfigPath, volumeName string) error {
|
||||
maxRetries := 180 // 15 minutes; NFS restores can take longer than 5 minutes
|
||||
for i := range maxRetries {
|
||||
cmd := exec.Command("kubectl", "get", "volumes.longhorn.io", volumeName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.status.state},{.status.restoreInitiated},{.status.robustness}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err == nil {
|
||||
parts := strings.Split(string(output), ",")
|
||||
if len(parts) == 3 {
|
||||
state := parts[0]
|
||||
restoreInitiated := parts[1]
|
||||
robustness := parts[2]
|
||||
|
||||
if state == "detached" || state == "attached" {
|
||||
if restoreInitiated == "true" {
|
||||
return nil
|
||||
}
|
||||
if robustness == "healthy" || robustness == "unknown" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if i%12 == 0 {
|
||||
slog.Info("waiting for volume to be ready", "component", "longhorn", "volume", volumeName,
|
||||
"state", state, "restoreInitiated", restoreInitiated, "robustness", robustness,
|
||||
"attempt", i, "maxRetries", maxRetries)
|
||||
}
|
||||
}
|
||||
} else if i%12 == 0 {
|
||||
slog.Info("waiting for volume to be ready", "component", "longhorn", "volume", volumeName, "attempt", i, "maxRetries", maxRetries)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for volume %s to be ready", volumeName)
|
||||
}
|
||||
|
||||
func (l *LonghornNativeStrategy) cleanupOldBackups(_, _, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPVInfo returns the PV name and Longhorn volume handle for a PVC.
|
||||
func (l *LonghornNativeStrategy) getPVInfo(kubeconfigPath, namespace, pvcName string) (string, string, error) {
|
||||
pvNameCmd := exec.Command("kubectl", "get", "pvc", pvcName, "-n", namespace,
|
||||
"-o", "jsonpath={.spec.volumeName}")
|
||||
tools.WithKubeconfig(pvNameCmd, kubeconfigPath)
|
||||
pvNameOutput, err := pvNameCmd.Output()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get PV name for PVC %s: %w", pvcName, err)
|
||||
}
|
||||
pvName := strings.TrimSpace(string(pvNameOutput))
|
||||
if pvName == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
handleCmd := exec.Command("kubectl", "get", "pv", pvName,
|
||||
"-o", "jsonpath={.spec.csi.volumeHandle}")
|
||||
tools.WithKubeconfig(handleCmd, kubeconfigPath)
|
||||
handleOutput, err := handleCmd.Output()
|
||||
if err != nil {
|
||||
return pvName, "", fmt.Errorf("failed to get volume handle for PV %s: %w", pvName, err)
|
||||
}
|
||||
|
||||
return pvName, strings.TrimSpace(string(handleOutput)), nil
|
||||
}
|
||||
|
||||
// deletePVIfExists deletes a PersistentVolume if it exists (handles Retain reclaim policy cleanup).
|
||||
func (l *LonghornNativeStrategy) deletePVIfExists(kubeconfigPath, pvName string) error {
|
||||
cmd := exec.Command("kubectl", "delete", "pv", pvName, "--ignore-not-found", "--timeout=60s")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to delete PV %s: %w, output: %s", pvName, err, output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteLonghornVolumeIfExists deletes a Longhorn Volume CR if it exists.
|
||||
func (l *LonghornNativeStrategy) deleteLonghornVolumeIfExists(kubeconfigPath, volumeName string) error {
|
||||
cmd := exec.Command("kubectl", "delete", "volumes.longhorn.io", volumeName,
|
||||
"-n", "longhorn-system", "--ignore-not-found", "--timeout=60s")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to delete Longhorn volume %s: %w, output: %s", volumeName, err, output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getLonghornVolumeSpec returns (exists, fromBackup) for a Longhorn Volume CR.
|
||||
func (l *LonghornNativeStrategy) getLonghornVolumeSpec(kubeconfigPath, volumeName string) (bool, string) {
|
||||
cmd := exec.Command("kubectl", "get", "volumes.longhorn.io", volumeName,
|
||||
"-n", "longhorn-system", "-o", "jsonpath={.spec.fromBackup}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, ""
|
||||
}
|
||||
return true, strings.TrimSpace(string(output))
|
||||
}
|
||||
|
||||
// waitForVolumeDeletion polls until the Longhorn Volume CR is gone (up to ~120s).
|
||||
func (l *LonghornNativeStrategy) waitForVolumeDeletion(kubeconfigPath, volumeName string) error {
|
||||
for range 60 {
|
||||
exists, _ := l.getLonghornVolumeSpec(kubeconfigPath, volumeName)
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for Longhorn volume %s to be deleted", volumeName)
|
||||
}
|
||||
|
||||
// createPVC creates a PVC pre-bound to a specific PV by name.
|
||||
func (l *LonghornNativeStrategy) createPVC(kubeconfigPath, pvcName, namespace, pvName, size, accessMode string) error {
|
||||
pvcYAML := fmt.Sprintf(`apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
spec:
|
||||
accessModes:
|
||||
- %s
|
||||
resources:
|
||||
requests:
|
||||
storage: %s
|
||||
storageClassName: longhorn
|
||||
volumeName: %s
|
||||
`, pvcName, namespace, accessMode, size, pvName)
|
||||
|
||||
cmd := exec.Command("kubectl", "apply", "-f", "-")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
cmd.Stdin = strings.NewReader(pvcYAML)
|
||||
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create PVC %s: %w, output: %s", pvcName, err, output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForPVCBound polls until the PVC reaches Bound status.
|
||||
func (l *LonghornNativeStrategy) waitForPVCBound(kubeconfigPath, namespace, pvcName string) error {
|
||||
for range 60 {
|
||||
cmd := exec.Command("kubectl", "get", "pvc", pvcName, "-n", namespace,
|
||||
"-o", "jsonpath={.status.phase}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
if output, err := cmd.Output(); err == nil && strings.TrimSpace(string(output)) == "Bound" {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for PVC %s in %s to be Bound", pvcName, namespace)
|
||||
}
|
||||
95
api/internal/backup/strategies/longhorn_native_test.go
Normal file
95
api/internal/backup/strategies/longhorn_native_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestRestoreVolumeNameCapped verifies that restore volume names are capped at 40 chars.
|
||||
// Longhorn enforces a 40-character limit via its mutating webhook; names exceeding
|
||||
// this are silently renamed, causing waitForVolume to poll the wrong name.
|
||||
func TestRestoreVolumeNameCapped(t *testing.T) {
|
||||
tests := []struct {
|
||||
pvcName string
|
||||
timestamp string
|
||||
wantLen int
|
||||
wantName string
|
||||
}{
|
||||
{
|
||||
// 41-char name that triggered the original bug: writefreely-data-restore-20260619t221555z
|
||||
// Truncated to 40 by dropping the trailing "z"
|
||||
pvcName: "writefreely-data",
|
||||
timestamp: "20260619T221555Z",
|
||||
wantLen: 40,
|
||||
wantName: "writefreely-data-restore-20260619t221555",
|
||||
},
|
||||
{
|
||||
// Short name that fits within 40 chars (no truncation)
|
||||
pvcName: "app-data",
|
||||
timestamp: "20260619T221555Z",
|
||||
wantLen: 33, // "app-data-restore-20260619t221555z" = 8+9+16 = 33
|
||||
wantName: "app-data-restore-20260619t221555z",
|
||||
},
|
||||
{
|
||||
// Exactly 40 chars (no truncation needed)
|
||||
pvcName: "writefreely-dat",
|
||||
timestamp: "20260619T221555Z",
|
||||
wantLen: 40, // "writefreely-dat-restore-20260619t221555z" = 40
|
||||
wantName: "writefreely-dat-restore-20260619t221555z",
|
||||
},
|
||||
{
|
||||
// Very long PVC name: first 40 chars happen to be exactly the PVC name itself
|
||||
pvcName: "my-very-long-pvc-name-that-is-quite-long",
|
||||
timestamp: "20260619T221555Z",
|
||||
wantLen: 40,
|
||||
wantName: "my-very-long-pvc-name-that-is-quite-long",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(fmt.Sprintf("%s/%s", tt.pvcName, tt.timestamp), func(t *testing.T) {
|
||||
desiredName := strings.ToLower(fmt.Sprintf("%s-restore-%s", tt.pvcName, tt.timestamp))
|
||||
restoreVolumeName := desiredName
|
||||
if len(desiredName) > 40 {
|
||||
restoreVolumeName = desiredName[:40]
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantName, restoreVolumeName)
|
||||
assert.LessOrEqual(t, len(restoreVolumeName), 40,
|
||||
"volume name must not exceed Longhorn's 40-char limit")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindVolumeByBackupURLParsing verifies that findVolumeByBackupURL correctly
|
||||
// parses kubectl jsonpath output to find the volume matching a backup URL.
|
||||
func TestFindVolumeByBackupURLParsing(t *testing.T) {
|
||||
// Simulate the jsonpath output format:
|
||||
// {range .items[*]}{.metadata.name} {.spec.fromBackup}{"\n"}{end}
|
||||
kubectlOutput := `writefreely-data-restore-202606-7b566524 nfs://192.168.8.222:/storage/nfs/test-cloud/backups?backup=writefreely-writefreely-data-snapshot-20260619t221555z&volume=pvc-68781fe9-7cb6-4df7-8732-9270f59cb06f
|
||||
other-volume nfs://192.168.8.222:/storage/nfs/test-cloud/backups?backup=some-other-backup&volume=pvc-abc123
|
||||
decidim-data-restore-20260609t151938z nfs://192.168.8.222:/storage/nfs/test-cloud/backups?backup=decidim-backup&volume=pvc-def456
|
||||
`
|
||||
|
||||
targetURL := "nfs://192.168.8.222:/storage/nfs/test-cloud/backups?backup=writefreely-writefreely-data-snapshot-20260619t221555z&volume=pvc-68781fe9-7cb6-4df7-8732-9270f59cb06f"
|
||||
|
||||
// Replicate the parsing logic from findVolumeByBackupURL
|
||||
found := ""
|
||||
for _, line := range strings.Split(kubectlOutput, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, " ", 2)
|
||||
if len(parts) == 2 && strings.TrimSpace(parts[1]) == targetURL {
|
||||
found = strings.TrimSpace(parts[0])
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, "writefreely-data-restore-202606-7b566524", found,
|
||||
"should find the volume matching the backup URL even when Longhorn renamed it")
|
||||
}
|
||||
423
api/internal/backup/strategies/mysql.go
Normal file
423
api/internal/backup/strategies/mysql.go
Normal file
@@ -0,0 +1,423 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// MySQLStrategy implements backup strategy for MySQL databases
|
||||
type MySQLStrategy struct {
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// NewMySQLStrategy creates a new MySQL backup strategy
|
||||
func NewMySQLStrategy(dataDir string) *MySQLStrategy {
|
||||
return &MySQLStrategy{
|
||||
dataDir: dataDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the strategy identifier
|
||||
func (m *MySQLStrategy) Name() string {
|
||||
return "mysql"
|
||||
}
|
||||
|
||||
// Backup creates a MySQL database backup, writing results to the plan
|
||||
func (m *MySQLStrategy) Backup(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("mysql")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for mysql in plan")
|
||||
}
|
||||
entry.Status = "backing_up"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, plan.Instance)
|
||||
|
||||
// Database name and user both default to app name (MySQL convention)
|
||||
dbName := plan.App
|
||||
dbUser := plan.App
|
||||
|
||||
// Record params in plan
|
||||
entry.Params = map[string]any{
|
||||
"podNamespace": "mysql",
|
||||
"dbName": dbName,
|
||||
"dbUser": dbUser,
|
||||
}
|
||||
|
||||
// Get MySQL root password from secret
|
||||
password, err := m.getMySQLPassword(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get MySQL password: %w", err)
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("mysql/%s/%s/%s.sql.gz", plan.Instance, plan.App, plan.Timestamp)
|
||||
|
||||
// Create mysqldump command that streams to stdout
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "mysql",
|
||||
"mysql-0", "--", "bash", "-c",
|
||||
fmt.Sprintf("mysqldump -uroot -p'%s' --single-transaction --routines --triggers --events %s",
|
||||
password, dbName))
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
// Use io.Pipe to stream and compress on the fly
|
||||
reader, writer := io.Pipe()
|
||||
gzWriter := gzip.NewWriter(writer)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
defer writer.Close()
|
||||
defer gzWriter.Close()
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("mysqldump failed: %v, stderr: %s", err, stderr.String())
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := gzWriter.Write(output); err != nil {
|
||||
errChan <- fmt.Errorf("compression failed: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
errChan <- nil
|
||||
}()
|
||||
|
||||
size, err := dest.Put(key, reader)
|
||||
if err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return fmt.Errorf("failed to upload backup: %w", err)
|
||||
}
|
||||
|
||||
if dumpErr := <-errChan; dumpErr != nil {
|
||||
return dumpErr
|
||||
}
|
||||
|
||||
entry.Backup = map[string]any{
|
||||
"location": key,
|
||||
"size": size,
|
||||
"database": dbName,
|
||||
"format": "sql",
|
||||
"compressed": true,
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore restores a MySQL database from backup into a staging database ({dbName}_restore).
|
||||
// The app is not touched here; Switch will atomically rename the staging DB into place.
|
||||
func (m *MySQLStrategy) Restore(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("mysql")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for mysql in plan")
|
||||
}
|
||||
entry.Status = "restoring"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, plan.Instance)
|
||||
|
||||
dbName, _ := entry.Params["dbName"].(string)
|
||||
if dbName == "" {
|
||||
return fmt.Errorf("database name not found in plan params")
|
||||
}
|
||||
|
||||
// Staging database name — always use _restore suffix, never colored names
|
||||
restoreDbName := dbName + "_restore"
|
||||
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in mysql strategy entry")
|
||||
}
|
||||
|
||||
password, err := m.getMySQLPassword(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get MySQL password: %w", err)
|
||||
}
|
||||
|
||||
// Get backup from destination
|
||||
compressedReader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve backup: %w", err)
|
||||
}
|
||||
defer compressedReader.Close()
|
||||
|
||||
// Decompress
|
||||
var reader io.Reader = compressedReader
|
||||
if compressed, _ := entry.Backup["compressed"].(bool); compressed {
|
||||
gzReader, err := gzip.NewReader(compressedReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
reader = gzReader
|
||||
}
|
||||
|
||||
// Drop and recreate staging database, grant access to the app user
|
||||
dbUser, _ := entry.Params["dbUser"].(string)
|
||||
if dbUser == "" {
|
||||
dbUser = dbName // fallback for plans created before dbUser was stored
|
||||
}
|
||||
|
||||
setupSQL := fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s; GRANT ALL PRIVILEGES ON %s.* TO '%s'@'%%'; FLUSH PRIVILEGES;",
|
||||
restoreDbName, restoreDbName, restoreDbName, dbUser)
|
||||
|
||||
setupCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -e \"%s\"", password, setupSQL))
|
||||
tools.WithKubeconfig(setupCmd, kubeconfigPath)
|
||||
|
||||
if output, err := setupCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to recreate database: %w, output: %s", err, output)
|
||||
}
|
||||
|
||||
// Restore database into staging
|
||||
restoreCmd := exec.Command("kubectl", "exec", "-i", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' %s", password, restoreDbName))
|
||||
tools.WithKubeconfig(restoreCmd, kubeconfigPath)
|
||||
restoreCmd.Stdin = reader
|
||||
|
||||
var stderr bytes.Buffer
|
||||
restoreCmd.Stderr = &stderr
|
||||
|
||||
if err := restoreCmd.Run(); err != nil {
|
||||
return fmt.Errorf("mysql restore failed: %w, stderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
// Record staging db name in plan for Switch phase
|
||||
entry.Restore = map[string]any{
|
||||
"dbName": restoreDbName,
|
||||
}
|
||||
entry.Status = "restored"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Switch atomically moves the restored database into place using a detect-then-reconcile
|
||||
// state machine. MySQL has no RENAME DATABASE, so it uses RENAME TABLE per table.
|
||||
// Steps: rename {db}→{db}_old, then rename {db}_restore→{db}.
|
||||
// Re-entrant: detects actual MySQL state so a crash between steps can be resumed.
|
||||
func (m *MySQLStrategy) Switch(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("mysql")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for mysql in plan")
|
||||
}
|
||||
entry.Status = "switching"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, plan.Instance)
|
||||
dbName, _ := entry.Params["dbName"].(string)
|
||||
if dbName == "" {
|
||||
return fmt.Errorf("database name not found in plan params")
|
||||
}
|
||||
restoreDbName := dbName + "_restore"
|
||||
oldDbName := dbName + "_old"
|
||||
|
||||
password, err := m.getMySQLPassword(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get MySQL password: %w", err)
|
||||
}
|
||||
|
||||
// Detect current state by querying MySQL directly (idempotent re-entry)
|
||||
currentExists := m.databaseExists(kubeconfigPath, password, dbName)
|
||||
restoreExists := m.databaseExists(kubeconfigPath, password, restoreDbName)
|
||||
oldExists := m.databaseExists(kubeconfigPath, password, oldDbName)
|
||||
|
||||
// Validate preconditions
|
||||
if !restoreExists && !oldExists {
|
||||
if currentExists {
|
||||
// Restore phase was skipped or plan is being re-run after cleanup; nothing to switch
|
||||
entry.Switch = map[string]any{"previousDbName": oldDbName}
|
||||
entry.Status = "switched"
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("restore database %s not found; restore phase may not have completed", restoreDbName)
|
||||
}
|
||||
|
||||
// Step 1: rename {db} → {db}_old (point of no return)
|
||||
if currentExists && !oldExists {
|
||||
if err := m.renameDatabase(kubeconfigPath, password, dbName, oldDbName); err != nil {
|
||||
return fmt.Errorf("failed to rename %s to %s: %w", dbName, oldDbName, err)
|
||||
}
|
||||
oldExists = true
|
||||
currentExists = false
|
||||
}
|
||||
|
||||
// Step 2: rename {db}_restore → {db}
|
||||
if restoreExists && !currentExists {
|
||||
if err := m.renameDatabase(kubeconfigPath, password, restoreDbName, dbName); err != nil {
|
||||
return fmt.Errorf("failed to rename %s to %s: %w", restoreDbName, dbName, err)
|
||||
}
|
||||
}
|
||||
|
||||
_ = oldExists // used above; may be true if step 1 was already done on re-entry
|
||||
entry.Switch = map[string]any{
|
||||
"previousDbName": oldDbName,
|
||||
}
|
||||
entry.Status = "switched"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup drops the previous active-color database
|
||||
func (m *MySQLStrategy) Cleanup(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("mysql")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for mysql in plan")
|
||||
}
|
||||
entry.Status = "cleaning_up"
|
||||
|
||||
previousDbName, _ := entry.Switch["previousDbName"].(string)
|
||||
if previousDbName == "" {
|
||||
entry.Status = "cleaned_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, plan.Instance)
|
||||
|
||||
password, err := m.getMySQLPassword(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get MySQL password: %w", err)
|
||||
}
|
||||
|
||||
dropCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -e 'DROP DATABASE IF EXISTS %s;'",
|
||||
password, previousDbName))
|
||||
tools.WithKubeconfig(dropCmd, kubeconfigPath)
|
||||
|
||||
if output, err := dropCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to drop database %s: %w, output: %s", previousDbName, err, output)
|
||||
}
|
||||
|
||||
entry.Status = "cleaned_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify checks if a MySQL backup can be restored
|
||||
func (m *MySQLStrategy) Verify(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("mysql")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for mysql in plan")
|
||||
}
|
||||
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in mysql strategy entry")
|
||||
}
|
||||
|
||||
reader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup not found in destination: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
compressed, _ := entry.Backup["compressed"].(bool)
|
||||
if compressed {
|
||||
header := make([]byte, 2)
|
||||
if _, err := io.ReadFull(reader, header); err != nil {
|
||||
return fmt.Errorf("failed to read backup header: %w", err)
|
||||
}
|
||||
|
||||
if header[0] != 0x1f || header[1] != 0x8b {
|
||||
return fmt.Errorf("invalid gzip format")
|
||||
}
|
||||
} else {
|
||||
header := make([]byte, 100)
|
||||
n, _ := reader.Read(header)
|
||||
headerStr := strings.ToLower(string(header[:n]))
|
||||
|
||||
if !strings.Contains(headerStr, "mysql") && !strings.Contains(headerStr, "--") &&
|
||||
!strings.Contains(headerStr, "database") {
|
||||
return fmt.Errorf("doesn't appear to be a MySQL dump")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// databaseExists returns true if the named MySQL database exists
|
||||
func (m *MySQLStrategy) databaseExists(kubeconfigPath, password, dbName string) bool {
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -sNe \"SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME='%s'\"",
|
||||
password, dbName))
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(string(output)) == dbName
|
||||
}
|
||||
|
||||
// renameDatabase renames a MySQL database by creating the target, moving all tables
|
||||
// via RENAME TABLE (MySQL has no native RENAME DATABASE), then dropping the source.
|
||||
func (m *MySQLStrategy) renameDatabase(kubeconfigPath, password, fromDB, toDB string) error {
|
||||
// Step 1: create the destination database
|
||||
createCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -e 'CREATE DATABASE IF NOT EXISTS %s;'", password, toDB))
|
||||
tools.WithKubeconfig(createCmd, kubeconfigPath)
|
||||
if output, err := createCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create database %s: %w, output: %s", toDB, err, output)
|
||||
}
|
||||
|
||||
// Step 2: list all tables in the source database
|
||||
listCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -sNe \"SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA='%s' AND TABLE_TYPE='BASE TABLE'\"",
|
||||
password, fromDB))
|
||||
tools.WithKubeconfig(listCmd, kubeconfigPath)
|
||||
listOutput, err := listCmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list tables in %s: %w", fromDB, err)
|
||||
}
|
||||
|
||||
// Step 3: RENAME TABLE for each table
|
||||
tables := strings.Fields(strings.TrimSpace(string(listOutput)))
|
||||
for _, table := range tables {
|
||||
renameSQL := fmt.Sprintf("RENAME TABLE %s.%s TO %s.%s;", fromDB, table, toDB, table)
|
||||
renameCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -e \"%s\"", password, renameSQL))
|
||||
tools.WithKubeconfig(renameCmd, kubeconfigPath)
|
||||
if output, err := renameCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to rename table %s.%s: %w, output: %s", fromDB, table, err, output)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: drop the now-empty source database
|
||||
dropCmd := exec.Command("kubectl", "exec", "-n", "mysql", "mysql-0", "--",
|
||||
"bash", "-c",
|
||||
fmt.Sprintf("mysql -uroot -p'%s' -e 'DROP DATABASE IF EXISTS %s;'", password, fromDB))
|
||||
tools.WithKubeconfig(dropCmd, kubeconfigPath)
|
||||
if output, err := dropCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to drop source database %s: %w, output: %s", fromDB, err, output)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getMySQLPassword retrieves the MySQL root password from Kubernetes secret
|
||||
func (m *MySQLStrategy) getMySQLPassword(kubeconfigPath string) (string, error) {
|
||||
cmd := exec.Command("kubectl", "get", "secret", "-n", "mysql", "mysql-secrets",
|
||||
"-o", "jsonpath={.data.rootPassword}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get MySQL secret: %w", err)
|
||||
}
|
||||
|
||||
decoded, err := base64.StdEncoding.DecodeString(string(output))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode password: %w", err)
|
||||
}
|
||||
|
||||
return string(decoded), nil
|
||||
}
|
||||
191
api/internal/backup/strategies/mysql_test.go
Normal file
191
api/internal/backup/strategies/mysql_test.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// MockMySQLDestination implements BackupDestination for testing
|
||||
type MockMySQLDestination struct {
|
||||
putData map[string][]byte
|
||||
getData map[string][]byte
|
||||
putError error
|
||||
getError error
|
||||
}
|
||||
|
||||
func NewMockMySQLDestination() *MockMySQLDestination {
|
||||
return &MockMySQLDestination{
|
||||
putData: make(map[string][]byte),
|
||||
getData: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockMySQLDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
if m.putError != nil {
|
||||
return 0, m.putError
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.putData[key] = data
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (m *MockMySQLDestination) Get(key string) (io.ReadCloser, error) {
|
||||
if m.getError != nil {
|
||||
return nil, m.getError
|
||||
}
|
||||
data, exists := m.getData[key]
|
||||
if !exists {
|
||||
data = []byte("mock backup data")
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
func (m *MockMySQLDestination) Delete(key string) error { return nil }
|
||||
|
||||
func (m *MockMySQLDestination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
var objects []btypes.BackupObject
|
||||
for key, data := range m.putData {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
LastModified: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (m *MockMySQLDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
return "https://mock.example.com/" + key, nil
|
||||
}
|
||||
|
||||
func (m *MockMySQLDestination) Type() string { return "mock" }
|
||||
|
||||
func TestMySQLStrategy_Name(t *testing.T) {
|
||||
s := NewMySQLStrategy("/tmp")
|
||||
assert.Equal(t, "mysql", s.Name())
|
||||
}
|
||||
|
||||
// TestMySQLDumpCommand_NoDatabasesFlag verifies that the mysqldump command does NOT
|
||||
// use the --databases flag.
|
||||
//
|
||||
// Root cause of the bug: --databases makes mysqldump emit "CREATE DATABASE writefreely;"
|
||||
// and "USE writefreely;" at the top of the dump. When this dump is piped into
|
||||
// "mysql ... writefreely_restore", the USE statement silently redirects all table
|
||||
// creation to the live writefreely database. After restore, writefreely_restore is
|
||||
// empty. SwitchApp then renames writefreely_restore (empty) → writefreely, destroying
|
||||
// the working database.
|
||||
//
|
||||
// Fix: omit --databases so the dump contains only the schema/data, no CREATE DATABASE
|
||||
// or USE statements. The mysql client then keeps all operations in writefreely_restore.
|
||||
func TestMySQLDumpCommand_NoDatabasesFlag(t *testing.T) {
|
||||
password := "testpass"
|
||||
dbName := "writefreely"
|
||||
|
||||
// This is the exact command template used by MySQLStrategy.Backup
|
||||
cmd := fmt.Sprintf("mysqldump -uroot -p'%s' --single-transaction --routines --triggers --events %s",
|
||||
password, dbName)
|
||||
|
||||
assert.NotContains(t, cmd, "--databases",
|
||||
"--databases flag causes mysqldump to emit USE <dbname> which redirects restore to the wrong database")
|
||||
assert.True(t, strings.HasSuffix(cmd, " "+dbName),
|
||||
"database name must be the last argument (no --databases prefix)")
|
||||
}
|
||||
|
||||
func TestMySQLStrategy_Verify(t *testing.T) {
|
||||
s := NewMySQLStrategy("/tmp")
|
||||
|
||||
// Build a valid gzip-compressed SQL dump
|
||||
var gzBuf bytes.Buffer
|
||||
gzw := gzip.NewWriter(&gzBuf)
|
||||
_, _ = gzw.Write([]byte("-- MySQL dump\nCREATE TABLE users (id INT);"))
|
||||
_ = gzw.Close()
|
||||
validGzip := gzBuf.Bytes()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
plan *btypes.RecoveryPlan
|
||||
destData map[string][]byte
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "valid gzip backup",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "writefreely",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "mysql",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "mysql/test-instance/writefreely/20260101T000000Z.sql.gz",
|
||||
"compressed": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
destData: map[string][]byte{"mysql/test-instance/writefreely/20260101T000000Z.sql.gz": validGzip},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid gzip header",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "writefreely",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "mysql",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "mysql/test-instance/writefreely/20260101T000000Z.sql.gz",
|
||||
"compressed": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
destData: map[string][]byte{"mysql/test-instance/writefreely/20260101T000000Z.sql.gz": []byte("not gzip")},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "no backup location",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "writefreely",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "mysql",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dest := NewMockMySQLDestination()
|
||||
dest.getData = tt.destData
|
||||
|
||||
err := s.Verify(tt.plan, dest)
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
535
api/internal/backup/strategies/postgres.go
Normal file
535
api/internal/backup/strategies/postgres.go
Normal file
@@ -0,0 +1,535 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// PostgreSQLStrategy implements backup strategy for PostgreSQL databases
|
||||
type PostgreSQLStrategy struct {
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// NewPostgreSQLStrategy creates a new PostgreSQL backup strategy
|
||||
func NewPostgreSQLStrategy(dataDir string) *PostgreSQLStrategy {
|
||||
return &PostgreSQLStrategy{
|
||||
dataDir: dataDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the strategy identifier
|
||||
func (p *PostgreSQLStrategy) Name() string {
|
||||
return "postgres"
|
||||
}
|
||||
|
||||
// Backup creates a PostgreSQL database backup, writing results to the plan
|
||||
func (p *PostgreSQLStrategy) Backup(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for postgres in plan")
|
||||
}
|
||||
entry.Status = "backing_up"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(p.dataDir, plan.Instance)
|
||||
|
||||
// Determine database name and user from config
|
||||
dbName := p.getDatabaseName(plan.Instance, plan.App)
|
||||
dbUser := p.getAppUser(plan.Instance, plan.App)
|
||||
|
||||
// Record params in plan for later phases
|
||||
entry.Params = map[string]interface{}{
|
||||
"podNamespace": "postgres",
|
||||
"podLabel": "app=postgres",
|
||||
"dbName": dbName,
|
||||
"dbUser": dbUser,
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("postgres/%s/%s/%s.dump", plan.Instance, plan.App, plan.Timestamp)
|
||||
|
||||
// Get the postgres pod name
|
||||
podName, err := p.getPostgresPod(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find postgres pod: %w", err)
|
||||
}
|
||||
|
||||
// Create pg_dump command that streams to stdout
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "postgres",
|
||||
podName, "--", "pg_dump",
|
||||
"-U", "postgres",
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--compress=9",
|
||||
"--dbname", dbName,
|
||||
)
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
reader, writer := io.Pipe()
|
||||
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = writer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
defer writer.Close()
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("pg_dump failed: %v, stderr: %s", err, stderr.String())
|
||||
} else {
|
||||
errChan <- nil
|
||||
}
|
||||
}()
|
||||
|
||||
size, err := dest.Put(key, reader)
|
||||
if err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return fmt.Errorf("failed to upload backup: %w", err)
|
||||
}
|
||||
|
||||
if pgErr := <-errChan; pgErr != nil {
|
||||
return pgErr
|
||||
}
|
||||
|
||||
// Also backup globals (users, roles, etc)
|
||||
globalsKey := fmt.Sprintf("postgres/%s/%s/%s-globals.sql", plan.Instance, plan.App, plan.Timestamp)
|
||||
if err := p.backupGlobals(kubeconfigPath, dest, globalsKey); err != nil {
|
||||
slog.Error("postgres globals backup failed", "component", "postgres", "error", err)
|
||||
globalsKey = ""
|
||||
}
|
||||
|
||||
// Write results to plan entry
|
||||
entry.Backup = map[string]interface{}{
|
||||
"location": key,
|
||||
"size": size,
|
||||
"database": dbName,
|
||||
"format": "custom",
|
||||
"globals": globalsKey,
|
||||
}
|
||||
entry.Status = "backed_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore restores a PostgreSQL database from backup into a staging database ({dbName}_restore).
|
||||
// The app is not touched here; Switch will atomically rename the staging DB into place.
|
||||
func (p *PostgreSQLStrategy) Restore(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for postgres in plan")
|
||||
}
|
||||
entry.Status = "restoring"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(p.dataDir, plan.Instance)
|
||||
|
||||
// Read params recorded during backup
|
||||
dbName, _ := entry.Params["dbName"].(string)
|
||||
if dbName == "" {
|
||||
return fmt.Errorf("database name not found in plan params")
|
||||
}
|
||||
dbUser, _ := entry.Params["dbUser"].(string)
|
||||
if dbUser == "" {
|
||||
dbUser = plan.App
|
||||
}
|
||||
|
||||
// Staging database name — always use _restore suffix, never colored names
|
||||
restoreDbName := dbName + "_restore"
|
||||
|
||||
// Get backup location
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in postgres strategy entry")
|
||||
}
|
||||
|
||||
podName, err := p.getPostgresPod(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find postgres pod: %w", err)
|
||||
}
|
||||
|
||||
// Get backup from destination
|
||||
reader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve backup: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Drop staging database if it already exists (idempotent re-entry)
|
||||
dropCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("DROP DATABASE IF EXISTS %s", restoreDbName))
|
||||
tools.WithKubeconfig(dropCmd, kubeconfigPath)
|
||||
if output, err := dropCmd.CombinedOutput(); err != nil {
|
||||
slog.Error("failed to drop staging database", "component", "postgres", "database", restoreDbName, "error", err, "output", string(output))
|
||||
}
|
||||
|
||||
// Create staging database
|
||||
createCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("CREATE DATABASE %s", restoreDbName))
|
||||
tools.WithKubeconfig(createCmd, kubeconfigPath)
|
||||
if output, err := createCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create database %s: %w, output: %s", restoreDbName, err, output)
|
||||
}
|
||||
|
||||
// Grant permissions to the app user
|
||||
if dbUser != "" && dbUser != "postgres" {
|
||||
grantCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", restoreDbName, dbUser))
|
||||
tools.WithKubeconfig(grantCmd, kubeconfigPath)
|
||||
if output, err := grantCmd.CombinedOutput(); err != nil {
|
||||
slog.Error("failed to grant privileges", "component", "postgres", "error", err, "output", string(output))
|
||||
}
|
||||
}
|
||||
|
||||
// Restore database using pg_restore
|
||||
pgRestoreCmd := exec.Command("kubectl", "exec", "-i", "-n", "postgres", podName, "--",
|
||||
"pg_restore", "-U", "postgres", "-d", restoreDbName, "--no-owner", "--clean", "--if-exists")
|
||||
tools.WithKubeconfig(pgRestoreCmd, kubeconfigPath)
|
||||
pgRestoreCmd.Stdin = reader
|
||||
|
||||
var stderr bytes.Buffer
|
||||
pgRestoreCmd.Stderr = &stderr
|
||||
|
||||
if err := pgRestoreCmd.Run(); err != nil {
|
||||
// pg_restore returns non-zero for warnings, check if database was actually restored
|
||||
checkCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", restoreDbName, "-c", "\\dt")
|
||||
tools.WithKubeconfig(checkCmd, kubeconfigPath)
|
||||
|
||||
if checkOutput, checkErr := checkCmd.Output(); checkErr != nil || !strings.Contains(string(checkOutput), "table") {
|
||||
return fmt.Errorf("pg_restore failed: %w, stderr: %s", err, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Transfer ownership and grant permissions after restore.
|
||||
// pg_restore runs as 'postgres' so all restored objects are owned by postgres.
|
||||
// Apps like Gitea and Immich require ownership (not just privileges) to run migrations.
|
||||
if dbUser != "" && dbUser != "postgres" {
|
||||
ownershipSQL := fmt.Sprintf(`
|
||||
DO $$
|
||||
DECLARE r RECORD;
|
||||
BEGIN
|
||||
FOR r IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' LOOP
|
||||
EXECUTE 'ALTER TABLE public.' || quote_ident(r.tablename) || ' OWNER TO %s';
|
||||
END LOOP;
|
||||
FOR r IN SELECT sequencename FROM pg_sequences WHERE schemaname = 'public' LOOP
|
||||
EXECUTE 'ALTER SEQUENCE public.' || quote_ident(r.sequencename) || ' OWNER TO %s';
|
||||
END LOOP;
|
||||
FOR r IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' LOOP
|
||||
EXECUTE 'ALTER INDEX public.' || quote_ident(r.indexname) || ' OWNER TO %s';
|
||||
END LOOP;
|
||||
END $$;
|
||||
ALTER SCHEMA public OWNER TO %s;`, dbUser, dbUser, dbUser, dbUser)
|
||||
|
||||
ownerCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", restoreDbName, "-c", ownershipSQL)
|
||||
tools.WithKubeconfig(ownerCmd, kubeconfigPath)
|
||||
if output, err := ownerCmd.CombinedOutput(); err != nil {
|
||||
slog.Error("failed to transfer ownership", "component", "postgres", "error", err, "output", string(output))
|
||||
}
|
||||
}
|
||||
|
||||
// Record staging db name in plan for Switch phase
|
||||
entry.Restore = map[string]interface{}{
|
||||
"dbName": restoreDbName,
|
||||
}
|
||||
entry.Status = "restored"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Switch atomically moves the restored database into place using a detect-then-reconcile
|
||||
// state machine. Steps: rename {db}→{db}_old, then rename {db}_restore→{db}.
|
||||
// Re-entrant: detects actual Postgres state so a crash between steps can be resumed.
|
||||
func (p *PostgreSQLStrategy) Switch(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for postgres in plan")
|
||||
}
|
||||
entry.Status = "switching"
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(p.dataDir, plan.Instance)
|
||||
dbName, _ := entry.Params["dbName"].(string)
|
||||
if dbName == "" {
|
||||
return fmt.Errorf("database name not found in plan params")
|
||||
}
|
||||
restoreDbName := dbName + "_restore"
|
||||
oldDbName := dbName + "_old"
|
||||
|
||||
podName, err := p.getPostgresPod(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find postgres pod: %w", err)
|
||||
}
|
||||
|
||||
// Detect current state by querying Postgres directly (idempotent re-entry)
|
||||
currentExists := p.databaseExists(kubeconfigPath, podName, dbName)
|
||||
restoreExists := p.databaseExists(kubeconfigPath, podName, restoreDbName)
|
||||
oldExists := p.databaseExists(kubeconfigPath, podName, oldDbName)
|
||||
|
||||
// Validate preconditions
|
||||
if !restoreExists && !oldExists {
|
||||
if currentExists {
|
||||
// Restore phase was skipped or plan is being re-run after cleanup; nothing to switch
|
||||
entry.Switch = map[string]interface{}{"previousDbName": oldDbName}
|
||||
entry.Status = "switched"
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("restore database %s not found; restore phase may not have completed", restoreDbName)
|
||||
}
|
||||
|
||||
// Step 1: rename {db} → {db}_old (point of no return)
|
||||
if currentExists && !oldExists {
|
||||
// Terminate connections and retry the rename — a stale connection can block
|
||||
// ALTER DATABASE RENAME even after scale-down. Retries with increasing delay.
|
||||
var renameErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
p.terminateConnections(kubeconfigPath, podName, dbName)
|
||||
if attempt > 0 {
|
||||
time.Sleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
renameCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("ALTER DATABASE %s RENAME TO %s", dbName, oldDbName))
|
||||
tools.WithKubeconfig(renameCmd, kubeconfigPath)
|
||||
output, err := renameCmd.CombinedOutput()
|
||||
if err == nil {
|
||||
renameErr = nil
|
||||
break
|
||||
}
|
||||
renameErr = fmt.Errorf("failed to rename %s to %s: %w, output: %s", dbName, oldDbName, err, output)
|
||||
}
|
||||
if renameErr != nil {
|
||||
return renameErr
|
||||
}
|
||||
slog.Info("postgres switch: renamed current db to old", "db", dbName, "old", oldDbName)
|
||||
oldExists = true
|
||||
currentExists = false
|
||||
}
|
||||
|
||||
// Step 2: rename {db}_restore → {db}
|
||||
if restoreExists && !currentExists {
|
||||
renameCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("ALTER DATABASE %s RENAME TO %s", restoreDbName, dbName))
|
||||
tools.WithKubeconfig(renameCmd, kubeconfigPath)
|
||||
if output, err := renameCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to rename %s to %s: %w, output: %s", restoreDbName, dbName, err, output)
|
||||
}
|
||||
slog.Info("postgres switch: renamed restore db to canonical", "restore", restoreDbName, "db", dbName)
|
||||
}
|
||||
|
||||
_ = oldExists // used above; may be true if step 1 was already done on re-entry
|
||||
entry.Switch = map[string]interface{}{
|
||||
"previousDbName": oldDbName,
|
||||
}
|
||||
entry.Status = "switched"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup drops the previous active-color database
|
||||
func (p *PostgreSQLStrategy) Cleanup(plan *btypes.RecoveryPlan) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for postgres in plan")
|
||||
}
|
||||
entry.Status = "cleaning_up"
|
||||
|
||||
previousDbName, _ := entry.Switch["previousDbName"].(string)
|
||||
if previousDbName == "" {
|
||||
// Nothing to clean up
|
||||
entry.Status = "cleaned_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
kubeconfigPath := tools.GetKubeconfigPath(p.dataDir, plan.Instance)
|
||||
podName, err := p.getPostgresPod(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find postgres pod: %w", err)
|
||||
}
|
||||
|
||||
// Terminate connections to the old database before dropping it
|
||||
p.terminateConnections(kubeconfigPath, podName, previousDbName)
|
||||
|
||||
// Drop the old database
|
||||
dropCmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("DROP DATABASE IF EXISTS %s", previousDbName))
|
||||
tools.WithKubeconfig(dropCmd, kubeconfigPath)
|
||||
if output, err := dropCmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to drop database %s: %w, output: %s", previousDbName, err, output)
|
||||
}
|
||||
|
||||
entry.Status = "cleaned_up"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify checks if a PostgreSQL backup can be restored
|
||||
func (p *PostgreSQLStrategy) Verify(plan *btypes.RecoveryPlan, dest btypes.BackupDestination) error {
|
||||
entry := plan.GetStrategyEntry("postgres")
|
||||
if entry == nil {
|
||||
return fmt.Errorf("no strategy entry for postgres in plan")
|
||||
}
|
||||
|
||||
location, _ := entry.Backup["location"].(string)
|
||||
if location == "" {
|
||||
return fmt.Errorf("no backup location in postgres strategy entry")
|
||||
}
|
||||
|
||||
reader, err := dest.Get(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup not found in destination: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// pg_dump custom format starts with "PGDMP"
|
||||
magic := make([]byte, 5)
|
||||
if _, err := io.ReadFull(reader, magic); err != nil {
|
||||
return fmt.Errorf("failed to read backup header: %w", err)
|
||||
}
|
||||
|
||||
if string(magic) != "PGDMP" {
|
||||
return fmt.Errorf("invalid PostgreSQL dump format")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupGlobals backs up PostgreSQL global objects (users, roles, etc)
|
||||
func (p *PostgreSQLStrategy) backupGlobals(kubeconfigPath string, dest btypes.BackupDestination, key string) error {
|
||||
podName, err := p.getPostgresPod(kubeconfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find postgres pod: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"pg_dumpall", "-U", "postgres", "--globals-only")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
reader, writer := io.Pipe()
|
||||
cmd.Stdout = writer
|
||||
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
defer writer.Close()
|
||||
if err := cmd.Run(); err != nil {
|
||||
errChan <- fmt.Errorf("pg_dumpall failed: %v, stderr: %s", err, stderr.String())
|
||||
} else {
|
||||
errChan <- nil
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := dest.Put(key, reader); err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return err
|
||||
}
|
||||
|
||||
return <-errChan
|
||||
}
|
||||
|
||||
// getDatabaseName determines the database name for the app from its per-app config file.
|
||||
func (p *PostgreSQLStrategy) getDatabaseName(instanceName, appName string) string {
|
||||
appConfig := p.readAppConfig(instanceName, appName)
|
||||
if appConfig == nil {
|
||||
return appName
|
||||
}
|
||||
if dbName, ok := appConfig["dbName"].(string); ok && dbName != "" {
|
||||
return dbName
|
||||
}
|
||||
if db, ok := appConfig["db"].(map[string]interface{}); ok {
|
||||
if dbName, ok := db["name"].(string); ok && dbName != "" {
|
||||
return dbName
|
||||
}
|
||||
}
|
||||
return appName
|
||||
}
|
||||
|
||||
// getAppUser retrieves the database user for the app from its per-app config file.
|
||||
func (p *PostgreSQLStrategy) getAppUser(instanceName, appName string) string {
|
||||
appConfig := p.readAppConfig(instanceName, appName)
|
||||
if appConfig != nil {
|
||||
if dbUser, ok := appConfig["dbUser"].(string); ok && dbUser != "" {
|
||||
return dbUser
|
||||
}
|
||||
if dbUsername, ok := appConfig["dbUsername"].(string); ok && dbUsername != "" {
|
||||
return dbUsername
|
||||
}
|
||||
if db, ok := appConfig["db"].(map[string]interface{}); ok {
|
||||
if dbUser, ok := db["user"].(string); ok && dbUser != "" {
|
||||
return dbUser
|
||||
}
|
||||
}
|
||||
}
|
||||
return appName
|
||||
}
|
||||
|
||||
// readAppConfig reads the per-app config file and returns its contents.
|
||||
func (p *PostgreSQLStrategy) readAppConfig(instanceName, appName string) map[string]interface{} {
|
||||
configPath := tools.GetAppConfigPath(p.dataDir, instanceName, appName)
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var config map[string]interface{}
|
||||
if err := yaml.Unmarshal(data, &config); err != nil {
|
||||
return nil
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// databaseExists returns true if the named Postgres database exists
|
||||
func (p *PostgreSQLStrategy) databaseExists(kubeconfigPath, podName, dbName string) bool {
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-tAc",
|
||||
fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname='%s'", dbName))
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(string(output)) == "1"
|
||||
}
|
||||
|
||||
// terminateConnections kills active connections to a database so it can be renamed/dropped
|
||||
func (p *PostgreSQLStrategy) terminateConnections(kubeconfigPath, podName, dbName string) {
|
||||
cmd := exec.Command("kubectl", "exec", "-n", "postgres", podName, "--",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-c",
|
||||
fmt.Sprintf("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='%s' AND pid <> pg_backend_pid()", dbName))
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
_, _ = cmd.CombinedOutput() // best effort
|
||||
}
|
||||
|
||||
// getPostgresPod finds the first running postgres pod
|
||||
func (p *PostgreSQLStrategy) getPostgresPod(kubeconfigPath string) (string, error) {
|
||||
cmd := exec.Command("kubectl", "get", "pods", "-n", "postgres",
|
||||
"-l", "app=postgres", "-o", "jsonpath={.items[0].metadata.name}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
cmd = exec.Command("kubectl", "get", "pods", "-n", "postgres",
|
||||
"-o", "jsonpath={.items[0].metadata.name}")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
output, err = cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get postgres pod: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
podName := strings.TrimSpace(string(output))
|
||||
if podName == "" {
|
||||
return "", fmt.Errorf("no postgres pod found")
|
||||
}
|
||||
|
||||
return podName, nil
|
||||
}
|
||||
294
api/internal/backup/strategies/postgres_test.go
Normal file
294
api/internal/backup/strategies/postgres_test.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package strategies
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
btypes "github.com/wild-cloud/wild-central/daemon/internal/backup/types"
|
||||
)
|
||||
|
||||
// MockPostgresDestination for testing
|
||||
type MockPostgresDestination struct {
|
||||
putData map[string][]byte
|
||||
getData map[string][]byte
|
||||
putError error
|
||||
getError error
|
||||
deleteKeys []string
|
||||
}
|
||||
|
||||
func NewMockPostgresDestination() *MockPostgresDestination {
|
||||
return &MockPostgresDestination{
|
||||
putData: make(map[string][]byte),
|
||||
getData: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) Put(key string, reader io.Reader) (int64, error) {
|
||||
if m.putError != nil {
|
||||
return 0, m.putError
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.putData[key] = data
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) Get(key string) (io.ReadCloser, error) {
|
||||
if m.getError != nil {
|
||||
return nil, m.getError
|
||||
}
|
||||
data, exists := m.getData[key]
|
||||
if !exists {
|
||||
data = []byte("mock backup data")
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) Delete(key string) error {
|
||||
m.deleteKeys = append(m.deleteKeys, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) List(prefix string) ([]btypes.BackupObject, error) {
|
||||
var objects []btypes.BackupObject
|
||||
for key, data := range m.putData {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
objects = append(objects, btypes.BackupObject{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
LastModified: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) GetURL(key string, expiry time.Duration) (string, error) {
|
||||
return "https://mock.example.com/" + key, nil
|
||||
}
|
||||
|
||||
func (m *MockPostgresDestination) Type() string {
|
||||
return "mock"
|
||||
}
|
||||
|
||||
func TestPostgreSQLStrategy_Name(t *testing.T) {
|
||||
s := NewPostgreSQLStrategy("/tmp")
|
||||
assert.Equal(t, "postgres", s.Name())
|
||||
}
|
||||
|
||||
func TestPostgreSQLStrategy_Verify(t *testing.T) {
|
||||
s := NewPostgreSQLStrategy("/tmp")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
plan *btypes.RecoveryPlan
|
||||
destData map[string][]byte
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "successful verification with PGDMP header",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "test-app",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "postgres",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "test/backup.dump",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
destData: map[string][]byte{
|
||||
"test/backup.dump": []byte("PGDMP\x00\x00\x00\x00"),
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid dump format",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "test-app",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "postgres",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{
|
||||
"location": "test/invalid.dump",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
destData: map[string][]byte{
|
||||
"test/invalid.dump": []byte("not a valid dump"),
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "missing backup location in plan",
|
||||
plan: &btypes.RecoveryPlan{
|
||||
App: "test-app",
|
||||
Instance: "test-instance",
|
||||
Strategies: []btypes.StrategyEntry{
|
||||
{
|
||||
Name: "postgres",
|
||||
Status: "backed_up",
|
||||
Backup: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dest := NewMockPostgresDestination()
|
||||
dest.getData = tt.destData
|
||||
|
||||
err := s.Verify(tt.plan, dest)
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeAppConfig writes a per-app config file at config/apps/{appName}/config.yaml.
|
||||
func writeAppConfig(t *testing.T, tmpDir, instanceName, appName, config string) {
|
||||
t.Helper()
|
||||
appConfigDir := filepath.Join(tmpDir, "instances", instanceName, "config", "apps", appName)
|
||||
err := os.MkdirAll(appConfigDir, 0755)
|
||||
assert.NoError(t, err)
|
||||
err = os.WriteFile(filepath.Join(appConfigDir, "config.yaml"), []byte(config), 0644)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestPostgreSQLStrategy_GetDatabaseName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config string
|
||||
appName string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "flat dbName key",
|
||||
config: "dbName: my_database\n",
|
||||
appName: "myapp",
|
||||
expected: "my_database",
|
||||
},
|
||||
{
|
||||
name: "nested db.name key",
|
||||
config: `namespace: e2e-test-app
|
||||
db:
|
||||
host: postgres
|
||||
name: e2e_test_app
|
||||
user: e2e_test_app
|
||||
`,
|
||||
appName: "e2e-test-app",
|
||||
expected: "e2e_test_app",
|
||||
},
|
||||
{
|
||||
name: "flat key takes precedence over nested",
|
||||
config: `dbName: flat_name
|
||||
db:
|
||||
name: nested_name
|
||||
`,
|
||||
appName: "myapp",
|
||||
expected: "flat_name",
|
||||
},
|
||||
{
|
||||
name: "no config falls back to appName",
|
||||
config: "namespace: myapp\n",
|
||||
appName: "myapp",
|
||||
expected: "myapp",
|
||||
},
|
||||
{
|
||||
name: "missing app falls back to appName",
|
||||
config: "",
|
||||
appName: "missing-app",
|
||||
expected: "missing-app",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
if tt.config != "" {
|
||||
writeAppConfig(t, tmpDir, "test-instance", tt.appName, tt.config)
|
||||
}
|
||||
s := &PostgreSQLStrategy{dataDir: tmpDir}
|
||||
result := s.getDatabaseName("test-instance", tt.appName)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgreSQLStrategy_GetAppUser(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config string
|
||||
appName string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "flat dbUser key",
|
||||
config: "dbUser: my_user\n",
|
||||
appName: "myapp",
|
||||
expected: "my_user",
|
||||
},
|
||||
{
|
||||
name: "flat dbUsername key",
|
||||
config: "dbUsername: my_username\n",
|
||||
appName: "myapp",
|
||||
expected: "my_username",
|
||||
},
|
||||
{
|
||||
name: "nested db.user key",
|
||||
config: `namespace: e2e-test-app
|
||||
db:
|
||||
host: postgres
|
||||
name: e2e_test_app
|
||||
user: e2e_test_app
|
||||
`,
|
||||
appName: "e2e-test-app",
|
||||
expected: "e2e_test_app",
|
||||
},
|
||||
{
|
||||
name: "flat key takes precedence over nested",
|
||||
config: `dbUser: flat_user
|
||||
db:
|
||||
user: nested_user
|
||||
`,
|
||||
appName: "myapp",
|
||||
expected: "flat_user",
|
||||
},
|
||||
{
|
||||
name: "no user config falls back to appName",
|
||||
config: "namespace: myapp\n",
|
||||
appName: "myapp",
|
||||
expected: "myapp",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
writeAppConfig(t, tmpDir, "test-instance", tt.appName, tt.config)
|
||||
s := &PostgreSQLStrategy{dataDir: tmpDir}
|
||||
result := s.getAppUser("test-instance", tt.appName)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
276
api/internal/backup/types/types.go
Normal file
276
api/internal/backup/types/types.go
Normal file
@@ -0,0 +1,276 @@
|
||||
// Package types provides shared types for the backup system
|
||||
package types
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Strategy defines the interface for backup strategies
|
||||
type Strategy interface {
|
||||
// Name returns the strategy identifier
|
||||
Name() string
|
||||
|
||||
// Backup creates a backup using this strategy, writing results to the plan's StrategyEntry
|
||||
Backup(plan *RecoveryPlan, dest BackupDestination) error
|
||||
|
||||
// Restore restores from backup using plan-driven coordination
|
||||
Restore(plan *RecoveryPlan, dest BackupDestination) error
|
||||
|
||||
// Switch performs the traffic cutover for this strategy's resources
|
||||
Switch(plan *RecoveryPlan) error
|
||||
|
||||
// Cleanup removes the previous active resources after a successful switch
|
||||
Cleanup(plan *RecoveryPlan) error
|
||||
|
||||
// Verify checks if a backup can be restored
|
||||
Verify(plan *RecoveryPlan, dest BackupDestination) error
|
||||
}
|
||||
|
||||
// RecoveryPlan is the coordination record for backup/restore operations.
|
||||
// It is created during backup, used during restore, switch, and cleanup phases.
|
||||
type RecoveryPlan struct {
|
||||
App string `yaml:"app" json:"app"`
|
||||
Instance string `yaml:"instance" json:"instance"`
|
||||
Timestamp string `yaml:"timestamp" json:"timestamp"`
|
||||
Version string `yaml:"version" json:"version,omitempty"`
|
||||
Status string `yaml:"status" json:"status"` // backing_up, backed_up, restoring, restored, switching, switched, cleaning_up, cleaned_up, failed
|
||||
Error string `yaml:"error" json:"error,omitempty"`
|
||||
Source RecoverySource `yaml:"source" json:"source"`
|
||||
Standby RecoveryStandby `yaml:"standby" json:"standby,omitempty"`
|
||||
Strategies []StrategyEntry `yaml:"strategies" json:"strategies"`
|
||||
Phases map[string]PhaseTime `yaml:"phases" json:"phases"`
|
||||
}
|
||||
|
||||
// RecoverySource describes the active deployment being backed up
|
||||
type RecoverySource struct {
|
||||
Namespace string `yaml:"namespace" json:"namespace"`
|
||||
AppDir string `yaml:"appDir" json:"appDir"`
|
||||
ConfigPath string `yaml:"configPath,omitempty" json:"configPath,omitempty"`
|
||||
SecretsPath string `yaml:"secretsPath,omitempty" json:"secretsPath,omitempty"`
|
||||
}
|
||||
|
||||
// RecoveryStandby describes the standby deployment target for restore
|
||||
type RecoveryStandby struct {
|
||||
Namespace string `yaml:"namespace" json:"namespace"`
|
||||
AppDir string `yaml:"appDir" json:"appDir"`
|
||||
}
|
||||
|
||||
// StrategyEntry tracks the state of a single strategy through the recovery lifecycle
|
||||
type StrategyEntry struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Status string `yaml:"status" json:"status"` // pending, backing_up, backed_up, restoring, restored, switching, switched, cleaning_up, cleaned_up, failed
|
||||
Params map[string]interface{} `yaml:"params,omitempty" json:"params,omitempty"`
|
||||
Backup map[string]interface{} `yaml:"backup,omitempty" json:"backup,omitempty"`
|
||||
Restore map[string]interface{} `yaml:"restore,omitempty" json:"restore,omitempty"`
|
||||
Switch map[string]interface{} `yaml:"switch,omitempty" json:"switch,omitempty"`
|
||||
Error string `yaml:"error,omitempty" json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// PhaseTime records timing for a recovery phase
|
||||
type PhaseTime struct {
|
||||
StartedAt *time.Time `yaml:"startedAt,omitempty" json:"startedAt,omitempty"`
|
||||
CompletedAt *time.Time `yaml:"completedAt,omitempty" json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// GetStrategyEntry returns the StrategyEntry for the named strategy, or nil
|
||||
func (p *RecoveryPlan) GetStrategyEntry(name string) *StrategyEntry {
|
||||
for i := range p.Strategies {
|
||||
if p.Strategies[i].Name == name {
|
||||
return &p.Strategies[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackupDestination defines the interface for backup storage destinations
|
||||
type BackupDestination interface {
|
||||
// Put uploads data to the destination, returns size written
|
||||
Put(key string, reader io.Reader) (int64, error)
|
||||
|
||||
// Get retrieves data from the destination
|
||||
Get(key string) (io.ReadCloser, error)
|
||||
|
||||
// Delete removes data from the destination
|
||||
Delete(key string) error
|
||||
|
||||
// List returns objects with the given prefix
|
||||
List(prefix string) ([]BackupObject, error)
|
||||
|
||||
// GetURL returns a pre-signed URL for direct access (optional)
|
||||
GetURL(key string, expiry time.Duration) (string, error)
|
||||
|
||||
// Type returns the destination type identifier
|
||||
Type() string
|
||||
}
|
||||
|
||||
// BackupObject represents an object in backup storage
|
||||
type BackupObject struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
LastModified time.Time `json:"lastModified"`
|
||||
}
|
||||
|
||||
// BackupInfo represents metadata about a backup (legacy, kept for compatibility)
|
||||
type BackupInfo struct {
|
||||
AppName string `json:"app_name"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Type string `json:"type"` // "full"
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Status string `json:"status"` // "completed", "failed", "in_progress"
|
||||
Error string `json:"error,omitempty"`
|
||||
Components []ComponentBackup `json:"components"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Verified bool `json:"verified"`
|
||||
VerifiedAt *time.Time `json:"verified_at,omitempty"`
|
||||
}
|
||||
|
||||
// ComponentBackup represents a single backup component (legacy, kept for compatibility)
|
||||
type ComponentBackup struct {
|
||||
Type string `json:"type"` // "postgres", "mysql", "pvc", "config"
|
||||
Name string `json:"name"` // Component identifier
|
||||
Size int64 `json:"size"`
|
||||
Location string `json:"location"` // Path in destination
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
// RestoreOptions configures restore behavior
|
||||
type RestoreOptions struct {
|
||||
Components []string `json:"components,omitempty"` // Specific components to restore
|
||||
SkipData bool `json:"skip_data"` // Skip data, restore only config
|
||||
AutoComplete bool `json:"auto_complete,omitempty"` // Chain restore→switch→cleanup into one operation
|
||||
}
|
||||
|
||||
// VerificationResult represents the result of backup verification
|
||||
type VerificationResult struct {
|
||||
Success bool `json:"success"`
|
||||
Duration float64 `json:"duration"` // seconds
|
||||
TestedAt time.Time `json:"testedAt"`
|
||||
Components []ComponentVerification `json:"components"`
|
||||
}
|
||||
|
||||
// ComponentVerification represents verification result for a single component
|
||||
type ComponentVerification struct {
|
||||
Type string `json:"type"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ProgressCallback is a function type for reporting backup/restore progress
|
||||
type ProgressCallback func(progress int, message string)
|
||||
|
||||
// BackupConfiguration represents instance-level backup configuration
|
||||
type BackupConfiguration struct {
|
||||
Destination DestinationConfig `yaml:"destination" json:"destination"`
|
||||
Retention RetentionPolicy `yaml:"retention" json:"retention"`
|
||||
Schedules []BackupSchedule `yaml:"schedules,omitempty" json:"schedules,omitempty"`
|
||||
Verification VerificationConfig `yaml:"verification" json:"verification"`
|
||||
}
|
||||
|
||||
// BackupSchedule defines a per-app or cluster backup schedule
|
||||
type BackupSchedule struct {
|
||||
ID string `yaml:"id" json:"id"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
TargetType string `yaml:"targetType" json:"target_type"` // "app" or "cluster"
|
||||
TargetName string `yaml:"targetName" json:"target_name"`
|
||||
Frequency string `yaml:"frequency" json:"frequency"` // "daily", "weekly", "monthly"
|
||||
Time string `yaml:"time" json:"time"` // "HH:MM" local time
|
||||
DayOfWeek int `yaml:"dayOfWeek" json:"day_of_week,omitempty"` // 0=Sun..6=Sat (weekly)
|
||||
DayOfMonth int `yaml:"dayOfMonth" json:"day_of_month,omitempty"` // 1-28 (monthly)
|
||||
Retention *ScheduleRetention `yaml:"retention,omitempty" json:"retention,omitempty"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
LastRun *time.Time `yaml:"lastRun,omitempty" json:"last_run,omitempty"`
|
||||
NextRun *time.Time `yaml:"nextRun,omitempty" json:"next_run,omitempty"`
|
||||
CreatedAt time.Time `yaml:"createdAt" json:"created_at"`
|
||||
UpdatedAt time.Time `yaml:"updatedAt" json:"updated_at"`
|
||||
}
|
||||
|
||||
// ScheduleRetention overrides the instance-level retention for a specific schedule
|
||||
type ScheduleRetention struct {
|
||||
KeepLast int `yaml:"keepLast" json:"keep_last"`
|
||||
KeepDays int `yaml:"keepDays" json:"keep_days"`
|
||||
}
|
||||
|
||||
// DestinationConfig configures where backups are stored
|
||||
type DestinationConfig struct {
|
||||
Type string `yaml:"type" json:"type"` // "s3", "azure", "nfs", "local"
|
||||
S3 *S3Config `yaml:"s3,omitempty" json:"s3,omitempty"`
|
||||
Azure *AzureConfig `yaml:"azure,omitempty" json:"azure,omitempty"`
|
||||
NFS *NFSConfig `yaml:"nfs,omitempty" json:"nfs,omitempty"`
|
||||
Local *LocalConfig `yaml:"local,omitempty" json:"local,omitempty"`
|
||||
}
|
||||
|
||||
// S3Config configures S3 backup destination
|
||||
type S3Config struct {
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Region string `yaml:"region" json:"region"`
|
||||
Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` // For S3-compatible services
|
||||
AccessKeyID string `yaml:"-" json:"-"` // Loaded from secrets.yaml
|
||||
SecretAccessKey string `yaml:"-" json:"-"` // Loaded from secrets.yaml
|
||||
}
|
||||
|
||||
// AzureConfig configures Azure Blob Storage destination
|
||||
type AzureConfig struct {
|
||||
Container string `yaml:"container" json:"container"`
|
||||
StorageAccount string `yaml:"storageAccount" json:"storageAccount"`
|
||||
AccessKey string `yaml:"-" json:"-"` // Loaded from secrets.yaml
|
||||
}
|
||||
|
||||
// NFSConfig configures NFS backup destination
|
||||
type NFSConfig struct {
|
||||
Server string `yaml:"server" json:"server"`
|
||||
Path string `yaml:"path" json:"path"`
|
||||
MountPoint string `yaml:"mountPoint,omitempty" json:"mountPoint,omitempty"`
|
||||
MountOptions string `yaml:"mountOptions,omitempty" json:"mountOptions,omitempty"`
|
||||
}
|
||||
|
||||
// LocalConfig configures local filesystem backup destination
|
||||
type LocalConfig struct {
|
||||
Path string `yaml:"path" json:"path"`
|
||||
}
|
||||
|
||||
// RetentionPolicy defines how long to keep backups
|
||||
type RetentionPolicy struct {
|
||||
Daily int `yaml:"daily" json:"daily"`
|
||||
Weekly int `yaml:"weekly" json:"weekly"`
|
||||
Monthly int `yaml:"monthly" json:"monthly"`
|
||||
Yearly int `yaml:"yearly" json:"yearly"`
|
||||
}
|
||||
|
||||
// VerificationConfig configures backup verification
|
||||
type VerificationConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Schedule string `yaml:"schedule" json:"schedule"` // Cron expression
|
||||
RandomSample bool `yaml:"randomSample" json:"randomSample"` // Test random backup each time
|
||||
}
|
||||
|
||||
// AppScheduleConfig is the operator-managed portion of a per-app backup schedule.
|
||||
// Stored in config/apps/{app}/schedule.yaml — one key per tier, no arrays, no IDs.
|
||||
type AppScheduleConfig struct {
|
||||
Daily *ScheduleTierConfig `yaml:"daily,omitempty" json:"daily,omitempty"`
|
||||
Weekly *ScheduleTierConfig `yaml:"weekly,omitempty" json:"weekly,omitempty"`
|
||||
Monthly *ScheduleTierConfig `yaml:"monthly,omitempty" json:"monthly,omitempty"`
|
||||
}
|
||||
|
||||
// ScheduleTierConfig holds the operator-set parameters for one backup tier.
|
||||
type ScheduleTierConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Time string `yaml:"time" json:"time"`
|
||||
DayOfWeek int `yaml:"dayOfWeek,omitempty" json:"day_of_week,omitempty"`
|
||||
DayOfMonth int `yaml:"dayOfMonth,omitempty" json:"day_of_month,omitempty"`
|
||||
Retention *ScheduleRetention `yaml:"retention,omitempty" json:"retention,omitempty"`
|
||||
}
|
||||
|
||||
// AppScheduleState is the daemon-written runtime state for a per-app schedule.
|
||||
// Stored in cache/schedules/{app}.yaml — regeneratable from backup/records/.
|
||||
type AppScheduleState struct {
|
||||
Daily *TierState `yaml:"daily,omitempty" json:"daily,omitempty"`
|
||||
Weekly *TierState `yaml:"weekly,omitempty" json:"weekly,omitempty"`
|
||||
Monthly *TierState `yaml:"monthly,omitempty" json:"monthly,omitempty"`
|
||||
}
|
||||
|
||||
// TierState holds the daemon-tracked last/next run times for one backup tier.
|
||||
type TierState struct {
|
||||
LastRun *time.Time `yaml:"lastRun,omitempty" json:"last_run,omitempty"`
|
||||
NextRun *time.Time `yaml:"nextRun,omitempty" json:"next_run,omitempty"`
|
||||
}
|
||||
@@ -4,18 +4,64 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/operations"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
|
||||
// nodeConfig is the subset of config/nodes/{hostname}.yaml fields needed by the cluster package.
|
||||
type nodeConfig struct {
|
||||
Role string `yaml:"role"`
|
||||
TargetIP string `yaml:"targetIp"`
|
||||
}
|
||||
|
||||
// readNodeConfig reads config/nodes/{hostname}.yaml and returns the parsed config.
|
||||
func readNodeConfig(dataDir, instanceName, hostname string) (*nodeConfig, error) {
|
||||
path := tools.GetNodeConfigPath(dataDir, instanceName, hostname)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read node config %s: %w", hostname, err)
|
||||
}
|
||||
var nc nodeConfig
|
||||
if err := yaml.Unmarshal(data, &nc); err != nil {
|
||||
return nil, fmt.Errorf("parse node config %s: %w", hostname, err)
|
||||
}
|
||||
return &nc, nil
|
||||
}
|
||||
|
||||
// controlPlaneIPs returns the targetIp of every controlplane node in config/nodes/.
|
||||
func controlPlaneIPs(dataDir, instanceName string) ([]string, error) {
|
||||
nodeDir := tools.GetNodeConfigDirPath(dataDir, instanceName)
|
||||
entries, err := os.ReadDir(nodeDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ips []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".yaml" {
|
||||
continue
|
||||
}
|
||||
hostname := strings.TrimSuffix(entry.Name(), ".yaml")
|
||||
nc, err := readNodeConfig(dataDir, instanceName, hostname)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if nc.Role == "controlplane" && nc.TargetIP != "" {
|
||||
ips = append(ips, nc.TargetIP)
|
||||
}
|
||||
}
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
// Manager handles cluster lifecycle operations
|
||||
type Manager struct {
|
||||
dataDir string
|
||||
@@ -43,7 +89,7 @@ type ClusterConfig struct {
|
||||
type NodeStatus struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Ready bool `json:"ready"`
|
||||
KubernetesReady bool `json:"kubernetes_ready"`
|
||||
KubernetesReady bool `json:"kubernetesReady"`
|
||||
Role string `json:"role"` // "control-plane" or "worker"
|
||||
}
|
||||
|
||||
@@ -51,56 +97,45 @@ type NodeStatus struct {
|
||||
type ClusterStatus struct {
|
||||
Status string `json:"status"` // ready, pending, error
|
||||
Nodes int `json:"nodes"`
|
||||
ControlPlaneNodes int `json:"control_plane_nodes"`
|
||||
WorkerNodes int `json:"worker_nodes"`
|
||||
KubernetesVersion string `json:"kubernetes_version"`
|
||||
TalosVersion string `json:"talos_version"`
|
||||
ControlPlaneNodes int `json:"controlPlaneNodes"`
|
||||
WorkerNodes int `json:"workerNodes"`
|
||||
KubernetesVersion string `json:"kubernetesVersion"`
|
||||
TalosVersion string `json:"talosVersion"`
|
||||
Services map[string]string `json:"services"`
|
||||
NodeStatuses map[string]NodeStatus `json:"node_statuses,omitempty"`
|
||||
}
|
||||
|
||||
// GetTalosDir returns the talos directory for an instance
|
||||
func (m *Manager) GetTalosDir(instanceName string) string {
|
||||
return tools.GetInstanceTalosPath(m.dataDir, instanceName)
|
||||
}
|
||||
|
||||
// GetGeneratedDir returns the generated config directory
|
||||
func (m *Manager) GetGeneratedDir(instanceName string) string {
|
||||
return filepath.Join(m.GetTalosDir(instanceName), "generated")
|
||||
NodeStatuses map[string]NodeStatus `json:"nodeStatuses,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateConfig generates initial cluster configuration using talosctl gen config
|
||||
func (m *Manager) GenerateConfig(instanceName string, config *ClusterConfig) error {
|
||||
generatedDir := m.GetGeneratedDir(instanceName)
|
||||
talosDir := tools.GetTalosDirPath(m.dataDir, instanceName)
|
||||
pkiPath := tools.GetTalosPkiPath(m.dataDir, instanceName)
|
||||
|
||||
// Check if already generated (idempotency)
|
||||
secretsFile := filepath.Join(generatedDir, "secrets.yaml")
|
||||
if storage.FileExists(secretsFile) {
|
||||
// Already generated
|
||||
if storage.FileExists(pkiPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure generated directory exists
|
||||
if err := storage.EnsureDir(generatedDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create generated directory: %w", err)
|
||||
slog.Info("generating cluster config", "component", "cluster", "instance", instanceName, "cluster", config.ClusterName, "vip", config.VIP)
|
||||
|
||||
if err := storage.EnsureDir(talosDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create talos directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate secrets
|
||||
cmd := exec.Command("talosctl", "gen", "secrets")
|
||||
cmd.Dir = generatedDir
|
||||
// Generate secrets → data/talos/pki.yaml
|
||||
cmd := exec.Command("talosctl", "gen", "secrets", "--output-file", pkiPath)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate secrets: %w\nOutput: %s", err, string(output))
|
||||
}
|
||||
|
||||
// Generate config with secrets
|
||||
// Generate config → data/talos/{talosconfig,controlplane.yaml,worker.yaml}
|
||||
endpoint := fmt.Sprintf("https://%s:6443", config.VIP)
|
||||
cmd = exec.Command("talosctl", "gen", "config",
|
||||
"--with-secrets", "secrets.yaml",
|
||||
"--with-secrets", pkiPath,
|
||||
"--output-dir", talosDir,
|
||||
config.ClusterName,
|
||||
endpoint,
|
||||
)
|
||||
cmd.Dir = generatedDir
|
||||
output, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate config: %w\nOutput: %s", err, string(output))
|
||||
@@ -117,9 +152,12 @@ func (m *Manager) Bootstrap(instanceName, nodeName string) (string, error) {
|
||||
return "", fmt.Errorf("failed to start bootstrap operation: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("starting cluster bootstrap", "component", "cluster", "instance", instanceName, "node", nodeName, "operationId", opID)
|
||||
|
||||
// Run bootstrap asynchronously
|
||||
go func() {
|
||||
if err := m.runBootstrapWithTracking(instanceName, nodeName, opID); err != nil {
|
||||
slog.Error("cluster bootstrap failed", "component", "cluster", "instance", instanceName, "node", nodeName, "error", err)
|
||||
_ = m.opsMgr.Update(instanceName, opID, "failed", err.Error(), 0)
|
||||
}
|
||||
}()
|
||||
@@ -130,26 +168,24 @@ func (m *Manager) Bootstrap(instanceName, nodeName string) (string, error) {
|
||||
// runBootstrapWithTracking runs the bootstrap process with detailed progress tracking
|
||||
func (m *Manager) runBootstrapWithTracking(instanceName, nodeName, opID string) error {
|
||||
ctx := context.Background()
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
yq := tools.NewYQ()
|
||||
|
||||
// Get node's target IP
|
||||
nodeIPRaw, err := yq.Get(configPath, fmt.Sprintf(".cluster.nodes.active.%s.targetIp", nodeName))
|
||||
// Get node's target IP from config/nodes/{nodeName}.yaml
|
||||
nc, err := readNodeConfig(m.dataDir, instanceName, nodeName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get node IP: %w", err)
|
||||
return fmt.Errorf("failed to get node config: %w", err)
|
||||
}
|
||||
|
||||
nodeIP := tools.CleanYQOutput(nodeIPRaw)
|
||||
if nodeIP == "" || nodeIP == "null" {
|
||||
if nc.TargetIP == "" {
|
||||
return fmt.Errorf("node %s does not have a target IP configured", nodeName)
|
||||
}
|
||||
nodeIP := nc.TargetIP
|
||||
|
||||
// Get VIP
|
||||
// Get VIP from config/instance.yaml
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
yq := tools.NewYQ()
|
||||
vipRaw, err := yq.Get(configPath, ".cluster.nodes.control.vip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get VIP: %w", err)
|
||||
}
|
||||
|
||||
vip := tools.CleanYQOutput(vipRaw)
|
||||
if vip == "" || vip == "null" {
|
||||
return fmt.Errorf("control plane VIP not configured")
|
||||
@@ -191,6 +227,7 @@ func (m *Manager) runBootstrapWithTracking(instanceName, nodeName, opID string)
|
||||
}
|
||||
|
||||
// Mark as completed
|
||||
slog.Info("cluster bootstrap completed", "component", "cluster", "instance", instanceName)
|
||||
_ = m.opsMgr.Update(instanceName, opID, "completed", "Bootstrap completed successfully", 100)
|
||||
return nil
|
||||
}
|
||||
@@ -385,7 +422,7 @@ func (m *Manager) retrieveKubeconfigFromCluster(instanceName, nodeIP string, tim
|
||||
tools.WithTalosconfig(cmdKubeconfig, talosconfigPath)
|
||||
|
||||
if output, err := cmdKubeconfig.CombinedOutput(); err == nil {
|
||||
log.Printf("Successfully retrieved kubeconfig for instance %s", instanceName)
|
||||
slog.Info("kubeconfig retrieved", "component", "cluster", "instance", instanceName)
|
||||
return nil
|
||||
} else {
|
||||
// Check if we've exceeded deadline
|
||||
@@ -424,13 +461,72 @@ func (m *Manager) RegenerateKubeconfig(instanceName string) error {
|
||||
return fmt.Errorf("control plane VIP not configured in cluster.nodes.control.vip")
|
||||
}
|
||||
|
||||
log.Printf("Regenerating kubeconfig for instance %s from cluster VIP %s", instanceName, vip)
|
||||
slog.Info("regenerating kubeconfig", "component", "cluster", "instance", instanceName, "vip", vip)
|
||||
// Use shorter timeout for manual regeneration (cluster should already be running)
|
||||
return m.retrieveKubeconfigFromCluster(instanceName, vip, 30*time.Second)
|
||||
}
|
||||
|
||||
// RenewTalosconfig regenerates the talosconfig client certificate from the existing PKI root.
|
||||
// Use this when the talosconfig cert has expired (1-year validity).
|
||||
func (m *Manager) RenewTalosconfig(instanceName string) error {
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
yq := tools.NewYQ()
|
||||
|
||||
vipRaw, err := yq.Get(configPath, ".cluster.nodes.control.vip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get VIP: %w", err)
|
||||
}
|
||||
vip := tools.CleanYQOutput(vipRaw)
|
||||
if vip == "" || vip == "null" {
|
||||
return fmt.Errorf("control plane VIP not configured")
|
||||
}
|
||||
|
||||
clusterNameRaw, err := yq.Get(configPath, ".cluster.name")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get cluster name: %w", err)
|
||||
}
|
||||
clusterName := tools.CleanYQOutput(clusterNameRaw)
|
||||
if clusterName == "" || clusterName == "null" {
|
||||
clusterName = instanceName
|
||||
}
|
||||
|
||||
pkiPath := tools.GetTalosPkiPath(m.dataDir, instanceName)
|
||||
if !storage.FileExists(pkiPath) {
|
||||
return fmt.Errorf("PKI file not found at %s — cluster has not been initialised", pkiPath)
|
||||
}
|
||||
|
||||
talosconfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
endpoint := fmt.Sprintf("https://%s:6443", vip)
|
||||
|
||||
slog.Info("renewing talosconfig", "component", "cluster", "instance", instanceName, "vip", vip)
|
||||
|
||||
cmd := exec.Command("talosctl", "gen", "config",
|
||||
"--with-secrets", pkiPath,
|
||||
"--output-types", "talosconfig",
|
||||
"--output", talosconfigPath,
|
||||
"--force",
|
||||
clusterName,
|
||||
endpoint,
|
||||
)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("talosctl gen config failed: %w\nOutput: %s", err, string(output))
|
||||
}
|
||||
|
||||
// Populate endpoint (gen config leaves it empty)
|
||||
cmdEndpoint := exec.Command("talosctl", "config", "endpoint", vip)
|
||||
tools.WithTalosconfig(cmdEndpoint, talosconfigPath)
|
||||
if output, err := cmdEndpoint.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to set endpoint: %w\nOutput: %s", err, string(output))
|
||||
}
|
||||
|
||||
slog.Info("talosconfig renewed", "component", "cluster", "instance", instanceName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigureEndpoints updates talosconfig to use VIP and retrieves kubeconfig
|
||||
func (m *Manager) ConfigureEndpoints(instanceName string, includeNodes bool) error {
|
||||
slog.Info("configuring cluster endpoints", "component", "cluster", "instance", instanceName, "includeNodes", includeNodes)
|
||||
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, instanceName)
|
||||
talosconfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
|
||||
@@ -452,14 +548,10 @@ func (m *Manager) ConfigureEndpoints(instanceName string, includeNodes bool) err
|
||||
|
||||
// Add control node IPs if requested
|
||||
if includeNodes {
|
||||
nodesRaw, err := yq.Exec("eval", ".cluster.nodes.active | to_entries | .[] | select(.value.role == \"controlplane\") | .value.targetIp", configPath)
|
||||
if err == nil {
|
||||
nodeIPs := strings.Split(strings.TrimSpace(string(nodesRaw)), "\n")
|
||||
for _, ip := range nodeIPs {
|
||||
ip = tools.CleanYQOutput(ip)
|
||||
if ip != "" && ip != "null" && ip != vip {
|
||||
endpoints = append(endpoints, ip)
|
||||
}
|
||||
nodeIPs, _ := controlPlaneIPs(m.dataDir, instanceName)
|
||||
for _, ip := range nodeIPs {
|
||||
if ip != vip {
|
||||
endpoints = append(endpoints, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -650,7 +742,7 @@ func (m *Manager) GetKubeconfig(instanceName string) (string, error) {
|
||||
|
||||
// GetTalosconfig returns the talosconfig for the cluster
|
||||
func (m *Manager) GetTalosconfig(instanceName string) (string, error) {
|
||||
talosconfigPath := filepath.Join(m.GetGeneratedDir(instanceName), "talosconfig")
|
||||
talosconfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
|
||||
if !storage.FileExists(talosconfigPath) {
|
||||
return "", fmt.Errorf("talosconfig not found - cluster may not be initialized")
|
||||
@@ -709,15 +801,17 @@ func (m *Manager) Reset(instanceName string, confirm bool) error {
|
||||
return fmt.Errorf("reset requires confirmation")
|
||||
}
|
||||
|
||||
slog.Info("resetting cluster", "component", "cluster", "instance", instanceName)
|
||||
|
||||
// This is a destructive operation
|
||||
// Real implementation would:
|
||||
// 1. Reset all nodes via talosctl reset
|
||||
// 2. Remove generated configs
|
||||
// 3. Clear node status in config.yaml
|
||||
|
||||
generatedDir := m.GetGeneratedDir(instanceName)
|
||||
if storage.FileExists(generatedDir) {
|
||||
if err := os.RemoveAll(generatedDir); err != nil {
|
||||
talosDir := tools.GetTalosDirPath(m.dataDir, instanceName)
|
||||
if storage.FileExists(talosDir) {
|
||||
if err := os.RemoveAll(talosDir); err != nil {
|
||||
return fmt.Errorf("failed to remove generated configs: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -725,38 +819,3 @@ func (m *Manager) Reset(instanceName string, confirm bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigureContext configures talosctl context for the cluster
|
||||
func (m *Manager) ConfigureContext(instanceName, clusterName string) error {
|
||||
talosconfigPath := filepath.Join(m.GetGeneratedDir(instanceName), "talosconfig")
|
||||
|
||||
if !storage.FileExists(talosconfigPath) {
|
||||
return fmt.Errorf("talosconfig not found")
|
||||
}
|
||||
|
||||
// Merge talosconfig into user's talosctl config
|
||||
cmd := exec.Command("talosctl", "config", "merge", talosconfigPath)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to merge talosconfig: %w\nOutput: %s", err, string(output))
|
||||
}
|
||||
|
||||
// Set context
|
||||
cmd = exec.Command("talosctl", "config", "context", clusterName)
|
||||
output, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set context: %w\nOutput: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasContext checks if talosctl context exists
|
||||
func (m *Manager) HasContext(clusterName string) (bool, error) {
|
||||
cmd := exec.Command("talosctl", "config", "contexts")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to list contexts: %w", err)
|
||||
}
|
||||
|
||||
return strings.Contains(string(output), clusterName), nil
|
||||
}
|
||||
|
||||
@@ -2,47 +2,25 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// GlobalConfig represents the main configuration structure
|
||||
type GlobalConfig struct {
|
||||
Wildcloud struct {
|
||||
Repository string `yaml:"repository,omitempty" json:"repository,omitempty"`
|
||||
CurrentPhase string `yaml:"currentPhase,omitempty" json:"currentPhase,omitempty"`
|
||||
CompletedPhases []string `yaml:"completedPhases,omitempty" json:"completedPhases,omitempty"`
|
||||
} `yaml:"wildcloud,omitempty" json:"wildcloud,omitempty"`
|
||||
Server struct {
|
||||
Port int `yaml:"port,omitempty" json:"port,omitempty"`
|
||||
Host string `yaml:"host,omitempty" json:"host,omitempty"`
|
||||
} `yaml:"server,omitempty" json:"server,omitempty"`
|
||||
Operator struct {
|
||||
Email string `yaml:"email,omitempty" json:"email,omitempty"`
|
||||
} `yaml:"operator,omitempty" json:"operator,omitempty"`
|
||||
Cloud struct {
|
||||
DNS struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
ExternalResolver string `yaml:"externalResolver,omitempty" json:"externalResolver,omitempty"`
|
||||
} `yaml:"dns,omitempty" json:"dns,omitempty"`
|
||||
Router struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
DynamicDns string `yaml:"dynamicDns,omitempty" json:"dynamicDns,omitempty"`
|
||||
} `yaml:"router,omitempty" json:"router,omitempty"`
|
||||
Dnsmasq struct {
|
||||
Interface string `yaml:"interface,omitempty" json:"interface,omitempty"`
|
||||
} `yaml:"dnsmasq,omitempty" json:"dnsmasq,omitempty"`
|
||||
} `yaml:"cloud,omitempty" json:"cloud,omitempty"`
|
||||
Cluster struct {
|
||||
EndpointIP string `yaml:"endpointIp,omitempty" json:"endpointIp,omitempty"`
|
||||
Nodes struct {
|
||||
Talos struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
} `yaml:"talos,omitempty" json:"talos,omitempty"`
|
||||
} `yaml:"nodes,omitempty" json:"nodes,omitempty"`
|
||||
} `yaml:"cluster,omitempty" json:"cluster,omitempty"`
|
||||
}
|
||||
|
||||
// LoadGlobalConfig loads configuration from the specified path
|
||||
@@ -57,14 +35,6 @@ func LoadGlobalConfig(configPath string) (*GlobalConfig, error) {
|
||||
return nil, fmt.Errorf("parsing config file: %w", err)
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
if config.Server.Port == 0 {
|
||||
config.Server.Port = 5055
|
||||
}
|
||||
if config.Server.Host == "" {
|
||||
config.Server.Host = "0.0.0.0"
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
@@ -89,8 +59,8 @@ func (c *GlobalConfig) IsEmpty() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if any essential fields are empty
|
||||
return c.Cloud.DNS.IP == "" || c.Cluster.Nodes.Talos.Version == ""
|
||||
// Check if essential fields are empty
|
||||
return c.Cloud.Router.IP == "" && c.Operator.Email == ""
|
||||
}
|
||||
|
||||
type NodeConfig struct {
|
||||
@@ -109,31 +79,12 @@ type InstanceConfig struct {
|
||||
Domain string `yaml:"domain" json:"domain"`
|
||||
InternalDomain string `yaml:"internalDomain" json:"internalDomain"`
|
||||
DHCPRange string `yaml:"dhcpRange" json:"dhcpRange"`
|
||||
DNS struct {
|
||||
IP string `yaml:"ip" json:"ip"`
|
||||
ExternalResolver string `yaml:"externalResolver" json:"externalResolver"`
|
||||
} `yaml:"dns" json:"dns"`
|
||||
Router struct {
|
||||
IP string `yaml:"ip" json:"ip"`
|
||||
DynamicDns string `yaml:"dynamicDns,omitempty" json:"dynamicDns,omitempty"`
|
||||
} `yaml:"router" json:"router"`
|
||||
Dnsmasq struct {
|
||||
Interface string `yaml:"interface" json:"interface"`
|
||||
} `yaml:"dnsmasq" json:"dnsmasq"`
|
||||
NFS struct {
|
||||
NFS struct {
|
||||
Host string `yaml:"host" json:"host"`
|
||||
MediaPath string `yaml:"mediaPath" json:"mediaPath"`
|
||||
StorageCapacity string `yaml:"storageCapacity" json:"storageCapacity"`
|
||||
} `yaml:"nfs" json:"nfs"`
|
||||
DockerRegistryHost string `yaml:"dockerRegistryHost" json:"dockerRegistryHost"`
|
||||
SMTP struct {
|
||||
Host string `yaml:"host" json:"host"`
|
||||
Port string `yaml:"port" json:"port"`
|
||||
User string `yaml:"user" json:"user"`
|
||||
From string `yaml:"from" json:"from"`
|
||||
TLS string `yaml:"tls" json:"tls"`
|
||||
StartTLS string `yaml:"startTls" json:"startTls"`
|
||||
} `yaml:"smtp" json:"smtp"`
|
||||
} `yaml:"cloud" json:"cloud"`
|
||||
Cluster struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
@@ -148,6 +99,9 @@ type InstanceConfig struct {
|
||||
ExternalDns struct {
|
||||
OwnerId string `yaml:"ownerId" json:"ownerId"`
|
||||
} `yaml:"externalDns" json:"externalDns"`
|
||||
InternalDns struct {
|
||||
ExternalResolver string `yaml:"externalResolver" json:"externalResolver"`
|
||||
} `yaml:"internalDns" json:"internalDns"`
|
||||
DockerRegistry struct {
|
||||
Storage string `yaml:"storage" json:"storage"`
|
||||
} `yaml:"dockerRegistry" json:"dockerRegistry"`
|
||||
@@ -192,3 +146,85 @@ func SaveCloudConfig(config *InstanceConfig, configPath string) error {
|
||||
|
||||
return os.WriteFile(configPath, data, 0644)
|
||||
}
|
||||
|
||||
// DeepMerge recursively merges src into dst, with src values taking precedence.
|
||||
// Nested maps are merged recursively; all other types are overwritten by src.
|
||||
func DeepMerge(dst, src map[string]interface{}) map[string]interface{} {
|
||||
result := make(map[string]interface{})
|
||||
for k, v := range dst {
|
||||
result[k] = v
|
||||
}
|
||||
for k, v := range src {
|
||||
if srcMap, ok := v.(map[string]interface{}); ok {
|
||||
if dstMap, ok := result[k].(map[string]interface{}); ok {
|
||||
result[k] = DeepMerge(dstMap, srcMap)
|
||||
continue
|
||||
}
|
||||
}
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// LoadMergedInstanceConfig loads the global config as a base and merges the
|
||||
// instance config on top. Returns the merged result as an untyped map suitable
|
||||
// for passing to gomplate as template context.
|
||||
// If the global config is missing, returns the instance config alone.
|
||||
// Assembles apps.* from config/apps/*/config.yaml files.
|
||||
func LoadMergedInstanceConfig(dataDir, instanceName string) (map[string]interface{}, error) {
|
||||
instanceConfigPath := tools.GetInstanceYamlPath(dataDir, instanceName)
|
||||
globalConfigPath := filepath.Join(dataDir, "config.yaml")
|
||||
|
||||
instanceData, err := os.ReadFile(instanceConfigPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading instance config %s: %w", instanceConfigPath, err)
|
||||
}
|
||||
|
||||
var instanceMap map[string]interface{}
|
||||
if err := yaml.Unmarshal(instanceData, &instanceMap); err != nil {
|
||||
return nil, fmt.Errorf("parsing instance config: %w", err)
|
||||
}
|
||||
if instanceMap == nil {
|
||||
instanceMap = make(map[string]interface{})
|
||||
}
|
||||
|
||||
// Assemble apps.* map from per-app config files
|
||||
appsConfigDir := filepath.Join(dataDir, "instances", instanceName, "config", "apps")
|
||||
if entries, err := os.ReadDir(appsConfigDir); err == nil {
|
||||
appsMap := make(map[string]interface{})
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
appName := entry.Name()
|
||||
appConfigPath := tools.GetAppConfigPath(dataDir, instanceName, appName)
|
||||
appData, err := os.ReadFile(appConfigPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var appConfig map[string]interface{}
|
||||
if err := yaml.Unmarshal(appData, &appConfig); err == nil && appConfig != nil {
|
||||
appsMap[appName] = appConfig
|
||||
}
|
||||
}
|
||||
if len(appsMap) > 0 {
|
||||
instanceMap["apps"] = appsMap
|
||||
}
|
||||
}
|
||||
|
||||
globalData, err := os.ReadFile(globalConfigPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
slog.Warn("no global config found, using instance config only", "path", globalConfigPath)
|
||||
return instanceMap, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reading global config %s: %w", globalConfigPath, err)
|
||||
}
|
||||
|
||||
var globalMap map[string]interface{}
|
||||
if err := yaml.Unmarshal(globalData, &globalMap); err != nil {
|
||||
return nil, fmt.Errorf("parsing global config: %w", err)
|
||||
}
|
||||
|
||||
return DeepMerge(globalMap, instanceMap), nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Test: LoadGlobalConfig loads valid configuration
|
||||
@@ -17,87 +19,42 @@ func TestLoadGlobalConfig(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "loads complete configuration",
|
||||
configYAML: `wildcloud:
|
||||
repository: "https://github.com/example/repo"
|
||||
currentPhase: "setup"
|
||||
completedPhases:
|
||||
- "phase1"
|
||||
- "phase2"
|
||||
server:
|
||||
port: 8080
|
||||
host: "localhost"
|
||||
operator:
|
||||
configYAML: `operator:
|
||||
email: "admin@example.com"
|
||||
cloud:
|
||||
dns:
|
||||
ip: "192.168.1.1"
|
||||
externalResolver: "8.8.8.8"
|
||||
router:
|
||||
ip: "192.168.1.254"
|
||||
dynamicDns: "example.dyndns.org"
|
||||
dnsmasq:
|
||||
interface: "eth0"
|
||||
cluster:
|
||||
endpointIp: "192.168.1.100"
|
||||
nodes:
|
||||
talos:
|
||||
version: "v1.8.0"
|
||||
`,
|
||||
verify: func(t *testing.T, config *GlobalConfig) {
|
||||
if config.Wildcloud.Repository != "https://github.com/example/repo" {
|
||||
t.Error("repository not loaded correctly")
|
||||
if config.Operator.Email != "admin@example.com" {
|
||||
t.Error("operator email not loaded correctly")
|
||||
}
|
||||
if config.Server.Port != 8080 {
|
||||
t.Error("port not loaded correctly")
|
||||
}
|
||||
if config.Cloud.DNS.IP != "192.168.1.1" {
|
||||
t.Error("DNS IP not loaded correctly")
|
||||
}
|
||||
if config.Cluster.EndpointIP != "192.168.1.100" {
|
||||
t.Error("endpoint IP not loaded correctly")
|
||||
if config.Cloud.Router.IP != "192.168.1.254" {
|
||||
t.Error("router IP not loaded correctly")
|
||||
}
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "applies default values",
|
||||
configYAML: `cloud:
|
||||
dns:
|
||||
ip: "192.168.1.1"
|
||||
cluster:
|
||||
nodes:
|
||||
talos:
|
||||
version: "v1.8.0"
|
||||
name: "loads minimal configuration",
|
||||
configYAML: `operator:
|
||||
email: "admin@example.com"
|
||||
`,
|
||||
verify: func(t *testing.T, config *GlobalConfig) {
|
||||
if config.Server.Port != 5055 {
|
||||
t.Errorf("default port not applied, got %d, want 5055", config.Server.Port)
|
||||
}
|
||||
if config.Server.Host != "0.0.0.0" {
|
||||
t.Errorf("default host not applied, got %q, want %q", config.Server.Host, "0.0.0.0")
|
||||
if config.Operator.Email != "admin@example.com" {
|
||||
t.Error("operator email not loaded correctly")
|
||||
}
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "preserves custom port and host",
|
||||
configYAML: `server:
|
||||
port: 9000
|
||||
host: "127.0.0.1"
|
||||
cloud:
|
||||
dns:
|
||||
ip: "192.168.1.1"
|
||||
cluster:
|
||||
nodes:
|
||||
talos:
|
||||
version: "v1.8.0"
|
||||
name: "loads empty configuration",
|
||||
configYAML: `{}
|
||||
`,
|
||||
verify: func(t *testing.T, config *GlobalConfig) {
|
||||
if config.Server.Port != 9000 {
|
||||
t.Errorf("custom port not preserved, got %d, want 9000", config.Server.Port)
|
||||
}
|
||||
if config.Server.Host != "127.0.0.1" {
|
||||
t.Errorf("custom host not preserved, got %q, want %q", config.Server.Host, "127.0.0.1")
|
||||
if config.Operator.Email != "" {
|
||||
t.Error("expected empty operator email")
|
||||
}
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -189,35 +146,24 @@ func TestSaveGlobalConfig(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "saves complete configuration",
|
||||
config: &GlobalConfig{
|
||||
Wildcloud: struct {
|
||||
Repository string `yaml:"repository,omitempty" json:"repository,omitempty"`
|
||||
CurrentPhase string `yaml:"currentPhase,omitempty" json:"currentPhase,omitempty"`
|
||||
CompletedPhases []string `yaml:"completedPhases,omitempty" json:"completedPhases,omitempty"`
|
||||
}{
|
||||
Repository: "https://github.com/example/repo",
|
||||
CurrentPhase: "setup",
|
||||
CompletedPhases: []string{"phase1", "phase2"},
|
||||
},
|
||||
Server: struct {
|
||||
Port int `yaml:"port,omitempty" json:"port,omitempty"`
|
||||
Host string `yaml:"host,omitempty" json:"host,omitempty"`
|
||||
}{
|
||||
Port: 8080,
|
||||
Host: "localhost",
|
||||
},
|
||||
},
|
||||
config: func() *GlobalConfig {
|
||||
cfg := &GlobalConfig{}
|
||||
cfg.Operator.Email = "admin@example.com"
|
||||
cfg.Cloud.Router.IP = "192.168.1.254"
|
||||
cfg.Cloud.Router.DynamicDns = "example.dyndns.org"
|
||||
return cfg
|
||||
}(),
|
||||
verify: func(t *testing.T, configPath string) {
|
||||
content, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read saved config: %v", err)
|
||||
}
|
||||
contentStr := string(content)
|
||||
if !strings.Contains(contentStr, "repository") {
|
||||
t.Error("saved config missing repository field")
|
||||
if !strings.Contains(contentStr, "admin@example.com") {
|
||||
t.Error("saved config missing operator email")
|
||||
}
|
||||
if !strings.Contains(contentStr, "8080") {
|
||||
t.Error("saved config missing port value")
|
||||
if !strings.Contains(contentStr, "192.168.1.254") {
|
||||
t.Error("saved config missing router IP")
|
||||
}
|
||||
},
|
||||
},
|
||||
@@ -313,101 +259,31 @@ func TestGlobalConfig_IsEmpty(t *testing.T) {
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "config with only DNS IP is empty",
|
||||
config: &GlobalConfig{
|
||||
Cloud: struct {
|
||||
DNS struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
ExternalResolver string `yaml:"externalResolver,omitempty" json:"externalResolver,omitempty"`
|
||||
} `yaml:"dns,omitempty" json:"dns,omitempty"`
|
||||
Router struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
DynamicDns string `yaml:"dynamicDns,omitempty" json:"dynamicDns,omitempty"`
|
||||
} `yaml:"router,omitempty" json:"router,omitempty"`
|
||||
Dnsmasq struct {
|
||||
Interface string `yaml:"interface,omitempty" json:"interface,omitempty"`
|
||||
} `yaml:"dnsmasq,omitempty" json:"dnsmasq,omitempty"`
|
||||
}{
|
||||
DNS: struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
ExternalResolver string `yaml:"externalResolver,omitempty" json:"externalResolver,omitempty"`
|
||||
}{
|
||||
IP: "192.168.1.1",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
name: "config with only router IP is not empty",
|
||||
config: func() *GlobalConfig {
|
||||
cfg := &GlobalConfig{}
|
||||
cfg.Cloud.Router.IP = "192.168.1.254"
|
||||
return cfg
|
||||
}(),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "config with only Talos version is empty",
|
||||
config: &GlobalConfig{
|
||||
Cluster: struct {
|
||||
EndpointIP string `yaml:"endpointIp,omitempty" json:"endpointIp,omitempty"`
|
||||
Nodes struct {
|
||||
Talos struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
} `yaml:"talos,omitempty" json:"talos,omitempty"`
|
||||
} `yaml:"nodes,omitempty" json:"nodes,omitempty"`
|
||||
}{
|
||||
Nodes: struct {
|
||||
Talos struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
} `yaml:"talos,omitempty" json:"talos,omitempty"`
|
||||
}{
|
||||
Talos: struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
}{
|
||||
Version: "v1.8.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
name: "config with only operator email is not empty",
|
||||
config: func() *GlobalConfig {
|
||||
cfg := &GlobalConfig{}
|
||||
cfg.Operator.Email = "admin@example.com"
|
||||
return cfg
|
||||
}(),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "config with both DNS IP and Talos version is not empty",
|
||||
config: &GlobalConfig{
|
||||
Cloud: struct {
|
||||
DNS struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
ExternalResolver string `yaml:"externalResolver,omitempty" json:"externalResolver,omitempty"`
|
||||
} `yaml:"dns,omitempty" json:"dns,omitempty"`
|
||||
Router struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
DynamicDns string `yaml:"dynamicDns,omitempty" json:"dynamicDns,omitempty"`
|
||||
} `yaml:"router,omitempty" json:"router,omitempty"`
|
||||
Dnsmasq struct {
|
||||
Interface string `yaml:"interface,omitempty" json:"interface,omitempty"`
|
||||
} `yaml:"dnsmasq,omitempty" json:"dnsmasq,omitempty"`
|
||||
}{
|
||||
DNS: struct {
|
||||
IP string `yaml:"ip,omitempty" json:"ip,omitempty"`
|
||||
ExternalResolver string `yaml:"externalResolver,omitempty" json:"externalResolver,omitempty"`
|
||||
}{
|
||||
IP: "192.168.1.1",
|
||||
},
|
||||
},
|
||||
Cluster: struct {
|
||||
EndpointIP string `yaml:"endpointIp,omitempty" json:"endpointIp,omitempty"`
|
||||
Nodes struct {
|
||||
Talos struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
} `yaml:"talos,omitempty" json:"talos,omitempty"`
|
||||
} `yaml:"nodes,omitempty" json:"nodes,omitempty"`
|
||||
}{
|
||||
Nodes: struct {
|
||||
Talos struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
} `yaml:"talos,omitempty" json:"talos,omitempty"`
|
||||
}{
|
||||
Talos: struct {
|
||||
Version string `yaml:"version,omitempty" json:"version,omitempty"`
|
||||
}{
|
||||
Version: "v1.8.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
name: "config with all fields is not empty",
|
||||
config: func() *GlobalConfig {
|
||||
cfg := &GlobalConfig{}
|
||||
cfg.Cloud.Router.IP = "192.168.1.254"
|
||||
cfg.Operator.Email = "admin@example.com"
|
||||
return cfg
|
||||
}(),
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
@@ -433,11 +309,6 @@ func TestLoadCloudConfig(t *testing.T) {
|
||||
{
|
||||
name: "loads complete instance configuration",
|
||||
configYAML: `cloud:
|
||||
router:
|
||||
ip: "192.168.1.254"
|
||||
dns:
|
||||
ip: "192.168.1.1"
|
||||
externalResolver: "8.8.8.8"
|
||||
dhcpRange: "192.168.1.100,192.168.1.200"
|
||||
baseDomain: "example.com"
|
||||
domain: "home"
|
||||
@@ -458,6 +329,9 @@ cluster:
|
||||
if config.Cloud.BaseDomain != "example.com" {
|
||||
t.Error("base domain not loaded correctly")
|
||||
}
|
||||
if config.Cloud.DHCPRange != "192.168.1.100,192.168.1.200" {
|
||||
t.Error("DHCP range not loaded correctly")
|
||||
}
|
||||
if config.Cluster.Name != "my-cluster" {
|
||||
t.Error("cluster name not loaded correctly")
|
||||
}
|
||||
@@ -611,29 +485,10 @@ func TestGlobalConfig_RoundTrip(t *testing.T) {
|
||||
configPath := filepath.Join(tempDir, "config.yaml")
|
||||
|
||||
// Create config with all fields
|
||||
original := &GlobalConfig{
|
||||
Wildcloud: struct {
|
||||
Repository string `yaml:"repository,omitempty" json:"repository,omitempty"`
|
||||
CurrentPhase string `yaml:"currentPhase,omitempty" json:"currentPhase,omitempty"`
|
||||
CompletedPhases []string `yaml:"completedPhases,omitempty" json:"completedPhases,omitempty"`
|
||||
}{
|
||||
Repository: "https://github.com/example/repo",
|
||||
CurrentPhase: "setup",
|
||||
CompletedPhases: []string{"phase1", "phase2"},
|
||||
},
|
||||
Server: struct {
|
||||
Port int `yaml:"port,omitempty" json:"port,omitempty"`
|
||||
Host string `yaml:"host,omitempty" json:"host,omitempty"`
|
||||
}{
|
||||
Port: 8080,
|
||||
Host: "localhost",
|
||||
},
|
||||
Operator: struct {
|
||||
Email string `yaml:"email,omitempty" json:"email,omitempty"`
|
||||
}{
|
||||
Email: "admin@example.com",
|
||||
},
|
||||
}
|
||||
original := &GlobalConfig{}
|
||||
original.Operator.Email = "admin@example.com"
|
||||
original.Cloud.Router.IP = "192.168.1.254"
|
||||
original.Cloud.Router.DynamicDns = "example.dyndns.org"
|
||||
|
||||
// Save config
|
||||
if err := SaveGlobalConfig(original, configPath); err != nil {
|
||||
@@ -647,15 +502,15 @@ func TestGlobalConfig_RoundTrip(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all fields match
|
||||
if loaded.Wildcloud.Repository != original.Wildcloud.Repository {
|
||||
t.Errorf("repository mismatch: got %q, want %q", loaded.Wildcloud.Repository, original.Wildcloud.Repository)
|
||||
}
|
||||
if loaded.Server.Port != original.Server.Port {
|
||||
t.Errorf("port mismatch: got %d, want %d", loaded.Server.Port, original.Server.Port)
|
||||
}
|
||||
if loaded.Operator.Email != original.Operator.Email {
|
||||
t.Errorf("email mismatch: got %q, want %q", loaded.Operator.Email, original.Operator.Email)
|
||||
}
|
||||
if loaded.Cloud.Router.IP != original.Cloud.Router.IP {
|
||||
t.Errorf("router IP mismatch: got %q, want %q", loaded.Cloud.Router.IP, original.Cloud.Router.IP)
|
||||
}
|
||||
if loaded.Cloud.Router.DynamicDns != original.Cloud.Router.DynamicDns {
|
||||
t.Errorf("dynamic DNS mismatch: got %q, want %q", loaded.Cloud.Router.DynamicDns, original.Cloud.Router.DynamicDns)
|
||||
}
|
||||
}
|
||||
|
||||
// Test: Round-trip save and load for instance config
|
||||
@@ -688,3 +543,157 @@ func TestInstanceConfig_RoundTrip(t *testing.T) {
|
||||
t.Errorf("cluster name mismatch: got %q, want %q", loaded.Cluster.Name, original.Cluster.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepMerge(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
dst map[string]interface{}
|
||||
src map[string]interface{}
|
||||
expected map[string]interface{}
|
||||
}{
|
||||
{
|
||||
name: "src overrides dst flat keys",
|
||||
dst: map[string]interface{}{"a": "1", "b": "2"},
|
||||
src: map[string]interface{}{"b": "3", "c": "4"},
|
||||
expected: map[string]interface{}{"a": "1", "b": "3", "c": "4"},
|
||||
},
|
||||
{
|
||||
name: "nested maps merge recursively",
|
||||
dst: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{
|
||||
"router": map[string]interface{}{"ip": "192.168.1.1"},
|
||||
},
|
||||
},
|
||||
src: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{
|
||||
"domain": "example.com",
|
||||
},
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{
|
||||
"router": map[string]interface{}{"ip": "192.168.1.1"},
|
||||
"domain": "example.com",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "src nested key overrides dst nested key",
|
||||
dst: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{"domain": "old.com"},
|
||||
},
|
||||
src: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{"domain": "new.com"},
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"cloud": map[string]interface{}{"domain": "new.com"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty src returns dst",
|
||||
dst: map[string]interface{}{"a": "1"},
|
||||
src: map[string]interface{}{},
|
||||
expected: map[string]interface{}{"a": "1"},
|
||||
},
|
||||
{
|
||||
name: "empty dst returns src",
|
||||
dst: map[string]interface{}{},
|
||||
src: map[string]interface{}{"a": "1"},
|
||||
expected: map[string]interface{}{"a": "1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := DeepMerge(tt.dst, tt.src)
|
||||
resultYAML, _ := yaml.Marshal(result)
|
||||
expectedYAML, _ := yaml.Marshal(tt.expected)
|
||||
if string(resultYAML) != string(expectedYAML) {
|
||||
t.Errorf("DeepMerge() =\n%s\nwant:\n%s", resultYAML, expectedYAML)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMergedInstanceConfig(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
instanceName := "test"
|
||||
instanceConfigDir := filepath.Join(dataDir, "instances", instanceName, "config")
|
||||
os.MkdirAll(instanceConfigDir, 0755)
|
||||
|
||||
globalConfig := `operator:
|
||||
email: test@example.com
|
||||
cloud:
|
||||
router:
|
||||
ip: 192.168.1.1
|
||||
`
|
||||
instanceConfig := `cluster:
|
||||
name: test
|
||||
nodes:
|
||||
control:
|
||||
vip: 192.168.1.100
|
||||
cloud:
|
||||
domain: cloud.example.com
|
||||
`
|
||||
|
||||
os.WriteFile(filepath.Join(dataDir, "config.yaml"), []byte(globalConfig), 0644)
|
||||
os.WriteFile(filepath.Join(instanceConfigDir, "instance.yaml"), []byte(instanceConfig), 0644)
|
||||
|
||||
merged, err := LoadMergedInstanceConfig(dataDir, instanceName)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMergedInstanceConfig() error: %v", err)
|
||||
}
|
||||
|
||||
cloud, ok := merged["cloud"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing cloud key in merged config")
|
||||
}
|
||||
|
||||
router, ok := cloud["router"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing cloud.router key — global config not merged")
|
||||
}
|
||||
if router["ip"] != "192.168.1.1" {
|
||||
t.Errorf("cloud.router.ip = %v, want 192.168.1.1", router["ip"])
|
||||
}
|
||||
|
||||
if cloud["domain"] != "cloud.example.com" {
|
||||
t.Errorf("cloud.domain = %v, want cloud.example.com", cloud["domain"])
|
||||
}
|
||||
|
||||
cluster, ok := merged["cluster"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing cluster key — instance config not merged")
|
||||
}
|
||||
if cluster["name"] != "test" {
|
||||
t.Errorf("cluster.name = %v, want test", cluster["name"])
|
||||
}
|
||||
|
||||
operator, ok := merged["operator"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing operator key — global config not merged")
|
||||
}
|
||||
if operator["email"] != "test@example.com" {
|
||||
t.Errorf("operator.email = %v, want test@example.com", operator["email"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMergedInstanceConfig_NoGlobalConfig(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
instanceConfigDir := filepath.Join(dataDir, "instances", "test", "config")
|
||||
os.MkdirAll(instanceConfigDir, 0755)
|
||||
|
||||
os.WriteFile(filepath.Join(instanceConfigDir, "instance.yaml"), []byte("cluster:\n name: test\n"), 0644)
|
||||
|
||||
merged, err := LoadMergedInstanceConfig(dataDir, "test")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMergedInstanceConfig() error: %v", err)
|
||||
}
|
||||
|
||||
cluster, ok := merged["cluster"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing cluster key")
|
||||
}
|
||||
if cluster["name"] != "test" {
|
||||
t.Errorf("cluster.name = %v, want test", cluster["name"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/network"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
@@ -20,8 +22,43 @@ func NewManager() *Manager {
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureGlobalConfig ensures a global config file exists with proper structure
|
||||
func (m *Manager) EnsureGlobalConfig(dataDir string) error {
|
||||
configPath := filepath.Join(dataDir, "config.yaml")
|
||||
|
||||
// Check if config already exists
|
||||
if storage.FileExists(configPath) {
|
||||
// Validate existing config
|
||||
if err := m.yq.Validate(configPath); err != nil {
|
||||
return fmt.Errorf("invalid config file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create config structure with detected defaults
|
||||
initialConfig := &GlobalConfig{}
|
||||
|
||||
// Detect network configuration
|
||||
netInfo, err := network.DetectNetworkInfo()
|
||||
if err != nil {
|
||||
slog.Info("network detection failed, using defaults", "component", "config", "error", err)
|
||||
} else {
|
||||
// Set detected values
|
||||
initialConfig.Cloud.Router.IP = netInfo.Gateway
|
||||
slog.Info("detected network", "component", "config", "gateway", netInfo.Gateway, "interface", netInfo.PrimaryInterface)
|
||||
}
|
||||
|
||||
// Ensure data directory exists
|
||||
if err := storage.EnsureDir(dataDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Save config using the model's save function
|
||||
return SaveGlobalConfig(initialConfig, configPath)
|
||||
}
|
||||
|
||||
// EnsureInstanceConfig ensures an instance config file exists with proper structure
|
||||
func (m *Manager) EnsureInstanceConfig(instancePath string) error {
|
||||
func (m *Manager) EnsureInstanceConfig(name string, instancePath string) error {
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
|
||||
// Check if config already exists
|
||||
@@ -33,18 +70,15 @@ func (m *Manager) EnsureInstanceConfig(instancePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create minimal config structure using InstanceConfig model
|
||||
initialConfig := &InstanceConfig{}
|
||||
// Initialize nested maps
|
||||
initialConfig.Cluster.Nodes.Active = make(map[string]NodeConfig)
|
||||
initialConfig.Apps = make(map[string]interface{})
|
||||
|
||||
// Ensure instance directory exists
|
||||
if err := storage.EnsureDir(instancePath, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Save config using the model's save function
|
||||
initialConfig := &InstanceConfig{}
|
||||
initialConfig.Cluster.Name = name
|
||||
initialConfig.Cluster.Nodes.Active = make(map[string]NodeConfig)
|
||||
initialConfig.Apps = make(map[string]interface{})
|
||||
return SaveCloudConfig(initialConfig, configPath)
|
||||
}
|
||||
|
||||
|
||||
@@ -13,11 +13,8 @@ import (
|
||||
// Test: NewManager creates manager successfully
|
||||
func TestNewManager(t *testing.T) {
|
||||
m := NewManager()
|
||||
if m == nil {
|
||||
t.Fatal("NewManager returned nil")
|
||||
}
|
||||
if m.yq == nil {
|
||||
t.Error("Manager.yq is nil")
|
||||
if m == nil || m.yq == nil {
|
||||
t.Fatal("NewManager returned nil or Manager.yq is nil")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +44,6 @@ cloud:
|
||||
dhcpRange: ""
|
||||
dns:
|
||||
ip: ""
|
||||
externalResolver: ""
|
||||
router:
|
||||
ip: ""
|
||||
dnsmasq:
|
||||
@@ -57,13 +53,6 @@ cloud:
|
||||
mediaPath: ""
|
||||
storageCapacity: ""
|
||||
dockerRegistryHost: ""
|
||||
smtp:
|
||||
host: ""
|
||||
port: ""
|
||||
user: ""
|
||||
from: ""
|
||||
tls: ""
|
||||
startTls: ""
|
||||
cluster:
|
||||
name: ""
|
||||
loadBalancerIp: ""
|
||||
@@ -114,7 +103,7 @@ apps: {}
|
||||
tt.setupFunc(t, instancePath)
|
||||
}
|
||||
|
||||
err := m.EnsureInstanceConfig(instancePath)
|
||||
err := m.EnsureInstanceConfig(tt.name, instancePath)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Error("expected error, got nil")
|
||||
@@ -831,7 +820,7 @@ func TestEnsureInstanceConfig_FilePermissions(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
m := NewManager()
|
||||
|
||||
if err := m.EnsureInstanceConfig(tempDir); err != nil {
|
||||
if err := m.EnsureInstanceConfig("my-wild-cloud", tempDir); err != nil {
|
||||
t.Fatalf("EnsureInstanceConfig failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -853,7 +842,7 @@ func TestEnsureInstanceConfig_Idempotent(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
// First call creates config
|
||||
if err := m.EnsureInstanceConfig(tempDir); err != nil {
|
||||
if err := m.EnsureInstanceConfig("my-wild-cloud", tempDir); err != nil {
|
||||
t.Fatalf("first EnsureInstanceConfig failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -864,7 +853,7 @@ func TestEnsureInstanceConfig_Idempotent(t *testing.T) {
|
||||
}
|
||||
|
||||
// Second call should not modify config
|
||||
if err := m.EnsureInstanceConfig(tempDir); err != nil {
|
||||
if err := m.EnsureInstanceConfig("my-wild-cloud", tempDir); err != nil {
|
||||
t.Fatalf("second EnsureInstanceConfig failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -883,7 +872,7 @@ func TestEnsureInstanceConfig_RequiredFields(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
m := NewManager()
|
||||
|
||||
if err := m.EnsureInstanceConfig(tempDir); err != nil {
|
||||
if err := m.EnsureInstanceConfig("my-wild-cloud", tempDir); err != nil {
|
||||
t.Fatalf("EnsureInstanceConfig failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -901,11 +890,7 @@ func TestEnsureInstanceConfig_RequiredFields(t *testing.T) {
|
||||
"domain:",
|
||||
"internalDomain:",
|
||||
"dhcpRange:",
|
||||
"dns:",
|
||||
"router:",
|
||||
"dnsmasq:",
|
||||
"nfs:",
|
||||
"smtp:",
|
||||
"cluster:",
|
||||
"loadBalancerIp:",
|
||||
"ipAddressPool:",
|
||||
|
||||
@@ -1,340 +0,0 @@
|
||||
// Package contracts contains API contracts for service management endpoints
|
||||
package contracts
|
||||
|
||||
import "time"
|
||||
|
||||
// ==============================
|
||||
// Request/Response Types
|
||||
// ==============================
|
||||
|
||||
// ServiceManifest represents basic service information
|
||||
type ServiceManifest struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Namespace string `json:"namespace"`
|
||||
ConfigReferences []string `json:"configReferences,omitempty"`
|
||||
ServiceConfig map[string]ConfigDefinition `json:"serviceConfig,omitempty"`
|
||||
}
|
||||
|
||||
// ConfigDefinition defines config that should be prompted during service setup
|
||||
type ConfigDefinition struct {
|
||||
Path string `json:"path"`
|
||||
Prompt string `json:"prompt"`
|
||||
Default string `json:"default"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
// PodStatus represents the status of a single pod
|
||||
type PodStatus struct {
|
||||
Name string `json:"name"` // Pod name
|
||||
Status string `json:"status"` // Pod phase: Running, Pending, Failed, etc.
|
||||
Ready string `json:"ready"` // Ready containers e.g. "1/1", "0/1"
|
||||
Restarts int `json:"restarts"` // Container restart count
|
||||
Age string `json:"age"` // Human-readable age e.g. "2h", "5m"
|
||||
Node string `json:"node"` // Node name where pod is scheduled
|
||||
IP string `json:"ip,omitempty"` // Pod IP if available
|
||||
Containers []string `json:"containers,omitempty"` // Container names in the pod
|
||||
}
|
||||
|
||||
// DetailedServiceStatus provides comprehensive service status
|
||||
type DetailedServiceStatus struct {
|
||||
Name string `json:"name"` // Service name
|
||||
Namespace string `json:"namespace"` // Kubernetes namespace
|
||||
DeploymentStatus string `json:"deploymentStatus"` // "Ready", "Progressing", "Degraded", "NotFound"
|
||||
Replicas ReplicaStatus `json:"replicas"` // Desired/current/ready replicas
|
||||
Pods []PodStatus `json:"pods"` // Pod details
|
||||
Config map[string]interface{} `json:"config,omitempty"` // Current config from config.yaml
|
||||
Manifest *ServiceManifest `json:"manifest,omitempty"` // Service manifest if available
|
||||
LastUpdated time.Time `json:"lastUpdated"` // Timestamp of status
|
||||
}
|
||||
|
||||
// ReplicaStatus tracks deployment replica counts
|
||||
type ReplicaStatus struct {
|
||||
Desired int32 `json:"desired"` // Desired replica count
|
||||
Current int32 `json:"current"` // Current replica count
|
||||
Ready int32 `json:"ready"` // Ready replica count
|
||||
Available int32 `json:"available"` // Available replica count
|
||||
}
|
||||
|
||||
// ServiceLogsRequest query parameters for log retrieval
|
||||
type ServiceLogsRequest struct {
|
||||
Container string `json:"container,omitempty"` // Specific container name (optional)
|
||||
Tail int `json:"tail,omitempty"` // Number of lines from end (default: 100)
|
||||
Follow bool `json:"follow,omitempty"` // Stream logs via SSE
|
||||
Previous bool `json:"previous,omitempty"` // Get previous container logs
|
||||
Since string `json:"since,omitempty"` // RFC3339 or duration string e.g. "10m"
|
||||
}
|
||||
|
||||
// ServiceLogsResponse for non-streaming log retrieval
|
||||
type ServiceLogsResponse struct {
|
||||
Service string `json:"service"` // Service name
|
||||
Namespace string `json:"namespace"` // Kubernetes namespace
|
||||
Container string `json:"container,omitempty"` // Container name if specified
|
||||
Lines []string `json:"lines"` // Log lines
|
||||
Truncated bool `json:"truncated"` // Whether logs were truncated
|
||||
Timestamp time.Time `json:"timestamp"` // Response timestamp
|
||||
}
|
||||
|
||||
// ServiceLogsSSEEvent for streaming logs via Server-Sent Events
|
||||
type ServiceLogsSSEEvent struct {
|
||||
Type string `json:"type"` // "log", "error", "end"
|
||||
Line string `json:"line,omitempty"` // Log line content
|
||||
Error string `json:"error,omitempty"` // Error message if type="error"
|
||||
Container string `json:"container,omitempty"` // Container source
|
||||
Timestamp time.Time `json:"timestamp"` // Event timestamp
|
||||
}
|
||||
|
||||
// ServiceConfigUpdate request to update service configuration
|
||||
type ServiceConfigUpdate struct {
|
||||
Config map[string]interface{} `json:"config"` // Configuration updates
|
||||
Redeploy bool `json:"redeploy"` // Trigger recompilation/redeployment
|
||||
Fetch bool `json:"fetch"` // Fetch fresh templates before redeployment
|
||||
}
|
||||
|
||||
// ServiceConfigResponse response after config update
|
||||
type ServiceConfigResponse struct {
|
||||
Service string `json:"service"` // Service name
|
||||
Namespace string `json:"namespace"` // Kubernetes namespace
|
||||
Config map[string]interface{} `json:"config"` // Updated configuration
|
||||
Redeployed bool `json:"redeployed"` // Whether service was redeployed
|
||||
Message string `json:"message"` // Success/info message
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// Error Response
|
||||
// ==============================
|
||||
|
||||
// ErrorResponse standard error format for all endpoints
|
||||
type ErrorResponse struct {
|
||||
Error ErrorDetail `json:"error"`
|
||||
}
|
||||
|
||||
// ErrorDetail contains error information
|
||||
type ErrorDetail struct {
|
||||
Code string `json:"code"` // Machine-readable error code
|
||||
Message string `json:"message"` // Human-readable error message
|
||||
Details map[string]interface{} `json:"details,omitempty"` // Additional error context
|
||||
}
|
||||
|
||||
// Standard error codes
|
||||
const (
|
||||
ErrCodeNotFound = "SERVICE_NOT_FOUND"
|
||||
ErrCodeInstanceNotFound = "INSTANCE_NOT_FOUND"
|
||||
ErrCodeInvalidRequest = "INVALID_REQUEST"
|
||||
ErrCodeKubectlFailed = "KUBECTL_FAILED"
|
||||
ErrCodeConfigInvalid = "CONFIG_INVALID"
|
||||
ErrCodeDeploymentFailed = "DEPLOYMENT_FAILED"
|
||||
ErrCodeStreamingError = "STREAMING_ERROR"
|
||||
ErrCodeInternalError = "INTERNAL_ERROR"
|
||||
)
|
||||
|
||||
// ==============================
|
||||
// API Endpoint Specifications
|
||||
// ==============================
|
||||
|
||||
/*
|
||||
1. GET /api/v1/instances/{name}/services/{service}/status
|
||||
|
||||
Purpose: Returns comprehensive service status including pods and health
|
||||
Response Codes:
|
||||
- 200 OK: Service status retrieved successfully
|
||||
- 404 Not Found: Instance or service not found
|
||||
- 500 Internal Server Error: kubectl command failed
|
||||
|
||||
Example Request:
|
||||
GET /api/v1/instances/production/services/nginx/status
|
||||
|
||||
Example Response (200 OK):
|
||||
{
|
||||
"name": "nginx",
|
||||
"namespace": "nginx",
|
||||
"deploymentStatus": "Ready",
|
||||
"replicas": {
|
||||
"desired": 3,
|
||||
"current": 3,
|
||||
"ready": 3,
|
||||
"available": 3
|
||||
},
|
||||
"pods": [
|
||||
{
|
||||
"name": "nginx-7c5464c66d-abc123",
|
||||
"status": "Running",
|
||||
"ready": "1/1",
|
||||
"restarts": 0,
|
||||
"age": "2h",
|
||||
"node": "worker-1",
|
||||
"ip": "10.42.1.5"
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"nginx.image": "nginx:1.21",
|
||||
"nginx.replicas": 3
|
||||
},
|
||||
"manifest": {
|
||||
"name": "nginx",
|
||||
"description": "NGINX web server",
|
||||
"namespace": "nginx"
|
||||
},
|
||||
"lastUpdated": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
|
||||
Example Error Response (404):
|
||||
{
|
||||
"error": {
|
||||
"code": "SERVICE_NOT_FOUND",
|
||||
"message": "Service nginx not found in instance production",
|
||||
"details": {
|
||||
"instance": "production",
|
||||
"service": "nginx"
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
/*
|
||||
2. GET /api/v1/instances/{name}/services/{service}/logs
|
||||
|
||||
Purpose: Retrieve or stream service logs
|
||||
Query Parameters:
|
||||
- container (string): Specific container name
|
||||
- tail (int): Number of lines from end (default: 100, max: 5000)
|
||||
- follow (bool): Stream logs via SSE (default: false)
|
||||
- previous (bool): Get previous container logs (default: false)
|
||||
- since (string): RFC3339 timestamp or duration (e.g. "10m")
|
||||
|
||||
Response Codes:
|
||||
- 200 OK: Logs retrieved successfully (or SSE stream started)
|
||||
- 400 Bad Request: Invalid query parameters
|
||||
- 404 Not Found: Instance, service, or container not found
|
||||
- 500 Internal Server Error: kubectl command failed
|
||||
|
||||
Example Request (buffered):
|
||||
GET /api/v1/instances/production/services/nginx/logs?tail=50
|
||||
|
||||
Example Response (200 OK):
|
||||
{
|
||||
"service": "nginx",
|
||||
"namespace": "nginx",
|
||||
"container": "nginx",
|
||||
"lines": [
|
||||
"2024/01/15 10:00:00 [notice] Configuration loaded",
|
||||
"2024/01/15 10:00:01 [info] Server started on port 80"
|
||||
],
|
||||
"truncated": false,
|
||||
"timestamp": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
|
||||
Example Request (streaming):
|
||||
GET /api/v1/instances/production/services/nginx/logs?follow=true
|
||||
Accept: text/event-stream
|
||||
|
||||
Example SSE Response:
|
||||
data: {"type":"log","line":"2024/01/15 10:00:00 [notice] Configuration loaded","container":"nginx","timestamp":"2024-01-15T10:30:00Z"}
|
||||
|
||||
data: {"type":"log","line":"2024/01/15 10:00:01 [info] Request from 10.0.0.1","container":"nginx","timestamp":"2024-01-15T10:30:01Z"}
|
||||
|
||||
data: {"type":"error","error":"Container restarting","timestamp":"2024-01-15T10:30:02Z"}
|
||||
|
||||
data: {"type":"end","timestamp":"2024-01-15T10:30:03Z"}
|
||||
*/
|
||||
|
||||
/*
|
||||
3. PATCH /api/v1/instances/{name}/services/{service}/config
|
||||
|
||||
Purpose: Update service configuration in config.yaml and optionally redeploy
|
||||
Request Body: ServiceConfigUpdate (JSON)
|
||||
Response Codes:
|
||||
- 200 OK: Configuration updated successfully
|
||||
- 400 Bad Request: Invalid configuration
|
||||
- 404 Not Found: Instance or service not found
|
||||
- 500 Internal Server Error: Update or deployment failed
|
||||
|
||||
Example Request:
|
||||
PATCH /api/v1/instances/production/services/nginx/config
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"config": {
|
||||
"nginx.image": "nginx:1.22",
|
||||
"nginx.replicas": 5,
|
||||
"nginx.resources.memory": "512Mi"
|
||||
},
|
||||
"redeploy": true
|
||||
}
|
||||
|
||||
Example Response (200 OK):
|
||||
{
|
||||
"service": "nginx",
|
||||
"namespace": "nginx",
|
||||
"config": {
|
||||
"nginx.image": "nginx:1.22",
|
||||
"nginx.replicas": 5,
|
||||
"nginx.resources.memory": "512Mi"
|
||||
},
|
||||
"redeployed": true,
|
||||
"message": "Service configuration updated and redeployed successfully"
|
||||
}
|
||||
|
||||
Example Error Response (400):
|
||||
{
|
||||
"error": {
|
||||
"code": "CONFIG_INVALID",
|
||||
"message": "Invalid configuration: nginx.replicas must be a positive integer",
|
||||
"details": {
|
||||
"field": "nginx.replicas",
|
||||
"value": -1,
|
||||
"constraint": "positive integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// ==============================
|
||||
// Validation Rules
|
||||
// ==============================
|
||||
|
||||
/*
|
||||
Query Parameter Validation:
|
||||
|
||||
ServiceLogsRequest:
|
||||
- tail: Must be between 1 and 5000 (default: 100)
|
||||
- since: Must be valid RFC3339 timestamp or Go duration string (e.g. "5m", "1h")
|
||||
- container: Must match existing container name if specified
|
||||
- follow: When true, response uses Server-Sent Events (SSE)
|
||||
- previous: Cannot be combined with follow=true
|
||||
|
||||
ServiceConfigUpdate:
|
||||
- config: Must be valid YAML-compatible structure
|
||||
- config keys: Must follow service's expected configuration schema
|
||||
- redeploy: When true, triggers kustomize recompilation and kubectl apply
|
||||
|
||||
Path Parameters:
|
||||
- instance name: Must match existing instance directory
|
||||
- service name: Must match installed service name
|
||||
*/
|
||||
|
||||
// ==============================
|
||||
// HTTP Status Code Summary
|
||||
// ==============================
|
||||
|
||||
/*
|
||||
200 OK:
|
||||
- Service status retrieved successfully
|
||||
- Logs retrieved successfully (non-streaming)
|
||||
- Configuration updated successfully
|
||||
|
||||
400 Bad Request:
|
||||
- Invalid query parameters
|
||||
- Invalid configuration in request body
|
||||
- Validation errors
|
||||
|
||||
404 Not Found:
|
||||
- Instance does not exist
|
||||
- Service not installed in instance
|
||||
- Container name not found (for logs)
|
||||
|
||||
500 Internal Server Error:
|
||||
- kubectl command execution failed
|
||||
- File system operations failed
|
||||
- Unexpected errors during processing
|
||||
*/
|
||||
@@ -2,7 +2,7 @@ package data
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
@@ -42,10 +42,10 @@ func (m *Manager) Initialize() error {
|
||||
} else {
|
||||
dataDir = filepath.Join(cwd, "data")
|
||||
}
|
||||
log.Printf("Running in development mode, using data directory: %s", dataDir)
|
||||
slog.Info("data directory configured", "component", "data", "mode", "development", "path", dataDir)
|
||||
} else {
|
||||
dataDir = "/var/lib/wild-central"
|
||||
log.Printf("Running in production mode, using data directory: %s", dataDir)
|
||||
slog.Info("data directory configured", "component", "data", "mode", "production", "path", dataDir)
|
||||
}
|
||||
|
||||
m.dataDir = dataDir
|
||||
@@ -60,7 +60,7 @@ func (m *Manager) Initialize() error {
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Data directory structure initialized at: %s", dataDir)
|
||||
slog.Info("data directory initialized", "component", "data", "path", dataDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package discovery
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/node"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/sse"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/storage"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
)
|
||||
@@ -19,18 +21,20 @@ type Manager struct {
|
||||
dataDir string
|
||||
nodeMgr *node.Manager
|
||||
talosctl *tools.Talosctl
|
||||
sseManager *sse.Manager
|
||||
discoveryMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewManager creates a new discovery manager
|
||||
func NewManager(dataDir string, instanceName string) *Manager {
|
||||
func NewManager(dataDir string, instanceName string, sseManager *sse.Manager) *Manager {
|
||||
// Get talosconfig path for the instance
|
||||
talosconfigPath := tools.GetTalosconfigPath(dataDir, instanceName)
|
||||
|
||||
return &Manager{
|
||||
dataDir: dataDir,
|
||||
nodeMgr: node.NewManager(dataDir, instanceName),
|
||||
talosctl: tools.NewTalosconfigWithConfig(talosconfigPath),
|
||||
dataDir: dataDir,
|
||||
nodeMgr: node.NewManager(dataDir, instanceName),
|
||||
talosctl: tools.NewTalosconfigWithConfig(talosconfigPath),
|
||||
sseManager: sseManager,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +115,8 @@ func (m *Manager) StartDiscovery(instanceName string, ipList []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Info("starting node discovery", "component", "discovery", "instance", instanceName, "addresses", len(ipList))
|
||||
|
||||
// Start discovery in background
|
||||
go m.runDiscovery(instanceName, ipList)
|
||||
|
||||
@@ -172,7 +178,15 @@ func (m *Manager) runDiscovery(instanceName string, ipList []string) {
|
||||
status.NodesFound = discoveredNodes
|
||||
_ = m.writeDiscoveryStatus(instanceName, status)
|
||||
m.discoveryMu.Unlock()
|
||||
|
||||
// Notify frontend of progress
|
||||
m.broadcastDiscoveryEvent(instanceName)
|
||||
}
|
||||
|
||||
// Notify frontend that discovery is complete
|
||||
m.broadcastDiscoveryEvent(instanceName)
|
||||
|
||||
slog.Info("node discovery completed", "component", "discovery", "instance", instanceName, "found", len(discoveredNodes))
|
||||
}
|
||||
|
||||
// probeNode attempts to detect if a node is running Talos in maintenance mode
|
||||
@@ -219,6 +233,30 @@ func (m *Manager) DiscoverNodes(instanceName string, ipList []string) ([]Discove
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// RemoveDiscoveredNode removes a node by IP from the discovery results
|
||||
func (m *Manager) RemoveDiscoveredNode(instanceName, ip string) {
|
||||
m.discoveryMu.Lock()
|
||||
defer m.discoveryMu.Unlock()
|
||||
|
||||
status, err := m.GetDiscoveryStatus(instanceName)
|
||||
if err != nil || len(status.NodesFound) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
filtered := make([]DiscoveredNode, 0, len(status.NodesFound))
|
||||
for _, n := range status.NodesFound {
|
||||
if n.IP != ip {
|
||||
filtered = append(filtered, n)
|
||||
}
|
||||
}
|
||||
|
||||
if len(filtered) < len(status.NodesFound) {
|
||||
status.NodesFound = filtered
|
||||
_ = m.writeDiscoveryStatus(instanceName, status)
|
||||
m.broadcastDiscoveryEvent(instanceName)
|
||||
}
|
||||
}
|
||||
|
||||
// ClearDiscoveryStatus removes discovery status file
|
||||
func (m *Manager) ClearDiscoveryStatus(instanceName string) error {
|
||||
statusPath := m.GetDiscoveryStatusPath(instanceName)
|
||||
@@ -230,6 +268,17 @@ func (m *Manager) ClearDiscoveryStatus(instanceName string) error {
|
||||
return os.Remove(statusPath)
|
||||
}
|
||||
|
||||
// broadcastDiscoveryEvent sends an SSE event to notify the frontend
|
||||
func (m *Manager) broadcastDiscoveryEvent(instanceName string) {
|
||||
if m.sseManager == nil {
|
||||
return
|
||||
}
|
||||
m.sseManager.Broadcast(&sse.Event{
|
||||
Type: "discovery:updated",
|
||||
InstanceName: instanceName,
|
||||
})
|
||||
}
|
||||
|
||||
// writeDiscoveryStatus writes discovery status to disk
|
||||
func (m *Manager) writeDiscoveryStatus(instanceName string, status *DiscoveryStatus) error {
|
||||
discoveryDir := m.GetDiscoveryDir(instanceName)
|
||||
|
||||
@@ -2,7 +2,7 @@ package dnsmasq
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
@@ -34,30 +34,26 @@ func (g *ConfigGenerator) GetConfigPath() string {
|
||||
}
|
||||
|
||||
// Generate creates a dnsmasq configuration from the app config
|
||||
// If the DNS IP or interface in the config don't match the current network,
|
||||
// it will auto-detect and use the current values
|
||||
// It auto-detects the Wild Central server's IP address
|
||||
func (g *ConfigGenerator) Generate(cfg *config.GlobalConfig, clouds []config.InstanceConfig) string {
|
||||
// Auto-detect network info to ensure we use the correct interface and IP
|
||||
netInfo, err := network.DetectNetworkInfo()
|
||||
// Get the Wild Central IP address
|
||||
dnsIP, err := network.GetWildCentralIP()
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to auto-detect network info, using config values: %v", err)
|
||||
// Fall back to config values if detection fails
|
||||
netInfo = &network.NetworkInfo{
|
||||
PrimaryIP: cfg.Cloud.DNS.IP,
|
||||
PrimaryInterface: cfg.Cloud.Dnsmasq.Interface,
|
||||
}
|
||||
slog.Error("failed to detect Wild Central IP", "component", "dnsmasq", "op", "generate", "error", err)
|
||||
// Fall back to empty string if detection fails
|
||||
dnsIP = ""
|
||||
}
|
||||
|
||||
// Use detected network info (this ensures dnsmasq works even if config is outdated)
|
||||
dnsIP := netInfo.PrimaryIP
|
||||
iface := netInfo.PrimaryInterface
|
||||
|
||||
resolution_section := ""
|
||||
for _, cloud := range clouds {
|
||||
// Point cloud domains to the cluster load balancer IP
|
||||
loadBalancerIP := cloud.Cluster.LoadBalancerIp
|
||||
loadBalancerIP := instanceLoadBalancerIP(cloud)
|
||||
if loadBalancerIP == "" {
|
||||
log.Printf("Warning: No load balancer IP configured for instance %s, skipping DNS config", cloud.Cluster.Name)
|
||||
slog.Info("no load balancer IP configured, adding commented DNS config", "component", "dnsmasq", "instance", cloud.Cluster.Name)
|
||||
// Add commented out entries for instances without load balancer
|
||||
resolution_section += fmt.Sprintf("# No load balancer IP configured for instance %s\n", cloud.Cluster.Name)
|
||||
resolution_section += fmt.Sprintf("# local=/%s/\n# address=/%s/<load-balancer-ip>\n", cloud.Cloud.InternalDomain, cloud.Cloud.InternalDomain)
|
||||
resolution_section += fmt.Sprintf("# address=/%s/<load-balancer-ip>\n", cloud.Cloud.Domain)
|
||||
continue
|
||||
}
|
||||
// Internal domain (.internal.cloud.example.tld) - local only, no external DNS
|
||||
@@ -71,7 +67,6 @@ func (g *ConfigGenerator) Generate(cfg *config.GlobalConfig, clouds []config.Ins
|
||||
template := `# Configuration file for dnsmasq.
|
||||
|
||||
# Basic Settings
|
||||
interface=%s
|
||||
listen-address=%s
|
||||
bind-interfaces
|
||||
domain-needed
|
||||
@@ -88,7 +83,6 @@ log-dhcp
|
||||
`
|
||||
|
||||
return fmt.Sprintf(template,
|
||||
iface,
|
||||
dnsIP,
|
||||
resolution_section,
|
||||
)
|
||||
@@ -98,7 +92,7 @@ log-dhcp
|
||||
func (g *ConfigGenerator) WriteConfig(cfg *config.GlobalConfig, clouds []config.InstanceConfig, configPath string) error {
|
||||
configContent := g.Generate(cfg, clouds)
|
||||
|
||||
log.Printf("Writing dnsmasq config to: %s", configPath)
|
||||
slog.Info("writing dnsmasq config", "component", "dnsmasq", "path", configPath)
|
||||
|
||||
if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing dnsmasq config: %w", err)
|
||||
@@ -115,6 +109,7 @@ func (g *ConfigGenerator) RestartService() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to restart dnsmasq: %w (output: %s)", err, string(output))
|
||||
}
|
||||
slog.Info("dnsmasq service restarted", "component", "dnsmasq")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -122,6 +117,7 @@ func (g *ConfigGenerator) RestartService() error {
|
||||
type ServiceStatus struct {
|
||||
Status string `json:"status"`
|
||||
PID int `json:"pid"`
|
||||
IP string `json:"ip"`
|
||||
ConfigFile string `json:"config_file"`
|
||||
InstancesConfigured int `json:"instances_configured"`
|
||||
LastRestart time.Time `json:"last_restart"`
|
||||
@@ -129,8 +125,16 @@ type ServiceStatus struct {
|
||||
|
||||
// GetStatus checks the status of the dnsmasq service
|
||||
func (g *ConfigGenerator) GetStatus() (*ServiceStatus, error) {
|
||||
// Get the Wild Central IP address
|
||||
dnsIP, err := network.GetWildCentralIP()
|
||||
if err != nil {
|
||||
slog.Error("failed to detect Wild Central IP", "component", "dnsmasq", "op", "getStatus", "error", err)
|
||||
dnsIP = ""
|
||||
}
|
||||
|
||||
status := &ServiceStatus{
|
||||
ConfigFile: g.configPath,
|
||||
IP: dnsIP,
|
||||
}
|
||||
|
||||
// Check if service is active
|
||||
@@ -171,12 +175,13 @@ func (g *ConfigGenerator) GetStatus() (*ServiceStatus, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Count instances in config
|
||||
// Count instances in config (both active and commented)
|
||||
if data, err := os.ReadFile(g.configPath); err == nil {
|
||||
// Count "local=/" occurrences (each instance has multiple)
|
||||
count := strings.Count(string(data), "local=/")
|
||||
// Each instance creates 2 "local=/" entries (domain and internal domain)
|
||||
status.InstancesConfigured = count / 2
|
||||
// Count both "local=/" and "# local=/" occurrences
|
||||
activeCount := strings.Count(string(data), "\nlocal=/")
|
||||
commentedCount := strings.Count(string(data), "\n# local=/")
|
||||
// Each instance creates 1 "local=/" entry (internal domain)
|
||||
status.InstancesConfigured = activeCount + commentedCount
|
||||
}
|
||||
|
||||
return status, nil
|
||||
@@ -197,7 +202,7 @@ func (g *ConfigGenerator) UpdateConfig(cfg *config.GlobalConfig, instances []con
|
||||
configContent := g.Generate(cfg, instances)
|
||||
|
||||
// Write config
|
||||
log.Printf("Writing dnsmasq config to: %s", g.configPath)
|
||||
slog.Info("writing dnsmasq config", "component", "dnsmasq", "path", g.configPath)
|
||||
if err := os.WriteFile(g.configPath, []byte(configContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing dnsmasq config: %w", err)
|
||||
}
|
||||
@@ -230,12 +235,12 @@ func (g *ConfigGenerator) ConfigureSystemDNS() error {
|
||||
return fmt.Errorf("failed to write resolved.conf: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Configured systemd-resolved to use DNS at %s", dnsIP)
|
||||
slog.Info("configured systemd-resolved", "component", "dnsmasq", "dnsIP", dnsIP)
|
||||
|
||||
// Restart systemd-resolved to apply changes (via polkit)
|
||||
cmd := exec.Command("systemctl", "restart", "systemd-resolved")
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
log.Printf("Warning: Failed to restart systemd-resolved: %v (output: %s)", err, string(output))
|
||||
slog.Error("failed to restart systemd-resolved", "component", "dnsmasq", "error", err, "output", string(output))
|
||||
// Don't return error - the config was written successfully
|
||||
}
|
||||
|
||||
|
||||
308
api/internal/dnsmasq/config_modular.go
Normal file
308
api/internal/dnsmasq/config_modular.go
Normal file
@@ -0,0 +1,308 @@
|
||||
package dnsmasq
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/config"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/network"
|
||||
)
|
||||
|
||||
const (
|
||||
instanceConfigDir = "/etc/dnsmasq.d/wild-cloud-instances"
|
||||
)
|
||||
|
||||
// GenerateMainConfig creates the main dnsmasq configuration with global settings
|
||||
// and a conf-dir directive to include per-instance configs
|
||||
func (g *ConfigGenerator) GenerateMainConfig(cfg *config.GlobalConfig) string {
|
||||
// Get the Wild Central IP address
|
||||
dnsIP, err := network.GetWildCentralIP()
|
||||
if err != nil {
|
||||
slog.Error("failed to detect Wild Central IP", "component", "dnsmasq", "error", err)
|
||||
// Fall back to empty string if detection fails
|
||||
dnsIP = ""
|
||||
}
|
||||
|
||||
template := `# Wild Cloud DNS Configuration (Main)
|
||||
# This file contains global settings. Instance-specific DNS entries are in:
|
||||
# /etc/dnsmasq.d/wild-cloud-instances/*.conf
|
||||
|
||||
# Basic Settings
|
||||
listen-address=%s
|
||||
bind-interfaces
|
||||
domain-needed
|
||||
bogus-priv
|
||||
no-resolv
|
||||
|
||||
# Include per-instance DNS configurations
|
||||
conf-dir=%s,*.conf
|
||||
|
||||
# Upstream DNS servers
|
||||
server=1.1.1.1
|
||||
server=8.8.8.8
|
||||
|
||||
# Logging
|
||||
log-queries
|
||||
log-dhcp
|
||||
`
|
||||
|
||||
return fmt.Sprintf(template,
|
||||
dnsIP,
|
||||
instanceConfigDir,
|
||||
)
|
||||
}
|
||||
|
||||
// instanceLoadBalancerIP returns the load balancer IP for an instance.
|
||||
// It checks cluster.loadBalancerIp first, then falls back to apps.metallb.loadBalancerIp,
|
||||
// which is where the normal app install flow writes it.
|
||||
func instanceLoadBalancerIP(instance config.InstanceConfig) string {
|
||||
if instance.Cluster.LoadBalancerIp != "" {
|
||||
return instance.Cluster.LoadBalancerIp
|
||||
}
|
||||
if metallb, ok := instance.Apps["metallb"].(map[string]interface{}); ok {
|
||||
if ip, ok := metallb["loadBalancerIp"].(string); ok {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GenerateInstanceConfig creates a DNS configuration for a single instance
|
||||
func (g *ConfigGenerator) GenerateInstanceConfig(instance config.InstanceConfig) string {
|
||||
var sb strings.Builder
|
||||
|
||||
fmt.Fprintf(&sb, "# DNS configuration for instance: %s\n", instance.Cluster.Name)
|
||||
sb.WriteString("# Generated by Wild Cloud\n\n")
|
||||
|
||||
loadBalancerIP := instanceLoadBalancerIP(instance)
|
||||
if loadBalancerIP == "" {
|
||||
sb.WriteString("# WARNING: No load balancer IP configured for this instance\n")
|
||||
sb.WriteString("# DNS entries are commented out until load balancer IP is configured\n\n")
|
||||
fmt.Fprintf(&sb, "# local=/%s/\n", instance.Cloud.InternalDomain)
|
||||
fmt.Fprintf(&sb, "# address=/%s/<load-balancer-ip>\n\n", instance.Cloud.InternalDomain)
|
||||
fmt.Fprintf(&sb, "# address=/%s/<load-balancer-ip>\n", instance.Cloud.Domain)
|
||||
} else {
|
||||
// Internal domain (.internal.cloud.example.tld) - local only, no external DNS
|
||||
sb.WriteString("# Internal domain (LAN-only)\n")
|
||||
fmt.Fprintf(&sb, "local=/%s/\n", instance.Cloud.InternalDomain)
|
||||
fmt.Fprintf(&sb, "address=/%s/%s\n\n", instance.Cloud.InternalDomain, loadBalancerIP)
|
||||
|
||||
// External domain (cloud.example.tld) - resolve to load balancer IP
|
||||
sb.WriteString("# Public domain (resolved locally to avoid external DNS)\n")
|
||||
fmt.Fprintf(&sb, "address=/%s/%s\n", instance.Cloud.Domain, loadBalancerIP)
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// ValidateConfig tests a dnsmasq configuration file for syntax errors
|
||||
func (g *ConfigGenerator) ValidateConfig(configPath string) error {
|
||||
// Use dnsmasq --test to validate the configuration
|
||||
cmd := exec.Command("dnsmasq", "--test", "-C", configPath)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("config validation failed: %w (output: %s)", err, string(output))
|
||||
}
|
||||
|
||||
// Check if output contains "syntax check OK"
|
||||
if !strings.Contains(string(output), "syntax check OK") {
|
||||
return fmt.Errorf("config validation did not report OK: %s", string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteInstanceConfig writes the DNS configuration for a single instance
|
||||
func (g *ConfigGenerator) WriteInstanceConfig(instanceName string, instance config.InstanceConfig) error {
|
||||
instanceFile := filepath.Join(instanceConfigDir, fmt.Sprintf("%s.conf", instanceName))
|
||||
configContent := g.GenerateInstanceConfig(instance)
|
||||
|
||||
// Ensure directory exists
|
||||
if err := os.MkdirAll(instanceConfigDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating instance config directory: %w", err)
|
||||
}
|
||||
|
||||
// Write to temp file first
|
||||
tempFile := instanceFile + ".tmp"
|
||||
if err := os.WriteFile(tempFile, []byte(configContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing temp instance config: %w", err)
|
||||
}
|
||||
|
||||
// Validate the temp config along with main config
|
||||
if err := g.ValidateWithInstance(tempFile); err != nil {
|
||||
os.Remove(tempFile) // Clean up temp file
|
||||
return fmt.Errorf("instance config validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Move temp file to final location (atomic operation)
|
||||
if err := os.Rename(tempFile, instanceFile); err != nil {
|
||||
os.Remove(tempFile) // Clean up temp file
|
||||
return fmt.Errorf("installing instance config: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("wrote instance DNS config", "component", "dnsmasq", "path", instanceFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateWithInstance validates the main config along with a specific instance config
|
||||
func (g *ConfigGenerator) ValidateWithInstance(instanceConfigPath string) error {
|
||||
// Create a temporary test directory
|
||||
tempDir, err := os.MkdirTemp("", "dnsmasq-test-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Copy main config to temp
|
||||
mainContent, err := os.ReadFile(g.configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading main config: %w", err)
|
||||
}
|
||||
|
||||
tempMainConfig := filepath.Join(tempDir, "main.conf")
|
||||
// Modify the conf-dir line to point to our temp instance dir
|
||||
tempInstanceDir := filepath.Join(tempDir, "instances")
|
||||
if err := os.MkdirAll(tempInstanceDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating temp instance dir: %w", err)
|
||||
}
|
||||
|
||||
modifiedContent := strings.ReplaceAll(
|
||||
string(mainContent),
|
||||
fmt.Sprintf("conf-dir=%s,*.conf", instanceConfigDir),
|
||||
fmt.Sprintf("conf-dir=%s,*.conf", tempInstanceDir),
|
||||
)
|
||||
|
||||
if err := os.WriteFile(tempMainConfig, []byte(modifiedContent), 0644); err != nil {
|
||||
return fmt.Errorf("writing temp main config: %w", err)
|
||||
}
|
||||
|
||||
// Copy instance config to temp
|
||||
instanceContent, err := os.ReadFile(instanceConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading instance config: %w", err)
|
||||
}
|
||||
|
||||
tempInstanceConfig := filepath.Join(tempInstanceDir, filepath.Base(instanceConfigPath))
|
||||
if err := os.WriteFile(tempInstanceConfig, instanceContent, 0644); err != nil {
|
||||
return fmt.Errorf("writing temp instance config: %w", err)
|
||||
}
|
||||
|
||||
// Validate the combined configuration
|
||||
return g.ValidateConfig(tempMainConfig)
|
||||
}
|
||||
|
||||
// RemoveInstanceConfig removes the DNS configuration for an instance
|
||||
func (g *ConfigGenerator) RemoveInstanceConfig(instanceName string) error {
|
||||
instanceFile := filepath.Join(instanceConfigDir, fmt.Sprintf("%s.conf", instanceName))
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(instanceFile); os.IsNotExist(err) {
|
||||
slog.Info("instance DNS config does not exist", "component", "dnsmasq", "path", instanceFile)
|
||||
return nil // Not an error, already removed
|
||||
}
|
||||
|
||||
// Remove the file
|
||||
if err := os.Remove(instanceFile); err != nil {
|
||||
return fmt.Errorf("removing instance config: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("removed instance DNS config", "component", "dnsmasq", "path", instanceFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReloadService sends a SIGHUP to dnsmasq to reload configuration
|
||||
// This is lighter weight than a full restart
|
||||
func (g *ConfigGenerator) ReloadService() error {
|
||||
// Use systemctl reload which sends SIGHUP
|
||||
cmd := exec.Command("systemctl", "reload", "dnsmasq.service")
|
||||
_, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
// If reload fails, try restart as fallback
|
||||
slog.Error("reload failed, attempting restart", "component", "dnsmasq", "error", err)
|
||||
return g.RestartService()
|
||||
}
|
||||
slog.Info("dnsmasq service reloaded", "component", "dnsmasq")
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateToModularConfig migrates from monolithic to modular configuration
|
||||
func (g *ConfigGenerator) UpdateToModularConfig(cfg *config.GlobalConfig, instanceNames []string, instances []config.InstanceConfig) error {
|
||||
slog.Info("migrating to modular configuration", "component", "dnsmasq")
|
||||
|
||||
// Ensure instance directory exists
|
||||
if err := os.MkdirAll(instanceConfigDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating instance config directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate and write instance configs first (but don't reload yet)
|
||||
for i, instance := range instances {
|
||||
instanceName := instanceNames[i]
|
||||
if err := g.WriteInstanceConfig(instanceName, instance); err != nil {
|
||||
slog.Error("failed to write instance config", "component", "dnsmasq", "instance", instanceName, "error", err)
|
||||
// Continue with other instances
|
||||
}
|
||||
}
|
||||
|
||||
// Generate new main config with conf-dir
|
||||
mainConfig := g.GenerateMainConfig(cfg)
|
||||
|
||||
// Write to temp file first
|
||||
tempFile := g.configPath + ".tmp"
|
||||
if err := os.WriteFile(tempFile, []byte(mainConfig), 0644); err != nil {
|
||||
return fmt.Errorf("writing temp main config: %w", err)
|
||||
}
|
||||
|
||||
// Validate the new config
|
||||
if err := g.ValidateConfig(tempFile); err != nil {
|
||||
os.Remove(tempFile)
|
||||
return fmt.Errorf("main config validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Backup current config
|
||||
backupFile := g.configPath + ".pre-modular"
|
||||
if err := os.Rename(g.configPath, backupFile); err != nil {
|
||||
os.Remove(tempFile)
|
||||
return fmt.Errorf("backing up current config: %w", err)
|
||||
}
|
||||
|
||||
// Install new config
|
||||
if err := os.Rename(tempFile, g.configPath); err != nil {
|
||||
// Try to restore backup
|
||||
_ = os.Rename(backupFile, g.configPath)
|
||||
return fmt.Errorf("installing new config: %w", err)
|
||||
}
|
||||
|
||||
// Reload dnsmasq
|
||||
if err := g.ReloadService(); err != nil {
|
||||
// Try to restore backup and reload
|
||||
slog.Error("reload failed, restoring backup", "component", "dnsmasq", "error", err)
|
||||
os.Remove(g.configPath)
|
||||
_ = os.Rename(backupFile, g.configPath)
|
||||
_ = g.ReloadService()
|
||||
return fmt.Errorf("reloading with new config: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("migrated to modular configuration", "component", "dnsmasq")
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateInstanceDNS updates DNS configuration for a single instance
|
||||
// This is called when instance configuration changes (e.g., domain names)
|
||||
func (g *ConfigGenerator) UpdateInstanceDNS(instanceName string, instance config.InstanceConfig) error {
|
||||
// Write the new instance config
|
||||
if err := g.WriteInstanceConfig(instanceName, instance); err != nil {
|
||||
return fmt.Errorf("writing instance DNS config: %w", err)
|
||||
}
|
||||
|
||||
// Reload dnsmasq to pick up changes
|
||||
if err := g.ReloadService(); err != nil {
|
||||
return fmt.Errorf("reloading dnsmasq: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("DNS updated for instance", "component", "dnsmasq", "instance", instanceName)
|
||||
return nil
|
||||
}
|
||||
292
api/internal/factory/factory.go
Normal file
292
api/internal/factory/factory.go
Normal file
@@ -0,0 +1,292 @@
|
||||
// Package factory provides a client for the Talos Image Factory API (factory.talos.dev).
|
||||
// It handles version discovery, schematic retrieval, and schematic-version compatibility validation.
|
||||
package factory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Client wraps the Talos Image Factory API
|
||||
type Client struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
cache *versionCache
|
||||
}
|
||||
|
||||
type versionCache struct {
|
||||
mu sync.RWMutex
|
||||
versions []string
|
||||
fetchedAt time.Time
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// Extension represents a Talos system extension available for a version
|
||||
type Extension struct {
|
||||
Name string `json:"name"`
|
||||
Ref string `json:"ref"`
|
||||
Digest string `json:"digest"`
|
||||
Author string `json:"author"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// Schematic represents a Talos Image Factory schematic
|
||||
type Schematic struct {
|
||||
Customization struct {
|
||||
SystemExtensions struct {
|
||||
OfficialExtensions []string `yaml:"officialExtensions"`
|
||||
} `yaml:"systemExtensions"`
|
||||
} `yaml:"customization"`
|
||||
}
|
||||
|
||||
// ValidationResult describes whether a schematic is compatible with a Talos version
|
||||
type ValidationResult struct {
|
||||
Valid bool `json:"valid"`
|
||||
MissingExtensions []string `json:"missingExtensions,omitempty"`
|
||||
}
|
||||
|
||||
// NewClient creates a new Image Factory client
|
||||
func NewClient() *Client {
|
||||
return &Client{
|
||||
baseURL: "https://factory.talos.dev",
|
||||
httpClient: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
cache: &versionCache{
|
||||
ttl: 1 * time.Hour,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetVersions returns all available Talos versions from the Image Factory.
|
||||
// Results are cached for 1 hour.
|
||||
func (c *Client) GetVersions() ([]string, error) {
|
||||
c.cache.mu.RLock()
|
||||
if c.cache.versions != nil && time.Since(c.cache.fetchedAt) < c.cache.ttl {
|
||||
versions := make([]string, len(c.cache.versions))
|
||||
copy(versions, c.cache.versions)
|
||||
c.cache.mu.RUnlock()
|
||||
return versions, nil
|
||||
}
|
||||
c.cache.mu.RUnlock()
|
||||
|
||||
resp, err := c.httpClient.Get(c.baseURL + "/versions")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching versions: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fetching versions: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var versions []string
|
||||
if err := json.NewDecoder(resp.Body).Decode(&versions); err != nil {
|
||||
return nil, fmt.Errorf("decoding versions: %w", err)
|
||||
}
|
||||
|
||||
c.cache.mu.Lock()
|
||||
c.cache.versions = versions
|
||||
c.cache.fetchedAt = time.Now()
|
||||
c.cache.mu.Unlock()
|
||||
|
||||
result := make([]string, len(versions))
|
||||
copy(result, versions)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetStableVersions returns recent stable Talos versions, filtered and sorted.
|
||||
// Excludes alpha/beta releases and keeps only the last 3 minor version lines.
|
||||
func (c *Client) GetStableVersions() ([]string, error) {
|
||||
all, err := c.GetVersions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return FilterStableVersions(all, 3), nil
|
||||
}
|
||||
|
||||
// FilterStableVersions filters out pre-release versions, keeps the last N minor
|
||||
// version lines, and sorts descending. Exported for testing.
|
||||
func FilterStableVersions(versions []string, minorLines int) []string {
|
||||
// Filter out pre-release versions
|
||||
var stable []string
|
||||
for _, v := range versions {
|
||||
if !strings.Contains(v, "-alpha") && !strings.Contains(v, "-beta") {
|
||||
stable = append(stable, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort descending by semver
|
||||
sort.Slice(stable, func(i, j int) bool {
|
||||
return CompareVersions(stable[i], stable[j]) > 0
|
||||
})
|
||||
|
||||
if minorLines <= 0 {
|
||||
return stable
|
||||
}
|
||||
|
||||
// Keep only the last N minor version lines
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
for _, v := range stable {
|
||||
minor := extractMinorLine(v)
|
||||
if minor == "" {
|
||||
continue
|
||||
}
|
||||
if !seen[minor] {
|
||||
if len(seen) >= minorLines {
|
||||
break
|
||||
}
|
||||
seen[minor] = true
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetExtensions returns official system extensions available for a Talos version
|
||||
func (c *Client) GetExtensions(version string) ([]Extension, error) {
|
||||
resp, err := c.httpClient.Get(fmt.Sprintf("%s/version/%s/extensions/official", c.baseURL, version))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching extensions: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fetching extensions for %s: status %d", version, resp.StatusCode)
|
||||
}
|
||||
|
||||
var extensions []Extension
|
||||
if err := json.NewDecoder(resp.Body).Decode(&extensions); err != nil {
|
||||
return nil, fmt.Errorf("decoding extensions: %w", err)
|
||||
}
|
||||
|
||||
return extensions, nil
|
||||
}
|
||||
|
||||
// GetSchematic retrieves a schematic by its content-addressed ID
|
||||
func (c *Client) GetSchematic(id string) (*Schematic, error) {
|
||||
resp, err := c.httpClient.Get(fmt.Sprintf("%s/schematics/%s", c.baseURL, id))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching schematic: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("schematic %s not found", id)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fetching schematic %s: status %d", id, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading schematic response: %w", err)
|
||||
}
|
||||
|
||||
var schematic Schematic
|
||||
if err := yaml.Unmarshal(body, &schematic); err != nil {
|
||||
return nil, fmt.Errorf("parsing schematic YAML: %w", err)
|
||||
}
|
||||
|
||||
return &schematic, nil
|
||||
}
|
||||
|
||||
// ValidateSchematic checks whether a schematic's extensions are available for a given Talos version.
|
||||
// An empty schematic (no extensions) is always valid.
|
||||
func (c *Client) ValidateSchematic(schematicID, version string) (*ValidationResult, error) {
|
||||
schematic, err := c.GetSchematic(schematicID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting schematic: %w", err)
|
||||
}
|
||||
|
||||
requiredExtensions := schematic.Customization.SystemExtensions.OfficialExtensions
|
||||
if len(requiredExtensions) == 0 {
|
||||
return &ValidationResult{Valid: true}, nil
|
||||
}
|
||||
|
||||
extensions, err := c.GetExtensions(version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting extensions for %s: %w", version, err)
|
||||
}
|
||||
|
||||
// Build set of available extension names
|
||||
// Schematics store short names like "siderolabs/gvisor" which match the Name field,
|
||||
// not the Ref field which is a full OCI reference like "ghcr.io/siderolabs/gvisor:v1.11.5"
|
||||
available := make(map[string]bool)
|
||||
for _, ext := range extensions {
|
||||
available[ext.Name] = true
|
||||
}
|
||||
|
||||
// Check each required extension
|
||||
var missing []string
|
||||
for _, name := range requiredExtensions {
|
||||
if !available[name] {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
|
||||
return &ValidationResult{
|
||||
Valid: len(missing) == 0,
|
||||
MissingExtensions: missing,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetInstallerImage returns the OCI installer image reference for a schematic and version.
|
||||
// Used with talosctl upgrade --image. OCI references must not include the https:// scheme.
|
||||
func (c *Client) GetInstallerImage(schematicID, version string) string {
|
||||
host := strings.TrimPrefix(strings.TrimPrefix(c.baseURL, "https://"), "http://")
|
||||
return fmt.Sprintf("%s/installer/%s:%s", host, schematicID, version)
|
||||
}
|
||||
|
||||
// CompareVersions compares two semver strings. Returns >0 if a > b, <0 if a < b, 0 if equal.
|
||||
func CompareVersions(a, b string) int {
|
||||
partsA := ParseVersion(a)
|
||||
partsB := ParseVersion(b)
|
||||
|
||||
for i := range 3 {
|
||||
if partsA[i] != partsB[i] {
|
||||
return partsA[i] - partsB[i]
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ParseVersion extracts major, minor, patch from a version string like "v1.12.3"
|
||||
func ParseVersion(v string) [3]int {
|
||||
v = strings.TrimPrefix(v, "v")
|
||||
// Strip any pre-release suffix
|
||||
if idx := strings.IndexByte(v, '-'); idx >= 0 {
|
||||
v = v[:idx]
|
||||
}
|
||||
var parts [3]int
|
||||
_, _ = fmt.Sscanf(v, "%d.%d.%d", &parts[0], &parts[1], &parts[2])
|
||||
return parts
|
||||
}
|
||||
|
||||
// MinorVersionGap returns the absolute difference in minor versions between two version strings.
|
||||
// For example, "v1.11.2" and "v1.13.0" returns 2.
|
||||
func MinorVersionGap(a, b string) int {
|
||||
partsA := ParseVersion(a)
|
||||
partsB := ParseVersion(b)
|
||||
gap := partsA[1] - partsB[1]
|
||||
if gap < 0 {
|
||||
gap = -gap
|
||||
}
|
||||
return gap
|
||||
}
|
||||
|
||||
// extractMinorLine returns the "major.minor" portion of a version (e.g., "v1.12.3" -> "1.12")
|
||||
func extractMinorLine(v string) string {
|
||||
parts := ParseVersion(v)
|
||||
return fmt.Sprintf("%d.%d", parts[0], parts[1])
|
||||
}
|
||||
149
api/internal/factory/factory_test.go
Normal file
149
api/internal/factory/factory_test.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package factory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFilterStableVersions_RemovesPreRelease(t *testing.T) {
|
||||
versions := []string{"v1.12.0", "v1.13.0-alpha.0", "v1.11.5", "v1.11.0-beta.1"}
|
||||
result := FilterStableVersions(versions, 10)
|
||||
|
||||
for _, v := range result {
|
||||
if v == "v1.13.0-alpha.0" || v == "v1.11.0-beta.1" {
|
||||
t.Errorf("pre-release version %s should have been filtered", v)
|
||||
}
|
||||
}
|
||||
if len(result) != 2 {
|
||||
t.Errorf("expected 2 versions, got %d: %v", len(result), result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterStableVersions_SortsDescending(t *testing.T) {
|
||||
versions := []string{"v1.10.0", "v1.12.0", "v1.11.5", "v1.11.0", "v1.12.3"}
|
||||
result := FilterStableVersions(versions, 10)
|
||||
|
||||
expected := []string{"v1.12.3", "v1.12.0", "v1.11.5", "v1.11.0", "v1.10.0"}
|
||||
if len(result) != len(expected) {
|
||||
t.Fatalf("expected %d versions, got %d: %v", len(expected), len(result), result)
|
||||
}
|
||||
for i, v := range result {
|
||||
if v != expected[i] {
|
||||
t.Errorf("position %d: expected %s, got %s", i, expected[i], v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterStableVersions_LimitsMinorLines(t *testing.T) {
|
||||
versions := []string{
|
||||
"v1.12.7", "v1.12.6", "v1.12.5",
|
||||
"v1.11.6", "v1.11.5", "v1.11.4",
|
||||
"v1.10.9", "v1.10.8",
|
||||
"v1.9.6", "v1.9.5",
|
||||
"v1.8.4",
|
||||
}
|
||||
result := FilterStableVersions(versions, 3)
|
||||
|
||||
// Should only include v1.12.x, v1.11.x, v1.10.x
|
||||
for _, v := range result {
|
||||
parts := ParseVersion(v)
|
||||
if parts[1] < 10 {
|
||||
t.Errorf("version %s should not be included (minor line %d.%d is outside top 3)", v, parts[0], parts[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Should have all versions from those 3 lines
|
||||
if len(result) != 8 {
|
||||
t.Errorf("expected 8 versions from 3 minor lines, got %d: %v", len(result), result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterStableVersions_EmptyInput(t *testing.T) {
|
||||
result := FilterStableVersions(nil, 3)
|
||||
if len(result) != 0 {
|
||||
t.Errorf("expected empty result, got %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterStableVersions_ZeroMinorLines(t *testing.T) {
|
||||
versions := []string{"v1.12.0", "v1.11.0"}
|
||||
result := FilterStableVersions(versions, 0)
|
||||
if len(result) != 2 {
|
||||
t.Errorf("expected all versions with minorLines=0, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareVersions(t *testing.T) {
|
||||
tests := []struct {
|
||||
a, b string
|
||||
want int // >0, <0, or 0
|
||||
}{
|
||||
{"v1.12.0", "v1.11.0", 1},
|
||||
{"v1.11.0", "v1.12.0", -1},
|
||||
{"v1.11.5", "v1.11.5", 0},
|
||||
{"v1.11.5", "v1.11.4", 1},
|
||||
{"v1.12.0", "v1.11.9", 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := CompareVersions(tt.a, tt.b)
|
||||
if (tt.want > 0 && got <= 0) || (tt.want < 0 && got >= 0) || (tt.want == 0 && got != 0) {
|
||||
t.Errorf("CompareVersions(%s, %s) = %d, want sign %d", tt.a, tt.b, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want [3]int
|
||||
}{
|
||||
{"v1.12.3", [3]int{1, 12, 3}},
|
||||
{"1.11.5", [3]int{1, 11, 5}},
|
||||
{"v1.14.0-alpha.0", [3]int{1, 14, 0}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := ParseVersion(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseVersion(%s) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMinorLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"v1.12.3", "1.12"},
|
||||
{"v1.11.0", "1.11"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := extractMinorLine(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("extractMinorLine(%s) = %s, want %s", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinorVersionGap(t *testing.T) {
|
||||
tests := []struct {
|
||||
a, b string
|
||||
want int
|
||||
}{
|
||||
{"v1.11.2", "v1.13.0", 2},
|
||||
{"v1.13.0", "v1.11.2", 2},
|
||||
{"v1.11.5", "v1.12.0", 1},
|
||||
{"v1.12.0", "v1.12.5", 0},
|
||||
{"v1.11.2", "v1.11.2", 0},
|
||||
{"v1.10.0", "v1.13.0", 3},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := MinorVersionGap(tt.a, tt.b)
|
||||
if got != tt.want {
|
||||
t.Errorf("MinorVersionGap(%s, %s) = %d, want %d", tt.a, tt.b, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package instance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
@@ -74,26 +75,47 @@ func (m *Manager) CreateInstance(name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
slog.Info("creating instance", "component", "instance", "name", name)
|
||||
|
||||
// Acquire lock for instance creation
|
||||
lockPath := tools.GetInstancesLockPath(m.dataDir)
|
||||
return storage.WithLock(lockPath, func() error {
|
||||
// Create instance directory
|
||||
if err := storage.EnsureDir(instancePath, 0755); err != nil {
|
||||
return fmt.Errorf("creating instance directory: %w", err)
|
||||
// Create instance directory and config/ subdirectory
|
||||
configDir := tools.GetInstanceConfigDirPath(m.dataDir, name)
|
||||
if err := storage.EnsureDir(configDir, 0755); err != nil {
|
||||
return fmt.Errorf("creating instance config directory: %w", err)
|
||||
}
|
||||
|
||||
// Create config file
|
||||
if err := m.configMgr.EnsureInstanceConfig(instancePath); err != nil {
|
||||
return fmt.Errorf("creating config file: %w", err)
|
||||
// Create config/instance.yaml
|
||||
configPath := tools.GetInstanceConfigPath(m.dataDir, name)
|
||||
if !storage.FileExists(configPath) {
|
||||
initialConfig := &config.InstanceConfig{}
|
||||
initialConfig.Cluster.Name = name
|
||||
initialConfig.Cluster.Nodes.Active = make(map[string]config.NodeConfig)
|
||||
initialConfig.Apps = make(map[string]interface{})
|
||||
if err := config.SaveCloudConfig(initialConfig, configPath); err != nil {
|
||||
return fmt.Errorf("creating config file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create secrets file
|
||||
if err := m.secretsMgr.EnsureSecretsFile(instancePath); err != nil {
|
||||
return fmt.Errorf("creating secrets file: %w", err)
|
||||
// Create config/secrets.yaml
|
||||
secretsPath := tools.GetInstanceSecretsPath(m.dataDir, name)
|
||||
if !storage.FileExists(secretsPath) {
|
||||
initialSecrets := `# Wild Cloud Instance Secrets
|
||||
# WARNING: This file contains sensitive data. Keep secure!
|
||||
cluster:
|
||||
talosSecrets: ""
|
||||
kubeconfig: ""
|
||||
cloudflare:
|
||||
token: ""
|
||||
`
|
||||
if err := storage.WriteFile(secretsPath, []byte(initialSecrets), 0600); err != nil {
|
||||
return fmt.Errorf("creating secrets file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create subdirectories
|
||||
subdirs := []string{"talos", "k8s", "logs", "backups"}
|
||||
// Create subdirectories (talos only; k8s, logs, backups removed per XDG layout)
|
||||
subdirs := []string{"talos"}
|
||||
for _, subdir := range subdirs {
|
||||
subdirPath := filepath.Join(instancePath, subdir)
|
||||
if err := storage.EnsureDir(subdirPath, 0755); err != nil {
|
||||
@@ -118,6 +140,8 @@ func (m *Manager) DeleteInstance(name string) error {
|
||||
return fmt.Errorf("instance %s does not exist", name)
|
||||
}
|
||||
|
||||
slog.Info("deleting instance", "component", "instance", "name", name)
|
||||
|
||||
// Clear context if this is the current instance
|
||||
currentContext, err := m.contextMgr.GetCurrentContext()
|
||||
if err == nil && currentContext == name {
|
||||
|
||||
@@ -22,10 +22,8 @@ func TestManager_CreateInstance(t *testing.T) {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
expectedDirs := []string{
|
||||
instancePath,
|
||||
filepath.Join(instancePath, "config"),
|
||||
filepath.Join(instancePath, "talos"),
|
||||
filepath.Join(instancePath, "k8s"),
|
||||
filepath.Join(instancePath, "logs"),
|
||||
filepath.Join(instancePath, "backups"),
|
||||
}
|
||||
|
||||
for _, dir := range expectedDirs {
|
||||
@@ -174,3 +172,4 @@ func TestManager_ValidateInstance(t *testing.T) {
|
||||
t.Logf("ValidateInstance failed (likely yq not installed): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
138
api/internal/logging/console.go
Normal file
138
api/internal/logging/console.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ANSI color codes
|
||||
const (
|
||||
dim = "\033[2m"
|
||||
red = "\033[31m"
|
||||
yellow = "\033[33m"
|
||||
cyan = "\033[36m"
|
||||
reset = "\033[0m"
|
||||
)
|
||||
|
||||
// ConsoleHandler formats log output for human readability on terminals.
|
||||
// It produces compact, color-coded lines:
|
||||
//
|
||||
// 20:15:54 INF daemon started addr=:5055
|
||||
// 20:15:54 ERR backup failed component=backup error="connection refused"
|
||||
type ConsoleHandler struct {
|
||||
w io.Writer
|
||||
level slog.Leveler
|
||||
attrs []slog.Attr
|
||||
mu *sync.Mutex
|
||||
}
|
||||
|
||||
// NewConsoleHandler creates a handler that writes human-friendly colored logs.
|
||||
func NewConsoleHandler(w io.Writer, opts *slog.HandlerOptions) *ConsoleHandler {
|
||||
level := slog.LevelInfo
|
||||
if opts != nil && opts.Level != nil {
|
||||
level = opts.Level.Level()
|
||||
}
|
||||
return &ConsoleHandler{
|
||||
w: w,
|
||||
level: level,
|
||||
mu: &sync.Mutex{},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ConsoleHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||
return level >= h.level.Level()
|
||||
}
|
||||
|
||||
func (h *ConsoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
// Time
|
||||
buf := []byte(dim + r.Time.Format("15:04:05") + reset + " ")
|
||||
|
||||
// Level badge
|
||||
switch {
|
||||
case r.Level >= slog.LevelError:
|
||||
buf = append(buf, red+"ERR"+reset+" "...)
|
||||
case r.Level >= slog.LevelWarn:
|
||||
buf = append(buf, yellow+"WRN"+reset+" "...)
|
||||
default:
|
||||
buf = append(buf, cyan+"INF"+reset+" "...)
|
||||
}
|
||||
|
||||
// Message
|
||||
buf = append(buf, r.Message...)
|
||||
|
||||
// Pre-set attrs (from slog.With)
|
||||
for _, a := range h.attrs {
|
||||
buf = appendAttr(buf, a)
|
||||
}
|
||||
|
||||
// Inline attrs
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
buf = appendAttr(buf, a)
|
||||
return true
|
||||
})
|
||||
|
||||
buf = append(buf, '\n')
|
||||
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
_, err := h.w.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *ConsoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||
return &ConsoleHandler{
|
||||
w: h.w,
|
||||
level: h.level,
|
||||
attrs: append(slices.Clone(h.attrs), attrs...),
|
||||
mu: h.mu,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ConsoleHandler) WithGroup(name string) slog.Handler {
|
||||
// Groups are rare in this codebase; treat as a prefixed attr set
|
||||
return &ConsoleHandler{
|
||||
w: h.w,
|
||||
level: h.level,
|
||||
attrs: append(slices.Clone(h.attrs), slog.String("group", name)),
|
||||
mu: h.mu,
|
||||
}
|
||||
}
|
||||
|
||||
func appendAttr(buf []byte, a slog.Attr) []byte {
|
||||
if a.Equal(slog.Attr{}) {
|
||||
return buf
|
||||
}
|
||||
v := a.Value.Resolve()
|
||||
buf = append(buf, ' ')
|
||||
buf = append(buf, dim...)
|
||||
buf = append(buf, a.Key...)
|
||||
buf = append(buf, '=')
|
||||
buf = append(buf, reset...)
|
||||
|
||||
s := v.String()
|
||||
if needsQuote(s) {
|
||||
buf = append(buf, fmt.Sprintf("%q", s)...)
|
||||
} else {
|
||||
buf = append(buf, s...)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func needsQuote(s string) bool {
|
||||
if s == "" {
|
||||
return true
|
||||
}
|
||||
for _, c := range s {
|
||||
if c <= ' ' || c == '"' || c == '\\' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Verify interface compliance at compile time.
|
||||
var _ slog.Handler = (*ConsoleHandler)(nil)
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
type NetworkInfo struct {
|
||||
PrimaryIP string `json:"primary_ip"`
|
||||
PrimaryInterface string `json:"primary_interface"`
|
||||
Gateway string `json:"gateway"`
|
||||
}
|
||||
|
||||
// DetectNetworkInfo detects the machine's primary network configuration
|
||||
@@ -28,6 +29,16 @@ func DetectNetworkInfo() (*NetworkInfo, error) {
|
||||
line := string(output)
|
||||
// Example output: "8.8.8.8 via 192.168.8.1 dev wlan0 src 192.168.8.152 uid 1000"
|
||||
|
||||
// Extract gateway (after "via")
|
||||
gateway := ""
|
||||
if idx := strings.Index(line, "via "); idx != -1 {
|
||||
rest := line[idx+4:]
|
||||
parts := strings.Fields(rest)
|
||||
if len(parts) > 0 {
|
||||
gateway = parts[0]
|
||||
}
|
||||
}
|
||||
|
||||
// Extract interface (after "dev")
|
||||
iface := ""
|
||||
if idx := strings.Index(line, "dev "); idx != -1 {
|
||||
@@ -56,6 +67,7 @@ func DetectNetworkInfo() (*NetworkInfo, error) {
|
||||
return &NetworkInfo{
|
||||
PrimaryIP: ip,
|
||||
PrimaryInterface: iface,
|
||||
Gateway: gateway,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -80,3 +92,13 @@ func ResolveDomain(domain string) (string, error) {
|
||||
// If no IPv4, return first IP
|
||||
return ips[0].String(), nil
|
||||
}
|
||||
|
||||
// GetWildCentralIP returns the IP address of the Wild Central server
|
||||
// This is the primary IP address that the server uses for network communication
|
||||
func GetWildCentralIP() (string, error) {
|
||||
netInfo, err := DetectNetworkInfo()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to detect Wild Central IP: %w", err)
|
||||
}
|
||||
return netInfo.PrimaryIP, nil
|
||||
}
|
||||
|
||||
@@ -3,15 +3,18 @@ package node
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/config"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/setup"
|
||||
"github.com/wild-cloud/wild-central/daemon/internal/tools"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Manager handles node configuration and state management
|
||||
@@ -75,83 +78,86 @@ func (m *Manager) GetInstancePath(instanceName string) string {
|
||||
return tools.GetInstancePath(m.dataDir, instanceName)
|
||||
}
|
||||
|
||||
// nodeConfigFile is the operator-authored portion stored in config/nodes/{hostname}.yaml
|
||||
type nodeConfigFile struct {
|
||||
Role string `yaml:"role"`
|
||||
Disk string `yaml:"disk"`
|
||||
Interface string `yaml:"interface,omitempty"`
|
||||
TargetIP string `yaml:"targetIp,omitempty"`
|
||||
SchematicID string `yaml:"schematicId,omitempty"`
|
||||
Version string `yaml:"version,omitempty"`
|
||||
Maintenance bool `yaml:"maintenance,omitempty"`
|
||||
}
|
||||
|
||||
// nodeCacheFile is the daemon-written state stored in cache/nodes/{hostname}.yaml
|
||||
type nodeCacheFile struct {
|
||||
Applied bool `yaml:"applied,omitempty"`
|
||||
Configured bool `yaml:"configured,omitempty"`
|
||||
CurrentIP string `yaml:"currentIp,omitempty"`
|
||||
}
|
||||
|
||||
// List returns all nodes for an instance
|
||||
func (m *Manager) List(instanceName string) ([]Node, error) {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
return m.listFromNodeFiles(instanceName)
|
||||
}
|
||||
|
||||
yq := tools.NewYQ()
|
||||
|
||||
// Get all node hostnames from cluster.nodes.active
|
||||
output, err := yq.Exec("eval", ".cluster.nodes.active | keys", configPath)
|
||||
if err != nil {
|
||||
// If the path doesn't exist or is null, return empty list
|
||||
if strings.Contains(err.Error(), "cannot get keys") || strings.Contains(err.Error(), "!!null") {
|
||||
return []Node{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read nodes: %w", err)
|
||||
}
|
||||
|
||||
// Parse hostnames (yq returns YAML array)
|
||||
hostnamesYAML := string(output)
|
||||
hostnamesYAML = strings.TrimSpace(hostnamesYAML)
|
||||
if hostnamesYAML == "" || hostnamesYAML == "null" || hostnamesYAML == "[]" {
|
||||
// listFromNodeFiles reads nodes from config/nodes/*.yaml + cache/nodes/*.yaml
|
||||
func (m *Manager) listFromNodeFiles(instanceName string) ([]Node, error) {
|
||||
configDir := tools.GetNodeConfigDirPath(m.dataDir, instanceName)
|
||||
entries, err := os.ReadDir(configDir)
|
||||
if os.IsNotExist(err) {
|
||||
return []Node{}, nil
|
||||
}
|
||||
|
||||
// Get hostnames line by line
|
||||
var hostnames []string
|
||||
for _, line := range strings.Split(hostnamesYAML, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" && line != "null" && line != "-" {
|
||||
// Remove leading "- " from YAML array
|
||||
hostname := line
|
||||
if len(hostname) > 2 && hostname[0:2] == "- " {
|
||||
hostname = hostname[2:]
|
||||
}
|
||||
if hostname != "" {
|
||||
hostnames = append(hostnames, hostname)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read node config dir: %w", err)
|
||||
}
|
||||
|
||||
// Get details for each node
|
||||
var nodes []Node
|
||||
for _, hostname := range hostnames {
|
||||
basePath := fmt.Sprintf(".cluster.nodes.active.%s", hostname)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".yaml" {
|
||||
continue
|
||||
}
|
||||
hostname := entry.Name()[:len(entry.Name())-5] // strip .yaml
|
||||
|
||||
// Get node fields
|
||||
role, _ := yq.Exec("eval", basePath+".role", configPath)
|
||||
targetIP, _ := yq.Exec("eval", basePath+".targetIp", configPath)
|
||||
currentIP, _ := yq.Exec("eval", basePath+".currentIp", configPath)
|
||||
disk, _ := yq.Exec("eval", basePath+".disk", configPath)
|
||||
iface, _ := yq.Exec("eval", basePath+".interface", configPath)
|
||||
version, _ := yq.Exec("eval", basePath+".version", configPath)
|
||||
schematicID, _ := yq.Exec("eval", basePath+".schematicId", configPath)
|
||||
maintenance, _ := yq.Exec("eval", basePath+".maintenance", configPath)
|
||||
configured, _ := yq.Exec("eval", basePath+".configured", configPath)
|
||||
applied, _ := yq.Exec("eval", basePath+".applied", configPath)
|
||||
data, err := os.ReadFile(filepath.Join(configDir, entry.Name()))
|
||||
if err != nil {
|
||||
slog.Warn("failed to read node config", "hostname", hostname, "error", err)
|
||||
continue
|
||||
}
|
||||
var cfg nodeConfigFile
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
slog.Warn("failed to parse node config", "hostname", hostname, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
node := Node{
|
||||
Hostname: hostname,
|
||||
Role: tools.CleanYQOutput(string(role)),
|
||||
TargetIP: tools.CleanYQOutput(string(targetIP)),
|
||||
CurrentIP: tools.CleanYQOutput(string(currentIP)),
|
||||
Disk: tools.CleanYQOutput(string(disk)),
|
||||
Interface: tools.CleanYQOutput(string(iface)),
|
||||
Version: tools.CleanYQOutput(string(version)),
|
||||
SchematicID: tools.CleanYQOutput(string(schematicID)),
|
||||
Maintenance: tools.CleanYQOutput(string(maintenance)) == "true",
|
||||
Configured: tools.CleanYQOutput(string(configured)) == "true",
|
||||
Applied: tools.CleanYQOutput(string(applied)) == "true",
|
||||
Role: cfg.Role,
|
||||
Disk: cfg.Disk,
|
||||
Interface: cfg.Interface,
|
||||
TargetIP: cfg.TargetIP,
|
||||
SchematicID: cfg.SchematicID,
|
||||
Version: cfg.Version,
|
||||
Maintenance: cfg.Maintenance,
|
||||
}
|
||||
|
||||
// Merge cache
|
||||
cacheData, err := os.ReadFile(tools.GetNodeCachePath(m.dataDir, instanceName, hostname))
|
||||
if err == nil {
|
||||
var cache nodeCacheFile
|
||||
if err := yaml.Unmarshal(cacheData, &cache); err == nil {
|
||||
node.Applied = cache.Applied
|
||||
node.Configured = cache.Configured
|
||||
node.CurrentIP = cache.CurrentIP
|
||||
}
|
||||
}
|
||||
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
|
||||
// Get returns a specific node by hostname
|
||||
func (m *Manager) Get(instanceName, hostname string) (*Node, error) {
|
||||
// Get all nodes
|
||||
@@ -170,9 +176,9 @@ func (m *Manager) Get(instanceName, hostname string) (*Node, error) {
|
||||
return nil, fmt.Errorf("node %s not found", hostname)
|
||||
}
|
||||
|
||||
// Add registers a new node in config.yaml
|
||||
// Add registers a new node
|
||||
func (m *Manager) Add(instanceName string, node *Node) error {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
slog.Info("adding node", "component", "node", "instance", instanceName, "hostname", node.Hostname, "role", node.Role)
|
||||
|
||||
// Validate node data
|
||||
if node.Hostname == "" {
|
||||
@@ -191,78 +197,67 @@ func (m *Manager) Add(instanceName string, node *Node) error {
|
||||
return fmt.Errorf("node %s already exists", node.Hostname)
|
||||
}
|
||||
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
yq := tools.NewYQ()
|
||||
|
||||
// If schematicId not provided, use instance-level default from cluster.nodes.talos.schematicId
|
||||
if node.SchematicID == "" {
|
||||
defaultSchematicID, err := yq.Get(configPath, ".cluster.nodes.talos.schematicId")
|
||||
if err == nil && defaultSchematicID != "" && defaultSchematicID != "null" {
|
||||
node.SchematicID = defaultSchematicID
|
||||
}
|
||||
}
|
||||
|
||||
// If version not provided, use instance-level default from cluster.nodes.talos.version
|
||||
if node.Version == "" {
|
||||
defaultVersion, err := yq.Get(configPath, ".cluster.nodes.talos.version")
|
||||
if err == nil && defaultVersion != "" && defaultVersion != "null" {
|
||||
node.Version = defaultVersion
|
||||
}
|
||||
}
|
||||
|
||||
// Set maintenance=true if currentIP provided (node in maintenance mode)
|
||||
if node.CurrentIP != "" {
|
||||
node.Maintenance = true
|
||||
}
|
||||
|
||||
// Add node to config.yaml
|
||||
// Path: cluster.nodes.active.{hostname}
|
||||
basePath := fmt.Sprintf("cluster.nodes.active.%s", node.Hostname)
|
||||
return m.addNodeFile(instanceName, node)
|
||||
}
|
||||
|
||||
// Set each field
|
||||
if err := yq.Set(configPath, basePath+".role", node.Role); err != nil {
|
||||
return fmt.Errorf("failed to set role: %w", err)
|
||||
}
|
||||
if err := yq.Set(configPath, basePath+".disk", node.Disk); err != nil {
|
||||
return fmt.Errorf("failed to set disk: %w", err)
|
||||
}
|
||||
if node.TargetIP != "" {
|
||||
if err := yq.Set(configPath, basePath+".targetIp", node.TargetIP); err != nil {
|
||||
return fmt.Errorf("failed to set targetIP: %w", err)
|
||||
}
|
||||
}
|
||||
if node.CurrentIP != "" {
|
||||
if err := yq.Set(configPath, basePath+".currentIp", node.CurrentIP); err != nil {
|
||||
return fmt.Errorf("failed to set currentIP: %w", err)
|
||||
}
|
||||
}
|
||||
if node.Interface != "" {
|
||||
if err := yq.Set(configPath, basePath+".interface", node.Interface); err != nil {
|
||||
return fmt.Errorf("failed to set interface: %w", err)
|
||||
}
|
||||
}
|
||||
if node.Version != "" {
|
||||
if err := yq.Set(configPath, basePath+".version", node.Version); err != nil {
|
||||
return fmt.Errorf("failed to set version: %w", err)
|
||||
}
|
||||
}
|
||||
if node.SchematicID != "" {
|
||||
if err := yq.Set(configPath, basePath+".schematicId", node.SchematicID); err != nil {
|
||||
return fmt.Errorf("failed to set schematicId: %w", err)
|
||||
}
|
||||
}
|
||||
if node.Maintenance {
|
||||
if err := yq.Set(configPath, basePath+".maintenance", "true"); err != nil {
|
||||
return fmt.Errorf("failed to set maintenance: %w", err)
|
||||
// addNodeFile writes a new node's config to config/nodes/{hostname}.yaml (new format)
|
||||
func (m *Manager) addNodeFile(instanceName string, node *Node) error {
|
||||
// Fill talos defaults from instance.yaml if missing
|
||||
if node.SchematicID == "" || node.Version == "" {
|
||||
instanceYaml := tools.GetInstanceYamlPath(m.dataDir, instanceName)
|
||||
if data, err := os.ReadFile(instanceYaml); err == nil {
|
||||
var root struct {
|
||||
Cluster struct {
|
||||
Nodes struct {
|
||||
Talos struct {
|
||||
SchematicID string `yaml:"schematicId"`
|
||||
Version string `yaml:"version"`
|
||||
} `yaml:"talos"`
|
||||
} `yaml:"nodes"`
|
||||
} `yaml:"cluster"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &root); err == nil {
|
||||
if node.SchematicID == "" {
|
||||
node.SchematicID = root.Cluster.Nodes.Talos.SchematicID
|
||||
}
|
||||
if node.Version == "" {
|
||||
node.Version = root.Cluster.Nodes.Talos.Version
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
configDir := tools.GetNodeConfigDirPath(m.dataDir, instanceName)
|
||||
if err := os.MkdirAll(configDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create node config dir: %w", err)
|
||||
}
|
||||
|
||||
cfg := nodeConfigFile{
|
||||
Role: node.Role,
|
||||
Disk: node.Disk,
|
||||
Interface: node.Interface,
|
||||
TargetIP: node.TargetIP,
|
||||
SchematicID: node.SchematicID,
|
||||
Version: node.Version,
|
||||
Maintenance: node.Maintenance,
|
||||
}
|
||||
data, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal node config: %w", err)
|
||||
}
|
||||
return os.WriteFile(tools.GetNodeConfigPath(m.dataDir, instanceName, node.Hostname), data, 0644)
|
||||
}
|
||||
|
||||
// Delete removes a node from config.yaml
|
||||
// If skipReset is false, the node will be reset before deletion (with 30s timeout)
|
||||
func (m *Manager) Delete(instanceName, nodeIdentifier string, skipReset bool) error {
|
||||
slog.Info("deleting node", "component", "node", "instance", instanceName, "node", nodeIdentifier, "skipReset", skipReset)
|
||||
|
||||
// Get node to find hostname
|
||||
node, err := m.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
@@ -290,28 +285,29 @@ func (m *Manager) Delete(instanceName, nodeIdentifier string, skipReset bool) er
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the k8s node object (best-effort — node may already be gone)
|
||||
kubeconfigPath := tools.GetKubeconfigPath(m.dataDir, instanceName)
|
||||
cmd := exec.Command("kubectl", "delete", "node", node.Hostname, "--ignore-not-found")
|
||||
tools.WithKubeconfig(cmd, kubeconfigPath)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
slog.Warn("failed to delete k8s node object (continuing)", "component", "node",
|
||||
"hostname", node.Hostname, "error", err, "output", strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Delete node from config.yaml
|
||||
return m.deleteFromConfig(instanceName, node.Hostname)
|
||||
}
|
||||
|
||||
// deleteFromConfig removes a node entry from config.yaml
|
||||
// deleteFromConfig removes a node entry from config
|
||||
func (m *Manager) deleteFromConfig(instanceName, hostname string) error {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
|
||||
// Delete node from config.yaml
|
||||
// Path: .cluster.nodes.active["hostname"]
|
||||
// Use bracket notation to safely handle hostnames with special characters
|
||||
nodePath := fmt.Sprintf(".cluster.nodes.active[\"%s\"]", hostname)
|
||||
|
||||
yq := tools.NewYQ()
|
||||
// Use yq to delete the node
|
||||
delExpr := fmt.Sprintf("del(%s)", nodePath)
|
||||
_, err := yq.Exec("eval", "-i", delExpr, configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete node: %w", err)
|
||||
configPath := tools.GetNodeConfigPath(m.dataDir, instanceName, hostname)
|
||||
if err := os.Remove(configPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to delete node config: %w", err)
|
||||
}
|
||||
cachePath := tools.GetNodeCachePath(m.dataDir, instanceName, hostname)
|
||||
if err := os.Remove(cachePath); err != nil && !os.IsNotExist(err) {
|
||||
slog.Warn("failed to delete node cache", "component", "node", "hostname", hostname, "error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -375,23 +371,11 @@ func (m *Manager) Apply(instanceName, nodeIdentifier string, opts ApplyOptions)
|
||||
return err
|
||||
}
|
||||
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
setupDir := filepath.Join(instancePath, "setup", "cluster-nodes")
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
yq := tools.NewYQ()
|
||||
setupDir := tools.GetNodeSetupCachePath(m.dataDir, instanceName)
|
||||
|
||||
// Ensure node has version and schematicId (use cluster defaults if missing)
|
||||
if node.Version == "" {
|
||||
defaultVersion, err := yq.Get(configPath, ".cluster.nodes.talos.version")
|
||||
if err == nil && defaultVersion != "" && defaultVersion != "null" {
|
||||
node.Version = defaultVersion
|
||||
}
|
||||
}
|
||||
if node.SchematicID == "" {
|
||||
defaultSchematicID, err := yq.Get(configPath, ".cluster.nodes.talos.schematicId")
|
||||
if err == nil && defaultSchematicID != "" && defaultSchematicID != "null" {
|
||||
node.SchematicID = defaultSchematicID
|
||||
}
|
||||
if node.Version == "" || node.SchematicID == "" {
|
||||
m.fillTalosDefaults(instanceName, node)
|
||||
}
|
||||
|
||||
// Always auto-extract templates from embedded files if they don't exist
|
||||
@@ -404,11 +388,10 @@ func (m *Manager) Apply(instanceName, nodeIdentifier string, opts ApplyOptions)
|
||||
|
||||
// Determine base configuration file (generated by cluster config generation)
|
||||
var baseConfig string
|
||||
baseConfigDir := filepath.Join(instancePath, "talos", "generated")
|
||||
if node.Role == "controlplane" {
|
||||
baseConfig = filepath.Join(baseConfigDir, "controlplane.yaml")
|
||||
baseConfig = tools.GetControlplaneConfigPath(m.dataDir, instanceName)
|
||||
} else {
|
||||
baseConfig = filepath.Join(baseConfigDir, "worker.yaml")
|
||||
baseConfig = tools.GetWorkerConfigPath(m.dataDir, instanceName)
|
||||
}
|
||||
|
||||
// Check if base config exists
|
||||
@@ -434,6 +417,8 @@ func (m *Manager) Apply(instanceName, nodeIdentifier string, opts ApplyOptions)
|
||||
return fmt.Errorf("failed to update node status: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("applying node config", "component", "node", "instance", instanceName, "hostname", node.Hostname, "role", node.Role)
|
||||
|
||||
// Apply configuration to node
|
||||
// Determine which IP to use and whether node is in maintenance mode
|
||||
//
|
||||
@@ -473,6 +458,7 @@ func (m *Manager) Apply(instanceName, nodeIdentifier string, opts ApplyOptions)
|
||||
return fmt.Errorf("failed to update node status: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("node config applied", "component", "node", "instance", instanceName, "hostname", node.Hostname, "ip", node.TargetIP)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -499,20 +485,37 @@ func (m *Manager) generateNodePatch(instanceName string, node *Node, setupDir st
|
||||
patchContent = strings.ReplaceAll(patchContent, "{{SCHEMATIC_ID}}", node.SchematicID)
|
||||
patchContent = strings.ReplaceAll(patchContent, "{{VERSION}}", node.Version)
|
||||
|
||||
// Stage 2: Process through gomplate with config.yaml context (like v.PoC does with wild-compile-template)
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
|
||||
// Use gomplate to process template with config context
|
||||
cmd := exec.Command("gomplate", "-c", ".="+configPath)
|
||||
cmd.Stdin = strings.NewReader(patchContent)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
// Stage 2: Process through gomplate with merged global+instance config context
|
||||
merged, err := config.LoadMergedInstanceConfig(m.dataDir, instanceName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to process template with gomplate: %w\nOutput: %s", err, string(output))
|
||||
return "", fmt.Errorf("failed to load merged config: %w", err)
|
||||
}
|
||||
|
||||
processedPatch := string(output)
|
||||
mergedYAML, err := yaml.Marshal(merged)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal merged config: %w", err)
|
||||
}
|
||||
|
||||
gomplate := tools.NewGomplate()
|
||||
processedPatch, err := gomplate.RenderTemplate(patchContent, string(mergedYAML))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to process template with gomplate: %w", err)
|
||||
}
|
||||
|
||||
// Wipe disk when node needs a fresh install (maintenance or never applied)
|
||||
if node.Maintenance || !node.Applied {
|
||||
var patchMap map[string]interface{}
|
||||
if err := yaml.Unmarshal([]byte(processedPatch), &patchMap); err == nil {
|
||||
if machine, ok := patchMap["machine"].(map[string]interface{}); ok {
|
||||
if install, ok := machine["install"].(map[string]interface{}); ok {
|
||||
install["wipe"] = true
|
||||
if patched, err := yaml.Marshal(patchMap); err == nil {
|
||||
processedPatch = string(patched)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create patch directory
|
||||
patchDir := filepath.Join(setupDir, "patch")
|
||||
@@ -594,130 +597,239 @@ func (m *Manager) extractEmbeddedTemplates(destDir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateNodeStatus updates node status flags in config.yaml
|
||||
// updateNodeStatus updates node status flags
|
||||
func (m *Manager) updateNodeStatus(instanceName string, node *Node) error {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
basePath := fmt.Sprintf("cluster.nodes.active.%s", node.Hostname)
|
||||
return m.updateNodeStatusFiles(instanceName, node)
|
||||
}
|
||||
|
||||
yq := tools.NewYQ()
|
||||
|
||||
// Update maintenance flag
|
||||
if node.Maintenance {
|
||||
if err := yq.Set(configPath, basePath+".maintenance", "true"); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := yq.Set(configPath, basePath+".maintenance", "false"); err != nil {
|
||||
return err
|
||||
}
|
||||
// updateNodeStatusFiles writes cache (applied/configured/currentIp) and updates maintenance in config (new format)
|
||||
func (m *Manager) updateNodeStatusFiles(instanceName string, node *Node) error {
|
||||
cacheDir := filepath.Dir(tools.GetNodeCachePath(m.dataDir, instanceName, node.Hostname))
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create node cache dir: %w", err)
|
||||
}
|
||||
cache := nodeCacheFile{
|
||||
Applied: node.Applied,
|
||||
Configured: node.Configured,
|
||||
CurrentIP: node.CurrentIP,
|
||||
}
|
||||
cacheData, err := yaml.Marshal(cache)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal node cache: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(tools.GetNodeCachePath(m.dataDir, instanceName, node.Hostname), cacheData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write node cache: %w", err)
|
||||
}
|
||||
|
||||
// Update currentIP (may have changed after application)
|
||||
if node.CurrentIP != "" {
|
||||
if err := yq.Set(configPath, basePath+".currentIp", node.CurrentIP); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update maintenance in config/nodes/{hostname}.yaml
|
||||
configPath := tools.GetNodeConfigPath(m.dataDir, instanceName, node.Hostname)
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read node config: %w", err)
|
||||
}
|
||||
|
||||
// Update configured flag
|
||||
configuredValue := "false"
|
||||
if node.Configured {
|
||||
configuredValue = "true"
|
||||
var cfg nodeConfigFile
|
||||
if err := yaml.Unmarshal(configData, &cfg); err != nil {
|
||||
return fmt.Errorf("failed to parse node config: %w", err)
|
||||
}
|
||||
if err := yq.Set(configPath, basePath+".configured", configuredValue); err != nil {
|
||||
return err
|
||||
cfg.Maintenance = node.Maintenance
|
||||
updatedData, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal node config: %w", err)
|
||||
}
|
||||
|
||||
// Update applied flag
|
||||
appliedValue := "false"
|
||||
if node.Applied {
|
||||
appliedValue = "true"
|
||||
}
|
||||
if err := yq.Set(configPath, basePath+".applied", appliedValue); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return os.WriteFile(configPath, updatedData, 0644)
|
||||
}
|
||||
|
||||
// Update modifies existing node configuration with partial updates
|
||||
func (m *Manager) Update(instanceName string, hostname string, updates map[string]interface{}) error {
|
||||
// Get existing node
|
||||
node, err := m.Get(instanceName, hostname)
|
||||
return m.updateNodeFile(instanceName, hostname, updates)
|
||||
}
|
||||
|
||||
// updateNodeFile applies partial updates to config/nodes/{hostname}.yaml (new format)
|
||||
func (m *Manager) updateNodeFile(instanceName, hostname string, updates map[string]interface{}) error {
|
||||
configPath := tools.GetNodeConfigPath(m.dataDir, instanceName, hostname)
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("node %s not found", hostname)
|
||||
return fmt.Errorf("node %s not found: %w", hostname, err)
|
||||
}
|
||||
var cfg nodeConfigFile
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return fmt.Errorf("failed to parse node config: %w", err)
|
||||
}
|
||||
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
configPath := filepath.Join(instancePath, "config.yaml")
|
||||
basePath := fmt.Sprintf("cluster.nodes.active.%s", hostname)
|
||||
yq := tools.NewYQ()
|
||||
|
||||
// Apply partial updates
|
||||
for key, value := range updates {
|
||||
switch key {
|
||||
case "target_ip":
|
||||
if strVal, ok := value.(string); ok {
|
||||
node.TargetIP = strVal
|
||||
if err := yq.Set(configPath, basePath+".targetIp", strVal); err != nil {
|
||||
return fmt.Errorf("failed to update targetIp: %w", err)
|
||||
}
|
||||
cfg.TargetIP = strVal
|
||||
}
|
||||
case "current_ip":
|
||||
if strVal, ok := value.(string); ok {
|
||||
node.CurrentIP = strVal
|
||||
node.Maintenance = true // Auto-set maintenance when currentIP changes
|
||||
if err := yq.Set(configPath, basePath+".currentIp", strVal); err != nil {
|
||||
return fmt.Errorf("failed to update currentIp: %w", err)
|
||||
}
|
||||
if err := yq.Set(configPath, basePath+".maintenance", "true"); err != nil {
|
||||
return fmt.Errorf("failed to set maintenance: %w", err)
|
||||
}
|
||||
}
|
||||
// currentIp is cache; update via cache file (handled by updateNodeStatus)
|
||||
case "disk":
|
||||
if strVal, ok := value.(string); ok {
|
||||
node.Disk = strVal
|
||||
if err := yq.Set(configPath, basePath+".disk", strVal); err != nil {
|
||||
return fmt.Errorf("failed to update disk: %w", err)
|
||||
}
|
||||
cfg.Disk = strVal
|
||||
}
|
||||
case "interface":
|
||||
if strVal, ok := value.(string); ok {
|
||||
node.Interface = strVal
|
||||
if err := yq.Set(configPath, basePath+".interface", strVal); err != nil {
|
||||
return fmt.Errorf("failed to update interface: %w", err)
|
||||
}
|
||||
cfg.Interface = strVal
|
||||
}
|
||||
case "version":
|
||||
if strVal, ok := value.(string); ok {
|
||||
cfg.Version = strVal
|
||||
}
|
||||
case "schematic_id":
|
||||
if strVal, ok := value.(string); ok {
|
||||
node.SchematicID = strVal
|
||||
if err := yq.Set(configPath, basePath+".schematicId", strVal); err != nil {
|
||||
return fmt.Errorf("failed to update schematicId: %w", err)
|
||||
}
|
||||
cfg.SchematicID = strVal
|
||||
}
|
||||
case "maintenance":
|
||||
if boolVal, ok := value.(bool); ok {
|
||||
node.Maintenance = boolVal
|
||||
if err := yq.Set(configPath, basePath+".maintenance", fmt.Sprintf("%t", boolVal)); err != nil {
|
||||
return fmt.Errorf("failed to update maintenance: %w", err)
|
||||
}
|
||||
cfg.Maintenance = boolVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
out, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal node config: %w", err)
|
||||
}
|
||||
return os.WriteFile(configPath, out, 0644)
|
||||
}
|
||||
|
||||
// FetchTemplates extracts patch templates from embedded files to instance
|
||||
func (m *Manager) FetchTemplates(instanceName string) error {
|
||||
instancePath := m.GetInstancePath(instanceName)
|
||||
destDir := filepath.Join(instancePath, "setup", "cluster-nodes", "patch.templates")
|
||||
destDir := filepath.Join(tools.GetNodeSetupCachePath(m.dataDir, instanceName), "patch.templates")
|
||||
return m.extractEmbeddedTemplates(destDir)
|
||||
}
|
||||
|
||||
// Reset resets a node to maintenance mode
|
||||
// fillTalosDefaults reads cluster talos version/schematicId defaults into node from instance.yaml
|
||||
func (m *Manager) fillTalosDefaults(instanceName string, node *Node) {
|
||||
instanceYaml := tools.GetInstanceYamlPath(m.dataDir, instanceName)
|
||||
if data, err := os.ReadFile(instanceYaml); err == nil {
|
||||
var root struct {
|
||||
Cluster struct {
|
||||
Nodes struct {
|
||||
Talos struct {
|
||||
SchematicID string `yaml:"schematicId"`
|
||||
Version string `yaml:"version"`
|
||||
} `yaml:"talos"`
|
||||
} `yaml:"nodes"`
|
||||
} `yaml:"cluster"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &root); err == nil {
|
||||
if node.SchematicID == "" {
|
||||
node.SchematicID = root.Cluster.Nodes.Talos.SchematicID
|
||||
}
|
||||
if node.Version == "" {
|
||||
node.Version = root.Cluster.Nodes.Talos.Version
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NodeHealth represents the health status of a node
|
||||
type NodeHealth struct {
|
||||
Node string `json:"node"`
|
||||
Services []tools.ServiceStatus `json:"services"`
|
||||
DmesgErrors []tools.DmesgError `json:"dmesgErrors"`
|
||||
Healthy bool `json:"healthy"`
|
||||
}
|
||||
|
||||
// Health checks node health by querying Talos service statuses and scanning dmesg for errors
|
||||
func (m *Manager) Health(instanceName, nodeIdentifier string) (*NodeHealth, error) {
|
||||
node, err := m.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("node not found: %w", err)
|
||||
}
|
||||
|
||||
if !node.Applied || node.Maintenance {
|
||||
return nil, fmt.Errorf("health check requires an applied, non-maintenance node")
|
||||
}
|
||||
|
||||
ip := node.TargetIP
|
||||
if ip == "" {
|
||||
return nil, fmt.Errorf("no IP address available for node %s", node.Hostname)
|
||||
}
|
||||
|
||||
// Fetch services and dmesg concurrently
|
||||
var services []tools.ServiceStatus
|
||||
var dmesgRaw string
|
||||
var svcErr, dmesgErr error
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
services, svcErr = m.talosctl.GetServices(ip)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
dmesgRaw, dmesgErr = m.talosctl.GetDmesg(ip)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if svcErr != nil {
|
||||
return nil, fmt.Errorf("failed to get services: %w", svcErr)
|
||||
}
|
||||
|
||||
var dmesgErrors []tools.DmesgError
|
||||
if dmesgErr == nil {
|
||||
dmesgErrors = tools.ParseDmesgErrors(dmesgRaw)
|
||||
}
|
||||
if dmesgErrors == nil {
|
||||
dmesgErrors = []tools.DmesgError{}
|
||||
}
|
||||
|
||||
// Compute overall health
|
||||
healthy := len(dmesgErrors) == 0
|
||||
for _, svc := range services {
|
||||
if !svc.Healthy && svc.HealthMessage != "" {
|
||||
healthy = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &NodeHealth{
|
||||
Node: node.Hostname,
|
||||
Services: services,
|
||||
DmesgErrors: dmesgErrors,
|
||||
Healthy: healthy,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Reboot restarts a node without wiping state
|
||||
func (m *Manager) Reboot(instanceName, nodeIdentifier string) error {
|
||||
slog.Info("rebooting node", "component", "node", "instance", instanceName, "node", nodeIdentifier)
|
||||
|
||||
node, err := m.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
return fmt.Errorf("node not found: %w", err)
|
||||
}
|
||||
|
||||
rebootIP := node.TargetIP
|
||||
if rebootIP == "" {
|
||||
rebootIP = node.CurrentIP
|
||||
}
|
||||
if rebootIP == "" {
|
||||
return fmt.Errorf("no IP address available for node %s", node.Hostname)
|
||||
}
|
||||
|
||||
if err := m.talosctl.Reboot(rebootIP); err != nil {
|
||||
return fmt.Errorf("failed to reboot node: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("node reboot initiated", "component", "node", "instance", instanceName, "hostname", node.Hostname, "ip", rebootIP)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reset resets a node to maintenance mode with resilient handling.
|
||||
// For control plane nodes, it first removes the etcd member from the cluster
|
||||
// via a healthy peer, then resets the node. The node stays in config.yaml
|
||||
// so it can be reconfigured and rejoined.
|
||||
func (m *Manager) Reset(instanceName, nodeIdentifier string) error {
|
||||
slog.Info("resetting node", "component", "node", "instance", instanceName, "node", nodeIdentifier)
|
||||
|
||||
// Get node
|
||||
node, err := m.Get(instanceName, nodeIdentifier)
|
||||
if err != nil {
|
||||
@@ -729,35 +841,75 @@ func (m *Manager) Reset(instanceName, nodeIdentifier string) error {
|
||||
if resetIP == "" {
|
||||
resetIP = node.TargetIP
|
||||
}
|
||||
if resetIP == "" {
|
||||
return fmt.Errorf("no IP address available for node %s", node.Hostname)
|
||||
}
|
||||
|
||||
// Execute reset command with graceful=false and reboot flags
|
||||
talosconfigPath := tools.GetTalosconfigPath(m.dataDir, instanceName)
|
||||
cmd := exec.Command("talosctl", "-n", resetIP, "--talosconfig", talosconfigPath, "reset", "--graceful=false", "--reboot")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
// Check if error is due to node rebooting (expected after reset command)
|
||||
outputStr := string(output)
|
||||
if strings.Contains(outputStr, "connection refused") || strings.Contains(outputStr, "Unavailable") {
|
||||
// This is expected - node is rebooting after successful reset
|
||||
// Continue with config cleanup
|
||||
} else {
|
||||
// Real error - return it
|
||||
return fmt.Errorf("failed to reset node: %w\nOutput: %s", err, outputStr)
|
||||
// For control plane nodes, remove from etcd first via a healthy peer
|
||||
if node.Role == "controlplane" {
|
||||
if err := m.removeEtcdMember(instanceName, node); err != nil {
|
||||
// Log but don't fail — the node may already be removed from etcd,
|
||||
// or etcd may be unreachable on this node. The reset should still proceed.
|
||||
slog.Warn("etcd member removal failed (continuing with reset)", "component", "node",
|
||||
"instance", instanceName, "hostname", node.Hostname, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update node status to maintenance mode, then remove from config
|
||||
// Reset the node via talosctl
|
||||
if err := m.talosctl.Reset(resetIP); err != nil {
|
||||
return fmt.Errorf("failed to reset node: %w", err)
|
||||
}
|
||||
|
||||
// Update node status to maintenance mode but keep in config for reconfiguration
|
||||
node.Maintenance = true
|
||||
node.Configured = false
|
||||
node.Applied = false
|
||||
node.CurrentIP = ""
|
||||
node.Version = ""
|
||||
if err := m.updateNodeStatus(instanceName, node); err != nil {
|
||||
return fmt.Errorf("failed to update node status: %w", err)
|
||||
}
|
||||
|
||||
// Remove node from config.yaml after successful reset
|
||||
if err := m.deleteFromConfig(instanceName, node.Hostname); err != nil {
|
||||
return fmt.Errorf("failed to remove node from config: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("node reset to maintenance mode", "component", "node",
|
||||
"instance", instanceName, "hostname", node.Hostname)
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeEtcdMember removes a control plane node's etcd member via a healthy peer.
|
||||
func (m *Manager) removeEtcdMember(instanceName string, targetNode *Node) error {
|
||||
// Find a healthy control plane peer to run the etcd remove-member command
|
||||
nodes, err := m.List(instanceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list nodes: %w", err)
|
||||
}
|
||||
|
||||
for _, peer := range nodes {
|
||||
// Skip the target node itself, non-control-plane nodes, and unapplied nodes
|
||||
if peer.Hostname == targetNode.Hostname || peer.Role != "controlplane" || !peer.Applied || peer.Maintenance {
|
||||
continue
|
||||
}
|
||||
|
||||
peerIP := peer.CurrentIP
|
||||
if peerIP == "" {
|
||||
peerIP = peer.TargetIP
|
||||
}
|
||||
if peerIP == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("removing etcd member via peer", "component", "node",
|
||||
"instance", instanceName, "target", targetNode.Hostname, "peer", peer.Hostname)
|
||||
|
||||
if err := m.talosctl.EtcdRemoveMember(peerIP, targetNode.Hostname); err != nil {
|
||||
slog.Warn("etcd remove-member failed via peer, trying next", "component", "node",
|
||||
"peer", peer.Hostname, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("etcd member removed successfully", "component", "node",
|
||||
"instance", instanceName, "target", targetNode.Hostname, "via", peer.Hostname)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("no healthy control plane peer available to remove etcd member for %s", targetNode.Hostname)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package operations
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
@@ -16,7 +17,23 @@ const totalBootstrapSteps = 7
|
||||
|
||||
// Manager handles async operation tracking
|
||||
type Manager struct {
|
||||
dataDir string
|
||||
dataDir string
|
||||
sseManager SSEManager // Optional SSE manager for broadcasting events
|
||||
}
|
||||
|
||||
// SSEManager is an interface for broadcasting SSE events
|
||||
type SSEManager interface {
|
||||
Broadcast(event *SSEEvent)
|
||||
}
|
||||
|
||||
// SSEEvent represents an SSE event (matching sse.Event structure)
|
||||
type SSEEvent struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
InstanceName string `json:"instanceName"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Data interface{} `json:"data"`
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// NewManager creates a new operations manager
|
||||
@@ -26,6 +43,11 @@ func NewManager(dataDir string) *Manager {
|
||||
}
|
||||
}
|
||||
|
||||
// SetSSEManager sets the SSE manager for broadcasting events
|
||||
func (m *Manager) SetSSEManager(sseManager SSEManager) {
|
||||
m.sseManager = sseManager
|
||||
}
|
||||
|
||||
// BootstrapProgress tracks detailed bootstrap progress
|
||||
type BootstrapProgress struct {
|
||||
CurrentStep int `json:"current_step"` // 0-6
|
||||
@@ -52,7 +74,7 @@ type Operation struct {
|
||||
Details *OperationDetails `json:"details,omitempty"` // Operation-specific details
|
||||
LogFile string `json:"logFile,omitempty"` // Path to output log file
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
EndedAt time.Time `json:"ended_at,omitempty"`
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"`
|
||||
}
|
||||
|
||||
// GetOperationsDir returns the operations directory for an instance
|
||||
@@ -94,6 +116,11 @@ func (m *Manager) Start(instanceName, opType, target string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
slog.Info("operation started", "component", "operations", "id", opID, "type", opType, "target", target, "instance", instanceName)
|
||||
|
||||
// Broadcast SSE event if manager is available
|
||||
m.broadcastOperationEvent("operation:started", op)
|
||||
|
||||
return opID, nil
|
||||
}
|
||||
|
||||
@@ -126,15 +153,50 @@ func (m *Manager) Update(instanceName, opID, status, message string, progress in
|
||||
return err
|
||||
}
|
||||
|
||||
oldStatus := op.Status
|
||||
op.Status = status
|
||||
op.Message = message
|
||||
op.Progress = progress
|
||||
|
||||
if status == "completed" || status == "failed" || status == "cancelled" {
|
||||
op.EndedAt = time.Now()
|
||||
now := time.Now()
|
||||
op.EndedAt = &now
|
||||
}
|
||||
|
||||
return m.writeOperation(op)
|
||||
if err := m.writeOperation(op); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Log terminal status transitions
|
||||
if oldStatus != status {
|
||||
switch status {
|
||||
case "completed":
|
||||
slog.Info("operation completed", "component", "operations", "id", op.ID, "type", op.Type, "target", op.Target, "instance", instanceName)
|
||||
case "failed":
|
||||
slog.Error("operation failed", "component", "operations", "id", op.ID, "type", op.Type, "target", op.Target, "instance", instanceName, "error", message)
|
||||
case "cancelled":
|
||||
slog.Info("operation cancelled", "component", "operations", "id", op.ID, "type", op.Type, "target", op.Target, "instance", instanceName)
|
||||
}
|
||||
}
|
||||
|
||||
// Broadcast appropriate SSE event based on status change
|
||||
if oldStatus != status {
|
||||
switch status {
|
||||
case "running":
|
||||
m.broadcastOperationEvent("operation:progress", op)
|
||||
case "completed":
|
||||
m.broadcastOperationEvent("operation:completed", op)
|
||||
case "failed":
|
||||
m.broadcastOperationEvent("operation:failed", op)
|
||||
case "cancelled":
|
||||
m.broadcastOperationEvent("operation:cancelled", op)
|
||||
}
|
||||
} else if oldStatus == "running" && status == "running" {
|
||||
// Progress update
|
||||
m.broadcastOperationEvent("operation:progress", op)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateStatus updates only the status
|
||||
@@ -144,13 +206,33 @@ func (m *Manager) UpdateStatus(instanceName, opID, status string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
oldStatus := op.Status
|
||||
op.Status = status
|
||||
|
||||
if status == "completed" || status == "failed" || status == "cancelled" {
|
||||
op.EndedAt = time.Now()
|
||||
now := time.Now()
|
||||
op.EndedAt = &now
|
||||
}
|
||||
|
||||
return m.writeOperation(op)
|
||||
if err := m.writeOperation(op); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Broadcast appropriate SSE event based on status change
|
||||
if oldStatus != status {
|
||||
switch status {
|
||||
case "running":
|
||||
m.broadcastOperationEvent("operation:progress", op)
|
||||
case "completed":
|
||||
m.broadcastOperationEvent("operation:completed", op)
|
||||
case "failed":
|
||||
m.broadcastOperationEvent("operation:failed", op)
|
||||
case "cancelled":
|
||||
m.broadcastOperationEvent("operation:cancelled", op)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateProgress updates operation progress
|
||||
@@ -160,12 +242,24 @@ func (m *Manager) UpdateProgress(instanceName, opID string, progress int, messag
|
||||
return err
|
||||
}
|
||||
|
||||
// Don't update progress on a cancelled operation
|
||||
if op.Status == "cancelled" {
|
||||
return nil
|
||||
}
|
||||
|
||||
op.Progress = progress
|
||||
if message != "" {
|
||||
op.Message = message
|
||||
}
|
||||
|
||||
return m.writeOperation(op)
|
||||
if err := m.writeOperation(op); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Broadcast progress event
|
||||
m.broadcastOperationEvent("operation:progress", op)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cancel requests operation cancellation
|
||||
@@ -223,6 +317,26 @@ func (m *Manager) Delete(instanceName, opID string) error {
|
||||
return os.Remove(opPath)
|
||||
}
|
||||
|
||||
// FailOrphaned marks all running/pending operations for an instance as failed.
|
||||
// Called on API startup to clean up operations that were interrupted by a restart.
|
||||
func (m *Manager) FailOrphaned(instanceName string) error {
|
||||
ops, err := m.List(instanceName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, op := range ops {
|
||||
if op.Status == "running" || op.Status == "pending" {
|
||||
slog.Info("failing orphaned operation", "component", "operations", "id", op.ID, "type", op.Type, "target", op.Target, "instance", instanceName)
|
||||
if err := m.Update(instanceName, op.ID, "failed", "API restarted while operation was in progress", op.Progress); err != nil {
|
||||
slog.Warn("failed to mark orphaned operation as failed", "component", "operations", "id", op.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup removes old completed/failed operations
|
||||
func (m *Manager) Cleanup(instanceName string, olderThan time.Duration) error {
|
||||
ops, err := m.List(instanceName)
|
||||
@@ -234,7 +348,7 @@ func (m *Manager) Cleanup(instanceName string, olderThan time.Duration) error {
|
||||
|
||||
for _, op := range ops {
|
||||
if (op.Status == "completed" || op.Status == "failed" || op.Status == "cancelled") &&
|
||||
!op.EndedAt.IsZero() && op.EndedAt.Before(cutoff) {
|
||||
op.EndedAt != nil && op.EndedAt.Before(cutoff) {
|
||||
_ = m.Delete(instanceName, op.ID)
|
||||
}
|
||||
}
|
||||
@@ -283,3 +397,33 @@ func (m *Manager) writeOperation(op *Operation) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// broadcastOperationEvent sends an SSE event for operation changes
|
||||
func (m *Manager) broadcastOperationEvent(eventType string, op *Operation) {
|
||||
if m.sseManager == nil || op == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create SSE event
|
||||
event := &SSEEvent{
|
||||
ID: fmt.Sprintf("op-%s-%d", op.ID, time.Now().UnixNano()),
|
||||
Type: eventType,
|
||||
InstanceName: op.Instance,
|
||||
Timestamp: time.Now(),
|
||||
Data: map[string]interface{}{
|
||||
"operation_id": op.ID,
|
||||
"type": op.Type,
|
||||
"target": op.Target,
|
||||
"status": op.Status,
|
||||
"progress": op.Progress,
|
||||
"message": op.Message,
|
||||
"details": op.Details,
|
||||
},
|
||||
Metadata: map[string]interface{}{
|
||||
"operation_type": op.Type,
|
||||
"target": op.Target,
|
||||
},
|
||||
}
|
||||
|
||||
m.sseManager.Broadcast(event)
|
||||
}
|
||||
|
||||
90
api/internal/operations/operations_test.go
Normal file
90
api/internal/operations/operations_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func setupTestManager(t *testing.T) (*Manager, string) {
|
||||
t.Helper()
|
||||
tmpDir := t.TempDir()
|
||||
instanceName := "test-cloud"
|
||||
|
||||
// Create the instances/test-cloud/operations directory
|
||||
opsDir := filepath.Join(tmpDir, "instances", instanceName, "operations")
|
||||
if err := os.MkdirAll(opsDir, 0755); err != nil {
|
||||
t.Fatalf("failed to create ops dir: %v", err)
|
||||
}
|
||||
|
||||
return NewManager(tmpDir), instanceName
|
||||
}
|
||||
|
||||
func TestFailOrphaned(t *testing.T) {
|
||||
m, instanceName := setupTestManager(t)
|
||||
|
||||
// Create operations in various states
|
||||
runningID, err := m.Start(instanceName, "backup", "myapp")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start operation: %v", err)
|
||||
}
|
||||
_ = m.Update(instanceName, runningID, "running", "Backing up", 50)
|
||||
|
||||
pendingID, err := m.Start(instanceName, "restore", "myapp")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start operation: %v", err)
|
||||
}
|
||||
|
||||
completedID, err := m.Start(instanceName, "backup", "otherapp")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start operation: %v", err)
|
||||
}
|
||||
_ = m.Update(instanceName, completedID, "completed", "Done", 100)
|
||||
|
||||
failedID, err := m.Start(instanceName, "deploy", "otherapp")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start operation: %v", err)
|
||||
}
|
||||
_ = m.Update(instanceName, failedID, "failed", "Something broke", 0)
|
||||
|
||||
// Run FailOrphaned
|
||||
if err := m.FailOrphaned(instanceName); err != nil {
|
||||
t.Fatalf("FailOrphaned failed: %v", err)
|
||||
}
|
||||
|
||||
// Running operation should now be failed
|
||||
op, _ := m.GetByInstance(instanceName, runningID)
|
||||
if op.Status != "failed" {
|
||||
t.Errorf("expected running op to be failed, got %s", op.Status)
|
||||
}
|
||||
if op.EndedAt == nil {
|
||||
t.Error("expected running op to have EndedAt set")
|
||||
}
|
||||
|
||||
// Pending operation should now be failed
|
||||
op, _ = m.GetByInstance(instanceName, pendingID)
|
||||
if op.Status != "failed" {
|
||||
t.Errorf("expected pending op to be failed, got %s", op.Status)
|
||||
}
|
||||
|
||||
// Completed operation should be unchanged
|
||||
op, _ = m.GetByInstance(instanceName, completedID)
|
||||
if op.Status != "completed" {
|
||||
t.Errorf("expected completed op to stay completed, got %s", op.Status)
|
||||
}
|
||||
|
||||
// Failed operation should be unchanged
|
||||
op, _ = m.GetByInstance(instanceName, failedID)
|
||||
if op.Status != "failed" {
|
||||
t.Errorf("expected already-failed op to stay failed, got %s", op.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailOrphaned_NoOperations(t *testing.T) {
|
||||
m, instanceName := setupTestManager(t)
|
||||
|
||||
// Should not error on empty operations directory
|
||||
if err := m.FailOrphaned(instanceName); err != nil {
|
||||
t.Fatalf("FailOrphaned on empty dir failed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package pxe
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -72,10 +72,6 @@ func (m *Manager) ListAssets(instanceName string) ([]Asset, error) {
|
||||
|
||||
if err == nil {
|
||||
asset.Size = info.Size()
|
||||
// Calculate SHA256 if file exists
|
||||
if hash, err := calculateSHA256(assetPath); err == nil {
|
||||
asset.SHA256 = hash
|
||||
}
|
||||
}
|
||||
|
||||
assets = append(assets, asset)
|
||||
@@ -145,6 +141,7 @@ func (m *Manager) DownloadAsset(instanceName, assetType, version, url string) er
|
||||
return fmt.Errorf("failed to move file: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("PXE asset downloaded", "component", "pxe", "instance", instanceName, "type", assetType, "version", version)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -204,18 +201,3 @@ func (m *Manager) DeleteAsset(instanceName, assetType string) error {
|
||||
return os.Remove(assetPath)
|
||||
}
|
||||
|
||||
// calculateSHA256 computes the SHA256 hash of a file
|
||||
func calculateSHA256(filePath string) (string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user