Compare commits
127 Commits
86eee0f916
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 51142081c9 | |||
| 02616eebbc | |||
| a61e3d8302 | |||
| 0450f8bdb0 | |||
| b4833560b5 | |||
| 81ee848f5f | |||
| 7f2bbe4d20 | |||
| 29f455a153 | |||
| 41ceab5a40 | |||
| 2ad119e214 | |||
| eb121453c8 | |||
| 67ffd6101b | |||
| 1f5c4e0756 | |||
| b5e87abad2 | |||
| fc6df89370 | |||
| 3893df4640 | |||
| f86c67aeee | |||
| 7dc4c1df84 | |||
| 10e0293559 | |||
| 9dd690bb3b | |||
| 8740b75841 | |||
| ce1cb9d157 | |||
| 8e83fee7b5 | |||
| a6d6468da8 | |||
| b87203c538 | |||
| 515330c436 | |||
| fdf401a0be | |||
| 3b77f2e5d0 | |||
| df9ec5b04c | |||
| 5788f53854 | |||
| 28da9c4096 | |||
| 6eb1b4add3 | |||
| 8633cbea26 | |||
| 3fb8209aef | |||
| 833edb2466 | |||
| c7e9e69345 | |||
| e79d871f36 | |||
| ad45fbe46e | |||
| 3cac7b61ac | |||
| 1d6c21764b | |||
| 6f392bef65 | |||
| 6035c547ae | |||
| a3a8fea071 | |||
| 0e2f48ab65 | |||
| b55950e24a | |||
| 025cd0336c | |||
| 4c76ae9f52 | |||
| 7a552715fe | |||
| 0f0de5dc30 | |||
| 1604f0f6f4 | |||
| af548f11c4 | |||
| c0eca3da55 | |||
| 212813e066 | |||
| ab5291b8d3 | |||
| 1929196287 | |||
| 9e75273fc5 | |||
| 5dd6ebb3db | |||
| 864a40e862 | |||
| 9412feaa58 | |||
| 0fe2aa439f | |||
| fe4ae9d147 | |||
| 0a3a3448d6 | |||
| 01b779c862 | |||
| 67d1af0baa | |||
| 4e8e4f75fd | |||
| 2be2d11b4a | |||
| b67ae7ff8b | |||
| ad2cc5585e | |||
| 8aa3f1d1f5 | |||
| 30cf93df27 | |||
| 01c4ccb316 | |||
| 00d4f2a70b | |||
| f8215a0c9a | |||
| ea9e3e41e3 | |||
| 5b5a5bc637 | |||
| c2e5a534aa | |||
| d0c23deb72 | |||
| 4c8ed24561 | |||
| 404b7f7500 | |||
| beca073ddc | |||
| 362a9dde62 | |||
| a73d34ac9f | |||
| 1c8cbd96ca | |||
| 86873bd035 | |||
| 2642f14e49 | |||
| bb62382e18 | |||
| c5865a5379 | |||
| 58d567f9bc | |||
| 2d9f453767 | |||
| 20a9f93378 | |||
| b78f102e9d | |||
| 8858e049ee | |||
| a75b97c4c0 | |||
| b4fc5a14b2 | |||
| a0fd33fcb8 | |||
| ef1761385e | |||
| 17215fd191 | |||
| 97753c3d3c | |||
| 994f7f66c4 | |||
| f76b6cad14 | |||
| 90e293facd | |||
| 50eb7cf2f3 | |||
| 26265be440 | |||
| 5faef1492d | |||
| 3045093dcf | |||
| 64a0ff8322 | |||
| 155ff63a73 | |||
| a9e40df44b | |||
| 98695f9888 | |||
| d8c0ee5d1e | |||
| 851f967f12 | |||
| c97c595253 | |||
| f5acdf2fff | |||
| e6ed7954de | |||
| 73e6bb7285 | |||
| d451d01fe2 | |||
| bfe17969e6 | |||
| 7cd87bc8a8 | |||
| 0babc57f57 | |||
| f41e59e695 | |||
| f3dcb8a3b9 | |||
| c2d7342a47 | |||
| 5ca2733ea9 | |||
| 8bff241b12 | |||
| 1e04da9de8 | |||
| 39b6cf3469 | |||
| 124421b85c |
88
CHANGELOG.md
Normal file
88
CHANGELOG.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Changelog
|
||||
|
||||
## 1.2.0 - 2026-05-28
|
||||
|
||||
Operations-focused release for more reliable production backups and maintenance.
|
||||
|
||||
### Added
|
||||
|
||||
- Staff-only updater page for checking configured Gitea releases, inspecting the installed git checkout, fetching tags, pulling the current branch, and running the native systemd updater.
|
||||
- Read-only control panel access level for authenticated non-staff users, with status pages visible and credentials, logs, configs, retention, and mutating actions kept staff-only.
|
||||
- Run completion notifications for email and webhooks, including recorded delivery history per run and target.
|
||||
- Dedicated hosts page with host cards, enabled/disabled filtering, and quick host/schedule/retention state controls.
|
||||
- Per-host rsync bandwidth limit overrides with inherit, unlimited, and explicit limit semantics.
|
||||
- Backup data totals by snapshot kind on dashboard and host detail pages, including unique/non-hardlinked data totals.
|
||||
|
||||
### Changed
|
||||
|
||||
- Real backup runs now default to verbose rsync progress output so the live run view behaves consistently with dry-runs.
|
||||
- Run progress panels are shared between dry-runs and real runs for more consistent status, timing, cancellation, and log display.
|
||||
- Incomplete snapshot cleanup now requires operator review before deletion.
|
||||
- Incomplete snapshot size reporting now prefers on-disk measurement when metadata is stale or missing.
|
||||
- Installer and environment examples now include optional updater configuration.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Remote preflight shell commands are now quoted correctly, including roots such as `/`.
|
||||
- Worker reconciliation now detects real rsync failures and stale/running process state more reliably.
|
||||
- Retention pruning and incomplete cleanup can delete snapshots containing restrictive directory modes preserved by rsync archive mode.
|
||||
- Snapshot data summaries no longer count incomplete metadata/log files as backup data when measuring from disk.
|
||||
- Filesystem SSH credential tests use writable test state without changing production defaults.
|
||||
|
||||
## 1.1.0 - 2026-05-21
|
||||
|
||||
UI-focused release for the Django control panel.
|
||||
|
||||
### Added
|
||||
|
||||
- Dedicated list pages for runs, snapshots, schedules, purged snapshots, and changelog navigation.
|
||||
- Dashboard priority panels for required action, next scheduled work, recent activity, and storage pressure.
|
||||
- Dashboard host cards with clearer backup activity, snapshot health, next run, and retention status.
|
||||
- Lightweight live refresh for active run detail pages, including status, timing, controls, and rsync log output.
|
||||
- Lightweight live refresh for dashboard priority and host status sections.
|
||||
- Current-page navigation states for primary and system navigation.
|
||||
- Responsive dashboard behavior for narrower screens.
|
||||
|
||||
### Changed
|
||||
|
||||
- Reworked the primary navigation around day-to-day operator workflows and moved admin/system links out of the main path.
|
||||
- Simplified legacy-facing labels and removed source-of-truth wording that no longer applies to the Django-first model.
|
||||
- Improved run and snapshot detail pages with clearer links between backup runs, snapshots, logs, and review actions.
|
||||
- Improved dashboard spacing and card layouts to reduce cramped or overlapping text.
|
||||
- Documented the Django-template-first partial refresh pattern for future UI work.
|
||||
|
||||
## 1.0.0 - 2026-05-21
|
||||
|
||||
Initial stable release of the Django-first pobsync control panel.
|
||||
|
||||
### Added
|
||||
|
||||
- Django control panel for hosts, global settings, schedules, SSH credentials, snapshots, runs, self-checks, and logs.
|
||||
- Native systemd installer and updater for production backup servers.
|
||||
- SQLite by default, with optional MariaDB support.
|
||||
- Scheduler and worker services for queued manual backups and scheduled backups.
|
||||
- Manual backup, dry-run, cancellation, verbose rsync logging, and run detail views.
|
||||
- Snapshot discovery for existing backup directories and SQL-backed snapshot records.
|
||||
- SQL retention planning and apply flow with base snapshot protection and incomplete snapshot visibility.
|
||||
- Explicit cleanup flow for incomplete snapshots, separate from normal retention pruning.
|
||||
- Purged snapshot audit overview with reason, action source, operator, host, kind, path, and timestamp.
|
||||
- Dashboard and host pages with backup health, latest run/snapshot, next run, and storage/stat summaries.
|
||||
- Review resolution for failed/warning runs and incomplete snapshot tasks so operational warnings can be acknowledged.
|
||||
- Worker heartbeat metadata and stale running-run reconciliation for queued backup workers.
|
||||
- SSH key generation, upload, edit, guarded delete, known_hosts management, and per-host key selection.
|
||||
- In-app changelog page sourced from this changelog.
|
||||
- Restore guidance on snapshot detail pages.
|
||||
|
||||
### Changed
|
||||
|
||||
- Django and the database are now the source of truth for configuration.
|
||||
- Docker Compose is documented as development and disposable test tooling rather than the primary production path.
|
||||
- The `pobsync` console entrypoint is now a maintainer layer around Django management commands.
|
||||
- Scheduled pruning is evaluated by the pobsync scheduler service and recorded through Django, not host cron.
|
||||
- Retention and incomplete cleanup now preserve audit history even after source snapshot records are removed.
|
||||
|
||||
### Removed
|
||||
|
||||
- Legacy YAML config import/export workflow.
|
||||
- Public short aliases for configuration commands.
|
||||
- Obsolete global config storage fields.
|
||||
@@ -10,7 +10,7 @@ RUN apt-get update \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY pyproject.toml README.md CHANGELOG.md ./
|
||||
COPY src ./src
|
||||
COPY manage.py ./
|
||||
COPY scripts/docker-entrypoint ./scripts/docker-entrypoint
|
||||
|
||||
135
README.md
135
README.md
@@ -43,6 +43,7 @@ The installer will, by default:
|
||||
- copy the checkout to `/opt/pobsync/app`
|
||||
- create `/opt/pobsync/venv`
|
||||
- write `/etc/pobsync/pobsync.env` if it does not exist
|
||||
- install `pobsync-manage`, a Django management wrapper that loads `/etc/pobsync/pobsync.env`
|
||||
- create `/var/lib/pobsync`, `/var/log/pobsync`, and the backup root
|
||||
- install Python dependencies
|
||||
- run migrations and collect static files
|
||||
@@ -55,6 +56,7 @@ Common overrides:
|
||||
```
|
||||
sudo scripts/install-systemd \
|
||||
--backup-root /mnt/backups/pobsync \
|
||||
--time-zone Europe/Amsterdam \
|
||||
--allowed-hosts backup.example.com,localhost,127.0.0.1 \
|
||||
--csrf-trusted-origins https://backup.example.com
|
||||
```
|
||||
@@ -64,6 +66,9 @@ installer to rewrite an existing `/etc/pobsync/pobsync.env`.
|
||||
Use `--non-interactive` for scripted installs. Use `--verbose` when you want to see the underlying apt, pip, Django, and
|
||||
systemd output.
|
||||
|
||||
Schedules are evaluated in `POBSYNC_TIME_ZONE`. The installer defaults this to the server timezone when it can detect
|
||||
one, otherwise `UTC`; override it with `--time-zone Europe/Amsterdam` or by editing `/etc/pobsync/pobsync.env`.
|
||||
|
||||
For MariaDB support, add:
|
||||
|
||||
```
|
||||
@@ -123,7 +128,21 @@ http://127.0.0.1:8010/
|
||||
Create a superuser if needed:
|
||||
|
||||
```
|
||||
sudo -u pobsync /opt/pobsync/venv/bin/python /opt/pobsync/app/manage.py createsuperuser
|
||||
sudo -u pobsync pobsync-manage createsuperuser
|
||||
```
|
||||
|
||||
The control panel supports two access levels. Django staff users can manage hosts, SSH keys, configs, retention,
|
||||
notifications, logs, and administrative actions. Normal authenticated users can view backup status pages such as the
|
||||
dashboard, hosts, runs, snapshots, schedules, purged history, changelog, and `/api/status/`, but cannot see SSH
|
||||
credentials or run mutating actions.
|
||||
|
||||
For other Django management commands on native installs, use `pobsync-manage` so the production environment file is
|
||||
loaded before Django starts:
|
||||
|
||||
```
|
||||
sudo -u pobsync pobsync-manage showmigrations pobsync_backend
|
||||
sudo -u pobsync pobsync-manage check
|
||||
sudo -u pobsync pobsync-manage check_pobsync_install
|
||||
```
|
||||
|
||||
The UI includes:
|
||||
@@ -139,12 +158,65 @@ The UI includes:
|
||||
- Django-managed SSH keys
|
||||
- `/self-check/` for runtime checks
|
||||
- `/logs/` for filtered pobsync service logs
|
||||
- `/updater/` for checking Gitea releases, pulling the git checkout, and running the native updater
|
||||
|
||||
## Bandwidth Limits
|
||||
|
||||
Global config can set an rsync bandwidth limit in KB/s. The default `0` means unlimited. Each host can inherit the
|
||||
global value, set `0` to explicitly run unlimited, or set its own limit for slower remote links.
|
||||
|
||||
For VPN-backed or remote backups, start conservatively and adjust after watching normal traffic:
|
||||
|
||||
- `2500` KB/s is roughly 20 Mbit/s
|
||||
- `5000` KB/s is roughly 40 Mbit/s
|
||||
- `10000` KB/s is roughly 80 Mbit/s
|
||||
|
||||
pobsync passes the effective value to rsync as `--bwlimit=<KB/s>` and shows it on the host detail and run detail pages.
|
||||
|
||||
## Restoring Data
|
||||
|
||||
pobsync treats restores as an explicit manual operation. The control panel shows restore guidance on each snapshot
|
||||
detail page, but it does not run restore commands for you yet. That is deliberate: restores should be inspected and
|
||||
tested before data is copied back into a live system.
|
||||
|
||||
Each snapshot directory contains:
|
||||
|
||||
```
|
||||
<snapshot>/data/ # backed-up filesystem contents
|
||||
<snapshot>/meta/ # metadata and rsync logs
|
||||
```
|
||||
|
||||
Use the `data/` directory as the rsync source. Start with a dry run and restore to a staging path first:
|
||||
|
||||
```
|
||||
rsync -aHAX --numeric-ids --info=progress2 --dry-run /backups/example.org/scheduled/<snapshot>/data/ /restore/example.org/
|
||||
rsync -aHAX --numeric-ids --info=progress2 /backups/example.org/scheduled/<snapshot>/data/ /restore/example.org/
|
||||
```
|
||||
|
||||
After validating the staged files, copy the specific files or directories back to the target machine. For a full-host
|
||||
restore, use another dry run before writing to the remote root:
|
||||
|
||||
```
|
||||
rsync -aHAX --numeric-ids --info=progress2 --dry-run /backups/example.org/scheduled/<snapshot>/data/ root@example.org:/
|
||||
```
|
||||
|
||||
For most incidents, prefer a targeted restore instead of copying the whole snapshot. Keep paths relative to the
|
||||
snapshot's `data/` directory:
|
||||
|
||||
```
|
||||
rsync -aHAX --numeric-ids --info=progress2 --dry-run /backups/example.org/scheduled/<snapshot>/data/etc/nginx/ /restore/example.org/etc/nginx/
|
||||
rsync -aHAX --numeric-ids --info=progress2 --dry-run /backups/example.org/scheduled/<snapshot>/data/home/example/site/public_html/index.php /restore/example.org/home/example/site/public_html/index.php
|
||||
```
|
||||
|
||||
Snapshots may use hardlinks for files that are unchanged between backups. That saves disk space and is safe for normal
|
||||
restore copies, but do not edit files inside snapshot directories. Treat snapshots as read-only and copy data out with
|
||||
rsync.
|
||||
|
||||
## SSH Keys
|
||||
|
||||
SSH keys can be managed from `/ssh-credentials/`. The recommended flow is to generate keys from Django or during the
|
||||
installer. pobsync stores the private key on disk under `POBSYNC_HOME`, keeps the public key visible in the UI, and lets
|
||||
you select a credential either as the global default or as a per-host override.
|
||||
installer. pobsync stores the private key on disk under the runtime state root (`POBSYNC_HOME`), keeps the public key
|
||||
visible in the UI, and lets you select a credential either as the global default or as a per-host override.
|
||||
|
||||
Generated private keys are stored at:
|
||||
|
||||
@@ -164,16 +236,67 @@ From a fresh checkout or the existing app directory:
|
||||
|
||||
```
|
||||
git pull
|
||||
sudo scripts/install-systemd --non-interactive
|
||||
sudo scripts/update-systemd
|
||||
```
|
||||
|
||||
The installer preserves an existing `/etc/pobsync/pobsync.env` unless you pass `--force-env`. It refreshes the installed
|
||||
app, Python dependencies, migrations, static files, and restarts the systemd services so new Django code is loaded.
|
||||
The updater is a thin wrapper around the installer for normal production deploys. It preserves the existing
|
||||
`/etc/pobsync/pobsync.env`, skips OS package installation, skips superuser creation, refreshes the installed app, updates
|
||||
Python dependencies, runs migrations, collects static files, and restarts the systemd services so new Django code is
|
||||
loaded.
|
||||
|
||||
The Django control panel also exposes an `/updater/` page for staff users. It can check a Gitea releases endpoint, run
|
||||
`git fetch`, run a fast-forward-only pull for the installed branch, and invoke the configured native update command.
|
||||
Configure these optional environment variables in `/etc/pobsync/pobsync.env`:
|
||||
|
||||
```
|
||||
POBSYNC_UPDATE_RELEASES_URL=https://code.example.test/api/v1/repos/owner/pobsync/releases
|
||||
POBSYNC_UPDATE_RELEASES_TOKEN=
|
||||
POBSYNC_UPDATE_GIT_REMOTE=origin
|
||||
POBSYNC_UPDATE_COMMAND=sudo -n scripts/update-systemd
|
||||
```
|
||||
|
||||
If the web service runs as the `pobsync` user, `POBSYNC_UPDATE_COMMAND` needs a matching sudoers rule or a different
|
||||
operator-approved command. Without that, the page still shows update status and command output, but the native update
|
||||
action will fail with a permission error instead of silently doing the wrong thing.
|
||||
|
||||
Use the full installer again when you intentionally want to change install-time settings, install OS packages, enable
|
||||
nginx, or rewrite the environment file:
|
||||
|
||||
```
|
||||
sudo scripts/install-systemd --non-interactive
|
||||
sudo scripts/install-systemd --force-env
|
||||
```
|
||||
|
||||
Then check:
|
||||
|
||||
```
|
||||
systemctl status pobsync-web pobsync-worker pobsync-scheduler
|
||||
sudo -u pobsync pobsync-manage check
|
||||
sudo -u pobsync pobsync-manage check_pobsync_install
|
||||
```
|
||||
|
||||
Restart services manually after environment or reverse proxy changes:
|
||||
|
||||
```
|
||||
sudo systemctl restart pobsync-web pobsync-worker pobsync-scheduler
|
||||
```
|
||||
|
||||
Inspect service logs with:
|
||||
|
||||
```
|
||||
journalctl -u pobsync-web -n 100 --no-pager
|
||||
journalctl -u pobsync-worker -f
|
||||
journalctl -u pobsync-scheduler -n 100 --no-pager
|
||||
```
|
||||
|
||||
Rollback to a previous revision by checking out the known-good commit or tag, then running the updater again:
|
||||
|
||||
```
|
||||
git switch master
|
||||
git pull
|
||||
git checkout <known-good-commit-or-tag>
|
||||
sudo scripts/update-systemd
|
||||
sudo -u pobsync pobsync-manage check_pobsync_install
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
24
deploy/bin/pobsync-manage
Normal file
24
deploy/bin/pobsync-manage
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
APP_DIR="@POBSYNC_APP_DIR@"
|
||||
VENV_DIR="@POBSYNC_VENV_DIR@"
|
||||
ENV_FILE="@POBSYNC_ENV_FILE@"
|
||||
SERVICE_USER="@POBSYNC_USER@"
|
||||
SERVICE_GROUP="@POBSYNC_GROUP@"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "pobsync environment file not found: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
export POBSYNC_ENV_FILE="$ENV_FILE"
|
||||
export POBSYNC_SERVICE_USER="$SERVICE_USER"
|
||||
export POBSYNC_SERVICE_GROUP="$SERVICE_GROUP"
|
||||
|
||||
cd "$APP_DIR"
|
||||
exec "$VENV_DIR/bin/python" "$APP_DIR/manage.py" "$@"
|
||||
@@ -7,9 +7,20 @@ POBSYNC_HOME=/var/lib/pobsync
|
||||
POBSYNC_BACKUP_ROOT=/backups
|
||||
POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
|
||||
POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
|
||||
POBSYNC_ENV_FILE=/etc/pobsync/pobsync.env
|
||||
POBSYNC_SERVICE_USER=pobsync
|
||||
POBSYNC_SERVICE_GROUP=pobsync
|
||||
|
||||
POBSYNC_WEB_BIND=127.0.0.1:8010
|
||||
POBSYNC_GUNICORN_WORKERS=2
|
||||
POBSYNC_GUNICORN_TIMEOUT=120
|
||||
POBSYNC_WORKER_INTERVAL=15
|
||||
POBSYNC_SCHEDULER_INTERVAL=60
|
||||
|
||||
# Optional UI updater integration.
|
||||
# Point this at the Gitea releases API endpoint, for example:
|
||||
# https://code.example.test/api/v1/repos/owner/pobsync/releases
|
||||
POBSYNC_UPDATE_RELEASES_URL=
|
||||
POBSYNC_UPDATE_RELEASES_TOKEN=
|
||||
POBSYNC_UPDATE_GIT_REMOTE=origin
|
||||
POBSYNC_UPDATE_COMMAND=sudo -n scripts/update-systemd
|
||||
|
||||
@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
|
||||
Group=@POBSYNC_GROUP@
|
||||
WorkingDirectory=@POBSYNC_APP_DIR@
|
||||
EnvironmentFile=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
|
||||
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
|
||||
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_scheduler --loop --interval "${POBSYNC_SCHEDULER_INTERVAL:-60}"'
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
|
||||
Group=@POBSYNC_GROUP@
|
||||
WorkingDirectory=@POBSYNC_APP_DIR@
|
||||
EnvironmentFile=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
|
||||
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
|
||||
ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py migrate --noinput
|
||||
ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py collectstatic --noinput --clear
|
||||
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/gunicorn pobsync_server.wsgi:application --bind "${POBSYNC_WEB_BIND:-127.0.0.1:8010}" --workers "${POBSYNC_GUNICORN_WORKERS:-2}" --timeout "${POBSYNC_GUNICORN_TIMEOUT:-120}"'
|
||||
|
||||
@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
|
||||
Group=@POBSYNC_GROUP@
|
||||
WorkingDirectory=@POBSYNC_APP_DIR@
|
||||
EnvironmentFile=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
|
||||
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
|
||||
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
|
||||
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_worker --loop --interval "${POBSYNC_WORKER_INTERVAL:-15}"'
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -47,6 +47,19 @@ pobsync django check
|
||||
python3 manage.py showmigrations pobsync_backend
|
||||
```
|
||||
|
||||
The short `pobsync` aliases are limited to operational actions that are useful while debugging a running install.
|
||||
Configuration aliases are intentionally not public commands; use the Django UI or explicit management commands instead.
|
||||
|
||||
## UI Refresh Pattern
|
||||
|
||||
The control panel stays Django-template-first. Pages that need live status should expose a small server-rendered partial
|
||||
view and opt into refresh with `data-refresh-url` and `data-refresh-interval` on the container that should be replaced.
|
||||
The shared script in `base.html` polls only those explicit regions, skips refreshes while the browser tab is hidden, and
|
||||
lets the partial response turn polling off with the `X-Pobsync-Refresh-Active: false` header.
|
||||
|
||||
Use this for operational status surfaces such as running backup details. Avoid refreshing form-heavy sections while an
|
||||
operator might be typing.
|
||||
|
||||
Worker and scheduler commands are normally run by systemd services:
|
||||
|
||||
```
|
||||
@@ -62,6 +75,14 @@ pobsync discover-snapshots --host <host>
|
||||
pobsync retention <host>
|
||||
```
|
||||
|
||||
For scripted configuration changes, call the Django management command explicitly so it is clear that this is an
|
||||
automation/debugging path rather than the normal UI workflow:
|
||||
|
||||
```
|
||||
pobsync django configure_pobsync_host <host> --address <host.example>
|
||||
pobsync django configure_pobsync_schedule <host> --schedule-expression "15 2 * * *"
|
||||
```
|
||||
|
||||
## Installer Development
|
||||
|
||||
The native installer is interactive by default when stdin is a terminal. It should keep every prompt backed by a command
|
||||
@@ -74,28 +95,16 @@ sudo scripts/install-systemd
|
||||
sudo scripts/install-systemd --non-interactive
|
||||
sudo scripts/install-systemd --verbose
|
||||
sudo scripts/install-systemd --create-superuser --superuser-username admin
|
||||
sudo scripts/update-systemd
|
||||
```
|
||||
|
||||
The installer should print a short completion summary with the control panel URL, Self Check reminder, and service log
|
||||
commands. Keep normal output user-facing: pobsync step names with OK, FAILED, or SKIPPED. Full apt, pip, Django, and
|
||||
systemd output belongs behind `--verbose` or in the failed step output.
|
||||
|
||||
## Migration Helpers
|
||||
|
||||
Import existing legacy YAML configs:
|
||||
|
||||
```
|
||||
python3 manage.py import_pobsync_configs --prefix /opt/pobsync
|
||||
```
|
||||
|
||||
Export SQL config to legacy runtime YAML for inspection or one-off compatibility:
|
||||
|
||||
```
|
||||
python3 manage.py export_pobsync_configs --prefix /opt/pobsync
|
||||
```
|
||||
|
||||
These commands are migration helpers, not the normal operating model. After import, review and continue operating from
|
||||
the Django control panel.
|
||||
The updater is intentionally a small wrapper around the installer for routine production deploys. It should stay
|
||||
non-interactive, preserve the existing environment file, skip OS package installation, skip superuser creation, and still
|
||||
run the Django/runtime refresh steps needed after a code update.
|
||||
|
||||
## Docker With SQLite
|
||||
|
||||
@@ -176,4 +185,3 @@ Next refactor targets:
|
||||
|
||||
- Move more snapshot lifecycle details into typed domain objects.
|
||||
- Replace remaining dictionary-shaped config at engine boundaries.
|
||||
- Remove legacy YAML import/export once production migration no longer needs it.
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "pobsync"
|
||||
version = "0.1.0"
|
||||
version = "1.2.0"
|
||||
description = "Pull-based rsync backup tool with hardlinked snapshots"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Deploy pobsync runtime into /opt/pobsync without pip/venv.
|
||||
# Copies python package sources into /opt/pobsync/lib and installs a stable entrypoint in /opt/pobsync/bin.
|
||||
|
||||
set -eu
|
||||
|
||||
PREFIX="/opt/pobsync"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [--prefix /opt/pobsync]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--prefix)
|
||||
[ $# -ge 2 ] || usage
|
||||
PREFIX="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
echo "Unknown arg: $1" >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Determine repo root from this script location
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
REPO_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
SRC_PKG="${REPO_ROOT}/src/pobsync"
|
||||
if [ ! -d "${SRC_PKG}" ]; then
|
||||
echo "ERROR: expected python package at ${SRC_PKG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BIN_DIR="${PREFIX}/bin"
|
||||
LIB_DIR="${PREFIX}/lib"
|
||||
DST_PKG="${LIB_DIR}/pobsync"
|
||||
BUILD_FILE="${DST_PKG}/_build.txt"
|
||||
|
||||
mkdir -p "${BIN_DIR}" "${LIB_DIR}"
|
||||
|
||||
# Copy code into /opt/pobsync/lib/pobsync
|
||||
# We use rsync if available (clean updates with --delete), otherwise fall back to cp -a.
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
rsync -a --delete \
|
||||
--exclude '__pycache__/' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude '*.pyo' \
|
||||
--exclude '*.pyd' \
|
||||
"${SRC_PKG}/" "${DST_PKG}/"
|
||||
else
|
||||
# Fallback: wipe + copy
|
||||
rm -rf "${DST_PKG}"
|
||||
mkdir -p "${DST_PKG}"
|
||||
cp -a "${SRC_PKG}/." "${DST_PKG}/"
|
||||
fi
|
||||
|
||||
# Write build info (best-effort)
|
||||
GIT_SHA="unknown"
|
||||
if command -v git >/dev/null 2>&1 && [ -d "${REPO_ROOT}/.git" ]; then
|
||||
GIT_SHA="$(cd "${REPO_ROOT}" && git rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||
fi
|
||||
|
||||
NOW_UTC="$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo unknown)"
|
||||
{
|
||||
echo "deployed_at_utc=${NOW_UTC}"
|
||||
echo "git_sha=${GIT_SHA}"
|
||||
echo "repo_root=${REPO_ROOT}"
|
||||
} > "${BUILD_FILE}"
|
||||
|
||||
# Install stable entrypoint that always runs code from /opt/pobsync/lib
|
||||
WRAPPER="${BIN_DIR}/pobsync"
|
||||
cat > "${WRAPPER}" <<EOF
|
||||
#!/bin/sh
|
||||
# managed-by=pobsync deploy
|
||||
set -eu
|
||||
|
||||
PREFIX="${PREFIX}"
|
||||
export PYTHONPATH="\${PREFIX}/lib"
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
exec /usr/bin/python3 -m pobsync "\$@"
|
||||
EOF
|
||||
|
||||
chmod 0755 "${WRAPPER}"
|
||||
|
||||
echo "OK"
|
||||
echo "- deployed package to ${DST_PKG}"
|
||||
echo "- wrote build info ${BUILD_FILE}"
|
||||
echo "- installed entrypoint ${WRAPPER}"
|
||||
|
||||
@@ -17,6 +17,7 @@ if [ -n "${POBSYNC_BACKUP_ROOT:-}" ]; then
|
||||
BACKUP_ROOT_EXPLICIT=1
|
||||
fi
|
||||
WEB_BIND=${POBSYNC_WEB_BIND:-127.0.0.1:8010}
|
||||
TIME_ZONE=${POBSYNC_TIME_ZONE:-}
|
||||
FORCE_ENV=0
|
||||
INSTALL_OS_PACKAGES=1
|
||||
WITH_NGINX=0
|
||||
@@ -74,6 +75,10 @@ while [ "$#" -gt 0 ]; do
|
||||
WEB_BIND=$2
|
||||
shift 2
|
||||
;;
|
||||
--time-zone)
|
||||
TIME_ZONE=$2
|
||||
shift 2
|
||||
;;
|
||||
--force-env)
|
||||
FORCE_ENV=1
|
||||
shift
|
||||
@@ -148,6 +153,38 @@ if [ -f "$ENV_FILE" ] && [ "$FORCE_ENV" -ne 1 ] && [ "$BACKUP_ROOT_EXPLICIT" -ne
|
||||
fi
|
||||
fi
|
||||
|
||||
detect_time_zone() {
|
||||
if [ -n "$TIME_ZONE" ]; then
|
||||
printf '%s\n' "$TIME_ZONE"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "${POBSYNC_TIME_ZONE:-}" ]; then
|
||||
printf '%s\n' "$POBSYNC_TIME_ZONE"
|
||||
return
|
||||
fi
|
||||
|
||||
if command -v timedatectl >/dev/null 2>&1; then
|
||||
detected=$(timedatectl show -p Timezone --value 2>/dev/null || true)
|
||||
if [ -n "$detected" ]; then
|
||||
printf '%s\n' "$detected"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -f /etc/timezone ]; then
|
||||
detected=$(sed -n '1p' /etc/timezone | tr -d '[:space:]')
|
||||
if [ -n "$detected" ]; then
|
||||
printf '%s\n' "$detected"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
|
||||
printf 'UTC\n'
|
||||
}
|
||||
|
||||
TIME_ZONE=$(detect_time_zone)
|
||||
|
||||
run_step() {
|
||||
label=$1
|
||||
shift
|
||||
@@ -261,6 +298,7 @@ if [ "$INTERACTIVE" -eq 1 ]; then
|
||||
SERVICE_GROUP=$(prompt_value "Service group" "$SERVICE_GROUP")
|
||||
BACKUP_ROOT=$(prompt_value "Backup storage path" "$BACKUP_ROOT")
|
||||
WEB_BIND=$(prompt_value "Gunicorn bind address" "$WEB_BIND")
|
||||
TIME_ZONE=$(prompt_value "Scheduler time zone" "$TIME_ZONE")
|
||||
ALLOWED_HOSTS=$(prompt_value "Allowed hosts" "$ALLOWED_HOSTS")
|
||||
CSRF_TRUSTED_ORIGINS=$(prompt_value "CSRF trusted origins, comma-separated or blank" "$CSRF_TRUSTED_ORIGINS")
|
||||
INSTALL_OS_PACKAGES=$(prompt_yes_no "Install required OS packages with apt-get" "$INSTALL_OS_PACKAGES")
|
||||
@@ -331,6 +369,12 @@ if ! command -v python3 >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! env POBSYNC_INSTALL_TIME_ZONE="$TIME_ZONE" python3 -c "import os; from zoneinfo import ZoneInfo; ZoneInfo(os.environ['POBSYNC_INSTALL_TIME_ZONE'])" >/dev/null 2>&1; then
|
||||
echo "Invalid time zone: $TIME_ZONE" >&2
|
||||
echo "Use an IANA timezone such as UTC or Europe/Amsterdam." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v rsync >/dev/null 2>&1; then
|
||||
echo "rsync is required." >&2
|
||||
exit 1
|
||||
@@ -416,14 +460,22 @@ POBSYNC_DJANGO_CSRF_TRUSTED_ORIGINS=$CSRF_TRUSTED_ORIGINS
|
||||
|
||||
POBSYNC_HOME=/var/lib/pobsync
|
||||
POBSYNC_BACKUP_ROOT=$BACKUP_ROOT
|
||||
POBSYNC_TIME_ZONE=$TIME_ZONE
|
||||
POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
|
||||
POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
|
||||
POBSYNC_ENV_FILE=$ENV_FILE
|
||||
POBSYNC_SERVICE_USER=$SERVICE_USER
|
||||
POBSYNC_SERVICE_GROUP=$SERVICE_GROUP
|
||||
|
||||
POBSYNC_WEB_BIND=$WEB_BIND
|
||||
POBSYNC_GUNICORN_WORKERS=2
|
||||
POBSYNC_GUNICORN_TIMEOUT=120
|
||||
POBSYNC_WORKER_INTERVAL=15
|
||||
POBSYNC_SCHEDULER_INTERVAL=60
|
||||
POBSYNC_UPDATE_RELEASES_URL=
|
||||
POBSYNC_UPDATE_RELEASES_TOKEN=
|
||||
POBSYNC_UPDATE_GIT_REMOTE=origin
|
||||
POBSYNC_UPDATE_COMMAND=sudo -n scripts/update-systemd
|
||||
EOF
|
||||
chmod 0640 "$ENV_FILE"
|
||||
chown "root:$SERVICE_GROUP" "$ENV_FILE"
|
||||
@@ -459,10 +511,23 @@ install_units() {
|
||||
|
||||
run_step "Install systemd units" install_units
|
||||
|
||||
install_manage_wrapper() {
|
||||
sed \
|
||||
-e "s|@POBSYNC_APP_DIR@|$APP_DIR|g" \
|
||||
-e "s|@POBSYNC_VENV_DIR@|$VENV_DIR|g" \
|
||||
-e "s|@POBSYNC_ENV_FILE@|$ENV_FILE|g" \
|
||||
-e "s|@POBSYNC_USER@|$SERVICE_USER|g" \
|
||||
-e "s|@POBSYNC_GROUP@|$SERVICE_GROUP|g" \
|
||||
"$APP_DIR/deploy/bin/pobsync-manage" > /usr/local/bin/pobsync-manage
|
||||
chmod 0755 /usr/local/bin/pobsync-manage
|
||||
}
|
||||
|
||||
run_step "Install manage wrapper" install_manage_wrapper
|
||||
|
||||
run_step "Reload systemd" systemctl daemon-reload
|
||||
run_step "Run database migrations" "$VENV_DIR/bin/python" "$APP_DIR/manage.py" migrate --noinput
|
||||
run_step "Ensure default SSH key" "$VENV_DIR/bin/python" "$APP_DIR/manage.py" ensure_pobsync_ssh_key --name default --set-global-default
|
||||
run_step "Collect static files" "$VENV_DIR/bin/python" "$APP_DIR/manage.py" collectstatic --noinput --clear
|
||||
run_step "Run database migrations" /usr/local/bin/pobsync-manage migrate --noinput
|
||||
run_step "Ensure default SSH key" /usr/local/bin/pobsync-manage ensure_pobsync_ssh_key --name default --set-global-default
|
||||
run_step "Collect static files" /usr/local/bin/pobsync-manage collectstatic --noinput --clear
|
||||
run_step "Finalize state permissions" chown -R "$SERVICE_USER:$SERVICE_GROUP" /var/lib/pobsync /var/log/pobsync
|
||||
|
||||
superuser_exists=$("$VENV_DIR/bin/python" -c "import os; os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pobsync_server.settings'); import django; django.setup(); from django.contrib.auth import get_user_model; print('yes' if get_user_model().objects.filter(is_superuser=True).exists() else 'no')")
|
||||
@@ -474,17 +539,17 @@ if [ "$CREATE_SUPERUSER" -eq 1 ]; then
|
||||
DJANGO_SUPERUSER_USERNAME="$SUPERUSER_USERNAME" \
|
||||
DJANGO_SUPERUSER_EMAIL="$SUPERUSER_EMAIL" \
|
||||
DJANGO_SUPERUSER_PASSWORD="$SUPERUSER_PASSWORD" \
|
||||
"$VENV_DIR/bin/python" "$APP_DIR/manage.py" createsuperuser --noinput
|
||||
/usr/local/bin/pobsync-manage createsuperuser --noinput
|
||||
run_step "Finalize superuser permissions" chown -R "$SERVICE_USER:$SERVICE_GROUP" /var/lib/pobsync /var/log/pobsync
|
||||
else
|
||||
note_step "Create Django superuser" "SKIPPED"
|
||||
echo "No superuser password was provided; create one later with:"
|
||||
echo " sudo -u $SERVICE_USER $VENV_DIR/bin/python $APP_DIR/manage.py createsuperuser"
|
||||
echo " sudo -u $SERVICE_USER pobsync-manage createsuperuser"
|
||||
fi
|
||||
elif [ "$superuser_exists" != "yes" ]; then
|
||||
note_step "Create Django superuser" "SKIPPED"
|
||||
echo "No Django superuser exists yet. Create one with:"
|
||||
echo " sudo -u $SERVICE_USER $VENV_DIR/bin/python $APP_DIR/manage.py createsuperuser"
|
||||
echo " sudo -u $SERVICE_USER pobsync-manage createsuperuser"
|
||||
else
|
||||
note_step "Create Django superuser" "SKIPPED"
|
||||
fi
|
||||
@@ -529,3 +594,5 @@ echo
|
||||
echo "Useful commands:"
|
||||
echo " systemctl status pobsync-web pobsync-worker pobsync-scheduler"
|
||||
echo " journalctl -u pobsync-worker -f"
|
||||
echo " sudo -u $SERVICE_USER pobsync-manage check"
|
||||
echo " sudo -u $SERVICE_USER pobsync-manage check_pobsync_install"
|
||||
|
||||
41
scripts/update-systemd
Executable file
41
scripts/update-systemd
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
|
||||
show_help() {
|
||||
cat <<'EOF'
|
||||
Usage: sudo scripts/update-systemd [options]
|
||||
|
||||
Refresh an existing native pobsync systemd install from the current checkout.
|
||||
|
||||
This is a thin, safer update wrapper around scripts/install-systemd. It keeps
|
||||
the install non-interactive, preserves the existing environment file, skips
|
||||
superuser creation, and skips OS package installation by default.
|
||||
|
||||
Common options are forwarded to install-systemd, for example:
|
||||
--source-dir PATH
|
||||
--app-dir PATH
|
||||
--venv-dir PATH
|
||||
--env-file PATH
|
||||
--service-user USER
|
||||
--service-group GROUP
|
||||
--install-extras mariadb
|
||||
--verbose
|
||||
|
||||
If OS packages need to be refreshed, run scripts/install-systemd directly.
|
||||
EOF
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
-h|--help)
|
||||
show_help
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
exec "$SCRIPT_DIR/install-systemd" \
|
||||
--non-interactive \
|
||||
--no-install-os-packages \
|
||||
--no-create-superuser \
|
||||
"$@"
|
||||
@@ -1,3 +1,2 @@
|
||||
__all__ = ["__version__"]
|
||||
__version__ = "0.1.0"
|
||||
|
||||
__version__ = "1.2.0"
|
||||
|
||||
@@ -6,11 +6,10 @@ from typing import Sequence
|
||||
|
||||
from django.core.management import execute_from_command_line
|
||||
|
||||
from pobsync import __version__
|
||||
|
||||
|
||||
COMMAND_ALIASES = {
|
||||
"configure-global": "configure_pobsync_global",
|
||||
"configure-host": "configure_pobsync_host",
|
||||
"schedule": "configure_pobsync_schedule",
|
||||
"backup": "run_pobsync_backup",
|
||||
"retention": "run_pobsync_retention",
|
||||
"discover-snapshots": "discover_pobsync_snapshots",
|
||||
@@ -29,11 +28,17 @@ Usage:
|
||||
|
||||
Commands:
|
||||
{commands}
|
||||
|
||||
Configuration is managed from the Django control panel. Use
|
||||
`pobsync django <management-command>` for automation or debugging.
|
||||
"""
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = list(sys.argv[1:] if argv is None else argv)
|
||||
if args and args[0] in {"--version", "version"}:
|
||||
print(f"pobsync {__version__}")
|
||||
return 0
|
||||
if not args or args[0] in {"-h", "--help", "help"}:
|
||||
print(_usage())
|
||||
return 0
|
||||
|
||||
@@ -4,9 +4,8 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
from ..config.source import ConfigSource, FileConfigSource
|
||||
from ..config.source import ConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..paths import PobsyncPaths
|
||||
from ..retention import Snapshot, apply_base_protection, build_retention_plan
|
||||
from ..snapshot_meta import iter_snapshot_dirs, read_snapshot_meta, resolve_host_root
|
||||
from ..util import sanitize_host
|
||||
@@ -40,10 +39,9 @@ def run_retention_plan(
|
||||
if kind not in {"scheduled", "manual", "all"}:
|
||||
raise ConfigError("kind must be scheduled, manual, or all")
|
||||
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
source = config_source or FileConfigSource(prefix=paths.home)
|
||||
cfg = source.effective_config_for_host(host)
|
||||
if config_source is None:
|
||||
raise ConfigError("A Django config source is required.")
|
||||
cfg = config_source.effective_config_for_host(host)
|
||||
|
||||
retention = cfg.get("retention")
|
||||
if not isinstance(retention, dict):
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from ..config.source import ConfigSource, FileConfigSource
|
||||
from ..config.source import ConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..lock import acquire_host_lock
|
||||
from ..paths import PobsyncPaths
|
||||
@@ -23,6 +23,7 @@ from ..util import ensure_dir, realpath_startswith, sanitize_host, write_yaml_at
|
||||
|
||||
|
||||
DEFAULT_DRY_RUN_TIMEOUT_SECONDS = 900
|
||||
RSYNC_PARTIAL_VANISHED_EXIT_CODE = 24
|
||||
|
||||
|
||||
def dry_run_log_path(host: str, run_id: int | None = None) -> Path:
|
||||
@@ -72,6 +73,24 @@ def classify_rsync_failure(exit_code: int | None, log_tail: list[str]) -> dict[s
|
||||
}
|
||||
|
||||
|
||||
def classify_rsync_warning(exit_code: int | None, log_tail: list[str]) -> dict[str, str] | None:
|
||||
joined_tail = "\n".join(log_tail).lower()
|
||||
if exit_code == RSYNC_PARTIAL_VANISHED_EXIT_CODE:
|
||||
return {
|
||||
"category": "vanished",
|
||||
"message": "Some source files vanished during rsync.",
|
||||
"hint": "This is common on live systems. The snapshot was kept, but review the rsync log if this happens often.",
|
||||
}
|
||||
if exit_code in (None, RSYNC_PARTIAL_VANISHED_EXIT_CODE) and (
|
||||
"file has vanished" in joined_tail or "vanished before it could be transferred" in joined_tail
|
||||
):
|
||||
return {
|
||||
"category": "vanished",
|
||||
"message": "Some source files vanished during rsync.",
|
||||
"hint": "This is common on live systems. The snapshot was kept, but review the rsync log if this happens often.",
|
||||
}
|
||||
return None
|
||||
|
||||
def _collect_run_stats(
|
||||
*,
|
||||
log_path: Path,
|
||||
@@ -158,13 +177,15 @@ def run_scheduled(
|
||||
run_id: int | None = None,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
verbose_output: bool = False,
|
||||
state_callback: Callable[[dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
|
||||
host = sanitize_host(host)
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
source = config_source or FileConfigSource(prefix=paths.home)
|
||||
cfg = source.effective_config_for_host(host)
|
||||
if config_source is None:
|
||||
raise ConfigError("A Django config source is required.")
|
||||
cfg = config_source.effective_config_for_host(host)
|
||||
|
||||
backup_root = cfg.get("backup_root")
|
||||
if not isinstance(backup_root, str) or not backup_root.startswith("/"):
|
||||
@@ -257,6 +278,7 @@ def run_scheduled(
|
||||
"exit_code": result.exit_code,
|
||||
"command": result.command,
|
||||
"log_tail": log_tail,
|
||||
"bwlimit_kbps": bwlimit_kbps,
|
||||
},
|
||||
}
|
||||
if result.exit_code != 0:
|
||||
@@ -315,21 +337,65 @@ def run_scheduled(
|
||||
"ended_at": None,
|
||||
"duration_seconds": None,
|
||||
"base": _base_meta_from_path(base_dir, link_dest),
|
||||
"rsync": {"exit_code": None, "command": cmd, "stats": {}},
|
||||
# Keep existing fields for future expansion / compatibility with current structure.
|
||||
"rsync": {"exit_code": None, "command": cmd, "stats": {}, "bwlimit_kbps": bwlimit_kbps},
|
||||
"overrides": {"includes": [], "excludes": [], "base": None},
|
||||
}
|
||||
|
||||
log_path.touch(exist_ok=True)
|
||||
write_yaml_atomic(meta_path, meta)
|
||||
if state_callback is not None:
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "preparing",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": None, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
result = run_rsync(cmd, log_path=log_path, timeout_seconds=timeout_seconds, cancel_check=cancel_check)
|
||||
def process_started(pid: int, pgid: int) -> None:
|
||||
if state_callback is None:
|
||||
return
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "rsync",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": None, "pid": pid, "pgid": pgid, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
run_rsync_kwargs: dict[str, Any] = {
|
||||
"log_path": log_path,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"cancel_check": cancel_check,
|
||||
}
|
||||
if state_callback is not None:
|
||||
run_rsync_kwargs["process_started"] = process_started
|
||||
result = run_rsync(cmd, **run_rsync_kwargs)
|
||||
log_tail = _read_log_tail(log_path)
|
||||
warning = classify_rsync_warning(result.exit_code, log_tail)
|
||||
successful_or_warning = result.exit_code == 0 or warning is not None
|
||||
if state_callback is not None:
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "finalizing",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": result.exit_code, "log_tail": log_tail, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
end_ts = utc_now()
|
||||
meta["ended_at"] = format_iso_z(end_ts)
|
||||
meta["duration_seconds"] = int((end_ts - ts).total_seconds())
|
||||
meta["rsync"]["exit_code"] = result.exit_code
|
||||
meta["status"] = "cancelled" if result.cancelled else ("success" if result.exit_code == 0 else "failed")
|
||||
meta["status"] = "cancelled" if result.cancelled else ("warning" if warning else ("success" if result.exit_code == 0 else "failed"))
|
||||
if warning is not None:
|
||||
meta["warning"] = warning
|
||||
meta["stats"] = _collect_run_stats(
|
||||
log_path=log_path,
|
||||
backup_root=Path(backup_root),
|
||||
@@ -349,8 +415,7 @@ def run_scheduled(
|
||||
"error": "rsync.log missing after execution",
|
||||
}
|
||||
|
||||
if result.exit_code != 0:
|
||||
log_tail = _read_log_tail(log_path)
|
||||
if not successful_or_warning:
|
||||
return {
|
||||
"ok": False,
|
||||
"dry_run": False,
|
||||
@@ -366,6 +431,7 @@ def run_scheduled(
|
||||
"exit_code": result.exit_code,
|
||||
"command": result.command,
|
||||
"log_tail": log_tail,
|
||||
"bwlimit_kbps": bwlimit_kbps,
|
||||
},
|
||||
"failure": classify_rsync_failure(result.exit_code, log_tail),
|
||||
}
|
||||
@@ -403,7 +469,10 @@ def run_scheduled(
|
||||
"host": host,
|
||||
"snapshot": str(final_dir),
|
||||
"base": str(base_dir) if base_dir else None,
|
||||
"rsync": {"exit_code": result.exit_code},
|
||||
"log": str(final_log_path),
|
||||
"status": meta["status"],
|
||||
"warning": warning,
|
||||
"rsync": {"exit_code": result.exit_code, "command": result.command, "log_tail": log_tail, "bwlimit_kbps": bwlimit_kbps},
|
||||
"verbose_output": bool(verbose_output),
|
||||
"duration_seconds": meta["duration_seconds"],
|
||||
"stats": meta["stats"],
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ..errors import ConfigError, ValidationError
|
||||
from ..validate import validate_dict
|
||||
from .schemas import GLOBAL_SCHEMA, HOST_SCHEMA
|
||||
|
||||
|
||||
def load_yaml_file(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise ConfigError(f"Missing config file: {path}")
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
raise ConfigError(f"Cannot read config file: {path}: {e}") from e
|
||||
|
||||
try:
|
||||
data = yaml.safe_load(raw)
|
||||
except yaml.YAMLError as e:
|
||||
raise ConfigError(f"Invalid YAML in {path}: {e}") from e
|
||||
|
||||
if data is None:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
raise ConfigError(f"Config root must be a mapping in {path}")
|
||||
return data
|
||||
|
||||
|
||||
def load_global_config(path: Path) -> dict[str, Any]:
|
||||
data = load_yaml_file(path)
|
||||
try:
|
||||
return validate_dict(data, GLOBAL_SCHEMA, path="global")
|
||||
except ValidationError as e:
|
||||
raise ConfigError(f"Invalid global config at {path}: {format_validation_error(e)}") from e
|
||||
|
||||
|
||||
def load_host_config(path: Path) -> dict[str, Any]:
|
||||
data = load_yaml_file(path)
|
||||
try:
|
||||
return validate_dict(data, HOST_SCHEMA, path="host")
|
||||
except ValidationError as e:
|
||||
raise ConfigError(f"Invalid host config at {path}: {format_validation_error(e)}") from e
|
||||
|
||||
|
||||
def format_validation_error(err: ValidationError) -> str:
|
||||
if err.path:
|
||||
return f"{err.path}: {err}"
|
||||
return str(err)
|
||||
|
||||
@@ -83,7 +83,6 @@ OUTPUT_SCHEMA = Schema(
|
||||
GLOBAL_SCHEMA = Schema(
|
||||
fields={
|
||||
"backup_root": FieldSpec(str, required=True),
|
||||
"pobsync_home": FieldSpec(str, required=False, default="/opt/pobsync"),
|
||||
"ssh": FieldSpec(dict, required=False, schema=SSH_SCHEMA),
|
||||
"rsync": FieldSpec(dict, required=False, schema=RSYNC_SCHEMA),
|
||||
"defaults": FieldSpec(dict, required=False, schema=DEFAULTS_SCHEMA),
|
||||
@@ -95,7 +94,6 @@ GLOBAL_SCHEMA = Schema(
|
||||
),
|
||||
"logging": FieldSpec(dict, required=False, schema=LOGGING_SCHEMA),
|
||||
"output": FieldSpec(dict, required=False, schema=OUTPUT_SCHEMA),
|
||||
# Used by `init-host` as a convenience default
|
||||
"retention_defaults": FieldSpec(
|
||||
dict,
|
||||
required=False,
|
||||
@@ -112,6 +110,7 @@ GLOBAL_SCHEMA = Schema(
|
||||
|
||||
HOST_RSYNC_SCHEMA = Schema(
|
||||
fields={
|
||||
"bwlimit_kbps": FieldSpec(int, required=False, min_value=0),
|
||||
"extra_args": FieldSpec(list, required=False, default=[], item=FieldSpec(str)),
|
||||
},
|
||||
allow_unknown=False,
|
||||
@@ -131,4 +130,3 @@ HOST_SCHEMA = Schema(
|
||||
},
|
||||
allow_unknown=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,22 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .load import load_global_config, load_host_config
|
||||
from .merge import build_effective_config
|
||||
|
||||
|
||||
class ConfigSource(Protocol):
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
"""Return the fully merged effective config for a host."""
|
||||
|
||||
|
||||
class FileConfigSource:
|
||||
def __init__(self, prefix: Path) -> None:
|
||||
self.prefix = prefix
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
global_cfg = load_global_config(self.prefix / "config" / "global.yaml")
|
||||
host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml")
|
||||
return build_effective_config(global_cfg, host_cfg)
|
||||
|
||||
@@ -8,14 +8,6 @@ from pathlib import Path
|
||||
class PobsyncPaths:
|
||||
home: Path # usually /opt/pobsync
|
||||
|
||||
@property
|
||||
def config_dir(self) -> Path:
|
||||
return self.home / "config"
|
||||
|
||||
@property
|
||||
def hosts_dir(self) -> Path:
|
||||
return self.config_dir / "hosts"
|
||||
|
||||
@property
|
||||
def state_dir(self) -> Path:
|
||||
return self.home / "state"
|
||||
@@ -28,11 +20,6 @@ class PobsyncPaths:
|
||||
def logs_dir(self) -> Path:
|
||||
return self.home / "logs"
|
||||
|
||||
@property
|
||||
def global_config_path(self) -> Path:
|
||||
return self.config_dir / "global.yaml"
|
||||
|
||||
@property
|
||||
def central_log_path(self) -> Path:
|
||||
return self.logs_dir / "pobsync.log"
|
||||
|
||||
|
||||
@@ -82,6 +82,7 @@ def run_rsync(
|
||||
log_path: Path,
|
||||
timeout_seconds: int,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
process_started: Callable[[int, int], None] | None = None,
|
||||
) -> RsyncResult:
|
||||
"""
|
||||
Run rsync and always write stdout/stderr to log_path.
|
||||
@@ -95,6 +96,8 @@ def run_rsync(
|
||||
|
||||
with log_path.open("ab") as f:
|
||||
process = subprocess.Popen(command, stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
if process_started is not None:
|
||||
process_started(process.pid, os.getpgid(process.pid))
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
exit_code = process.poll()
|
||||
|
||||
39
src/pobsync_backend/access.py
Normal file
39
src/pobsync_backend/access.py
Normal file
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
|
||||
|
||||
def can_view_status(user) -> bool:
|
||||
return bool(user.is_authenticated)
|
||||
|
||||
|
||||
def can_manage_control_panel(user) -> bool:
|
||||
return bool(user.is_authenticated and user.is_staff)
|
||||
|
||||
|
||||
def status_view_required(view_func: Callable[..., HttpResponse]) -> Callable[..., HttpResponse]:
|
||||
return login_required(view_func)
|
||||
|
||||
|
||||
def control_panel_admin_required(view_func: Callable[..., HttpResponse]) -> Callable[..., HttpResponse]:
|
||||
@login_required
|
||||
@wraps(view_func)
|
||||
def wrapper(request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
|
||||
if not can_manage_control_panel(request.user):
|
||||
raise PermissionDenied
|
||||
return view_func(request, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def access_context(request: HttpRequest) -> dict[str, Any]:
|
||||
return {
|
||||
"can_view_status": can_view_status(request.user),
|
||||
"can_manage_control_panel": can_manage_control_panel(request.user),
|
||||
}
|
||||
@@ -6,7 +6,17 @@ from django.urls import reverse
|
||||
from django.utils.html import format_html
|
||||
from django.utils.http import urlencode
|
||||
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord, SshCredential
|
||||
from .models import (
|
||||
BackupRun,
|
||||
GlobalConfig,
|
||||
HostConfig,
|
||||
NotificationDelivery,
|
||||
NotificationTarget,
|
||||
PurgedSnapshot,
|
||||
ScheduleConfig,
|
||||
SnapshotRecord,
|
||||
SshCredential,
|
||||
)
|
||||
|
||||
|
||||
@admin.register(SshCredential)
|
||||
@@ -34,7 +44,7 @@ class GlobalConfigAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at")
|
||||
readonly_fields = ("created_at", "updated_at")
|
||||
fieldsets = (
|
||||
(None, {"fields": ("name", "backup_root", "pobsync_home")}),
|
||||
(None, {"fields": ("name", "backup_root")}),
|
||||
("SSH", {"fields": ("default_ssh_credential", "ssh_user", "ssh_port", "ssh_options")}),
|
||||
(
|
||||
"Rsync",
|
||||
@@ -50,7 +60,6 @@ class GlobalConfigAdmin(admin.ModelAdmin):
|
||||
),
|
||||
("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}),
|
||||
("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
|
||||
("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
@@ -74,9 +83,9 @@ class HostConfigAdmin(admin.ModelAdmin):
|
||||
(None, {"fields": ("host", "address", "enabled")}),
|
||||
("SSH override", {"fields": ("ssh_credential", "ssh_user", "ssh_port")}),
|
||||
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
|
||||
("Rsync override", {"fields": ("rsync_extra_args",)}),
|
||||
("Rsync override", {"fields": ("rsync_extra_args", "rsync_bwlimit_kbps")}),
|
||||
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
|
||||
("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}),
|
||||
("Runtime state", {"fields": ("config",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
@@ -137,6 +146,38 @@ class BackupRunAdmin(admin.ModelAdmin):
|
||||
return format_html('<a href="{}">{}</a>', url, obj.snapshot.dirname)
|
||||
|
||||
|
||||
@admin.register(NotificationTarget)
|
||||
class NotificationTargetAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "channel", "enabled", "last_status", "last_sent_at", "updated_at")
|
||||
list_filter = ("enabled", "channel", "last_status")
|
||||
search_fields = ("name", "email_to", "webhook_url", "notes")
|
||||
readonly_fields = ("created_at", "updated_at", "last_status", "last_error", "last_sent_at")
|
||||
fieldsets = (
|
||||
(None, {"fields": ("name", "enabled", "channel", "statuses")}),
|
||||
("Email", {"fields": ("email_to",)}),
|
||||
("Webhook", {"fields": ("webhook_url", "webhook_headers")}),
|
||||
("State", {"fields": ("last_status", "last_error", "last_sent_at"), "classes": ("collapse",)}),
|
||||
("Notes", {"fields": ("notes",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
|
||||
@admin.register(NotificationDelivery)
|
||||
class NotificationDeliveryAdmin(admin.ModelAdmin):
|
||||
list_display = ("target", "run", "status", "created_at")
|
||||
list_filter = ("status", "target__channel", "created_at")
|
||||
search_fields = ("target__name", "run__host__host", "error")
|
||||
readonly_fields = ("target", "run", "status", "error", "payload", "created_at")
|
||||
list_select_related = ("target", "run", "run__host")
|
||||
date_hierarchy = "created_at"
|
||||
|
||||
def has_add_permission(self, request) -> bool:
|
||||
return False
|
||||
|
||||
def has_change_permission(self, request, obj=None) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@admin.register(SnapshotRecord)
|
||||
class SnapshotRecordAdmin(admin.ModelAdmin):
|
||||
list_display = ("host", "kind", "dirname", "status", "base_link", "backup_run_count_link", "started_at", "discovered_at")
|
||||
@@ -174,6 +215,16 @@ class SnapshotRecordAdmin(admin.ModelAdmin):
|
||||
return format_html('<a href="{}">{}</a>', url, count)
|
||||
|
||||
|
||||
@admin.register(PurgedSnapshot)
|
||||
class PurgedSnapshotAdmin(admin.ModelAdmin):
|
||||
list_display = ("host_name", "kind", "dirname", "action", "reason", "triggered_by", "purged_at")
|
||||
list_filter = ("action", "kind", "purged_at")
|
||||
search_fields = ("host_name", "dirname", "path", "reason", "triggered_by")
|
||||
list_select_related = ("host",)
|
||||
readonly_fields = ("purged_at",)
|
||||
date_hierarchy = "purged_at"
|
||||
|
||||
|
||||
@admin.register(ScheduleConfig)
|
||||
class ScheduleConfigAdmin(admin.ModelAdmin):
|
||||
list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at")
|
||||
|
||||
@@ -2,16 +2,16 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.admin.views.decorators import staff_member_required
|
||||
from django.db import connection
|
||||
from django.db.models import Count
|
||||
from django.http import JsonResponse
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import BackupRun, HostConfig, ScheduleConfig, SnapshotRecord
|
||||
from .access import control_panel_admin_required, status_view_required
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@control_panel_admin_required
|
||||
def api_index(request) -> JsonResponse:
|
||||
return JsonResponse(
|
||||
{
|
||||
@@ -26,7 +26,7 @@ def api_index(request) -> JsonResponse:
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@status_view_required
|
||||
def status(request) -> JsonResponse:
|
||||
latest_run = BackupRun.objects.select_related("host", "snapshot").order_by("-created_at").first()
|
||||
latest_schedule = ScheduleConfig.objects.select_related("host").order_by("-last_started_at", "-updated_at").first()
|
||||
@@ -55,7 +55,7 @@ def status(request) -> JsonResponse:
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@control_panel_admin_required
|
||||
def hosts(request) -> JsonResponse:
|
||||
host_qs = (
|
||||
HostConfig.objects.annotate(snapshot_count=Count("snapshots", distinct=True), run_count=Count("runs", distinct=True))
|
||||
@@ -65,7 +65,7 @@ def hosts(request) -> JsonResponse:
|
||||
return JsonResponse({"ok": True, "hosts": [_host_payload(host) for host in host_qs]})
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@control_panel_admin_required
|
||||
def snapshots(request) -> JsonResponse:
|
||||
snapshot_qs = SnapshotRecord.objects.select_related("host", "base").order_by("host__host", "-started_at", "dirname")
|
||||
host_filter = request.GET.get("host")
|
||||
@@ -78,7 +78,7 @@ def snapshots(request) -> JsonResponse:
|
||||
return JsonResponse({"ok": True, "snapshots": [_snapshot_payload(snapshot) for snapshot in snapshot_qs[:limit]]})
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@control_panel_admin_required
|
||||
def runs(request) -> JsonResponse:
|
||||
run_qs = BackupRun.objects.select_related("host", "snapshot").order_by("-created_at")
|
||||
host_filter = request.GET.get("host")
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
import os
|
||||
import socket
|
||||
from datetime import timedelta, timezone as datetime_timezone
|
||||
from pathlib import Path
|
||||
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync.commands.run_scheduled import DEFAULT_DRY_RUN_TIMEOUT_SECONDS, classify_rsync_failure, dry_run_log_path, run_scheduled
|
||||
from pobsync.commands.run_scheduled import (
|
||||
DEFAULT_DRY_RUN_TIMEOUT_SECONDS,
|
||||
classify_rsync_failure,
|
||||
classify_rsync_warning,
|
||||
dry_run_log_path,
|
||||
run_scheduled,
|
||||
)
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import BackupRun, HostConfig
|
||||
from pobsync_backend.notifications import notify_backup_run_completed
|
||||
from pobsync_backend.retention import run_sql_retention_apply
|
||||
from pobsync_backend.snapshot_discovery import infer_snapshot_kind, upsert_snapshot_record
|
||||
|
||||
@@ -18,7 +27,7 @@ def queue_backup_run(
|
||||
host: HostConfig,
|
||||
run_type: str = BackupRun.RunType.MANUAL,
|
||||
dry_run: bool = False,
|
||||
verbose_output: bool = False,
|
||||
verbose_output: bool = True,
|
||||
prune: bool = False,
|
||||
prune_max_delete: int = 10,
|
||||
prune_protect_bases: bool = False,
|
||||
@@ -64,6 +73,7 @@ def execute_backup_run(
|
||||
run_id=run.id,
|
||||
cancel_check=lambda: _run_cancel_requested(run.id),
|
||||
verbose_output=bool(dry_run or verbose_output),
|
||||
state_callback=lambda state: _record_running_state(run.id, state),
|
||||
)
|
||||
except Exception as exc:
|
||||
run.refresh_from_db()
|
||||
@@ -76,11 +86,14 @@ def execute_backup_run(
|
||||
"type": type(exc).__name__,
|
||||
}
|
||||
run.save(update_fields=["status", "ended_at", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
raise
|
||||
|
||||
run.refresh_from_db()
|
||||
if result.get("cancelled") or run.status == BackupRun.Status.CANCELLED:
|
||||
run.status = BackupRun.Status.CANCELLED
|
||||
elif result.get("status") == BackupRun.Status.WARNING:
|
||||
run.status = BackupRun.Status.WARNING
|
||||
else:
|
||||
run.status = BackupRun.Status.SUCCESS if result.get("ok") else BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
@@ -107,11 +120,12 @@ def execute_backup_run(
|
||||
protect_bases=bool(prune_protect_bases),
|
||||
yes=True,
|
||||
max_delete=int(prune_max_delete),
|
||||
action=run.run_type,
|
||||
acquire_lock=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
result["prune"] = {"ok": False, "error": str(exc), "type": type(exc).__name__}
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.status = BackupRun.Status.WARNING
|
||||
run.result = result
|
||||
run.snapshot = snapshot_record
|
||||
run.save(
|
||||
@@ -125,7 +139,6 @@ def execute_backup_run(
|
||||
"result",
|
||||
],
|
||||
)
|
||||
raise
|
||||
|
||||
run.snapshot = snapshot_record
|
||||
run.result = result
|
||||
@@ -140,6 +153,7 @@ def execute_backup_run(
|
||||
"result",
|
||||
],
|
||||
)
|
||||
notify_backup_run_completed(run)
|
||||
return run
|
||||
|
||||
|
||||
@@ -159,10 +173,10 @@ def claim_next_queued_run() -> BackupRun | None:
|
||||
return run
|
||||
|
||||
|
||||
def reconcile_running_runs(*, grace_seconds: int = 300) -> int:
|
||||
def reconcile_running_runs(*, grace_seconds: int = 300, stale_worker_seconds: int = 24 * 60 * 60) -> int:
|
||||
reconciled = 0
|
||||
for run in BackupRun.objects.select_related("host").filter(status=BackupRun.Status.RUNNING).order_by("started_at", "id"):
|
||||
if _reconcile_running_run(run=run, grace_seconds=grace_seconds):
|
||||
if _reconcile_running_run(run=run, grace_seconds=grace_seconds, stale_worker_seconds=stale_worker_seconds):
|
||||
reconciled += 1
|
||||
return reconciled
|
||||
|
||||
@@ -177,7 +191,9 @@ def requested_options(run: BackupRun) -> dict[str, object]:
|
||||
def _running_result(*, run: BackupRun, dry_run: bool) -> dict[str, object]:
|
||||
result = dict(run.result) if isinstance(run.result, dict) else {}
|
||||
execution = {
|
||||
**_worker_execution_details(),
|
||||
"started_at": (run.started_at or timezone.now()).isoformat(),
|
||||
"heartbeat_at": timezone.now().isoformat(),
|
||||
}
|
||||
if dry_run:
|
||||
execution["log"] = str(dry_run_log_path(run.host.host, run_id=run.id))
|
||||
@@ -186,24 +202,143 @@ def _running_result(*, run: BackupRun, dry_run: bool) -> dict[str, object]:
|
||||
|
||||
|
||||
def _run_cancel_requested(run_id: int) -> bool:
|
||||
return BackupRun.objects.filter(id=run_id, status=BackupRun.Status.CANCELLED).exists()
|
||||
try:
|
||||
run = BackupRun.objects.only("id", "status", "result").get(id=run_id)
|
||||
except BackupRun.DoesNotExist:
|
||||
return True
|
||||
if run.status == BackupRun.Status.CANCELLED:
|
||||
return True
|
||||
if run.status == BackupRun.Status.RUNNING:
|
||||
_refresh_run_heartbeat(run)
|
||||
return False
|
||||
|
||||
|
||||
def _reconcile_running_run(*, run: BackupRun, grace_seconds: int) -> bool:
|
||||
def _record_running_state(run_id: int, state: dict[str, object]) -> None:
|
||||
try:
|
||||
run = BackupRun.objects.only("id", "status", "result", "snapshot_path", "rsync_exit_code").get(id=run_id)
|
||||
except BackupRun.DoesNotExist:
|
||||
return
|
||||
if run.status != BackupRun.Status.RUNNING:
|
||||
return
|
||||
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
|
||||
incoming_rsync = state.get("rsync") if isinstance(state.get("rsync"), dict) else {}
|
||||
|
||||
log_path = state.get("log")
|
||||
snapshot_path = state.get("snapshot")
|
||||
phase = state.get("phase")
|
||||
if isinstance(phase, str) and phase:
|
||||
execution["phase"] = phase
|
||||
if isinstance(log_path, str) and log_path:
|
||||
execution["log"] = log_path
|
||||
if isinstance(snapshot_path, str) and snapshot_path:
|
||||
execution["snapshot"] = snapshot_path
|
||||
run.snapshot_path = snapshot_path
|
||||
if incoming_rsync:
|
||||
result["rsync"] = {**rsync, **incoming_rsync}
|
||||
exit_code = incoming_rsync.get("exit_code")
|
||||
if isinstance(exit_code, int):
|
||||
run.rsync_exit_code = exit_code
|
||||
result["execution"] = {
|
||||
**execution,
|
||||
"worker_pid": os.getpid(),
|
||||
"worker_host": socket.gethostname(),
|
||||
"heartbeat_at": timezone.now().isoformat(),
|
||||
}
|
||||
run.result = result
|
||||
run.save(update_fields=["snapshot_path", "rsync_exit_code", "result"])
|
||||
|
||||
|
||||
def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_seconds: int) -> bool:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
requested = result.get("requested") if isinstance(result.get("requested"), dict) else {}
|
||||
if not requested.get("dry_run"):
|
||||
return False
|
||||
|
||||
log_path = _execution_log_path(result)
|
||||
log_tail = _read_log_tail(log_path) if log_path is not None else []
|
||||
terminal_log = _terminal_rsync_log(log_tail)
|
||||
timed_out = _running_dry_run_timed_out(run=run, grace_seconds=grace_seconds)
|
||||
if not terminal_log and not timed_out:
|
||||
exit_code = _exit_code_from_log(log_tail)
|
||||
stale_worker = _running_worker_timed_out(run=run, stale_worker_seconds=stale_worker_seconds)
|
||||
if not requested.get("dry_run"):
|
||||
if terminal_log:
|
||||
failure = classify_rsync_failure(exit_code or 255, log_tail)
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
"host": run.host.host,
|
||||
"log": str(log_path) if log_path else "",
|
||||
"failure": failure,
|
||||
"rsync": {
|
||||
**(result.get("rsync") if isinstance(result.get("rsync"), dict) else {}),
|
||||
"exit_code": exit_code or 255,
|
||||
"log_tail": log_tail,
|
||||
},
|
||||
}
|
||||
)
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.rsync_exit_code = exit_code or 255
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
if _running_rsync_process_missing(run=run, grace_seconds=grace_seconds):
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
"host": run.host.host,
|
||||
"log": str(log_path) if log_path else "",
|
||||
"failure": {
|
||||
"category": "rsync_process",
|
||||
"message": "The rsync process is no longer running while the backup is still marked running.",
|
||||
"hint": "Check the rsync log and pobsync-worker.service logs before retrying the backup.",
|
||||
},
|
||||
"rsync": {
|
||||
**(result.get("rsync") if isinstance(result.get("rsync"), dict) else {}),
|
||||
"exit_code": exit_code or 255,
|
||||
"log_tail": log_tail,
|
||||
},
|
||||
}
|
||||
)
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.rsync_exit_code = exit_code or 255
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
if stale_worker:
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
"host": run.host.host,
|
||||
"failure": {
|
||||
"category": "worker",
|
||||
"message": "The worker heartbeat stopped before the run finished.",
|
||||
"hint": "Check pobsync-worker.service logs before retrying the backup.",
|
||||
},
|
||||
}
|
||||
)
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
return False
|
||||
|
||||
exit_code = _exit_code_from_log(log_tail) or (124 if timed_out else 255)
|
||||
timed_out = _running_dry_run_timed_out(run=run, grace_seconds=grace_seconds)
|
||||
if not terminal_log and not timed_out and not stale_worker:
|
||||
return False
|
||||
|
||||
exit_code = exit_code or (124 if timed_out or stale_worker else 255)
|
||||
failure = classify_rsync_failure(exit_code, log_tail)
|
||||
if stale_worker and not terminal_log:
|
||||
failure = {
|
||||
"category": "worker",
|
||||
"message": "The worker heartbeat stopped before the dry-run finished.",
|
||||
"hint": "Check pobsync-worker.service logs before retrying the dry-run.",
|
||||
}
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
@@ -224,9 +359,34 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int) -> bool:
|
||||
run.rsync_exit_code = exit_code
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
|
||||
|
||||
def _worker_execution_details() -> dict[str, object]:
|
||||
return {
|
||||
"worker_pid": os.getpid(),
|
||||
"worker_host": socket.gethostname(),
|
||||
"claimed_at": timezone.now().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _refresh_run_heartbeat(run: BackupRun, *, interval_seconds: int = 30) -> None:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
heartbeat_at = _parse_iso_datetime(execution.get("heartbeat_at"))
|
||||
if heartbeat_at is not None and timezone.now() < heartbeat_at + timedelta(seconds=interval_seconds):
|
||||
return
|
||||
result["execution"] = {
|
||||
**execution,
|
||||
"worker_pid": os.getpid(),
|
||||
"worker_host": socket.gethostname(),
|
||||
"heartbeat_at": timezone.now().isoformat(),
|
||||
}
|
||||
run.result = result
|
||||
run.save(update_fields=["result"])
|
||||
|
||||
|
||||
def _execution_log_path(result: dict[str, object]) -> Path | None:
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
log = execution.get("log") or result.get("log")
|
||||
@@ -245,6 +405,9 @@ def _read_log_tail(log_path: Path | None, *, max_lines: int = 40) -> list[str]:
|
||||
|
||||
|
||||
def _terminal_rsync_log(log_tail: list[str]) -> bool:
|
||||
warning = classify_rsync_warning(_exit_code_from_log(log_tail), log_tail)
|
||||
if warning is not None:
|
||||
return False
|
||||
return any(line.startswith("rsync error:") for line in log_tail)
|
||||
|
||||
|
||||
@@ -252,6 +415,8 @@ def _exit_code_from_log(log_tail: list[str]) -> int | None:
|
||||
for line in reversed(log_tail):
|
||||
if "code 255" in line:
|
||||
return 255
|
||||
if "code 24" in line:
|
||||
return 24
|
||||
if "code 124" in line:
|
||||
return 124
|
||||
if "code 12" in line:
|
||||
@@ -267,3 +432,55 @@ def _running_dry_run_timed_out(*, run: BackupRun, grace_seconds: int) -> bool:
|
||||
if not isinstance(timeout_seconds, int) or timeout_seconds <= 0:
|
||||
timeout_seconds = DEFAULT_DRY_RUN_TIMEOUT_SECONDS
|
||||
return timezone.now() >= run.started_at + timedelta(seconds=timeout_seconds + grace_seconds)
|
||||
|
||||
|
||||
def _running_worker_timed_out(*, run: BackupRun, stale_worker_seconds: int) -> bool:
|
||||
if stale_worker_seconds <= 0:
|
||||
return False
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
heartbeat_at = _parse_iso_datetime(execution.get("heartbeat_at"))
|
||||
if heartbeat_at is None:
|
||||
heartbeat_at = run.started_at
|
||||
if heartbeat_at is None:
|
||||
return False
|
||||
return timezone.now() >= heartbeat_at + timedelta(seconds=stale_worker_seconds)
|
||||
|
||||
|
||||
def _running_rsync_process_missing(*, run: BackupRun, grace_seconds: int) -> bool:
|
||||
if grace_seconds <= 0:
|
||||
return False
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
if execution.get("phase") != "rsync":
|
||||
return False
|
||||
rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
|
||||
pid = rsync.get("pid")
|
||||
if not isinstance(pid, int) or pid <= 0:
|
||||
return False
|
||||
heartbeat_at = _parse_iso_datetime(execution.get("heartbeat_at")) or run.started_at
|
||||
if heartbeat_at is None or timezone.now() < heartbeat_at + timedelta(seconds=grace_seconds):
|
||||
return False
|
||||
return not _process_exists(pid)
|
||||
|
||||
|
||||
def _process_exists(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _parse_iso_datetime(value: object):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = timezone.datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if timezone.is_naive(parsed):
|
||||
return timezone.make_aware(parsed, timezone=datetime_timezone.utc)
|
||||
return parsed
|
||||
|
||||
@@ -17,7 +17,7 @@ CRITICAL_ROOT_EXCLUDES = ("/proc/***", "/sys/***", "/dev/***", "/run/***", "/tmp
|
||||
def collect_global_config_checks(global_config: GlobalConfig) -> list[SelfCheck]:
|
||||
checks = [
|
||||
_absolute_path_check("Global backup root", global_config.backup_root),
|
||||
_absolute_path_check("Global pobsync home", global_config.pobsync_home),
|
||||
_absolute_path_check("Runtime state root", settings.POBSYNC_HOME),
|
||||
_runtime_backup_root_check(global_config),
|
||||
_rsync_binary_check(global_config.rsync_binary),
|
||||
_rsync_recursion_check(
|
||||
@@ -97,7 +97,7 @@ def _runtime_backup_root_check(global_config: GlobalConfig) -> SelfCheck:
|
||||
return SelfCheck(
|
||||
"Runtime backup root",
|
||||
"warning",
|
||||
"Database backup root differs from runtime POBSYNC_BACKUP_ROOT.",
|
||||
"Database backup root differs from the runtime backup root.",
|
||||
f"database={global_config.backup_root} runtime={settings.POBSYNC_BACKUP_ROOT}",
|
||||
)
|
||||
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
|
||||
from pobsync.config.schemas import GLOBAL_SCHEMA, HOST_SCHEMA
|
||||
from pobsync.paths import PobsyncPaths
|
||||
from pobsync.util import write_yaml_atomic
|
||||
from pobsync.validate import validate_dict
|
||||
|
||||
from .models import GlobalConfig, HostConfig
|
||||
@@ -17,10 +14,9 @@ class ConfigRepositoryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
|
||||
def _global_runtime_data(global_config: GlobalConfig) -> dict[str, Any]:
|
||||
data = {
|
||||
"backup_root": global_config.backup_root,
|
||||
"pobsync_home": global_config.pobsync_home,
|
||||
"ssh": {
|
||||
"user": global_config.ssh_user,
|
||||
"port": global_config.ssh_port,
|
||||
@@ -48,7 +44,7 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
|
||||
return validate_dict(data, GLOBAL_SCHEMA, path="global")
|
||||
|
||||
|
||||
def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
def _host_runtime_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {
|
||||
"host": host_config.host,
|
||||
"address": host_config.address,
|
||||
@@ -72,55 +68,34 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
data["excludes_replace"] = list(host_config.excludes_replace or [])
|
||||
else:
|
||||
data["excludes_add"] = list(host_config.excludes_add or [])
|
||||
if host_config.rsync_extra_args:
|
||||
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
|
||||
if host_config.rsync_extra_args or host_config.rsync_bwlimit_kbps is not None:
|
||||
data["rsync"] = {}
|
||||
if host_config.rsync_extra_args:
|
||||
data["rsync"]["extra_args"] = list(host_config.rsync_extra_args or [])
|
||||
if host_config.rsync_bwlimit_kbps is not None:
|
||||
data["rsync"]["bwlimit_kbps"] = host_config.rsync_bwlimit_kbps
|
||||
return validate_dict(data, HOST_SCHEMA, path="host")
|
||||
|
||||
|
||||
def global_config_object_data(global_config: GlobalConfig) -> dict[str, Any]:
|
||||
return _global_runtime_data(global_config)
|
||||
|
||||
|
||||
def host_config_object_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
return _host_runtime_data(host_config)
|
||||
|
||||
|
||||
def global_config_data(name: str = "default") -> dict[str, Any]:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
|
||||
return _global_yaml_data(global_config)
|
||||
raise ConfigRepositoryError(f"Missing global config {name!r}") from exc
|
||||
return _global_runtime_data(global_config)
|
||||
|
||||
|
||||
def host_config_data(host: str) -> dict[str, Any]:
|
||||
try:
|
||||
host_config = HostConfig.objects.get(host=host, enabled=True)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
|
||||
return _host_yaml_data(host_config)
|
||||
|
||||
|
||||
def export_global_config(prefix: Path, name: str = "default") -> Path:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
|
||||
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
write_yaml_atomic(paths.global_config_path, _global_yaml_data(global_config))
|
||||
return paths.global_config_path
|
||||
|
||||
|
||||
def export_host_config(prefix: Path, host: str) -> Path:
|
||||
try:
|
||||
host_config = HostConfig.objects.get(host=host, enabled=True)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
|
||||
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
target = paths.hosts_dir / f"{host_config.host}.yaml"
|
||||
write_yaml_atomic(target, _host_yaml_data(host_config))
|
||||
return target
|
||||
|
||||
|
||||
def export_runtime_configs(prefix: Path, host: str | None = None) -> list[Path]:
|
||||
written = [export_global_config(prefix)]
|
||||
hosts = HostConfig.objects.filter(enabled=True).order_by("host")
|
||||
if host is not None:
|
||||
hosts = hosts.filter(host=host)
|
||||
for host_config in hosts:
|
||||
written.append(export_host_config(prefix, host_config.host))
|
||||
return written
|
||||
raise ConfigRepositoryError(f"Missing enabled host {host!r}") from exc
|
||||
return _host_runtime_data(host_config)
|
||||
|
||||
9
src/pobsync_backend/context_processors.py
Normal file
9
src/pobsync_backend/context_processors.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from .access import access_context
|
||||
|
||||
|
||||
def pobsync_access(request: HttpRequest) -> dict[str, object]:
|
||||
return access_context(request)
|
||||
@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
|
||||
from django import forms
|
||||
from django.conf import settings
|
||||
|
||||
from .models import GlobalConfig, HostConfig, ScheduleConfig, SshCredential
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, NotificationTarget, ScheduleConfig, SshCredential
|
||||
from .scheduler import parse_cron_expr
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ class HostConfigForm(forms.ModelForm):
|
||||
"excludes_add",
|
||||
"excludes_replace",
|
||||
"rsync_extra_args",
|
||||
"rsync_bwlimit_kbps",
|
||||
"retention_daily",
|
||||
"retention_weekly",
|
||||
"retention_monthly",
|
||||
@@ -70,6 +71,7 @@ class HostConfigForm(forms.ModelForm):
|
||||
"ssh_user": "Leave empty to use the global SSH user.",
|
||||
"ssh_port": "Leave empty to use the global SSH port.",
|
||||
"source_root": "Leave empty to use the global default source root.",
|
||||
"rsync_bwlimit_kbps": "Leave empty to inherit the global limit. Use 0 for unlimited on this host.",
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +114,7 @@ class GlobalConfigForm(forms.ModelForm):
|
||||
help_texts = {
|
||||
"name": "Usually 'default'. The backup engine currently reads the default config.",
|
||||
"default_ssh_credential": "Optional. Used by hosts without their own SSH credential.",
|
||||
"rsync_bwlimit_kbps": "Rsync bandwidth limit in KB/s. Use 0 for unlimited.",
|
||||
"default_source_root": "Used by hosts without a custom source root.",
|
||||
"default_destination_subdir": "Optional subdirectory below each snapshot.",
|
||||
}
|
||||
@@ -119,7 +122,6 @@ class GlobalConfigForm(forms.ModelForm):
|
||||
def save(self, commit: bool = True):
|
||||
instance = super().save(commit=False)
|
||||
instance.backup_root = settings.POBSYNC_BACKUP_ROOT
|
||||
instance.pobsync_home = settings.POBSYNC_HOME
|
||||
if commit:
|
||||
instance.save()
|
||||
self.save_m2m()
|
||||
@@ -151,6 +153,62 @@ class ManualBackupForm(forms.Form):
|
||||
)
|
||||
|
||||
|
||||
class NotificationTargetForm(forms.ModelForm):
|
||||
TERMINAL_STATUS_CHOICES = (
|
||||
(BackupRun.Status.SUCCESS, BackupRun.Status.SUCCESS.label),
|
||||
(BackupRun.Status.WARNING, BackupRun.Status.WARNING.label),
|
||||
(BackupRun.Status.FAILED, BackupRun.Status.FAILED.label),
|
||||
(BackupRun.Status.CANCELLED, BackupRun.Status.CANCELLED.label),
|
||||
)
|
||||
|
||||
statuses = forms.MultipleChoiceField(
|
||||
choices=TERMINAL_STATUS_CHOICES,
|
||||
widget=forms.CheckboxSelectMultiple,
|
||||
initial=[choice[0] for choice in TERMINAL_STATUS_CHOICES],
|
||||
help_text="Send notifications for these terminal run statuses.",
|
||||
)
|
||||
email_to = forms.CharField(
|
||||
widget=forms.Textarea,
|
||||
required=False,
|
||||
help_text="One recipient per line, or comma-separated.",
|
||||
)
|
||||
webhook_headers = forms.JSONField(
|
||||
required=False,
|
||||
widget=forms.Textarea(attrs={"rows": 4}),
|
||||
help_text='Optional JSON object with extra headers, for example {"Authorization": "Bearer ..."}.',
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = NotificationTarget
|
||||
fields = (
|
||||
"name",
|
||||
"enabled",
|
||||
"channel",
|
||||
"statuses",
|
||||
"email_to",
|
||||
"webhook_url",
|
||||
"webhook_headers",
|
||||
"notes",
|
||||
)
|
||||
widgets = {
|
||||
"notes": forms.Textarea,
|
||||
}
|
||||
|
||||
def clean(self):
|
||||
cleaned_data = super().clean()
|
||||
channel = cleaned_data.get("channel")
|
||||
if channel == NotificationTarget.Channel.EMAIL and not cleaned_data.get("email_to", "").strip():
|
||||
self.add_error("email_to", "Email targets need at least one recipient.")
|
||||
if channel == NotificationTarget.Channel.WEBHOOK and not cleaned_data.get("webhook_url"):
|
||||
self.add_error("webhook_url", "Webhook targets need a URL.")
|
||||
return cleaned_data
|
||||
|
||||
def clean_email_to(self) -> str:
|
||||
value = self.cleaned_data.get("email_to", "")
|
||||
recipients = [line.strip() for line in value.replace(",", "\n").splitlines() if line.strip()]
|
||||
return "\n".join(recipients)
|
||||
|
||||
|
||||
class SshCredentialForm(forms.ModelForm):
|
||||
private_key_file = forms.FileField(
|
||||
required=False,
|
||||
@@ -193,7 +251,7 @@ class SshCredentialForm(forms.ModelForm):
|
||||
if not raw_private_key.strip():
|
||||
if self.instance and self.instance.pk and self.instance.key_path:
|
||||
return self.instance.private_key
|
||||
raise forms.ValidationError("Paste a private key, upload a private key file, or generate a key from Django.")
|
||||
raise forms.ValidationError("Paste a private key, upload a private key file, or generate a key in pobsync.")
|
||||
|
||||
private_key = normalize_private_key(raw_private_key)
|
||||
public_key = validate_ssh_private_key(private_key)
|
||||
@@ -249,12 +307,18 @@ class RetentionApplyForm(forms.Form):
|
||||
kind = forms.ChoiceField(choices=(("scheduled", "Scheduled"), ("manual", "Manual"), ("all", "All")))
|
||||
protect_bases = forms.BooleanField(required=False)
|
||||
max_delete = forms.IntegerField(min_value=0, initial=10)
|
||||
confirm_delete_count = forms.IntegerField(min_value=0)
|
||||
confirm_host = forms.CharField()
|
||||
|
||||
def __init__(self, *args, host_name: str, **kwargs) -> None:
|
||||
def __init__(self, *args, host_name: str, expected_delete_count: int | None = None, **kwargs) -> None:
|
||||
self.host_name = host_name
|
||||
self.expected_delete_count = expected_delete_count
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["confirm_host"].help_text = f"Type {host_name} to confirm deletion."
|
||||
if expected_delete_count is not None:
|
||||
self.fields["confirm_delete_count"].help_text = (
|
||||
f"Type {expected_delete_count} to confirm the current number of planned deletions."
|
||||
)
|
||||
|
||||
def clean_confirm_host(self) -> str:
|
||||
value = self.cleaned_data["confirm_host"].strip()
|
||||
@@ -262,6 +326,42 @@ class RetentionApplyForm(forms.Form):
|
||||
raise forms.ValidationError(f"Type {self.host_name} to confirm.")
|
||||
return value
|
||||
|
||||
def clean_confirm_delete_count(self) -> int:
|
||||
value = self.cleaned_data["confirm_delete_count"]
|
||||
if self.expected_delete_count is not None and value != self.expected_delete_count:
|
||||
raise forms.ValidationError(f"Type {self.expected_delete_count} to confirm the delete count.")
|
||||
return value
|
||||
|
||||
|
||||
class IncompleteCleanupForm(forms.Form):
|
||||
max_delete = forms.IntegerField(min_value=0, initial=0)
|
||||
confirm_delete_count = forms.IntegerField(min_value=0)
|
||||
confirm_host = forms.CharField()
|
||||
|
||||
def __init__(self, *args, host_name: str, expected_delete_count: int, **kwargs) -> None:
|
||||
self.host_name = host_name
|
||||
self.expected_delete_count = expected_delete_count
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["confirm_host"].help_text = f"Type {host_name} to confirm incomplete snapshot cleanup."
|
||||
self.fields["confirm_delete_count"].help_text = (
|
||||
f"Type {expected_delete_count} to confirm the current number of incomplete snapshots."
|
||||
)
|
||||
self.fields["max_delete"].help_text = (
|
||||
f"Must be at least {expected_delete_count} for the incomplete snapshots shown here."
|
||||
)
|
||||
|
||||
def clean_confirm_host(self) -> str:
|
||||
value = self.cleaned_data["confirm_host"].strip()
|
||||
if value != self.host_name:
|
||||
raise forms.ValidationError(f"Type {self.host_name} to confirm.")
|
||||
return value
|
||||
|
||||
def clean_confirm_delete_count(self) -> int:
|
||||
value = self.cleaned_data["confirm_delete_count"]
|
||||
if value != self.expected_delete_count:
|
||||
raise forms.ValidationError(f"Type {self.expected_delete_count} to confirm the incomplete count.")
|
||||
return value
|
||||
|
||||
|
||||
class ScheduleConfigForm(forms.ModelForm):
|
||||
cron_expr = forms.CharField(
|
||||
@@ -277,7 +377,6 @@ class ScheduleConfigForm(forms.ModelForm):
|
||||
model = ScheduleConfig
|
||||
fields = (
|
||||
"cron_expr",
|
||||
"user",
|
||||
"enabled",
|
||||
"prune",
|
||||
"prune_max_delete",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pobsync_backend.self_check import collect_self_checks, summarize_self_checks
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Run pobsync runtime self checks for native installs and updates."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--fail-on-warning",
|
||||
action="store_true",
|
||||
help="Exit with an error when warnings are present.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
checks = collect_self_checks()
|
||||
summary = summarize_self_checks(checks)
|
||||
|
||||
for check in checks:
|
||||
line = f"[{check.status.upper()}] {check.name}: {check.message}"
|
||||
if check.detail:
|
||||
line = f"{line} ({check.detail})"
|
||||
if check.status == "failed":
|
||||
self.stderr.write(line)
|
||||
elif check.status == "warning":
|
||||
self.stderr.write(line)
|
||||
else:
|
||||
self.stdout.write(line)
|
||||
|
||||
self.stdout.write(
|
||||
"Summary: "
|
||||
f"{summary['ok']} ok, "
|
||||
f"{summary['warning']} warning(s), "
|
||||
f"{summary['failed']} failed, "
|
||||
f"{summary['skipped']} skipped"
|
||||
)
|
||||
|
||||
if summary["failed"]:
|
||||
raise CommandError("pobsync install self check failed.")
|
||||
if options["fail_on_warning"] and summary["warning"]:
|
||||
raise CommandError("pobsync install self check reported warnings.")
|
||||
@@ -1,9 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pobsync.config.retention import parse_retention
|
||||
@@ -13,12 +11,11 @@ from pobsync_backend.models import GlobalConfig
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create or update the SQL-backed global pobsync configuration."
|
||||
help = "Create or update the default global backup configuration."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("--name", default="default")
|
||||
parser.add_argument("--backup-root", required=True)
|
||||
parser.add_argument("--pobsync-home", default=settings.POBSYNC_HOME)
|
||||
parser.add_argument("--ssh-user", default="root")
|
||||
parser.add_argument("--ssh-port", type=int, default=22)
|
||||
parser.add_argument("--source-root", default="/")
|
||||
@@ -30,11 +27,9 @@ class Command(BaseCommand):
|
||||
if not is_absolute_non_root(backup_root):
|
||||
raise CommandError("--backup-root must be an absolute path and must not be '/'")
|
||||
|
||||
pobsync_home = str(Path(options["pobsync_home"]))
|
||||
retention = parse_retention(options["retention"])
|
||||
defaults = {
|
||||
"backup_root": backup_root,
|
||||
"pobsync_home": pobsync_home,
|
||||
"ssh_user": options["ssh_user"],
|
||||
"ssh_port": options["ssh_port"],
|
||||
"ssh_options": ["-oBatchMode=yes", "-oStrictHostKeyChecking=accept-new"],
|
||||
@@ -53,8 +48,8 @@ class Command(BaseCommand):
|
||||
}
|
||||
|
||||
if GlobalConfig.objects.filter(name=options["name"]).exists() and not options["force"]:
|
||||
raise CommandError(f"GlobalConfig {options['name']!r} already exists; use --force to update")
|
||||
raise CommandError(f"Global config {options['name']!r} already exists; use --force to update")
|
||||
|
||||
_obj, created = GlobalConfig.objects.update_or_create(name=options["name"], defaults=defaults)
|
||||
action = "Created" if created else "Updated"
|
||||
self.stdout.write(self.style.SUCCESS(f"{action} GlobalConfig {options['name']!r}."))
|
||||
self.stdout.write(self.style.SUCCESS(f"{action} global config {options['name']!r}."))
|
||||
|
||||
@@ -10,7 +10,7 @@ from pobsync_backend.models import GlobalConfig, HostConfig
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create or update a SQL-backed host pobsync configuration."
|
||||
help = "Create or update a host backup configuration."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("host")
|
||||
@@ -22,6 +22,12 @@ class Command(BaseCommand):
|
||||
parser.add_argument("--exclude-add", action="append", default=[])
|
||||
parser.add_argument("--exclude-replace", action="append", default=None)
|
||||
parser.add_argument("--rsync-extra-arg", action="append", default=[])
|
||||
parser.add_argument(
|
||||
"--rsync-bwlimit-kbps",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Host rsync bandwidth limit in KB/s. Omit to inherit global; set 0 for unlimited.",
|
||||
)
|
||||
parser.add_argument("--retention", default=None)
|
||||
parser.add_argument("--disabled", action="store_true")
|
||||
parser.add_argument("--force", action="store_true", help="Update existing host")
|
||||
@@ -29,7 +35,7 @@ class Command(BaseCommand):
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
host = sanitize_host(options["host"])
|
||||
if HostConfig.objects.filter(host=host).exists() and not options["force"]:
|
||||
raise CommandError(f"HostConfig {host!r} already exists; use --force to update")
|
||||
raise CommandError(f"Host {host!r} already exists; use --force to update")
|
||||
|
||||
retention = self._retention(options["retention"])
|
||||
defaults = {
|
||||
@@ -42,6 +48,7 @@ class Command(BaseCommand):
|
||||
"excludes_add": [] if options["exclude_replace"] is not None else list(options["exclude_add"]),
|
||||
"excludes_replace": options["exclude_replace"],
|
||||
"rsync_extra_args": list(options["rsync_extra_arg"]),
|
||||
"rsync_bwlimit_kbps": options["rsync_bwlimit_kbps"],
|
||||
"retention_daily": retention["daily"],
|
||||
"retention_weekly": retention["weekly"],
|
||||
"retention_monthly": retention["monthly"],
|
||||
@@ -49,7 +56,7 @@ class Command(BaseCommand):
|
||||
}
|
||||
_obj, created = HostConfig.objects.update_or_create(host=host, defaults=defaults)
|
||||
action = "Created" if created else "Updated"
|
||||
self.stdout.write(self.style.SUCCESS(f"{action} HostConfig {host!r}."))
|
||||
self.stdout.write(self.style.SUCCESS(f"{action} host {host!r}."))
|
||||
|
||||
def _retention(self, value: str | None) -> dict[str, int]:
|
||||
if value:
|
||||
|
||||
@@ -9,12 +9,16 @@ from pobsync_backend.scheduler import parse_cron_expr
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create, update, disable, or remove a SQL-backed pobsync schedule."
|
||||
help = "Create, update, disable, or remove a scheduler-managed host schedule."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("host")
|
||||
parser.add_argument("--cron", help='Cron expression, e.g. "15 2 * * *"')
|
||||
parser.add_argument("--user", default="root")
|
||||
parser.add_argument(
|
||||
"--schedule-expression",
|
||||
"--cron",
|
||||
dest="schedule_expression",
|
||||
help='Five-field schedule expression, e.g. "15 2 * * *"',
|
||||
)
|
||||
parser.add_argument("--prune", action="store_true")
|
||||
parser.add_argument("--prune-max-delete", type=int, default=10)
|
||||
parser.add_argument("--prune-protect-bases", action="store_true")
|
||||
@@ -25,25 +29,25 @@ class Command(BaseCommand):
|
||||
try:
|
||||
host = HostConfig.objects.get(host=options["host"])
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise CommandError(f"Missing HostConfig {options['host']!r}") from exc
|
||||
raise CommandError(f"Missing host {options['host']!r}") from exc
|
||||
|
||||
if options["delete"]:
|
||||
deleted, _details = ScheduleConfig.objects.filter(host=host).delete()
|
||||
self.stdout.write(self.style.SUCCESS(f"Deleted {deleted} schedule row(s) for {host.host!r}."))
|
||||
return
|
||||
|
||||
if not options["cron"]:
|
||||
raise CommandError("--cron is required unless --delete is used")
|
||||
schedule_expression = options["schedule_expression"]
|
||||
if not schedule_expression:
|
||||
raise CommandError("--schedule-expression is required unless --delete is used")
|
||||
try:
|
||||
parse_cron_expr(options["cron"])
|
||||
parse_cron_expr(schedule_expression)
|
||||
except ValueError as exc:
|
||||
raise CommandError(str(exc)) from exc
|
||||
|
||||
schedule, created = ScheduleConfig.objects.update_or_create(
|
||||
host=host,
|
||||
defaults={
|
||||
"cron_expr": options["cron"],
|
||||
"user": options["user"],
|
||||
"cron_expr": schedule_expression,
|
||||
"enabled": not options["disabled"],
|
||||
"prune": bool(options["prune"]),
|
||||
"prune_max_delete": int(options["prune_max_delete"]),
|
||||
|
||||
@@ -20,14 +20,14 @@ class Command(BaseCommand):
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name="default")
|
||||
except GlobalConfig.DoesNotExist as exc:
|
||||
raise CommandError("Missing GlobalConfig 'default'") from exc
|
||||
raise CommandError("Missing default global config") from exc
|
||||
|
||||
host = None
|
||||
if options["host"]:
|
||||
try:
|
||||
host = HostConfig.objects.get(host=options["host"], enabled=True)
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise CommandError(f"Missing enabled HostConfig {options['host']!r}") from exc
|
||||
raise CommandError(f"Missing enabled host {options['host']!r}") from exc
|
||||
|
||||
kind = normalize_kind(options["kind"])
|
||||
kinds = ["scheduled", "manual", "incomplete"] if kind == "all" else [kind]
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from pobsync_backend.config_repository import export_runtime_configs
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Export Django database configs to pobsync runtime YAML files."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
|
||||
parser.add_argument("--host", default=None, help="Export only one enabled host")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
written = export_runtime_configs(prefix=Path(options["prefix"]), host=options["host"])
|
||||
for path in written:
|
||||
self.stdout.write(str(path))
|
||||
self.stdout.write(self.style.SUCCESS(f"Exported {len(written)} config file(s)."))
|
||||
@@ -1,81 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from pobsync.config.load import load_global_config, load_host_config
|
||||
from pobsync.paths import PobsyncPaths
|
||||
from pobsync_backend.models import GlobalConfig, HostConfig
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Import pobsync YAML configs into the Django database."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
paths = PobsyncPaths(home=Path(options["prefix"]))
|
||||
if not paths.global_config_path.exists():
|
||||
raise CommandError(f"Missing global config: {paths.global_config_path}")
|
||||
|
||||
global_cfg = load_global_config(paths.global_config_path)
|
||||
global_ssh = global_cfg.get("ssh") or {}
|
||||
global_rsync = global_cfg.get("rsync") or {}
|
||||
global_defaults = global_cfg.get("defaults") or {}
|
||||
retention_defaults = global_cfg.get("retention_defaults") or {}
|
||||
GlobalConfig.objects.update_or_create(
|
||||
name="default",
|
||||
defaults={
|
||||
"backup_root": global_cfg["backup_root"],
|
||||
"pobsync_home": global_cfg.get("pobsync_home", str(paths.home)),
|
||||
"ssh_user": global_ssh.get("user") or "root",
|
||||
"ssh_port": global_ssh.get("port") or 22,
|
||||
"ssh_options": global_ssh.get("options") or [],
|
||||
"rsync_binary": global_rsync.get("binary") or "rsync",
|
||||
"rsync_args": global_rsync.get("args") or [],
|
||||
"rsync_extra_args": global_rsync.get("extra_args") or [],
|
||||
"rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0,
|
||||
"rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0,
|
||||
"default_source_root": global_defaults.get("source_root") or "/",
|
||||
"default_destination_subdir": global_defaults.get("destination_subdir") or "",
|
||||
"excludes_default": global_cfg.get("excludes_default") or [],
|
||||
"retention_daily": retention_defaults.get("daily", 14),
|
||||
"retention_weekly": retention_defaults.get("weekly", 8),
|
||||
"retention_monthly": retention_defaults.get("monthly", 12),
|
||||
"retention_yearly": retention_defaults.get("yearly", 0),
|
||||
"data": global_cfg,
|
||||
},
|
||||
)
|
||||
|
||||
count = 0
|
||||
for host_path in sorted(paths.hosts_dir.glob("*.yaml")):
|
||||
host_cfg = load_host_config(host_path)
|
||||
host_ssh = host_cfg.get("ssh") or {}
|
||||
host_rsync = host_cfg.get("rsync") or {}
|
||||
host_retention = host_cfg.get("retention") or {}
|
||||
HostConfig.objects.update_or_create(
|
||||
host=host_cfg["host"],
|
||||
defaults={
|
||||
"address": host_cfg["address"],
|
||||
"ssh_user": host_ssh.get("user") or "",
|
||||
"ssh_port": host_ssh.get("port"),
|
||||
"source_root": host_cfg.get("source_root") or "",
|
||||
"includes": host_cfg.get("includes") or [],
|
||||
"excludes_add": host_cfg.get("excludes_add") or [],
|
||||
"excludes_replace": host_cfg.get("excludes_replace"),
|
||||
"rsync_extra_args": host_rsync.get("extra_args") or [],
|
||||
"retention_daily": host_retention.get("daily", 14),
|
||||
"retention_weekly": host_retention.get("weekly", 8),
|
||||
"retention_monthly": host_retention.get("monthly", 12),
|
||||
"retention_yearly": host_retention.get("yearly", 0),
|
||||
"config": host_cfg,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
count += 1
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"Imported global config and {count} host config(s)."))
|
||||
@@ -16,9 +16,10 @@ class Command(BaseCommand):
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("host", help="Host to back up")
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Run rsync --dry-run")
|
||||
parser.add_argument("--verbose-rsync", action="store_true", help="Write itemized rsync output to the run log")
|
||||
parser.add_argument("--quiet-rsync", action="store_true", help="Skip default rsync progress output for real runs")
|
||||
parser.add_argument("--prune", action="store_true", help="Apply retention after a successful run")
|
||||
parser.add_argument("--prune-max-delete", type=int, default=10)
|
||||
parser.add_argument("--prune-protect-bases", action="store_true")
|
||||
@@ -30,8 +31,9 @@ class Command(BaseCommand):
|
||||
try:
|
||||
host = HostConfig.objects.get(host=host_name, enabled=True)
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
|
||||
raise CommandError(f"Missing enabled host {host_name!r}") from exc
|
||||
|
||||
verbose_output = bool(options["dry_run"] or options["verbose_rsync"] or not options["quiet_rsync"])
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
run_type=BackupRun.RunType.MANUAL if options["manual"] else BackupRun.RunType.SCHEDULED,
|
||||
@@ -39,7 +41,7 @@ class Command(BaseCommand):
|
||||
result={
|
||||
"requested": {
|
||||
"dry_run": bool(options["dry_run"]),
|
||||
"verbose_output": bool(options["dry_run"] or options["verbose_rsync"]),
|
||||
"verbose_output": verbose_output,
|
||||
"prune": bool(options["prune"]),
|
||||
"prune_max_delete": int(options["prune_max_delete"]),
|
||||
"prune_protect_bases": bool(options["prune_protect_bases"]),
|
||||
@@ -50,7 +52,7 @@ class Command(BaseCommand):
|
||||
run=run,
|
||||
prefix=paths.home,
|
||||
dry_run=bool(options["dry_run"]),
|
||||
verbose_output=bool(options["dry_run"] or options["verbose_rsync"]),
|
||||
verbose_output=verbose_output,
|
||||
prune=bool(options["prune"]),
|
||||
prune_max_delete=int(options["prune_max_delete"]),
|
||||
prune_protect_bases=bool(options["prune_protect_bases"]),
|
||||
@@ -60,5 +62,8 @@ class Command(BaseCommand):
|
||||
if run.status == BackupRun.Status.SUCCESS:
|
||||
self.stdout.write(self.style.SUCCESS(f"Backup completed for {host.host}."))
|
||||
return
|
||||
if run.status == BackupRun.Status.WARNING:
|
||||
self.stdout.write(self.style.WARNING(f"Backup completed with warnings for {host.host}; run id={run.id}"))
|
||||
return
|
||||
|
||||
raise CommandError(f"Backup failed for {host.host}; run id={run.id}")
|
||||
|
||||
@@ -12,11 +12,11 @@ from pobsync_backend.retention import run_sql_retention_apply, run_sql_retention
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Plan or apply retention using SQL-backed pobsync configuration."
|
||||
help = "Plan or apply retention using the Django backup configuration."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("host")
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME)
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
|
||||
parser.add_argument("--kind", default="scheduled", choices=["scheduled", "manual", "all"])
|
||||
parser.add_argument("--protect-bases", action="store_true")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
@@ -36,6 +36,7 @@ class Command(BaseCommand):
|
||||
protect_bases=bool(options["protect_bases"]),
|
||||
yes=True,
|
||||
max_delete=int(options["max_delete"]),
|
||||
action="cli",
|
||||
)
|
||||
else:
|
||||
result = run_sql_retention_plan(
|
||||
|
||||
@@ -10,7 +10,7 @@ from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync_backend.models import ScheduleConfig
|
||||
from pobsync_backend.models import BackupRun, ScheduleConfig
|
||||
from pobsync_backend.scheduler import due_key, is_due
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ class Command(BaseCommand):
|
||||
help = "Run due pobsync schedules from the Django database."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
|
||||
parser.add_argument("--once", action="store_true", help="Check once and exit")
|
||||
parser.add_argument("--loop", action="store_true", help="Keep checking schedules")
|
||||
parser.add_argument("--interval", type=int, default=60, help="Loop interval in seconds")
|
||||
@@ -52,12 +52,13 @@ class Command(BaseCommand):
|
||||
if not is_due(schedule.cron_expr, now):
|
||||
continue
|
||||
|
||||
schedule_started_at = timezone.now()
|
||||
with transaction.atomic():
|
||||
locked = ScheduleConfig.objects.select_for_update().get(pk=schedule.pk)
|
||||
if locked.last_due_key == current_due_key:
|
||||
continue
|
||||
locked.last_due_key = current_due_key
|
||||
locked.last_started_at = timezone.now()
|
||||
locked.last_started_at = schedule_started_at
|
||||
locked.last_status = "running"
|
||||
locked.save(update_fields=["last_due_key", "last_started_at", "last_status", "updated_at"])
|
||||
|
||||
@@ -72,6 +73,7 @@ class Command(BaseCommand):
|
||||
prune_max_delete=schedule.prune_max_delete,
|
||||
prune_protect_bases=schedule.prune_protect_bases,
|
||||
)
|
||||
status = _latest_scheduled_run_status(host_id=schedule.host_id, started_at=schedule_started_at) or status
|
||||
except Exception as exc:
|
||||
status = "failed"
|
||||
self.stderr.write(f"{schedule.host.host}: {type(exc).__name__}: {exc}")
|
||||
@@ -83,3 +85,16 @@ class Command(BaseCommand):
|
||||
ran += 1
|
||||
|
||||
return ran
|
||||
|
||||
|
||||
def _latest_scheduled_run_status(*, host_id: int, started_at) -> str | None:
|
||||
run = (
|
||||
BackupRun.objects.filter(
|
||||
host_id=host_id,
|
||||
run_type=BackupRun.RunType.SCHEDULED,
|
||||
created_at__gte=started_at,
|
||||
)
|
||||
.order_by("-created_at", "-id")
|
||||
.first()
|
||||
)
|
||||
return run.status if run is not None else None
|
||||
|
||||
@@ -15,10 +15,16 @@ class Command(BaseCommand):
|
||||
help = "Run queued pobsync backup jobs from the Django database."
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
|
||||
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
|
||||
parser.add_argument("--once", action="store_true", help="Process one queued run and exit")
|
||||
parser.add_argument("--loop", action="store_true", help="Keep checking for queued runs")
|
||||
parser.add_argument("--interval", type=int, default=15, help="Loop interval in seconds")
|
||||
parser.add_argument(
|
||||
"--stale-running-seconds",
|
||||
type=int,
|
||||
default=24 * 60 * 60,
|
||||
help="Mark running runs failed after this many seconds without a worker heartbeat; use 0 to disable",
|
||||
)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
if not options["once"] and not options["loop"]:
|
||||
@@ -26,14 +32,14 @@ class Command(BaseCommand):
|
||||
|
||||
paths = PobsyncPaths(home=Path(options["prefix"]))
|
||||
while True:
|
||||
count = self._run_once(prefix=paths.home)
|
||||
count = self._run_once(prefix=paths.home, stale_running_seconds=int(options["stale_running_seconds"]))
|
||||
self.stdout.write(f"Ran {count} queued backup run(s).")
|
||||
if options["once"]:
|
||||
return
|
||||
time.sleep(max(1, int(options["interval"])))
|
||||
|
||||
def _run_once(self, *, prefix: Path) -> int:
|
||||
reconciled = reconcile_running_runs()
|
||||
def _run_once(self, *, prefix: Path, stale_running_seconds: int = 24 * 60 * 60) -> int:
|
||||
reconciled = reconcile_running_runs(stale_worker_seconds=stale_running_seconds)
|
||||
run = claim_next_queued_run()
|
||||
if run is None:
|
||||
return reconciled
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("pobsync_backend", "0007_filesystem_ssh_credentials"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="backuprun",
|
||||
name="status",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("queued", "Queued"),
|
||||
("running", "Running"),
|
||||
("success", "Success"),
|
||||
("warning", "Warning"),
|
||||
("failed", "Failed"),
|
||||
("cancelled", "Cancelled"),
|
||||
],
|
||||
default="queued",
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0008_alter_backuprun_status"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name="scheduleconfig",
|
||||
name="user",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0009_remove_scheduleconfig_user"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name="globalconfig",
|
||||
name="pobsync_home",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0010_remove_globalconfig_pobsync_home"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name="globalconfig",
|
||||
name="data",
|
||||
),
|
||||
]
|
||||
30
src/pobsync_backend/migrations/0012_review_state.py
Normal file
30
src/pobsync_backend/migrations/0012_review_state.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0011_remove_globalconfig_data"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="backuprun",
|
||||
name="reviewed_at",
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="backuprun",
|
||||
name="reviewed_by",
|
||||
field=models.CharField(blank=True, max_length=150),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="snapshotrecord",
|
||||
name="reviewed_at",
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="snapshotrecord",
|
||||
name="reviewed_by",
|
||||
field=models.CharField(blank=True, max_length=150),
|
||||
),
|
||||
]
|
||||
50
src/pobsync_backend/migrations/0013_purgedsnapshot.py
Normal file
50
src/pobsync_backend/migrations/0013_purgedsnapshot.py
Normal file
@@ -0,0 +1,50 @@
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0012_review_state"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="PurgedSnapshot",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("host_name", models.CharField(max_length=255)),
|
||||
("kind", models.CharField(max_length=16)),
|
||||
("dirname", models.CharField(max_length=255)),
|
||||
("path", models.CharField(max_length=1024)),
|
||||
("reason", models.CharField(blank=True, max_length=512)),
|
||||
(
|
||||
"action",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("manual", "Manual"),
|
||||
("scheduled", "Scheduled"),
|
||||
("cli", "CLI"),
|
||||
("incomplete_cleanup", "Incomplete cleanup"),
|
||||
],
|
||||
max_length=32,
|
||||
),
|
||||
),
|
||||
("triggered_by", models.CharField(blank=True, max_length=150)),
|
||||
("metadata", models.JSONField(blank=True, default=dict)),
|
||||
("purged_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"host",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="purged_snapshots",
|
||||
to="pobsync_backend.hostconfig",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["-purged_at", "host_name", "dirname"],
|
||||
},
|
||||
),
|
||||
]
|
||||
18
src/pobsync_backend/migrations/0014_host_bwlimit_override.py
Normal file
18
src/pobsync_backend/migrations/0014_host_bwlimit_override.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.14 on 2026-05-22 22:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('pobsync_backend', '0013_purgedsnapshot'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='hostconfig',
|
||||
name='rsync_bwlimit_kbps',
|
||||
field=models.PositiveIntegerField(blank=True, null=True),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,54 @@
|
||||
# Generated by Django 5.2.14 on 2026-05-28 19:11
|
||||
|
||||
import django.db.models.deletion
|
||||
import pobsync_backend.models
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('pobsync_backend', '0014_host_bwlimit_override'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='NotificationTarget',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('name', models.CharField(max_length=128, unique=True)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('channel', models.CharField(choices=[('email', 'Email'), ('webhook', 'Webhook')], max_length=16)),
|
||||
('statuses', models.JSONField(blank=True, default=pobsync_backend.models.default_notification_statuses)),
|
||||
('email_to', models.TextField(blank=True)),
|
||||
('webhook_url', models.URLField(blank=True, max_length=1024)),
|
||||
('webhook_headers', models.JSONField(blank=True, default=dict)),
|
||||
('notes', models.TextField(blank=True)),
|
||||
('last_status', models.CharField(blank=True, max_length=16)),
|
||||
('last_error', models.TextField(blank=True)),
|
||||
('last_sent_at', models.DateTimeField(blank=True, null=True)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['name'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='NotificationDelivery',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('status', models.CharField(choices=[('sent', 'Sent'), ('failed', 'Failed'), ('skipped', 'Skipped')], max_length=16)),
|
||||
('error', models.TextField(blank=True)),
|
||||
('payload', models.JSONField(blank=True, default=dict)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notification_deliveries', to='pobsync_backend.backuprun')),
|
||||
('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='pobsync_backend.notificationtarget')),
|
||||
],
|
||||
options={
|
||||
'verbose_name_plural': 'notification deliveries',
|
||||
'ordering': ['-created_at', 'target__name'],
|
||||
'constraints': [models.UniqueConstraint(fields=('target', 'run'), name='unique_notification_delivery_per_target_run')],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -14,7 +14,6 @@ class TimestampedModel(models.Model):
|
||||
class GlobalConfig(TimestampedModel):
|
||||
name = models.CharField(max_length=64, default="default", unique=True)
|
||||
backup_root = models.CharField(max_length=512)
|
||||
pobsync_home = models.CharField(max_length=512, default="/opt/pobsync")
|
||||
default_ssh_credential = models.ForeignKey(
|
||||
"SshCredential",
|
||||
on_delete=models.SET_NULL,
|
||||
@@ -37,7 +36,6 @@ class GlobalConfig(TimestampedModel):
|
||||
retention_weekly = models.PositiveIntegerField(default=8)
|
||||
retention_monthly = models.PositiveIntegerField(default=12)
|
||||
retention_yearly = models.PositiveIntegerField(default=0)
|
||||
data = models.JSONField(default=dict, blank=True)
|
||||
|
||||
class Meta:
|
||||
verbose_name = "global config"
|
||||
@@ -65,6 +63,7 @@ class HostConfig(TimestampedModel):
|
||||
excludes_add = models.JSONField(default=list, blank=True)
|
||||
excludes_replace = models.JSONField(null=True, blank=True)
|
||||
rsync_extra_args = models.JSONField(default=list, blank=True)
|
||||
rsync_bwlimit_kbps = models.PositiveIntegerField(null=True, blank=True)
|
||||
retention_daily = models.PositiveIntegerField(default=14)
|
||||
retention_weekly = models.PositiveIntegerField(default=8)
|
||||
retention_monthly = models.PositiveIntegerField(default=12)
|
||||
@@ -105,6 +104,7 @@ class BackupRun(models.Model):
|
||||
QUEUED = "queued", "Queued"
|
||||
RUNNING = "running", "Running"
|
||||
SUCCESS = "success", "Success"
|
||||
WARNING = "warning", "Warning"
|
||||
FAILED = "failed", "Failed"
|
||||
CANCELLED = "cancelled", "Cancelled"
|
||||
|
||||
@@ -125,6 +125,8 @@ class BackupRun(models.Model):
|
||||
rsync_exit_code = models.IntegerField(null=True, blank=True)
|
||||
result = models.JSONField(default=dict, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
reviewed_at = models.DateTimeField(null=True, blank=True)
|
||||
reviewed_by = models.CharField(max_length=150, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
@@ -133,6 +135,63 @@ class BackupRun(models.Model):
|
||||
return f"{self.host} {self.run_type} {self.status}"
|
||||
|
||||
|
||||
def default_notification_statuses() -> list[str]:
|
||||
return [
|
||||
BackupRun.Status.SUCCESS,
|
||||
BackupRun.Status.WARNING,
|
||||
BackupRun.Status.FAILED,
|
||||
BackupRun.Status.CANCELLED,
|
||||
]
|
||||
|
||||
|
||||
class NotificationTarget(TimestampedModel):
|
||||
class Channel(models.TextChoices):
|
||||
EMAIL = "email", "Email"
|
||||
WEBHOOK = "webhook", "Webhook"
|
||||
|
||||
name = models.CharField(max_length=128, unique=True)
|
||||
enabled = models.BooleanField(default=True)
|
||||
channel = models.CharField(max_length=16, choices=Channel.choices)
|
||||
statuses = models.JSONField(default=default_notification_statuses, blank=True)
|
||||
email_to = models.TextField(blank=True)
|
||||
webhook_url = models.URLField(max_length=1024, blank=True)
|
||||
webhook_headers = models.JSONField(default=dict, blank=True)
|
||||
notes = models.TextField(blank=True)
|
||||
last_status = models.CharField(max_length=16, blank=True)
|
||||
last_error = models.TextField(blank=True)
|
||||
last_sent_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["name"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class NotificationDelivery(models.Model):
|
||||
class Status(models.TextChoices):
|
||||
SENT = "sent", "Sent"
|
||||
FAILED = "failed", "Failed"
|
||||
SKIPPED = "skipped", "Skipped"
|
||||
|
||||
target = models.ForeignKey(NotificationTarget, on_delete=models.CASCADE, related_name="deliveries")
|
||||
run = models.ForeignKey(BackupRun, on_delete=models.CASCADE, related_name="notification_deliveries")
|
||||
status = models.CharField(max_length=16, choices=Status.choices)
|
||||
error = models.TextField(blank=True)
|
||||
payload = models.JSONField(default=dict, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["target", "run"], name="unique_notification_delivery_per_target_run"),
|
||||
]
|
||||
ordering = ["-created_at", "target__name"]
|
||||
verbose_name_plural = "notification deliveries"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.target} run {self.run_id} {self.status}"
|
||||
|
||||
|
||||
class SnapshotRecord(models.Model):
|
||||
class Kind(models.TextChoices):
|
||||
SCHEDULED = "scheduled", "Scheduled"
|
||||
@@ -159,6 +218,8 @@ class SnapshotRecord(models.Model):
|
||||
ended_at = models.DateTimeField(null=True, blank=True)
|
||||
metadata = models.JSONField(default=dict, blank=True)
|
||||
discovered_at = models.DateTimeField(auto_now_add=True)
|
||||
reviewed_at = models.DateTimeField(null=True, blank=True)
|
||||
reviewed_by = models.CharField(max_length=150, blank=True)
|
||||
|
||||
class Meta:
|
||||
constraints = [
|
||||
@@ -170,10 +231,34 @@ class SnapshotRecord(models.Model):
|
||||
return f"{self.host}/{self.kind}/{self.dirname}"
|
||||
|
||||
|
||||
class PurgedSnapshot(models.Model):
|
||||
class Action(models.TextChoices):
|
||||
MANUAL = "manual", "Manual"
|
||||
SCHEDULED = "scheduled", "Scheduled"
|
||||
CLI = "cli", "CLI"
|
||||
INCOMPLETE_CLEANUP = "incomplete_cleanup", "Incomplete cleanup"
|
||||
|
||||
host = models.ForeignKey(HostConfig, on_delete=models.SET_NULL, null=True, blank=True, related_name="purged_snapshots")
|
||||
host_name = models.CharField(max_length=255)
|
||||
kind = models.CharField(max_length=16)
|
||||
dirname = models.CharField(max_length=255)
|
||||
path = models.CharField(max_length=1024)
|
||||
reason = models.CharField(max_length=512, blank=True)
|
||||
action = models.CharField(max_length=32, choices=Action.choices)
|
||||
triggered_by = models.CharField(max_length=150, blank=True)
|
||||
metadata = models.JSONField(default=dict, blank=True)
|
||||
purged_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-purged_at", "host_name", "dirname"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.host_name}/{self.kind}/{self.dirname}"
|
||||
|
||||
|
||||
class ScheduleConfig(TimestampedModel):
|
||||
host = models.OneToOneField(HostConfig, on_delete=models.CASCADE, related_name="schedule")
|
||||
cron_expr = models.CharField(max_length=128)
|
||||
user = models.CharField(max_length=64, default="root")
|
||||
enabled = models.BooleanField(default=True)
|
||||
prune = models.BooleanField(default=False)
|
||||
prune_max_delete = models.PositiveIntegerField(default=10)
|
||||
|
||||
168
src/pobsync_backend/notifications.py
Normal file
168
src/pobsync_backend/notifications.py
Normal file
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.mail import send_mail
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import BackupRun, NotificationDelivery, NotificationTarget
|
||||
|
||||
|
||||
TERMINAL_RUN_STATUSES = {
|
||||
BackupRun.Status.SUCCESS,
|
||||
BackupRun.Status.WARNING,
|
||||
BackupRun.Status.FAILED,
|
||||
BackupRun.Status.CANCELLED,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryResult:
|
||||
target: NotificationTarget
|
||||
delivery: NotificationDelivery
|
||||
sent: bool
|
||||
|
||||
|
||||
def notify_backup_run_completed(run: BackupRun) -> list[DeliveryResult]:
|
||||
if run.status not in TERMINAL_RUN_STATUSES:
|
||||
return []
|
||||
|
||||
targets = [target for target in NotificationTarget.objects.filter(enabled=True) if _target_wants_status(target, run.status)]
|
||||
return [_notify_target(target=target, run=run) for target in targets]
|
||||
|
||||
|
||||
def _target_wants_status(target: NotificationTarget, status: str) -> bool:
|
||||
statuses = target.statuses
|
||||
if not isinstance(statuses, list):
|
||||
return False
|
||||
return status in {str(item) for item in statuses}
|
||||
|
||||
|
||||
def _notify_target(*, target: NotificationTarget, run: BackupRun) -> DeliveryResult:
|
||||
payload = _run_payload(run)
|
||||
delivery, created = NotificationDelivery.objects.get_or_create(
|
||||
target=target,
|
||||
run=run,
|
||||
defaults={
|
||||
"status": NotificationDelivery.Status.SKIPPED,
|
||||
"payload": payload,
|
||||
},
|
||||
)
|
||||
if not created:
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
||||
|
||||
try:
|
||||
if target.channel == NotificationTarget.Channel.EMAIL:
|
||||
_send_email(target=target, run=run, payload=payload)
|
||||
elif target.channel == NotificationTarget.Channel.WEBHOOK:
|
||||
_send_webhook(target=target, payload=payload)
|
||||
else:
|
||||
raise ValueError(f"Unsupported notification channel: {target.channel}")
|
||||
except Exception as exc:
|
||||
delivery.status = NotificationDelivery.Status.FAILED
|
||||
delivery.error = str(exc)
|
||||
delivery.save(update_fields=["status", "error"])
|
||||
target.last_status = NotificationDelivery.Status.FAILED
|
||||
target.last_error = str(exc)
|
||||
target.save(update_fields=["last_status", "last_error", "updated_at"])
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
||||
|
||||
delivery.status = NotificationDelivery.Status.SENT
|
||||
delivery.save(update_fields=["status"])
|
||||
target.last_status = NotificationDelivery.Status.SENT
|
||||
target.last_error = ""
|
||||
target.last_sent_at = timezone.now()
|
||||
target.save(update_fields=["last_status", "last_error", "last_sent_at", "updated_at"])
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=True)
|
||||
|
||||
|
||||
def _send_email(*, target: NotificationTarget, run: BackupRun, payload: dict[str, Any]) -> None:
|
||||
recipients = [line.strip() for line in target.email_to.replace(",", "\n").splitlines() if line.strip()]
|
||||
if not recipients:
|
||||
raise ValueError("Email notification target has no recipients.")
|
||||
|
||||
subject = f"pobsync {run.status}: {run.host.host} run {run.id}"
|
||||
message = _email_message(payload)
|
||||
from_email = getattr(settings, "DEFAULT_FROM_EMAIL", "") or "pobsync@localhost"
|
||||
sent = send_mail(subject, message, from_email, recipients, fail_silently=False)
|
||||
if sent == 0:
|
||||
raise ValueError("Django email backend reported zero sent messages.")
|
||||
|
||||
|
||||
def _send_webhook(*, target: NotificationTarget, payload: dict[str, Any]) -> None:
|
||||
if not target.webhook_url:
|
||||
raise ValueError("Webhook notification target has no URL.")
|
||||
|
||||
headers = {"Content-Type": "application/json", **_string_headers(target.webhook_headers)}
|
||||
request = urllib.request.Request(
|
||||
target.webhook_url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
if response.status >= 400:
|
||||
raise ValueError(f"Webhook returned HTTP {response.status}.")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise ValueError(f"Webhook returned HTTP {exc.code}.") from exc
|
||||
|
||||
|
||||
def _string_headers(headers: object) -> dict[str, str]:
|
||||
if not isinstance(headers, dict):
|
||||
return {}
|
||||
return {str(key): str(value) for key, value in headers.items() if str(key).strip()}
|
||||
|
||||
|
||||
def _run_payload(run: BackupRun) -> dict[str, Any]:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
failure = result.get("failure") if isinstance(result.get("failure"), dict) else {}
|
||||
prune = result.get("prune") if isinstance(result.get("prune"), dict) else {}
|
||||
return {
|
||||
"event": "backup_run.completed",
|
||||
"run": {
|
||||
"id": run.id,
|
||||
"host": run.host.host,
|
||||
"type": run.run_type,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"ended_at": run.ended_at.isoformat() if run.ended_at else None,
|
||||
"snapshot": run.snapshot_path,
|
||||
"rsync_exit_code": run.rsync_exit_code,
|
||||
},
|
||||
"failure": {
|
||||
"category": failure.get("category"),
|
||||
"message": failure.get("message") or result.get("error"),
|
||||
"hint": failure.get("hint"),
|
||||
},
|
||||
"prune": {
|
||||
"ok": prune.get("ok") if prune else None,
|
||||
"error": prune.get("error") if prune else "",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _email_message(payload: dict[str, Any]) -> str:
|
||||
run = payload["run"]
|
||||
lines = [
|
||||
f"Host: {run['host']}",
|
||||
f"Run: {run['id']}",
|
||||
f"Type: {run['type']}",
|
||||
f"Status: {run['status']}",
|
||||
f"Started: {run['started_at'] or '-'}",
|
||||
f"Ended: {run['ended_at'] or '-'}",
|
||||
f"Snapshot: {run['snapshot'] or '-'}",
|
||||
f"Rsync exit code: {run['rsync_exit_code'] if run['rsync_exit_code'] is not None else '-'}",
|
||||
]
|
||||
failure = payload.get("failure") if isinstance(payload.get("failure"), dict) else {}
|
||||
if failure.get("message"):
|
||||
lines.extend(["", f"Failure: {failure['message']}"])
|
||||
prune = payload.get("prune") if isinstance(payload.get("prune"), dict) else {}
|
||||
if prune.get("error"):
|
||||
lines.extend(["", f"Retention: {prune['error']}"])
|
||||
return "\n".join(lines)
|
||||
232
src/pobsync_backend/preflight.py
Normal file
232
src/pobsync_backend/preflight.py
Normal file
@@ -0,0 +1,232 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from pobsync.config.merge import build_effective_config
|
||||
from pobsync.rsync import build_ssh_command
|
||||
|
||||
from .config_repository import global_config_object_data, host_config_object_data
|
||||
from .config_source import DjangoConfigSource
|
||||
from .host_ops import collect_host_checks
|
||||
from .models import GlobalConfig, HostConfig
|
||||
from .self_check import SelfCheck
|
||||
|
||||
|
||||
DRY_RUN_BLOCKING_CHECKS = {
|
||||
"Host global config",
|
||||
"Host address",
|
||||
"Host SSH key file",
|
||||
"Host effective source root",
|
||||
"Host effective SSH user",
|
||||
"Host effective SSH port",
|
||||
"Host effective SSH credential",
|
||||
"Host effective rsync recursion",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupGate:
|
||||
state: str
|
||||
message: str
|
||||
checks: list[SelfCheck]
|
||||
real_blockers: list[SelfCheck]
|
||||
dry_run_blockers: list[SelfCheck]
|
||||
warnings: list[SelfCheck]
|
||||
|
||||
@property
|
||||
def can_queue_real(self) -> bool:
|
||||
return not self.real_blockers
|
||||
|
||||
@property
|
||||
def can_queue_dry_run(self) -> bool:
|
||||
return not self.dry_run_blockers
|
||||
|
||||
|
||||
def collect_backup_gate(host: HostConfig, global_config: GlobalConfig | None = None) -> BackupGate:
|
||||
checks = collect_host_checks(host, global_config)
|
||||
remote_preflight_check = _remote_preflight_self_check(host)
|
||||
if remote_preflight_check is not None:
|
||||
checks.append(remote_preflight_check)
|
||||
real_blockers = [check for check in checks if check.status == "failed"]
|
||||
dry_run_blockers = [check for check in real_blockers if check.name in DRY_RUN_BLOCKING_CHECKS]
|
||||
warnings = [check for check in checks if check.status == "warning"]
|
||||
|
||||
if real_blockers:
|
||||
state = "blocked"
|
||||
message = "Real backups are blocked until failed host checks are resolved."
|
||||
elif warnings:
|
||||
state = "warning"
|
||||
message = "Backups can run, but review the warnings first."
|
||||
else:
|
||||
state = "ready"
|
||||
message = "This host is ready for backup runs."
|
||||
|
||||
return BackupGate(
|
||||
state=state,
|
||||
message=message,
|
||||
checks=checks,
|
||||
real_blockers=real_blockers,
|
||||
dry_run_blockers=dry_run_blockers,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
def run_remote_preflight(host: HostConfig, *, timeout_seconds: int = 20) -> dict[str, Any]:
|
||||
config = DjangoConfigSource().effective_config_for_host(host.host)
|
||||
ssh_cfg = config.get("ssh", {}) or {}
|
||||
rsync_cfg = config.get("rsync", {}) or {}
|
||||
address = str(config.get("address") or host.address)
|
||||
user = str(ssh_cfg.get("user") or "root")
|
||||
source_root = str(config.get("source_root") or (config.get("defaults", {}) or {}).get("source_root") or "/")
|
||||
rsync_binary = str(rsync_cfg.get("binary") or "rsync")
|
||||
target = f"{user}@{address}"
|
||||
ssh_cmd = build_ssh_command(ssh_cfg)
|
||||
|
||||
checks = [
|
||||
_run_remote_check(
|
||||
name="SSH reachability",
|
||||
command=[*ssh_cmd, "-oBatchMode=yes", target, "true"],
|
||||
timeout_seconds=timeout_seconds,
|
||||
),
|
||||
_run_remote_check(
|
||||
name="Remote rsync",
|
||||
command=[
|
||||
*ssh_cmd,
|
||||
"-oBatchMode=yes",
|
||||
target,
|
||||
_remote_shell_command(f"command -v {shlex.quote(rsync_binary)} >/dev/null"),
|
||||
],
|
||||
timeout_seconds=timeout_seconds,
|
||||
),
|
||||
_run_remote_check(
|
||||
name="Remote source root",
|
||||
command=[
|
||||
*ssh_cmd,
|
||||
"-oBatchMode=yes",
|
||||
target,
|
||||
_remote_shell_command(f"test -e {shlex.quote(source_root)} && test -r {shlex.quote(source_root)}"),
|
||||
],
|
||||
timeout_seconds=timeout_seconds,
|
||||
),
|
||||
]
|
||||
result = {
|
||||
"ok": all(check["ok"] for check in checks),
|
||||
"checks": checks,
|
||||
"target": target,
|
||||
"source_root": source_root,
|
||||
"rsync_binary": rsync_binary,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
}
|
||||
host.config = {**(host.config or {}), "last_preflight": result}
|
||||
host.save(update_fields=["config", "updated_at"])
|
||||
return result
|
||||
|
||||
|
||||
def _remote_shell_command(script: str) -> str:
|
||||
return f"sh -lc {shlex.quote(script)}"
|
||||
|
||||
|
||||
def effective_host_config_preview(host: HostConfig, global_config: GlobalConfig) -> dict[str, Any]:
|
||||
config = build_effective_config(global_config_object_data(global_config), host_config_object_data(host))
|
||||
credential = host.ssh_credential or global_config.default_ssh_credential
|
||||
ssh = config.get("ssh", {}) or {}
|
||||
rsync = config.get("rsync", {}) or {}
|
||||
retention = config.get("retention", {}) or {}
|
||||
|
||||
return {
|
||||
"source_root": config.get("source_root", ""),
|
||||
"destination_subdir": (config.get("defaults", {}) or {}).get("destination_subdir", ""),
|
||||
"includes": list(config.get("includes") or []),
|
||||
"excludes": list(config.get("excludes_effective") or []),
|
||||
"ssh": {
|
||||
"user": ssh.get("user", ""),
|
||||
"port": ssh.get("port", ""),
|
||||
"options": list(ssh.get("options") or []),
|
||||
"credential": str(credential) if credential else "",
|
||||
},
|
||||
"rsync": {
|
||||
"binary": rsync.get("binary", ""),
|
||||
"args": list(rsync.get("args_effective") or []),
|
||||
"timeout_seconds": rsync.get("timeout_seconds", 0),
|
||||
"bwlimit_kbps": rsync.get("bwlimit_kbps", 0),
|
||||
},
|
||||
"retention": {
|
||||
"daily": retention.get("daily", 0),
|
||||
"weekly": retention.get("weekly", 0),
|
||||
"monthly": retention.get("monthly", 0),
|
||||
"yearly": retention.get("yearly", 0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _run_remote_check(*, name: str, command: list[str], timeout_seconds: int) -> dict[str, Any]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {
|
||||
"name": name,
|
||||
"ok": False,
|
||||
"exit_code": 124,
|
||||
"message": f"{name} timed out after {timeout_seconds}s.",
|
||||
"detail": _clip_output((exc.stderr or exc.stdout or "").strip()),
|
||||
}
|
||||
except OSError as exc:
|
||||
return {
|
||||
"name": name,
|
||||
"ok": False,
|
||||
"exit_code": None,
|
||||
"message": f"{name} could not start.",
|
||||
"detail": str(exc),
|
||||
}
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"ok": result.returncode == 0,
|
||||
"exit_code": result.returncode,
|
||||
"message": f"{name} passed." if result.returncode == 0 else f"{name} failed.",
|
||||
"detail": _clip_output((result.stderr or result.stdout or "").strip()),
|
||||
}
|
||||
|
||||
|
||||
def _remote_preflight_self_check(host: HostConfig) -> SelfCheck | None:
|
||||
preflight = (host.config or {}).get("last_preflight")
|
||||
if not isinstance(preflight, dict):
|
||||
return SelfCheck(
|
||||
"Remote preflight",
|
||||
"warning",
|
||||
"No remote connection preflight has been run yet.",
|
||||
"Run connection preflight before the first real backup.",
|
||||
)
|
||||
checks = preflight.get("checks")
|
||||
if not isinstance(checks, list):
|
||||
return SelfCheck("Remote preflight", "failed", "Stored remote preflight result is invalid.")
|
||||
failed = [str(check.get("name", "unknown")) for check in checks if isinstance(check, dict) and not check.get("ok")]
|
||||
if failed:
|
||||
return SelfCheck(
|
||||
"Remote preflight",
|
||||
"failed",
|
||||
"Remote connection preflight failed.",
|
||||
", ".join(failed),
|
||||
)
|
||||
return SelfCheck(
|
||||
"Remote preflight",
|
||||
"ok",
|
||||
"Remote connection preflight passed.",
|
||||
f"{preflight.get('target', '')} {preflight.get('source_root', '')}".strip(),
|
||||
)
|
||||
|
||||
|
||||
def _clip_output(value: str, *, max_chars: int = 800) -> str:
|
||||
if len(value) <= max_chars:
|
||||
return value
|
||||
return f"{value[:max_chars]}..."
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import stat
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -11,7 +12,7 @@ from pobsync.paths import PobsyncPaths
|
||||
from pobsync.retention import Snapshot, apply_base_protection, build_retention_plan
|
||||
from pobsync.util import sanitize_host
|
||||
|
||||
from .models import HostConfig, SnapshotRecord
|
||||
from .models import HostConfig, PurgedSnapshot, SnapshotRecord
|
||||
|
||||
|
||||
def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict[str, Any]:
|
||||
@@ -22,6 +23,7 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
|
||||
host_config = _enabled_host_config(host)
|
||||
retention = _retention_for_host(host_config)
|
||||
snapshots = _snapshots_for_retention(host_config=host_config, kind=kind)
|
||||
incomplete_items = _incomplete_snapshot_items_for_host(host_config)
|
||||
|
||||
plan = build_retention_plan(
|
||||
snapshots=snapshots,
|
||||
@@ -35,6 +37,7 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
|
||||
keep, reasons = apply_base_protection(snapshots=snapshots, keep=keep, reasons=reasons)
|
||||
|
||||
delete = [snapshot for snapshot in snapshots if snapshot.dirname not in keep]
|
||||
keep_items = [snapshot for snapshot in snapshots if snapshot.dirname in keep]
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
@@ -44,7 +47,11 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
|
||||
"retention": retention,
|
||||
"source": "sql",
|
||||
"keep": sorted(keep),
|
||||
"delete": [_snapshot_to_delete_item(snapshot) for snapshot in delete],
|
||||
"keep_items": [_snapshot_to_item(snapshot, reasons=reasons.get(snapshot.dirname, [])) for snapshot in keep_items],
|
||||
"delete": [_snapshot_to_item(snapshot, reasons=["outside retention policy"]) for snapshot in delete],
|
||||
"incomplete": incomplete_items,
|
||||
"incomplete_reviewed_count": sum(1 for item in incomplete_items if item["reviewed"]),
|
||||
"incomplete_unreviewed_count": sum(1 for item in incomplete_items if not item["reviewed"]),
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
@@ -57,6 +64,8 @@ def run_sql_retention_apply(
|
||||
protect_bases: bool,
|
||||
yes: bool,
|
||||
max_delete: int,
|
||||
action: str = PurgedSnapshot.Action.MANUAL,
|
||||
triggered_by: str = "",
|
||||
acquire_lock: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
host = sanitize_host(host)
|
||||
@@ -70,8 +79,11 @@ def run_sql_retention_apply(
|
||||
def _do_apply() -> dict[str, Any]:
|
||||
plan = run_sql_retention_plan(host=host, kind=kind, protect_bases=bool(protect_bases))
|
||||
delete_list = plan.get("delete") or []
|
||||
incomplete_list = plan.get("incomplete") or []
|
||||
if not isinstance(delete_list, list):
|
||||
raise ConfigError("Invalid retention plan output: delete is not a list")
|
||||
if not isinstance(incomplete_list, list):
|
||||
raise ConfigError("Invalid retention plan output: incomplete is not a list")
|
||||
if max_delete == 0 and len(delete_list) > 0:
|
||||
raise ConfigError("Deletion blocked by --max-delete=0")
|
||||
if len(delete_list) > max_delete:
|
||||
@@ -89,17 +101,29 @@ def run_sql_retention_apply(
|
||||
if snap_kind not in {"scheduled", "manual"}:
|
||||
raise ConfigError(f"Refusing to delete unsupported snapshot kind: {snap_kind!r}")
|
||||
|
||||
path = Path(snap_path)
|
||||
path = _snapshot_delete_path(path=Path(snap_path), dirname=dirname)
|
||||
_validate_snapshot_delete_path(host=host, kind=snap_kind, path=path, dirname=dirname)
|
||||
reason = str(item.get("reason") or "outside retention policy")
|
||||
if not path.exists():
|
||||
actions.append(f"skip missing {snap_kind}/{dirname}")
|
||||
continue
|
||||
if not path.is_dir():
|
||||
raise ConfigError(f"Refusing to delete non-directory path: {snap_path}")
|
||||
raise ConfigError(f"Refusing to delete non-directory path: {path}")
|
||||
|
||||
shutil.rmtree(path)
|
||||
_remove_snapshot_tree(path)
|
||||
_record_purged_snapshot(
|
||||
host_config=_enabled_host_config(host),
|
||||
kind=snap_kind,
|
||||
dirname=dirname,
|
||||
path=path,
|
||||
reason=reason,
|
||||
action=action,
|
||||
triggered_by=triggered_by,
|
||||
metadata={"source": "retention", "protect_bases": bool(protect_bases), "retention_kind": kind},
|
||||
)
|
||||
SnapshotRecord.objects.filter(host__host=host, kind=snap_kind, dirname=dirname).delete()
|
||||
actions.append(f"deleted {snap_kind} {dirname}")
|
||||
deleted.append({"dirname": dirname, "kind": snap_kind, "path": snap_path})
|
||||
deleted.append({"dirname": dirname, "kind": snap_kind, "path": str(path), "reason": reason})
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
@@ -108,6 +132,8 @@ def run_sql_retention_apply(
|
||||
"protect_bases": bool(protect_bases),
|
||||
"max_delete": max_delete,
|
||||
"source": "sql",
|
||||
"planned_delete_count": len(delete_list),
|
||||
"incomplete_ignored_count": len(incomplete_list),
|
||||
"deleted": deleted,
|
||||
"actions": actions,
|
||||
}
|
||||
@@ -118,11 +144,98 @@ def run_sql_retention_apply(
|
||||
return _do_apply()
|
||||
|
||||
|
||||
def run_incomplete_cleanup(
|
||||
*,
|
||||
prefix: Path,
|
||||
host: str,
|
||||
yes: bool,
|
||||
max_delete: int,
|
||||
triggered_by: str = "",
|
||||
acquire_lock: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
host = sanitize_host(host)
|
||||
if not yes:
|
||||
raise ConfigError("Refusing to delete incomplete snapshots without --yes")
|
||||
if max_delete < 0:
|
||||
raise ConfigError("--max-delete must be >= 0")
|
||||
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
def _do_cleanup() -> dict[str, Any]:
|
||||
host_config = _enabled_host_config(host)
|
||||
unreviewed_count = _unreviewed_incomplete_count(host_config)
|
||||
if unreviewed_count:
|
||||
raise ConfigError(
|
||||
f"Refusing to delete {unreviewed_count} incomplete snapshot(s) that have not been reviewed."
|
||||
)
|
||||
|
||||
incomplete_list = [
|
||||
_snapshot_to_item(snapshot, reasons=["manual incomplete cleanup"])
|
||||
for snapshot in _reviewed_incomplete_snapshots_for_host(host_config)
|
||||
]
|
||||
if max_delete == 0 and len(incomplete_list) > 0:
|
||||
raise ConfigError("Incomplete cleanup blocked by --max-delete=0")
|
||||
if len(incomplete_list) > max_delete:
|
||||
raise ConfigError(
|
||||
f"Refusing to delete {len(incomplete_list)} incomplete snapshots (exceeds --max-delete={max_delete})"
|
||||
)
|
||||
|
||||
actions: list[str] = []
|
||||
deleted: list[dict[str, Any]] = []
|
||||
|
||||
for item in incomplete_list:
|
||||
dirname = item["dirname"]
|
||||
snap_path = Path(item["path"])
|
||||
path = _snapshot_delete_path(path=snap_path, dirname=dirname)
|
||||
_validate_incomplete_delete_path(host=host, path=path, dirname=dirname)
|
||||
|
||||
if not path.exists():
|
||||
actions.append(f"skip missing incomplete/{dirname}")
|
||||
elif not path.is_dir():
|
||||
raise ConfigError(f"Refusing to delete non-directory path: {path}")
|
||||
else:
|
||||
_remove_snapshot_tree(path)
|
||||
actions.append(f"deleted incomplete {dirname}")
|
||||
|
||||
_record_purged_snapshot(
|
||||
host_config=host_config,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=dirname,
|
||||
path=path,
|
||||
reason="manual incomplete cleanup",
|
||||
action=PurgedSnapshot.Action.INCOMPLETE_CLEANUP,
|
||||
triggered_by=triggered_by,
|
||||
metadata={"source": "incomplete_cleanup"},
|
||||
)
|
||||
SnapshotRecord.objects.filter(
|
||||
host__host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=dirname,
|
||||
).delete()
|
||||
deleted.append({"dirname": dirname, "kind": SnapshotRecord.Kind.INCOMPLETE, "path": str(path)})
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"host": host,
|
||||
"kind": SnapshotRecord.Kind.INCOMPLETE,
|
||||
"max_delete": max_delete,
|
||||
"source": "sql",
|
||||
"planned_delete_count": len(incomplete_list),
|
||||
"deleted": deleted,
|
||||
"actions": actions,
|
||||
}
|
||||
|
||||
if acquire_lock:
|
||||
with acquire_host_lock(paths.locks_dir, host, command="incomplete-cleanup"):
|
||||
return _do_cleanup()
|
||||
return _do_cleanup()
|
||||
|
||||
|
||||
def _enabled_host_config(host: str) -> HostConfig:
|
||||
try:
|
||||
return HostConfig.objects.get(host=host, enabled=True)
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise ConfigError(f"Missing enabled HostConfig {host!r}") from exc
|
||||
raise ConfigError(f"Missing enabled host {host!r}") from exc
|
||||
|
||||
|
||||
def _retention_for_host(host_config: HostConfig) -> dict[str, int]:
|
||||
@@ -145,6 +258,39 @@ def _snapshots_for_retention(*, host_config: HostConfig, kind: str) -> list[Snap
|
||||
return [_snapshot_from_record(record) for record in records]
|
||||
|
||||
|
||||
def _incomplete_snapshot_items_for_host(host_config: HostConfig) -> list[dict[str, Any]]:
|
||||
records = (
|
||||
SnapshotRecord.objects.filter(host=host_config, kind=SnapshotRecord.Kind.INCOMPLETE)
|
||||
.select_related("base")
|
||||
.order_by("-started_at", "dirname")
|
||||
)
|
||||
return [
|
||||
_snapshot_record_to_item(record, reasons=["incomplete snapshot; excluded from retention cleanup"])
|
||||
for record in records
|
||||
]
|
||||
|
||||
|
||||
def _reviewed_incomplete_snapshots_for_host(host_config: HostConfig) -> list[Snapshot]:
|
||||
records = (
|
||||
SnapshotRecord.objects.filter(
|
||||
host=host_config,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
reviewed_at__isnull=False,
|
||||
)
|
||||
.select_related("base")
|
||||
.order_by("-started_at", "dirname")
|
||||
)
|
||||
return [_snapshot_from_record(record) for record in records]
|
||||
|
||||
|
||||
def _unreviewed_incomplete_count(host_config: HostConfig) -> int:
|
||||
return SnapshotRecord.objects.filter(
|
||||
host=host_config,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
reviewed_at__isnull=True,
|
||||
).count()
|
||||
|
||||
|
||||
def _snapshot_from_record(record: SnapshotRecord) -> Snapshot:
|
||||
return Snapshot(
|
||||
kind=record.kind,
|
||||
@@ -172,11 +318,114 @@ def _base_meta_from_record(record: SnapshotRecord) -> dict[str, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _snapshot_to_delete_item(snapshot: Snapshot) -> dict[str, Any]:
|
||||
def _snapshot_to_item(snapshot: Snapshot, *, reasons: list[str]) -> dict[str, Any]:
|
||||
return {
|
||||
"dirname": snapshot.dirname,
|
||||
"kind": snapshot.kind,
|
||||
"path": snapshot.path,
|
||||
"dt": snapshot.dt.isoformat(),
|
||||
"status": snapshot.status,
|
||||
"reasons": reasons,
|
||||
"reason": ", ".join(reasons),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot_record_to_item(record: SnapshotRecord, *, reasons: list[str]) -> dict[str, Any]:
|
||||
item = _snapshot_to_item(_snapshot_from_record(record), reasons=reasons)
|
||||
item["reviewed"] = record.reviewed_at is not None
|
||||
item["reviewed_at"] = record.reviewed_at.isoformat() if record.reviewed_at else ""
|
||||
item["reviewed_by"] = record.reviewed_by
|
||||
return item
|
||||
|
||||
|
||||
def _snapshot_delete_path(*, path: Path, dirname: str) -> Path:
|
||||
if path.name == "data" and path.parent.name == dirname:
|
||||
return path.parent
|
||||
return path
|
||||
|
||||
|
||||
def _record_purged_snapshot(
|
||||
*,
|
||||
host_config: HostConfig,
|
||||
kind: str,
|
||||
dirname: str,
|
||||
path: Path,
|
||||
reason: str,
|
||||
action: str,
|
||||
triggered_by: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> None:
|
||||
PurgedSnapshot.objects.create(
|
||||
host=host_config,
|
||||
host_name=host_config.host,
|
||||
kind=kind,
|
||||
dirname=dirname,
|
||||
path=str(path),
|
||||
reason=reason,
|
||||
action=action,
|
||||
triggered_by=triggered_by,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def _validate_incomplete_delete_path(*, host: str, path: Path, dirname: str) -> None:
|
||||
path_parts = path.parts
|
||||
if path.name != dirname or ".incomplete" not in path_parts or host not in path_parts:
|
||||
raise ConfigError(f"Refusing to delete unexpected incomplete snapshot path: {path}")
|
||||
incomplete_index = path_parts.index(".incomplete")
|
||||
if incomplete_index == 0 or path_parts[incomplete_index - 1] != host:
|
||||
raise ConfigError(f"Refusing to delete incomplete snapshot outside host backup root: {path}")
|
||||
|
||||
|
||||
def _validate_snapshot_delete_path(*, host: str, kind: str, path: Path, dirname: str) -> None:
|
||||
if kind not in {SnapshotRecord.Kind.SCHEDULED, SnapshotRecord.Kind.MANUAL}:
|
||||
raise ConfigError(f"Refusing to delete unsupported snapshot kind: {kind!r}")
|
||||
path_parts = path.parts
|
||||
if path.name != dirname or kind not in path_parts or host not in path_parts:
|
||||
raise ConfigError(f"Refusing to delete unexpected snapshot path: {path}")
|
||||
kind_index = path_parts.index(kind)
|
||||
if kind_index == 0 or path_parts[kind_index - 1] != host:
|
||||
raise ConfigError(f"Refusing to delete snapshot outside host backup root: {path}")
|
||||
|
||||
|
||||
def _remove_snapshot_tree(path: Path) -> None:
|
||||
_make_snapshot_tree_user_removable(path)
|
||||
shutil.rmtree(path, onexc=_retry_remove_with_user_permissions)
|
||||
|
||||
|
||||
def _make_snapshot_tree_user_removable(path: Path) -> None:
|
||||
stack = [path]
|
||||
while stack:
|
||||
directory = stack.pop()
|
||||
if directory.is_symlink():
|
||||
continue
|
||||
_make_path_user_removable(directory)
|
||||
try:
|
||||
children = list(directory.iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for child in children:
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
stack.append(child)
|
||||
|
||||
|
||||
def _retry_remove_with_user_permissions(function: Any, path: str, excinfo: BaseException) -> None:
|
||||
failed_path = Path(path)
|
||||
_make_path_user_removable(failed_path)
|
||||
function(path)
|
||||
|
||||
|
||||
def _make_path_user_removable(path: Path) -> None:
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
except OSError:
|
||||
return
|
||||
wanted = stat.S_IRUSR | stat.S_IWUSR
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
wanted |= stat.S_IXUSR
|
||||
if mode & wanted == wanted:
|
||||
return
|
||||
try:
|
||||
path.chmod(mode | wanted)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pwd
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -28,6 +29,7 @@ class SelfCheck:
|
||||
def collect_self_checks() -> list[SelfCheck]:
|
||||
checks: list[SelfCheck] = []
|
||||
checks.extend(_django_checks())
|
||||
checks.extend(_install_checks())
|
||||
checks.extend(_path_checks())
|
||||
checks.extend(_binary_checks())
|
||||
checks.extend(_database_checks())
|
||||
@@ -36,6 +38,10 @@ def collect_self_checks() -> list[SelfCheck]:
|
||||
return checks
|
||||
|
||||
|
||||
def _native_runtime_available() -> bool:
|
||||
return Path("/run/systemd/system").exists() and shutil.which("systemctl") is not None
|
||||
|
||||
|
||||
def summarize_self_checks(checks: list[SelfCheck]) -> dict[str, int]:
|
||||
return {
|
||||
"ok": sum(1 for check in checks if check.status == "ok"),
|
||||
@@ -70,10 +76,17 @@ def _django_checks() -> list[SelfCheck]:
|
||||
|
||||
def _path_checks() -> list[SelfCheck]:
|
||||
checks = []
|
||||
checks.append(_path_check("POBSYNC_HOME", Path(settings.POBSYNC_HOME), must_be_absolute=True, must_be_writable=True))
|
||||
checks.append(
|
||||
_path_check(
|
||||
"POBSYNC_BACKUP_ROOT",
|
||||
"State root",
|
||||
Path(settings.POBSYNC_HOME),
|
||||
must_be_absolute=True,
|
||||
must_be_writable=True,
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
_path_check(
|
||||
"Backup root",
|
||||
Path(settings.POBSYNC_BACKUP_ROOT),
|
||||
must_be_absolute=True,
|
||||
must_exist=True,
|
||||
@@ -91,18 +104,105 @@ def _path_checks() -> list[SelfCheck]:
|
||||
)
|
||||
db_settings = settings.DATABASES["default"]
|
||||
if db_settings["ENGINE"] == "django.db.backends.sqlite3":
|
||||
sqlite_path = Path(str(db_settings["NAME"]))
|
||||
checks.append(
|
||||
_path_check(
|
||||
"SQLite directory",
|
||||
Path(str(db_settings["NAME"])).parent,
|
||||
sqlite_path.parent,
|
||||
must_be_absolute=True,
|
||||
must_exist=True,
|
||||
must_be_writable=True,
|
||||
)
|
||||
)
|
||||
checks.append(_sqlite_database_check(sqlite_path))
|
||||
return checks
|
||||
|
||||
|
||||
def _install_checks() -> list[SelfCheck]:
|
||||
if not _native_runtime_available() and not Path(settings.POBSYNC_ENV_FILE).exists():
|
||||
return [
|
||||
SelfCheck(
|
||||
"Environment file",
|
||||
"skipped",
|
||||
"Native environment file is not configured in this runtime.",
|
||||
"This is expected inside Docker or local development.",
|
||||
),
|
||||
SelfCheck(
|
||||
"Service user",
|
||||
"skipped",
|
||||
"Native service user check is not available in this runtime.",
|
||||
"This is expected inside Docker or local development.",
|
||||
),
|
||||
SelfCheck(
|
||||
"Backup root owner",
|
||||
"skipped",
|
||||
"Native backup root ownership check is not available in this runtime.",
|
||||
"This is expected inside Docker or local development.",
|
||||
),
|
||||
]
|
||||
|
||||
checks = [_env_file_check(Path(settings.POBSYNC_ENV_FILE)), _service_user_check()]
|
||||
checks.append(_backup_root_owner_check(Path(settings.POBSYNC_BACKUP_ROOT)))
|
||||
return checks
|
||||
|
||||
|
||||
def _env_file_check(path: Path) -> SelfCheck:
|
||||
if not path.is_absolute():
|
||||
return SelfCheck("Environment file", "failed", f"{path} is not absolute.")
|
||||
if not path.exists():
|
||||
return SelfCheck("Environment file", "failed", f"{path} does not exist.")
|
||||
if not path.is_file():
|
||||
return SelfCheck("Environment file", "failed", f"{path} is not a regular file.")
|
||||
if not os.access(path, os.R_OK):
|
||||
return SelfCheck("Environment file", "failed", f"{path} is not readable by this process.")
|
||||
return SelfCheck("Environment file", "ok", str(path))
|
||||
|
||||
|
||||
def _service_user_check() -> SelfCheck:
|
||||
expected_user = settings.POBSYNC_SERVICE_USER
|
||||
try:
|
||||
current_user = pwd.getpwuid(os.geteuid()).pw_name
|
||||
except KeyError:
|
||||
return SelfCheck("Service user", "failed", f"Current uid {os.geteuid()} has no passwd entry.")
|
||||
if current_user != expected_user:
|
||||
return SelfCheck(
|
||||
"Service user",
|
||||
"warning",
|
||||
f"Current process runs as {current_user}, expected {expected_user}.",
|
||||
"Run terminal checks with sudo -u <service-user> pobsync-manage check_pobsync_install.",
|
||||
)
|
||||
return SelfCheck("Service user", "ok", current_user)
|
||||
|
||||
|
||||
def _backup_root_owner_check(path: Path) -> SelfCheck:
|
||||
if not path.exists():
|
||||
return SelfCheck("Backup root owner", "failed", f"{path} does not exist.")
|
||||
expected_user = settings.POBSYNC_SERVICE_USER
|
||||
try:
|
||||
owner = pwd.getpwuid(path.stat().st_uid).pw_name
|
||||
except KeyError:
|
||||
return SelfCheck("Backup root owner", "warning", f"{path} owner uid {path.stat().st_uid} has no passwd entry.")
|
||||
if owner != expected_user:
|
||||
return SelfCheck(
|
||||
"Backup root owner",
|
||||
"warning",
|
||||
f"{path} is owned by {owner}, expected {expected_user}.",
|
||||
)
|
||||
return SelfCheck("Backup root owner", "ok", f"{path} owner={owner}")
|
||||
|
||||
|
||||
def _sqlite_database_check(path: Path) -> SelfCheck:
|
||||
if not path.is_absolute():
|
||||
return SelfCheck("SQLite database", "failed", f"{path} is not absolute.")
|
||||
if not path.exists():
|
||||
return SelfCheck("SQLite database", "warning", f"{path} does not exist yet.")
|
||||
if not path.is_file():
|
||||
return SelfCheck("SQLite database", "failed", f"{path} is not a regular file.")
|
||||
if not os.access(path, os.R_OK | os.W_OK):
|
||||
return SelfCheck("SQLite database", "failed", f"{path} is not readable and writable by this process.")
|
||||
return SelfCheck("SQLite database", "ok", str(path))
|
||||
|
||||
|
||||
def _path_check(
|
||||
name: str,
|
||||
path: Path,
|
||||
@@ -166,19 +266,19 @@ def _config_checks() -> list[SelfCheck]:
|
||||
message = "Default global config exists."
|
||||
if global_config.backup_root != settings.POBSYNC_BACKUP_ROOT:
|
||||
status = "warning"
|
||||
message = "Global config backup root differs from runtime POBSYNC_BACKUP_ROOT."
|
||||
message = "Saved backup root differs from the active backup root."
|
||||
return [
|
||||
SelfCheck(
|
||||
"Global config",
|
||||
status,
|
||||
message,
|
||||
f"database={global_config.backup_root} runtime={settings.POBSYNC_BACKUP_ROOT}",
|
||||
f"saved={global_config.backup_root} active={settings.POBSYNC_BACKUP_ROOT}",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _systemd_checks() -> list[SelfCheck]:
|
||||
if not Path("/run/systemd/system").exists() or shutil.which("systemctl") is None:
|
||||
if not _native_runtime_available():
|
||||
return [
|
||||
SelfCheck(
|
||||
"Systemd services",
|
||||
|
||||
@@ -5,15 +5,16 @@ from typing import Any, Iterable
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync.run_stats import filesystem_capacity
|
||||
from pobsync.run_stats import filesystem_capacity, tree_usage
|
||||
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
||||
|
||||
|
||||
def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: GlobalConfig | None) -> dict[str, Any]:
|
||||
hosts = list(hosts)
|
||||
runs = list(
|
||||
BackupRun.objects.select_related("host", "snapshot")
|
||||
.filter(status=BackupRun.Status.SUCCESS)
|
||||
.filter(status__in=_COMPLETED_BACKUP_STATUSES)
|
||||
.order_by("-started_at", "-created_at")[:100]
|
||||
)
|
||||
real_runs = [_run_summary(run) for run in runs if _is_real_run(run)]
|
||||
@@ -21,6 +22,7 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
||||
|
||||
for host in hosts:
|
||||
host.stats_summary = collect_host_stats(host=host)
|
||||
backup_data = _sum_backup_data_by_kind(host.stats_summary["backup_data"] for host in hosts)
|
||||
|
||||
literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in real_runs]
|
||||
literal_values = [value for value in literal_values if value is not None]
|
||||
@@ -37,6 +39,8 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
||||
available = _int_at(capacity, "available_bytes")
|
||||
daily_literal = _average_daily_literal(real_runs)
|
||||
|
||||
link_dest_savings_ratio = round(total_matched / savings_basis, 4) if savings_basis else None
|
||||
|
||||
return {
|
||||
"runs_sampled": len(real_runs),
|
||||
"avg_duration_seconds": _average(duration_values),
|
||||
@@ -44,33 +48,40 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
||||
"avg_literal_data_bytes": avg_literal,
|
||||
"total_literal_data_bytes": total_literal,
|
||||
"total_matched_data_bytes": total_matched,
|
||||
"link_dest_savings_ratio": round(total_matched / savings_basis, 4) if savings_basis else None,
|
||||
"link_dest_savings_ratio": link_dest_savings_ratio,
|
||||
"link_dest_savings_percent": round(link_dest_savings_ratio * 100, 1) if link_dest_savings_ratio is not None else None,
|
||||
"estimated_runs_until_full": int(available / avg_literal) if available and avg_literal else None,
|
||||
"estimated_days_until_full": int(available / daily_literal) if available and daily_literal else None,
|
||||
"capacity": capacity,
|
||||
"backup_data": backup_data,
|
||||
}
|
||||
|
||||
|
||||
def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]:
|
||||
runs = list(host.runs.select_related("snapshot").filter(status=BackupRun.Status.SUCCESS).order_by("-started_at", "-created_at")[:50])
|
||||
runs = list(host.runs.select_related("snapshot").order_by("-started_at", "-created_at")[:50])
|
||||
real_runs = [_run_summary(run) for run in runs if _is_real_run(run)]
|
||||
real_runs = [run for run in real_runs if run["has_stats"]][:limit]
|
||||
completed_real_runs = [run for run in real_runs if run["status"] in _COMPLETED_BACKUP_STATUSES]
|
||||
trend_runs = [run for run in completed_real_runs if run["has_stats"]][:limit]
|
||||
latest_snapshot = host.snapshots.order_by("-started_at", "-discovered_at", "-id").first()
|
||||
latest_snapshot_stats = _snapshot_summary(latest_snapshot) if latest_snapshot else {}
|
||||
backup_data = _backup_data_by_kind(host)
|
||||
|
||||
literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in real_runs]
|
||||
literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in trend_runs]
|
||||
literal_values = [value for value in literal_values if value is not None]
|
||||
matched_values = [_int_at(run, "rsync", "matched_data_bytes") for run in real_runs]
|
||||
matched_values = [_int_at(run, "rsync", "matched_data_bytes") for run in trend_runs]
|
||||
matched_values = [value for value in matched_values if value is not None]
|
||||
max_literal = max(literal_values) if literal_values else 0
|
||||
max_matched = max(matched_values) if matched_values else 0
|
||||
|
||||
return {
|
||||
"runs": [_with_bar_percentages(run, max_literal=max_literal, max_matched=max_matched) for run in real_runs],
|
||||
"latest_run": real_runs[0] if real_runs else {},
|
||||
"runs": [_with_bar_percentages(run, max_literal=max_literal, max_matched=max_matched) for run in trend_runs],
|
||||
"latest_run": completed_real_runs[0] if completed_real_runs else {},
|
||||
"latest_good_run": _first_run_with_status(real_runs, {BackupRun.Status.SUCCESS}),
|
||||
"latest_problem_run": _first_run_with_status(real_runs, {BackupRun.Status.WARNING, BackupRun.Status.FAILED}),
|
||||
"latest_snapshot": latest_snapshot_stats,
|
||||
"backup_data": backup_data,
|
||||
"avg_literal_data_bytes": _average(literal_values),
|
||||
"avg_daily_literal_data_bytes": _average_daily_literal(real_runs),
|
||||
"avg_daily_literal_data_bytes": _average_daily_literal(trend_runs),
|
||||
"total_literal_data_bytes": sum(literal_values),
|
||||
"total_matched_data_bytes": sum(matched_values),
|
||||
}
|
||||
@@ -87,6 +98,8 @@ def _run_summary(run: BackupRun) -> dict[str, Any]:
|
||||
"ended_at": run.ended_at,
|
||||
"snapshot": run.snapshot,
|
||||
"snapshot_path": run.snapshot_path,
|
||||
"status": run.status,
|
||||
"reviewed_at": run.reviewed_at,
|
||||
"has_stats": bool(stats),
|
||||
"duration_seconds": _int_at(stats, "duration_seconds"),
|
||||
"rsync": stats.get("rsync") if isinstance(stats.get("rsync"), dict) else {},
|
||||
@@ -94,6 +107,65 @@ def _run_summary(run: BackupRun) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _backup_data_by_kind(host: HostConfig) -> dict[str, Any]:
|
||||
rows: dict[str, dict[str, int]] = {
|
||||
SnapshotRecord.Kind.SCHEDULED: _empty_snapshot_data_row(),
|
||||
SnapshotRecord.Kind.MANUAL: _empty_snapshot_data_row(),
|
||||
SnapshotRecord.Kind.INCOMPLETE: _empty_snapshot_data_row(),
|
||||
}
|
||||
total = _empty_snapshot_data_row()
|
||||
|
||||
for snapshot in host.snapshots.all():
|
||||
summary = _snapshot_summary(snapshot)
|
||||
row = rows.setdefault(snapshot.kind, _empty_snapshot_data_row())
|
||||
allocated = summary.get("allocated_size_bytes") or summary.get("apparent_size_bytes") or 0
|
||||
apparent = summary.get("apparent_size_bytes") or 0
|
||||
unique_apparent = summary.get("unique_apparent_size_bytes") or 0
|
||||
row["count"] += 1
|
||||
row["allocated_size_bytes"] += int(allocated)
|
||||
row["apparent_size_bytes"] += int(apparent)
|
||||
row["unique_apparent_size_bytes"] += int(unique_apparent)
|
||||
total["count"] += 1
|
||||
total["allocated_size_bytes"] += int(allocated)
|
||||
total["apparent_size_bytes"] += int(apparent)
|
||||
total["unique_apparent_size_bytes"] += int(unique_apparent)
|
||||
|
||||
return {
|
||||
"scheduled": rows[SnapshotRecord.Kind.SCHEDULED],
|
||||
"manual": rows[SnapshotRecord.Kind.MANUAL],
|
||||
"incomplete": rows[SnapshotRecord.Kind.INCOMPLETE],
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
def _empty_snapshot_data_row() -> dict[str, int]:
|
||||
return {
|
||||
"count": 0,
|
||||
"allocated_size_bytes": 0,
|
||||
"apparent_size_bytes": 0,
|
||||
"unique_apparent_size_bytes": 0,
|
||||
}
|
||||
|
||||
|
||||
def _sum_backup_data_by_kind(rows: Iterable[dict[str, dict[str, int]]]) -> dict[str, dict[str, int]]:
|
||||
total_rows: dict[str, dict[str, int]] = {
|
||||
"scheduled": _empty_snapshot_data_row(),
|
||||
"manual": _empty_snapshot_data_row(),
|
||||
"incomplete": _empty_snapshot_data_row(),
|
||||
"total": _empty_snapshot_data_row(),
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
for kind, values in row.items():
|
||||
total_row = total_rows.setdefault(kind, _empty_snapshot_data_row())
|
||||
total_row["count"] += values.get("count", 0)
|
||||
total_row["allocated_size_bytes"] += values.get("allocated_size_bytes", 0)
|
||||
total_row["apparent_size_bytes"] += values.get("apparent_size_bytes", 0)
|
||||
total_row["unique_apparent_size_bytes"] += values.get("unique_apparent_size_bytes", 0)
|
||||
|
||||
return total_rows
|
||||
|
||||
|
||||
def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
|
||||
if snapshot is None:
|
||||
return {}
|
||||
@@ -101,18 +173,43 @@ def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
|
||||
stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
||||
storage = stats.get("storage") if isinstance(stats.get("storage"), dict) else {}
|
||||
snapshot_storage = storage.get("snapshot") if isinstance(storage.get("snapshot"), dict) else {}
|
||||
if snapshot.kind == SnapshotRecord.Kind.INCOMPLETE:
|
||||
snapshot_storage = _snapshot_storage_from_filesystem(snapshot)
|
||||
else:
|
||||
has_recorded_size = (
|
||||
_int_at(snapshot_storage, "allocated_size_bytes") is not None
|
||||
or _int_at(snapshot_storage, "apparent_size_bytes") is not None
|
||||
)
|
||||
if not has_recorded_size:
|
||||
snapshot_storage = _snapshot_storage_from_filesystem(snapshot)
|
||||
apparent_size = _int_at(snapshot_storage, "apparent_size_bytes")
|
||||
hardlinked_apparent = _int_at(snapshot_storage, "hardlinked_apparent_size_bytes") or 0
|
||||
return {
|
||||
"id": snapshot.id,
|
||||
"dirname": snapshot.dirname,
|
||||
"kind": snapshot.kind,
|
||||
"status": snapshot.status,
|
||||
"started_at": snapshot.started_at,
|
||||
"apparent_size_bytes": _int_at(snapshot_storage, "apparent_size_bytes"),
|
||||
"apparent_size_bytes": apparent_size,
|
||||
"allocated_size_bytes": _int_at(snapshot_storage, "allocated_size_bytes"),
|
||||
"hardlinked_files": _int_at(snapshot_storage, "hardlinked_files"),
|
||||
"hardlinked_apparent_size_bytes": hardlinked_apparent,
|
||||
"unique_apparent_size_bytes": max((apparent_size or 0) - hardlinked_apparent, 0),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot_storage_from_filesystem(snapshot: SnapshotRecord) -> dict[str, Any]:
|
||||
if not snapshot.path:
|
||||
return {}
|
||||
snapshot_path = Path(snapshot.path)
|
||||
data_path = snapshot_path / "data"
|
||||
if snapshot_path.name == "data":
|
||||
return tree_usage(snapshot_path)
|
||||
if data_path.exists():
|
||||
return tree_usage(data_path)
|
||||
return tree_usage(snapshot_path)
|
||||
|
||||
|
||||
def _is_real_run(run: BackupRun) -> bool:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
if result.get("dry_run") is True:
|
||||
@@ -121,6 +218,13 @@ def _is_real_run(run: BackupRun) -> bool:
|
||||
return requested.get("dry_run") is not True
|
||||
|
||||
|
||||
def _first_run_with_status(runs: list[dict[str, Any]], statuses: set[str]) -> dict[str, Any]:
|
||||
for run in runs:
|
||||
if run["status"] in statuses and run.get("reviewed_at") is None:
|
||||
return run
|
||||
return {}
|
||||
|
||||
|
||||
def _capacity_from_system(global_config: GlobalConfig | None) -> dict[str, Any]:
|
||||
if global_config is None or not global_config.backup_root:
|
||||
return {}
|
||||
@@ -198,3 +302,6 @@ def _int_at(data: dict[str, Any], *keys: str) -> int | None:
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
return None
|
||||
|
||||
|
||||
_COMPLETED_BACKUP_STATUSES = [BackupRun.Status.SUCCESS, BackupRun.Status.WARNING]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
46
src/pobsync_backend/templates/pobsync_backend/changelog.html
Normal file
46
src/pobsync_backend/templates/pobsync_backend/changelog.html
Normal file
@@ -0,0 +1,46 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Changelog - pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Release notes</div>
|
||||
<h1>Changelog</h1>
|
||||
<div class="page-subtitle">Installed release notes rendered from the repository changelog.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Changelog actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<div class="stack spaced">
|
||||
<div><strong>Installed version:</strong> {{ app_version }}</div>
|
||||
<div class="muted">Changelog file: {{ changelog_path }}</div>
|
||||
{% if missing %}
|
||||
<div class="status warning">missing</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="stack">
|
||||
{% for block in changelog_blocks %}
|
||||
{% if block.kind == "heading" %}
|
||||
{% if block.level == 1 %}
|
||||
<h2>{{ block.text }}</h2>
|
||||
{% else %}
|
||||
<h3>{{ block.text }}</h3>
|
||||
{% endif %}
|
||||
{% elif block.kind == "list" %}
|
||||
<ul>
|
||||
{% for item in block.items %}
|
||||
<li>{{ item }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% else %}
|
||||
<p>{{ block.text }}</p>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -3,130 +3,108 @@
|
||||
{% block title %}pobsync dashboard{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Dashboard</h1>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Control panel</div>
|
||||
<h1>Dashboard</h1>
|
||||
<div class="page-subtitle">Backup health, required action, storage pressure, and recent activity in one place.</div>
|
||||
</div>
|
||||
{% if can_manage_control_panel %}
|
||||
<section class="actions" aria-label="Dashboard actions">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">New host</a>
|
||||
<a class="button-link secondary" href="{% url 'edit_global_config' %}">{% if global_config %}Edit global config{% else %}Create global config{% endif %}</a>
|
||||
</section>
|
||||
{% endif %}
|
||||
</header>
|
||||
|
||||
<section class="actions" aria-label="Dashboard actions">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">New host</a>
|
||||
<a class="button-link secondary" href="{% url 'edit_global_config' %}">{% if global_config %}Edit global config{% else %}Create global config{% endif %}</a>
|
||||
</section>
|
||||
|
||||
{% if not global_config or not counts.hosts %}
|
||||
<section class="panel">
|
||||
<h2>Setup</h2>
|
||||
{% if not global_config %}
|
||||
<p class="muted">No default global config exists yet. Create it first so discovery, backups, and retention know where pobsync stores snapshots.</p>
|
||||
<div class="actions inline">
|
||||
<a class="button-link" href="{% url 'edit_global_config' %}">Create global config</a>
|
||||
</div>
|
||||
{% elif not counts.hosts %}
|
||||
<p class="muted">Global config is ready. Add the first host to make this dashboard useful.</p>
|
||||
<div class="actions inline">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">Add first host</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% if can_manage_control_panel %}
|
||||
{% if not global_config or not counts.hosts %}
|
||||
<section class="panel">
|
||||
<h2>Setup</h2>
|
||||
{% if not global_config %}
|
||||
<p class="muted">No default global config exists yet. Create it first so discovery, backups, and retention know where pobsync stores snapshots.</p>
|
||||
<div class="actions inline">
|
||||
<a class="button-link" href="{% url 'edit_global_config' %}">Create global config</a>
|
||||
</div>
|
||||
{% elif not counts.hosts %}
|
||||
<p class="muted">Global config is ready. Add the first host to make this dashboard useful.</p>
|
||||
<div class="actions inline">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">Add first host</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
<section class="grid" aria-label="Summary">
|
||||
<div class="metric"><div class="label">Global Configs</div><div class="value">{{ counts.global_configs }}</div></div>
|
||||
<div class="metric"><div class="label">Hosts</div><div class="value">{{ counts.enabled_hosts }}/{{ counts.hosts }}</div></div>
|
||||
<div class="metric"><div class="label">Schedules</div><div class="value">{{ counts.enabled_schedules }}/{{ counts.schedules }}</div></div>
|
||||
<div class="metric"><div class="label">Snapshots</div><div class="value">{{ counts.snapshots }}</div></div>
|
||||
<div class="metric"><div class="label">Runs</div><div class="value">{{ counts.runs }}</div></div>
|
||||
<div class="metric"><div class="label">Running</div><div class="value">{{ counts.running_runs }}</div></div>
|
||||
<div class="metric"><div class="label">Failed</div><div class="value">{{ counts.failed_runs }}</div></div>
|
||||
<div
|
||||
data-refresh-url="{% url 'dashboard_priority_live' %}"
|
||||
data-refresh-interval="10000"
|
||||
data-refresh-active="true"
|
||||
aria-live="polite"
|
||||
>
|
||||
{% include "pobsync_backend/partials/dashboard_priority.html" %}
|
||||
</div>
|
||||
|
||||
<section class="grid dashboard-summary-grid" aria-label="Summary">
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}"><div class="label">Hosts</div><div class="value">{{ counts.enabled_hosts }}/{{ counts.hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'schedules_list' %}"><div class="label">Schedules</div><div class="value">{{ counts.enabled_schedules }}/{{ counts.schedules }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'snapshots_list' %}"><div class="label">Snapshots</div><div class="value">{{ counts.snapshots }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'runs_list' %}"><div class="label">Runs</div><div class="value">{{ counts.runs }}</div></a>
|
||||
<a class="metric metric-link {% if counts.warning_runs %}warning{% endif %}" href="{% url 'runs_list' %}?status=warning&review=needed"><div class="label">Warnings</div><div class="value">{{ counts.warning_runs }}</div></a>
|
||||
<a class="metric metric-link {% if counts.failed_runs %}failed{% endif %}" href="{% url 'runs_list' %}?status=failed&review=needed"><div class="label">Failed</div><div class="value">{{ counts.failed_runs }}</div></a>
|
||||
</section>
|
||||
|
||||
{% if stats_summary.runs_sampled %}
|
||||
<section class="grid" aria-label="Backup trends">
|
||||
<div class="metric"><div class="label">Backup Root Used</div><div class="value">{{ stats_summary.capacity.used_percent|default:"" }}{% if stats_summary.capacity.used_percent is not None %}%{% endif %}</div></div>
|
||||
<div class="metric"><div class="label">Available</div><div class="value">{{ stats_summary.capacity.available_bytes|filesizeformat }}</div></div>
|
||||
<div class="metric"><div class="label">Avg New Data</div><div class="value">{{ stats_summary.avg_literal_data_bytes|filesizeformat }}</div></div>
|
||||
<div class="metric"><div class="label">Avg Daily New</div><div class="value">{{ stats_summary.avg_daily_literal_data_bytes|filesizeformat }}</div></div>
|
||||
<div class="metric"><div class="label">Avg Duration</div><div class="value">{{ stats_summary.avg_duration_seconds|default:"" }}{% if stats_summary.avg_duration_seconds is not None %}s{% endif %}</div></div>
|
||||
<div class="metric"><div class="label">Link-Dest Savings</div><div class="value">{{ stats_summary.link_dest_savings_ratio|default:"" }}</div></div>
|
||||
<div class="metric"><div class="label">Runs Until Full</div><div class="value">{{ stats_summary.estimated_runs_until_full|default:"" }}</div></div>
|
||||
<div class="metric"><div class="label">Days Until Full</div><div class="value">{{ stats_summary.estimated_days_until_full|default:"" }}</div></div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Hosts</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Address</th>
|
||||
<th>Enabled</th>
|
||||
<th>Snapshots</th>
|
||||
<th>Latest Snapshot</th>
|
||||
<th>Latest Run</th>
|
||||
<th>Next Run</th>
|
||||
<th>New Data</th>
|
||||
<th>Runs</th>
|
||||
<th>Retention</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for host in hosts %}
|
||||
<tr>
|
||||
<td><a href="{% url 'host_detail' host.host %}">{{ host.host }}</a></td>
|
||||
<td>{{ host.address }}</td>
|
||||
<td>{{ host.enabled|yesno:"yes,no" }}</td>
|
||||
<td>{{ host.snapshot_count }}</td>
|
||||
<td>
|
||||
{% if host.latest_snapshot %}
|
||||
<a href="{% url 'snapshot_detail' host.latest_snapshot.id %}">{{ host.latest_snapshot.dirname }}</a>
|
||||
<div class="muted">{{ host.latest_snapshot.kind }} {{ host.latest_snapshot.status }}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if host.stats_summary.latest_run.id %}
|
||||
<a href="{% url 'run_detail' host.stats_summary.latest_run.id %}">Run {{ host.stats_summary.latest_run.id }}</a>
|
||||
<div class="muted">{{ host.stats_summary.latest_run.run_type }} {{ host.stats_summary.latest_run.duration_seconds|default:"" }}{% if host.stats_summary.latest_run.duration_seconds is not None %}s{% endif %}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{% if host.next_run_at %}{{ host.next_run_at }}{% else %}<span class="muted">none</span>{% endif %}</td>
|
||||
<td>{{ host.stats_summary.latest_run.rsync.literal_data_bytes|filesizeformat }}</td>
|
||||
<td>{{ host.run_count }}</td>
|
||||
<td>d{{ host.retention_daily }} w{{ host.retention_weekly }} m{{ host.retention_monthly }} y{{ host.retention_yearly }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="10" class="muted">No hosts configured yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<section class="panel dashboard-trends-panel">
|
||||
<h2>Backup Trends</h2>
|
||||
{% if stats_summary.runs_sampled %}
|
||||
<div class="insight-grid" aria-label="Backup trends">
|
||||
<div class="insight-item">
|
||||
<div class="label">Runway</div>
|
||||
<div class="value">
|
||||
{% if stats_summary.estimated_days_until_full %}
|
||||
{{ stats_summary.estimated_days_until_full }} days
|
||||
{% elif stats_summary.estimated_runs_until_full %}
|
||||
{{ stats_summary.estimated_runs_until_full }} runs
|
||||
{% else %}
|
||||
unknown
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="muted">Estimated from average new data per day.</div>
|
||||
</div>
|
||||
<div class="insight-item">
|
||||
<div class="label">New Data</div>
|
||||
<div class="value">{{ stats_summary.avg_daily_literal_data_bytes|filesizeformat }}/day</div>
|
||||
<div class="muted">{{ stats_summary.avg_literal_data_bytes|filesizeformat }} per backup on average.</div>
|
||||
</div>
|
||||
<div class="insight-item">
|
||||
<div class="label">Link-Dest Savings</div>
|
||||
<div class="value">
|
||||
{% if stats_summary.link_dest_savings_percent is not None %}
|
||||
{{ stats_summary.link_dest_savings_percent|floatformat:1 }}%
|
||||
{% else %}
|
||||
unknown
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="muted">{{ stats_summary.total_matched_data_bytes|filesizeformat }} reused across sampled runs.</div>
|
||||
</div>
|
||||
<div class="insight-item">
|
||||
<div class="label">Average Duration</div>
|
||||
<div class="value">{{ stats_summary.avg_duration_seconds|default:"unknown" }}{% if stats_summary.avg_duration_seconds is not None %}s{% endif %}</div>
|
||||
<div class="muted">Based on {{ stats_summary.runs_sampled }} completed backup run{{ stats_summary.runs_sampled|pluralize }} with stats.</div>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">No completed backup runs with stats yet. This section will show disk usage, growth estimates, and link-dest savings after the first real backup finishes.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Latest Runs</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Status</th>
|
||||
<th>Started</th>
|
||||
<th>Ended</th>
|
||||
<th>Snapshot</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for run in latest_runs %}
|
||||
<tr>
|
||||
<td><a href="{% url 'host_detail' run.host.host %}">{{ run.host.host }}</a></td>
|
||||
<td><a href="{% url 'run_detail' run.id %}"><span class="status {{ run.status }}">{{ run.status }}</span></a></td>
|
||||
<td>{{ run.started_at|default:"" }}</td>
|
||||
<td>{{ run.ended_at|default:"" }}</td>
|
||||
<td>{% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}<span class="muted">{{ run.snapshot_path }}</span>{% endif %}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">No backup runs recorded yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
<div
|
||||
data-refresh-url="{% url 'dashboard_hosts_live' %}"
|
||||
data-refresh-interval="15000"
|
||||
data-refresh-active="true"
|
||||
aria-live="polite"
|
||||
>
|
||||
{% include "pobsync_backend/partials/dashboard_hosts.html" %}
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@@ -3,17 +3,22 @@
|
||||
{% block title %}Global Config{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{% if global_config %}Global Config{% else %}Create Global Config{% endif %}</h1>
|
||||
|
||||
<section class="actions" aria-label="Global config actions">
|
||||
<a class="button-link" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Configuration</div>
|
||||
<h1>{% if global_config %}Global Config{% else %}Create Global Config{% endif %}</h1>
|
||||
<div class="page-subtitle">Defaults used by hosts unless a host overrides them explicitly.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Global config actions">
|
||||
<a class="button-link" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if global_config %}Edit Global Config{% else %}Create Global Config{% endif %}</h2>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Backup root:</strong> {{ backup_root }}</div>
|
||||
<div class="muted">This path comes from the runtime environment and is written back when the config is saved.</div>
|
||||
<div class="muted">This path is managed by the service environment and is saved with the config.</div>
|
||||
</div>
|
||||
<form method="post" class="form-grid">
|
||||
{% csrf_token %}
|
||||
@@ -28,8 +33,9 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">Save global config</button>
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
@@ -3,24 +3,176 @@
|
||||
{% block title %}{{ host.host }} | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{{ host.host }}</h1>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Host</div>
|
||||
<h1>{{ host.host }}</h1>
|
||||
<div class="page-subtitle">{{ host.address }} · {{ host.enabled|yesno:"enabled,disabled" }}</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="actions" aria-label="Host actions">
|
||||
<a class="button-link" href="{% url 'edit_host_config' host.host %}">Edit config</a>
|
||||
<form method="post" action="{% url 'discover_host_snapshots' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit">Discover snapshots</button>
|
||||
</form>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}">Plan retention</a>
|
||||
<a class="button-link" href="{% url 'edit_host_schedule' host.host %}">Edit schedule</a>
|
||||
<form method="post" action="{% url 'prepare_host_directories' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Prepare directories</button>
|
||||
</form>
|
||||
<form method="post" action="{% url 'scan_host_known_key' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Scan SSH host key</button>
|
||||
</form>
|
||||
{% if retention_warning.has_warning %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Retention Warnings</h2>
|
||||
<div class="stack">
|
||||
{% if retention_warning.prune_exceeded %}
|
||||
<div>
|
||||
Scheduled pruning would delete {{ retention_warning.delete_count }} snapshot(s), above max delete
|
||||
{{ retention_warning.max_delete }}. Scheduled pruning will refuse this plan until the limit or retention
|
||||
selection is adjusted.
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if retention_warning.incomplete_count %}
|
||||
<div>
|
||||
{{ retention_warning.incomplete_count }} incomplete snapshot(s) exist. Retention does not delete incomplete
|
||||
snapshots automatically; inspect them before cleanup.
|
||||
</div>
|
||||
{% if can_manage_control_panel %}
|
||||
<form method="post" action="{% url 'resolve_host_incomplete_reviews' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Mark incomplete reviewed</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if retention_warning.error %}
|
||||
<div>{{ retention_warning.error }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="host-control-grid" aria-label="Host control workspace">
|
||||
<article class="panel host-control-panel">
|
||||
<h2>Host Status</h2>
|
||||
<div class="host-control-primary">
|
||||
<div>
|
||||
{% if host.enabled %}
|
||||
<span class="status ok">enabled</span>
|
||||
{% else %}
|
||||
<span class="status failed">disabled</span>
|
||||
{% endif %}
|
||||
<span class="muted">{{ host.address }}</span>
|
||||
</div>
|
||||
{% if active_run %}
|
||||
<a class="status-summary {{ active_run.status }}" href="{% url 'run_detail' active_run.id %}">
|
||||
<span class="status {{ active_run.status }}">{{ active_run.status }}</span>
|
||||
<strong>Run {{ active_run.id }} is active.</strong>
|
||||
</a>
|
||||
{% elif counts.failed_runs %}
|
||||
<a class="status-summary failed" href="{% url 'runs_list' %}?host={{ host.host }}&status=failed&review=needed">
|
||||
<span class="status failed">failed</span>
|
||||
<strong>{{ counts.failed_runs }} failed run{{ counts.failed_runs|pluralize }} need review.</strong>
|
||||
</a>
|
||||
{% elif retention_warning.has_warning %}
|
||||
<span class="status-summary warning">
|
||||
<span class="status warning">warning</span>
|
||||
<strong>Retention needs attention.</strong>
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="status-summary success">
|
||||
<span class="status ok">ok</span>
|
||||
<strong>No active blockers for this host.</strong>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="host-control-meta">
|
||||
<div><span class="label">Snapshots</span><strong>{{ counts.snapshots }}</strong></div>
|
||||
<div><span class="label">Runs</span><strong>{{ counts.runs }}</strong></div>
|
||||
<div><span class="label">Incomplete</span><strong>{{ counts.incomplete_snapshots }}</strong></div>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
{% if can_manage_control_panel %}
|
||||
<article class="panel host-control-panel">
|
||||
<h2>Backup Control</h2>
|
||||
<div class="operator-state">
|
||||
{% if active_run %}
|
||||
<span class="status {{ active_run.status }}">{{ active_run.status }}</span>
|
||||
<a href="{% url 'run_detail' active_run.id %}">Run {{ active_run.id }}</a>
|
||||
{% elif has_global_config and host.enabled %}
|
||||
<span class="status {{ backup_gate.state }}">{{ backup_gate.state }}</span>
|
||||
<span class="muted">{{ backup_gate.message }}</span>
|
||||
{% elif not host.enabled %}
|
||||
<span class="status failed">disabled</span>
|
||||
{% elif not has_global_config %}
|
||||
<span class="status failed">missing global config</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<section class="actions inline" aria-label="Quick backup actions">
|
||||
<form class="inline-form" method="post" action="{% url 'queue_manual_backup' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="dry_run" value="on">
|
||||
<input type="hidden" name="verbose_output" value="on">
|
||||
<input type="hidden" name="prune_max_delete" value="10">
|
||||
<button type="submit" class="secondary" {% if not can_queue_dry_run %}disabled{% endif %}>Queue dry-run</button>
|
||||
</form>
|
||||
<form class="inline-form" method="post" action="{% url 'queue_manual_backup' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="verbose_output" value="on">
|
||||
<input type="hidden" name="prune_max_delete" value="10">
|
||||
<button type="submit" {% if not can_queue_real_backup %}disabled{% endif %}>Queue backup</button>
|
||||
</form>
|
||||
</section>
|
||||
{% if active_run %}
|
||||
<p class="muted">Wait for the active run to finish, or cancel it from the run detail page.</p>
|
||||
{% elif not can_queue_dry_run or not can_queue_real_backup %}
|
||||
{% if not has_global_config %}
|
||||
<p class="muted">Create the default global config before queueing backups.</p>
|
||||
{% elif not host.enabled %}
|
||||
<p class="muted">Enable this host before queueing backups.</p>
|
||||
{% elif backup_gate.real_blockers %}
|
||||
<p class="muted">Real backups are blocked by failed preflight checks. Dry-runs may still be available when storage-only checks fail.</p>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endif %}
|
||||
|
||||
<article class="panel host-control-panel">
|
||||
<h2>
|
||||
Schedule
|
||||
{% if can_manage_control_panel %}
|
||||
<a class="button-link secondary compact" href="{% url 'edit_host_schedule' host.host %}">Edit schedule</a>
|
||||
{% endif %}
|
||||
</h2>
|
||||
{% if schedule %}
|
||||
<div class="host-control-meta">
|
||||
<div><span class="label">Schedule expression</span><strong>{{ schedule.cron_expr }}</strong></div>
|
||||
<div><span class="label">Next run</span><strong>{% if next_run_at %}{{ next_run_at|date:"Y-m-d H:i T" }}{% else %}none{% endif %}</strong></div>
|
||||
<div><span class="label">Timezone</span><strong>{{ scheduler_timezone }}</strong></div>
|
||||
<div><span class="label">Prune</span><strong>{{ schedule.prune|yesno:"yes,no" }}</strong></div>
|
||||
<div><span class="label">Last status</span><strong>{{ schedule.last_status|default:"none" }}</strong></div>
|
||||
</div>
|
||||
<p class="muted">Evaluated by the pobsync scheduler service.</p>
|
||||
{% else %}
|
||||
<p class="muted">No schedule configured.</p>
|
||||
{% if can_manage_control_panel %}
|
||||
<a class="button-link secondary compact" href="{% url 'edit_host_schedule' host.host %}">Add schedule</a>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</article>
|
||||
|
||||
<article class="panel host-control-panel">
|
||||
<h2>Current Activity</h2>
|
||||
{% if latest_runs %}
|
||||
{% with run=latest_runs.0 %}
|
||||
<a class="activity-row" href="{% url 'run_detail' run.id %}">
|
||||
<span class="status {{ run.status }}">{{ run.status }}</span>
|
||||
<span>
|
||||
<strong>Run {{ run.id }}</strong>
|
||||
<span class="muted">{{ run.run_type }} · {{ run.started_at|default:run.created_at }}</span>
|
||||
</span>
|
||||
</a>
|
||||
{% endwith %}
|
||||
{% else %}
|
||||
<p class="muted">No backup runs recorded for this host.</p>
|
||||
{% endif %}
|
||||
{% if stats_summary.latest_run.duration_seconds is not None %}
|
||||
<div class="host-control-meta">
|
||||
<div><span class="label">Latest duration</span><strong>{{ stats_summary.latest_run.duration_seconds }}s</strong></div>
|
||||
<div><span class="label">New data</span><strong>{{ stats_summary.latest_run.rsync.literal_data_bytes|filesizeformat }}</strong></div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="grid" aria-label="Host summary">
|
||||
@@ -32,52 +184,34 @@
|
||||
<div class="metric"><div class="label">Incomplete</div><div class="value">{{ counts.incomplete_snapshots }}</div></div>
|
||||
</section>
|
||||
|
||||
<div class="two-col">
|
||||
<section class="panel">
|
||||
<h2>Config</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Address:</strong> {{ host.address }}</div>
|
||||
<div><strong>Enabled:</strong> {{ host.enabled|yesno:"yes,no" }}</div>
|
||||
<div><strong>SSH key:</strong> {{ host.ssh_credential|default:"global default" }}</div>
|
||||
<div><strong>SSH:</strong> {{ host.ssh_user|default:"global" }}{% if host.ssh_port %}:{{ host.ssh_port }}{% endif %}</div>
|
||||
<div><strong>Source:</strong> {{ host.source_root|default:"global default" }}</div>
|
||||
<div><strong>Retention:</strong> daily {{ host.retention_daily }}, weekly {{ host.retention_weekly }}, monthly {{ host.retention_monthly }}, yearly {{ host.retention_yearly }}</div>
|
||||
<section class="panel">
|
||||
<h2>Backup Data</h2>
|
||||
<section class="grid" aria-label="Host backup data totals">
|
||||
<div class="metric">
|
||||
<div class="label">Scheduled</div>
|
||||
<div class="value">{{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Manual</div>
|
||||
<div class="value">{{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Incomplete</div>
|
||||
<div class="value">{{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">measured from disk</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Total</div>
|
||||
<div class="value">{{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Schedule</h2>
|
||||
{% if schedule %}
|
||||
<div class="stack">
|
||||
<div><strong>Schedule expression:</strong> {{ schedule.cron_expr }}</div>
|
||||
<div class="muted">Evaluated by the pobsync scheduler service.</div>
|
||||
<div><strong>Enabled:</strong> {{ schedule.enabled|yesno:"yes,no" }}</div>
|
||||
<div><strong>Next run:</strong> {{ next_run_at|default:"" }}</div>
|
||||
<div><strong>Prune:</strong> {{ schedule.prune|yesno:"yes,no" }}</div>
|
||||
<div><strong>Last status:</strong> {{ schedule.last_status|default:"" }}</div>
|
||||
<div><strong>Last started:</strong> {{ schedule.last_started_at|default:"" }}</div>
|
||||
<div><strong>Last finished:</strong> {{ schedule.last_finished_at|default:"" }}</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">No schedule configured.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Snapshot Discovery</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Backup root:</strong> {{ discovery.backup_root|default:"" }}</div>
|
||||
<div><strong>Host root:</strong> {{ discovery.host_root|default:"" }}</div>
|
||||
<div><strong>Status:</strong> {{ discovery.message }}</div>
|
||||
{% if discovery.kind_counts %}
|
||||
<div><strong>On disk:</strong>
|
||||
scheduled {{ discovery.kind_counts.scheduled|default:0 }},
|
||||
manual {{ discovery.kind_counts.manual|default:0 }},
|
||||
incomplete {{ discovery.kind_counts.incomplete|default:0 }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<p class="muted">
|
||||
Main totals use allocated snapshot size. Unique values estimate non-hardlinked visible data; incomplete
|
||||
snapshots are measured from disk because their metadata can be stale.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
{% if stats_summary.runs %}
|
||||
@@ -129,75 +263,219 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Host Check</h2>
|
||||
{% if can_manage_control_panel %}
|
||||
<section class="panel">
|
||||
<h2>Host Check</h2>
|
||||
<section class="grid" aria-label="Host check summary">
|
||||
<div class="metric"><div class="label">OK</div><div class="value">{{ host_check_summary.ok }}</div></div>
|
||||
<div class="metric"><div class="label">Warnings</div><div class="value">{{ host_check_summary.warning }}</div></div>
|
||||
<div class="metric"><div class="label">Failed</div><div class="value">{{ host_check_summary.failed }}</div></div>
|
||||
<div class="metric"><div class="label">Skipped</div><div class="value">{{ host_check_summary.skipped }}</div></div>
|
||||
</section>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Check</th>
|
||||
<th>Message</th>
|
||||
<th>Detail</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for check in host_checks %}
|
||||
<tr>
|
||||
<td><span class="status {{ check.status }}">{{ check.status }}</span></td>
|
||||
<td>{{ check.name }}</td>
|
||||
<td>{{ check.message }}</td>
|
||||
<td class="muted">{{ check.detail }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Backup Control</h2>
|
||||
<div class="operator-state">
|
||||
{% if active_run %}
|
||||
<span class="status {{ active_run.status }}">{{ active_run.status }}</span>
|
||||
<a href="{% url 'run_detail' active_run.id %}">Run {{ active_run.id }}</a>
|
||||
{% elif can_queue_backup %}
|
||||
<span class="status success">ready</span>
|
||||
{% elif not host.enabled %}
|
||||
<span class="status failed">disabled</span>
|
||||
{% elif not has_global_config %}
|
||||
<span class="status failed">missing global config</span>
|
||||
{% endif %}
|
||||
<div class="record-list">
|
||||
{% for check in host_checks %}
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<strong>{{ check.name }}</strong>
|
||||
<span class="muted">{{ check.message }}</span>
|
||||
</div>
|
||||
<span class="status {{ check.status }}">{{ check.status }}</span>
|
||||
</div>
|
||||
{% if check.detail %}
|
||||
<div class="record-fact">
|
||||
<span class="label">Detail</span>
|
||||
<span class="muted">{{ check.detail }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="actions inline" aria-label="Quick backup actions">
|
||||
<form class="inline-form" method="post" action="{% url 'queue_manual_backup' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="dry_run" value="on">
|
||||
<input type="hidden" name="verbose_output" value="on">
|
||||
<input type="hidden" name="prune_max_delete" value="10">
|
||||
<button type="submit" class="secondary" {% if not can_queue_backup %}disabled{% endif %}>Queue dry-run</button>
|
||||
</form>
|
||||
<form class="inline-form" method="post" action="{% url 'queue_manual_backup' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="prune_max_delete" value="10">
|
||||
<button type="submit" {% if not can_queue_backup %}disabled{% endif %}>Queue backup</button>
|
||||
</form>
|
||||
<div class="panel-grid">
|
||||
<section class="panel">
|
||||
<h2>Configuration</h2>
|
||||
<div class="host-control-meta">
|
||||
<div><span class="label">Address</span><strong>{{ host.address }}</strong></div>
|
||||
{% if can_manage_control_panel %}
|
||||
<div><span class="label">SSH key</span><strong>{{ host.ssh_credential|default:"global default" }}</strong></div>
|
||||
<div><span class="label">SSH</span><strong>{{ host.ssh_user|default:"global" }}{% if host.ssh_port %}:{{ host.ssh_port }}{% endif %}</strong></div>
|
||||
{% endif %}
|
||||
<div><span class="label">Backup source</span><strong>{{ host.source_root|default:"global default" }}</strong></div>
|
||||
<div><span class="label">Retention</span><strong>d{{ host.retention_daily }} w{{ host.retention_weekly }} m{{ host.retention_monthly }} y{{ host.retention_yearly }}</strong></div>
|
||||
</div>
|
||||
{% if can_manage_control_panel %}
|
||||
<div class="actions inline">
|
||||
<a class="button-link secondary compact" href="{% url 'edit_host_config' host.host %}">Edit config</a>
|
||||
<a class="button-link secondary compact" href="{% url 'host_retention_plan' host.host %}">Plan retention</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
{% if not can_queue_backup %}
|
||||
{% if not has_global_config %}
|
||||
<p class="muted">Create the default global config before queueing backups.</p>
|
||||
{% elif not host.enabled %}
|
||||
<p class="muted">Enable this host before queueing backups.</p>
|
||||
{% if can_manage_control_panel %}
|
||||
<section class="panel">
|
||||
<h2>Connection Preflight & SSH</h2>
|
||||
{% if last_preflight %}
|
||||
<div class="host-control-meta">
|
||||
<div>
|
||||
<span class="label">Preflight</span>
|
||||
<strong>
|
||||
<span class="status {% if last_preflight.ok %}ok{% else %}failed{% endif %}">
|
||||
{% if last_preflight.ok %}ok{% else %}failed{% endif %}
|
||||
</span>
|
||||
</strong>
|
||||
</div>
|
||||
<div><span class="label">Target</span><strong>{{ last_preflight.target }}</strong></div>
|
||||
<div><span class="label">Backup source</span><strong>{{ last_preflight.source_root }}</strong></div>
|
||||
<div><span class="label">Remote rsync</span><strong>{{ last_preflight.rsync_binary }}</strong></div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">No connection preflight recorded yet.</p>
|
||||
{% endif %}
|
||||
<div class="actions inline">
|
||||
<form method="post" action="{% url 'run_host_preflight' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary compact">Run connection preflight</button>
|
||||
</form>
|
||||
<form method="post" action="{% url 'scan_host_known_key' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary compact">Scan SSH host key</button>
|
||||
</form>
|
||||
</div>
|
||||
{% if last_preflight.checks %}
|
||||
<div class="activity-list">
|
||||
{% for check in last_preflight.checks %}
|
||||
<div class="activity-row">
|
||||
<span class="status {% if check.ok %}ok{% else %}failed{% endif %}">
|
||||
{% if check.ok %}ok{% else %}failed{% endif %}
|
||||
</span>
|
||||
<span>
|
||||
<strong>{{ check.name }}</strong>
|
||||
<span class="muted">{{ check.message }}{% if check.detail %} · {{ check.detail }}{% endif %}</span>
|
||||
</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<h3>Advanced Options</h3>
|
||||
<section class="panel">
|
||||
<h2>Snapshot Storage</h2>
|
||||
<div class="host-control-meta">
|
||||
<div><span class="label">Backup root</span><strong>{{ discovery.backup_root|default:"" }}</strong></div>
|
||||
<div><span class="label">Host root</span><strong>{{ discovery.host_root|default:"" }}</strong></div>
|
||||
<div><span class="label">Status</span><strong>{{ discovery.message }}</strong></div>
|
||||
{% if discovery.kind_counts %}
|
||||
<div>
|
||||
<span class="label">On disk</span>
|
||||
<strong>
|
||||
scheduled {{ discovery.kind_counts.scheduled|default:0 }},
|
||||
manual {{ discovery.kind_counts.manual|default:0 }},
|
||||
incomplete {{ discovery.kind_counts.incomplete|default:0 }}
|
||||
</strong>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if can_manage_control_panel %}
|
||||
<div class="actions inline">
|
||||
<form method="post" action="{% url 'discover_host_snapshots' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary compact">Discover snapshots</button>
|
||||
</form>
|
||||
<form method="post" action="{% url 'prepare_host_directories' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary compact">Prepare directories</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{% if effective_config %}
|
||||
<section class="panel">
|
||||
<h2>Effective Config</h2>
|
||||
<p class="muted">Runtime settings after global defaults and host overrides are combined.</p>
|
||||
<div class="record-list">
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<strong>Backup target</strong>
|
||||
<span class="muted">Source and destination used by rsync.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="record-facts">
|
||||
<div class="record-fact"><span class="label">Backup source:</span><strong>{{ effective_config.source_root }}</strong></div>
|
||||
<div class="record-fact"><span class="label">Destination subdir:</span><strong>{{ effective_config.destination_subdir|default:"none" }}</strong></div>
|
||||
</div>
|
||||
</article>
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<strong>Connection</strong>
|
||||
<span class="muted">SSH and rsync execution settings.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="record-facts">
|
||||
<div class="record-fact"><span class="label">SSH:</span><strong>{{ effective_config.ssh.user }}@{{ host.address }}:{{ effective_config.ssh.port }}</strong></div>
|
||||
<div class="record-fact"><span class="label">SSH key:</span><strong>{{ effective_config.ssh.credential|default:"none selected" }}</strong></div>
|
||||
<div class="record-fact"><span class="label">SSH options:</span><span>{{ effective_config.ssh.options|join:" " }}</span></div>
|
||||
<div class="record-fact"><span class="label">Rsync binary:</span><strong>{{ effective_config.rsync.binary }}</strong></div>
|
||||
<div class="record-fact"><span class="label">Rsync args:</span><span>{{ effective_config.rsync.args|join:" " }}</span></div>
|
||||
<div class="record-fact"><span class="label">Timeout:</span><strong>{{ effective_config.rsync.timeout_seconds }}s</strong></div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Bandwidth limit:</span>
|
||||
<strong>{% if effective_config.rsync.bwlimit_kbps %}{{ effective_config.rsync.bwlimit_kbps }} KB/s{% else %}unlimited{% endif %}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<strong>Selection & retention</strong>
|
||||
<span class="muted">Include/exclude rules and retention counts.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="record-facts">
|
||||
<div class="record-fact">
|
||||
<span class="label">Retention:</span>
|
||||
<strong>
|
||||
d{{ effective_config.retention.daily }}
|
||||
w{{ effective_config.retention.weekly }}
|
||||
m{{ effective_config.retention.monthly }}
|
||||
y{{ effective_config.retention.yearly }}
|
||||
</strong>
|
||||
</div>
|
||||
<div class="record-fact"><span class="label">Includes:</span><strong>{{ effective_config.includes|length }}</strong></div>
|
||||
<div class="record-fact"><span class="label">Excludes:</span><strong>{{ effective_config.excludes|length }}</strong></div>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<div class="stack">
|
||||
{% if effective_config.includes %}
|
||||
<pre>{{ effective_config.includes|join:" " }}</pre>
|
||||
{% else %}
|
||||
<div class="muted">No include rules configured.</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="stack">
|
||||
{% if effective_config.excludes %}
|
||||
<pre>{{ effective_config.excludes|join:" " }}</pre>
|
||||
{% else %}
|
||||
<div class="muted">No exclude rules configured.</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if can_manage_control_panel %}
|
||||
<section class="panel">
|
||||
<h2>Backup Options</h2>
|
||||
<p class="muted">Use this when the quick actions above need a custom label, include/exclude override, or prune limit.</p>
|
||||
<form method="post" action="{% url 'queue_manual_backup' host.host %}" class="form-grid">
|
||||
{% csrf_token %}
|
||||
{{ manual_backup_form.non_field_errors }}
|
||||
@@ -211,65 +489,96 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<button type="submit" {% if not can_queue_backup %}disabled{% endif %}>Queue with options</button>
|
||||
<div class="form-actions">
|
||||
<button type="submit" {% if not can_queue_dry_run and not can_queue_real_backup %}disabled{% endif %}>Queue with options</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Latest Runs <a class="button-link secondary compact" href="{% url 'runs_list' %}?host={{ host.host }}">View all</a></h2>
|
||||
<div class="record-list">
|
||||
{% for run in latest_runs %}
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<a href="{% url 'run_detail' run.id %}">Run {{ run.id }}</a>
|
||||
<span class="muted">{{ run.run_type }}{% if run.result.duration_seconds %} · {{ run.result.duration_seconds }}s{% endif %}</span>
|
||||
</div>
|
||||
<span class="status {{ run.status }}">{{ run.status }}</span>
|
||||
</div>
|
||||
<div class="record-facts">
|
||||
<div class="record-fact">
|
||||
<span class="label">Started</span>
|
||||
<strong>{{ run.started_at|default:run.created_at }}</strong>
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Ended</span>
|
||||
<strong>{{ run.ended_at|default:"running or queued" }}</strong>
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Snapshot</span>
|
||||
{% if run.snapshot %}
|
||||
<strong><a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a></strong>
|
||||
{% elif run.snapshot_path %}
|
||||
<span class="muted">{{ run.snapshot_path }}</span>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Base</span>
|
||||
<span class="muted">{{ run.base_path|default:"none" }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
{% empty %}
|
||||
<p class="muted">No backup runs recorded for this host.</p>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Latest Runs</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Started</th>
|
||||
<th>Ended</th>
|
||||
<th>Snapshot</th>
|
||||
<th>Base</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for run in latest_runs %}
|
||||
<tr>
|
||||
<td><a href="{% url 'run_detail' run.id %}"><span class="status {{ run.status }}">{{ run.status }}</span></a></td>
|
||||
<td>{{ run.started_at|default:"" }}</td>
|
||||
<td>{{ run.ended_at|default:"" }}</td>
|
||||
<td>{% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}<span class="muted">{{ run.snapshot_path }}</span>{% endif %}</td>
|
||||
<td>{{ run.base_path|default:"" }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">No backup runs recorded for this host.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Snapshots</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Kind</th>
|
||||
<th>Status</th>
|
||||
<th>Started</th>
|
||||
<th>Dirname</th>
|
||||
<th>Base</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for snapshot in snapshots %}
|
||||
<tr>
|
||||
<td>{{ snapshot.kind }}</td>
|
||||
<td>{{ snapshot.status }}</td>
|
||||
<td>{{ snapshot.started_at|default:"" }}</td>
|
||||
<td><a href="{% url 'snapshot_detail' snapshot.id %}">{{ snapshot.dirname }}</a></td>
|
||||
<td>{% if snapshot.base %}<a href="{% url 'snapshot_detail' snapshot.base.id %}">{{ snapshot.base.dirname }}</a>{% else %}{{ snapshot.base_dirname }}{% endif %}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">No snapshots discovered for this host.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<h2>Snapshots <a class="button-link secondary compact" href="{% url 'snapshots_list' %}?host={{ host.host }}">View all</a></h2>
|
||||
<div class="record-list">
|
||||
{% for snapshot in snapshots %}
|
||||
<article class="record-card">
|
||||
<div class="record-card-header">
|
||||
<div class="record-title">
|
||||
<a href="{% url 'snapshot_detail' snapshot.id %}">{{ snapshot.dirname }}</a>
|
||||
<span class="muted">{{ snapshot.kind }}</span>
|
||||
</div>
|
||||
<span class="status {{ snapshot.status }}">{{ snapshot.status }}</span>
|
||||
</div>
|
||||
<div class="record-facts">
|
||||
<div class="record-fact">
|
||||
<span class="label">Started</span>
|
||||
<strong>{{ snapshot.started_at|default:"unknown" }}</strong>
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Ended</span>
|
||||
<strong>{{ snapshot.ended_at|default:"unknown" }}</strong>
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Base</span>
|
||||
{% if snapshot.base %}
|
||||
<strong><a href="{% url 'snapshot_detail' snapshot.base.id %}">{{ snapshot.base.dirname }}</a></strong>
|
||||
{% elif snapshot.base_dirname %}
|
||||
<span class="muted">{{ snapshot.base_dirname }}</span>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Path</span>
|
||||
<span class="muted">{{ snapshot.path|default:"not recorded" }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
{% empty %}
|
||||
<p class="muted">No snapshots discovered for this host.</p>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
@@ -3,15 +3,20 @@
|
||||
{% block title %}{% if host %}Config | {{ host.host }}{% else %}New Host{% endif %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{% if host %}Config: {{ host.host }}{% else %}New Host{% endif %}</h1>
|
||||
|
||||
<section class="actions" aria-label="Config actions">
|
||||
{% if host %}
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
{% else %}
|
||||
<a class="button-link" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
{% endif %}
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Configuration</div>
|
||||
<h1>{% if host %}Config: {{ host.host }}{% else %}New Host{% endif %}</h1>
|
||||
<div class="page-subtitle">Host-specific backup, retention, SSH, include, and exclude settings.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Config actions">
|
||||
{% if host %}
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
{% else %}
|
||||
<a class="button-link" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
{% endif %}
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if host %}Edit Host Config{% else %}Create Host Config{% endif %}</h2>
|
||||
@@ -28,8 +33,13 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">{% if host %}Save config{% else %}Create host{% endif %}</button>
|
||||
{% if host %}
|
||||
<a class="button-link secondary" href="{% url 'host_detail' host.host %}">Cancel</a>
|
||||
{% else %}
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Cancel</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Hosts | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Inventory</div>
|
||||
<h1>Hosts</h1>
|
||||
<div class="page-subtitle">Configured backup targets, schedules, retention state, and host-level controls.</div>
|
||||
</div>
|
||||
{% if can_manage_control_panel %}
|
||||
<section class="actions" aria-label="Host actions">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">New host</a>
|
||||
</section>
|
||||
{% endif %}
|
||||
</header>
|
||||
|
||||
<section class="grid dashboard-summary-grid" aria-label="Host summary">
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}"><div class="label">Showing</div><div class="value">{{ counts.hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}?enabled=yes"><div class="label">Enabled</div><div class="value">{{ counts.enabled_hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}?enabled=no"><div class="label">Disabled</div><div class="value">{{ counts.disabled_hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'dashboard' %}"><div class="label">Total</div><div class="value">{{ total_count }}</div></a>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form class="filter-form" method="get">
|
||||
<div class="field">
|
||||
<label for="enabled">Host state</label>
|
||||
<select id="enabled" name="enabled">
|
||||
<option value="" {% if selected_enabled == "" %}selected{% endif %}>All hosts</option>
|
||||
<option value="yes" {% if selected_enabled == "yes" %}selected{% endif %}>Enabled only</option>
|
||||
<option value="no" {% if selected_enabled == "no" %}selected{% endif %}>Disabled only</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'hosts_list' %}">Reset</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
{% include "pobsync_backend/partials/dashboard_hosts.html" %}
|
||||
{% endblock %}
|
||||
@@ -3,15 +3,20 @@
|
||||
{% block title %}Logs | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Logs</h1>
|
||||
|
||||
<section class="actions" aria-label="Log actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Operations</div>
|
||||
<h1>Logs</h1>
|
||||
<div class="page-subtitle">Filter pobsync service logs by unit, priority, host, run, or message content.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Log actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filter</h2>
|
||||
<form method="get" class="form-grid">
|
||||
<form method="get" class="filter-form">
|
||||
<div class="field">
|
||||
<label for="unit">Unit</label>
|
||||
<select id="unit" name="unit">
|
||||
@@ -29,12 +34,29 @@
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="window">Time window</label>
|
||||
<select id="window" name="window">
|
||||
{% for value, label in time_windows.items %}
|
||||
<option value="{{ value }}" {% if selected_window == value %}selected{% endif %}>{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="host">Host contains</label>
|
||||
<input id="host" name="host" value="{{ host_filter }}" placeholder="web-01.example.test">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="run">Run</label>
|
||||
<input id="run" name="run" value="{{ run_filter }}" inputmode="numeric" placeholder="12">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="q">Message contains</label>
|
||||
<input id="q" name="q" value="{{ query }}">
|
||||
</div>
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">Filter logs</button>
|
||||
<a class="button-link secondary" href="{% url 'logs' %}">Clear</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}{{ title }} | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Reports</div>
|
||||
<h1>{{ title }}</h1>
|
||||
<div class="page-subtitle">Choose which completed backup statuses should trigger an email or webhook report.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Notification target form actions">
|
||||
<a class="button-link" href="{% url 'notification_targets' %}">Back to notifications</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if target %}Edit Target{% else %}Create Target{% endif %}</h2>
|
||||
<form method="post" class="form-grid">
|
||||
{% csrf_token %}
|
||||
{{ form.non_field_errors }}
|
||||
|
||||
{% for field in form %}
|
||||
<div class="field">
|
||||
{{ field.errors }}
|
||||
<label for="{{ field.id_for_label }}">{{ field.label }}</label>
|
||||
{{ field }}
|
||||
{% if field.help_text %}<div class="helptext">{{ field.help_text }}</div>{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="form-actions">
|
||||
<button type="submit">{{ submit_label }}</button>
|
||||
<a class="button-link secondary" href="{% url 'notification_targets' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,91 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Notifications | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Reports</div>
|
||||
<h1>Notifications</h1>
|
||||
<div class="page-subtitle">Send email or webhook reports when backup runs finish.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Notification actions">
|
||||
<a class="button-link" href="{% url 'create_notification_target' %}">New target</a>
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Targets</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Channel</th>
|
||||
<th>Status</th>
|
||||
<th>Events</th>
|
||||
<th>Destination</th>
|
||||
<th>Last delivery</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for target in targets %}
|
||||
<tr>
|
||||
<td><a href="{% url 'edit_notification_target' target.id %}">{{ target.name }}</a></td>
|
||||
<td>{{ target.get_channel_display }}</td>
|
||||
<td><span class="status {% if target.enabled %}ok{% else %}skipped{% endif %}">{{ target.enabled|yesno:"enabled,disabled" }}</span></td>
|
||||
<td>{{ target.statuses|join:", " }}</td>
|
||||
<td>
|
||||
{% if target.channel == "email" %}
|
||||
{{ target.email_to|linebreaksbr }}
|
||||
{% else %}
|
||||
<code>{{ target.webhook_url|truncatechars:70 }}</code>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if target.last_status %}
|
||||
<span class="status {{ target.last_status }}">{{ target.last_status }}</span>
|
||||
{% if target.last_error %}<div class="muted">{{ target.last_error|truncatechars:90 }}</div>{% endif %}
|
||||
{% if target.last_sent_at %}<div class="muted">{{ target.last_sent_at }}</div>{% endif %}
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><a class="button-link secondary" href="{% url 'edit_notification_target' target.id %}">Edit</a></td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="7" class="muted">No notification targets configured yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Recent Deliveries</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Target</th>
|
||||
<th>Run</th>
|
||||
<th>Status</th>
|
||||
<th>Created</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for delivery in deliveries %}
|
||||
<tr>
|
||||
<td>{{ delivery.target.name }}</td>
|
||||
<td><a href="{% url 'run_detail' delivery.run.id %}">Run {{ delivery.run.id }}</a> {{ delivery.run.host.host }}</td>
|
||||
<td><span class="status {{ delivery.status }}">{{ delivery.status }}</span></td>
|
||||
<td>{{ delivery.created_at }}</td>
|
||||
<td class="muted">{{ delivery.error|default:"" }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">No notification deliveries recorded yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,178 @@
|
||||
<section class="panel dashboard-hosts-panel" id="hosts">
|
||||
<h2>Hosts</h2>
|
||||
<div class="host-list">
|
||||
{% for host in hosts %}
|
||||
<article class="host-card">
|
||||
<div class="host-card-header">
|
||||
<div class="host-card-title">
|
||||
<a href="{% url 'host_detail' host.host %}">{{ host.host }}</a>
|
||||
<span class="muted">{{ host.address }}</span>
|
||||
</div>
|
||||
<div class="host-card-status">
|
||||
<span class="status {% if host.enabled %}ok{% else %}skipped{% endif %}">{{ host.enabled|yesno:"enabled,disabled" }}</span>
|
||||
{% if host.queued_run_count %}
|
||||
<span class="status queued">queued {{ host.queued_run_count }}</span>
|
||||
{% endif %}
|
||||
{% if host.running_run_count %}
|
||||
<span class="status running">running {{ host.running_run_count }}</span>
|
||||
{% endif %}
|
||||
{% if host.warning_run_count %}
|
||||
<span class="status warning">warning {{ host.warning_run_count }}</span>
|
||||
{% endif %}
|
||||
{% if host.failed_run_count %}
|
||||
<span class="status failed">failed {{ host.failed_run_count }}</span>
|
||||
{% endif %}
|
||||
{% if show_host_controls %}
|
||||
{% if host.schedule %}
|
||||
<span class="status {% if host.schedule.enabled %}ok{% else %}skipped{% endif %}">schedule {{ host.schedule.enabled|yesno:"on,paused" }}</span>
|
||||
<span class="status {% if host.schedule.prune %}ok{% else %}skipped{% endif %}">retention {{ host.schedule.prune|yesno:"on,paused" }}</span>
|
||||
{% else %}
|
||||
<span class="status skipped">no schedule</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-layout">
|
||||
<div class="host-card-section">
|
||||
<div class="host-card-section-title">Backup activity</div>
|
||||
<div class="host-card-timeline">
|
||||
<div class="host-card-item">
|
||||
<div class="label">Latest Snapshot</div>
|
||||
<div class="value">
|
||||
{% if host.latest_snapshot %}
|
||||
<a href="{% url 'snapshot_detail' host.latest_snapshot.id %}">{{ host.latest_snapshot.dirname }}</a>
|
||||
<div class="muted">{{ host.latest_snapshot.kind }} {{ host.latest_snapshot.status }}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-item">
|
||||
<div class="label">Last Good Backup</div>
|
||||
<div class="value">
|
||||
{% if host.stats_summary.latest_good_run.id %}
|
||||
<a href="{% url 'run_detail' host.stats_summary.latest_good_run.id %}">Run {{ host.stats_summary.latest_good_run.id }}</a>
|
||||
<div class="muted">{{ host.stats_summary.latest_good_run.run_type }} {{ host.stats_summary.latest_good_run.duration_seconds|default:"" }}{% if host.stats_summary.latest_good_run.duration_seconds is not None %}s{% endif %}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-item">
|
||||
<div class="label">Latest Issue</div>
|
||||
<div class="value">
|
||||
{% if host.stats_summary.latest_problem_run.id %}
|
||||
<a href="{% url 'run_detail' host.stats_summary.latest_problem_run.id %}">Run {{ host.stats_summary.latest_problem_run.id }}</a>
|
||||
<div><span class="status {{ host.stats_summary.latest_problem_run.status }}">{{ host.stats_summary.latest_problem_run.status }}</span></div>
|
||||
<div class="muted">{{ host.stats_summary.latest_problem_run.run_type }} {{ host.stats_summary.latest_problem_run.duration_seconds|default:"" }}{% if host.stats_summary.latest_problem_run.duration_seconds is not None %}s{% endif %}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-item">
|
||||
<div class="label">Next Run</div>
|
||||
<div class="value">
|
||||
{% if host.next_run_at %}
|
||||
{{ host.next_run_at|date:"Y-m-d H:i T" }}
|
||||
<div class="muted">{{ scheduler_timezone }}</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-section">
|
||||
<div class="host-card-section-title">Snapshot health</div>
|
||||
<div class="host-card-stats">
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Snapshots</div>
|
||||
<div class="value">{{ host.snapshot_count }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Runs</div>
|
||||
<div class="value">{{ host.run_count }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">New Data</div>
|
||||
<div class="value">{{ host.stats_summary.latest_run.rsync.literal_data_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Retention</div>
|
||||
<div class="value">d{{ host.retention_daily }} w{{ host.retention_weekly }} m{{ host.retention_monthly }} y{{ host.retention_yearly }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Scheduled data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Manual data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Incomplete data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">measured from disk</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Total data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if host.retention_warning.has_warning %}
|
||||
<div class="host-card-warning">
|
||||
<span class="status warning">retention</span>
|
||||
{% if host.retention_warning.prune_exceeded %}
|
||||
Scheduled prune would delete {{ host.retention_warning.delete_count }} snapshot(s), above max {{ host.retention_warning.max_delete }}.
|
||||
{% endif %}
|
||||
{% if host.retention_warning.incomplete_count %}
|
||||
{{ host.retention_warning.incomplete_count }} incomplete snapshot(s) need review.
|
||||
<form method="post" action="{% url 'resolve_host_incomplete_reviews' host.host %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Mark reviewed</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if host.retention_warning.error %}
|
||||
{{ host.retention_warning.error }}
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if show_host_controls %}
|
||||
<div class="host-card-actions">
|
||||
<a class="button-link compact secondary" href="{% url 'host_detail' host.host %}">Open</a>
|
||||
<a class="button-link compact secondary" href="{% url 'edit_host_config' host.host %}">Edit config</a>
|
||||
<a class="button-link compact secondary" href="{% url 'edit_host_schedule' host.host %}">{% if host.schedule %}Edit schedule{% else %}Create schedule{% endif %}</a>
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.enabled %}disable_host{% else %}enable_host{% endif %}">
|
||||
<button class="compact {% if host.enabled %}secondary{% endif %}" type="submit">{{ host.enabled|yesno:"Disable host,Enable host" }}</button>
|
||||
</form>
|
||||
{% if host.schedule %}
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.schedule.enabled %}disable_schedule{% else %}enable_schedule{% endif %}">
|
||||
<button class="compact secondary" type="submit">{{ host.schedule.enabled|yesno:"Pause schedule,Resume schedule" }}</button>
|
||||
</form>
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.schedule.prune %}disable_prune{% else %}enable_prune{% endif %}">
|
||||
<button class="compact secondary" type="submit">{{ host.schedule.prune|yesno:"Pause retention,Resume retention" }}</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% empty %}
|
||||
<p class="muted">No hosts configured yet.</p>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
@@ -0,0 +1,152 @@
|
||||
<section class="dashboard-priority-grid" aria-label="Operator priorities">
|
||||
<article class="panel priority-panel dashboard-panel-required">
|
||||
<h2>Required Action</h2>
|
||||
{% if action_items %}
|
||||
<div class="action-list">
|
||||
{% for item in action_items %}
|
||||
<a class="action-row {{ item.status }}" href="{{ item.url }}">
|
||||
<span class="status {{ item.status }}">{{ item.label }}</span>
|
||||
<span>
|
||||
<strong>{{ item.host.host }}</strong>
|
||||
<span class="muted">{{ item.message }}</span>
|
||||
</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% elif counts.hosts %}
|
||||
<p><span class="status ok">ok</span> No queued, running, unreviewed warning/failed runs, or retention warnings.</p>
|
||||
{% else %}
|
||||
<p class="muted">Add a host to start tracking backup status here.</p>
|
||||
{% endif %}
|
||||
{% if counts.running_runs or counts.queued_runs %}
|
||||
<div class="operator-state">
|
||||
{% if counts.running_runs %}
|
||||
<a class="status-summary running" href="{% url 'runs_list' %}?status=running">
|
||||
<span class="status running">running</span>
|
||||
<strong>{{ counts.running_runs }} backup run{{ counts.running_runs|pluralize }} in progress.</strong>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if counts.queued_runs %}
|
||||
<a class="status-summary queued" href="{% url 'runs_list' %}?status=queued">
|
||||
<span class="status queued">queued</span>
|
||||
<strong>{{ counts.queued_runs }} backup run{{ counts.queued_runs|pluralize }} waiting.</strong>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
|
||||
<article class="panel priority-panel dashboard-panel-schedules">
|
||||
<h2>Next Scheduled Work <a class="button-link secondary compact" href="{% url 'schedules_list' %}">View all</a></h2>
|
||||
{% if next_schedule_rows %}
|
||||
<div class="schedule-list">
|
||||
{% for row in next_schedule_rows %}
|
||||
<a class="schedule-row" href="{% url 'host_detail' row.schedule.host.host %}">
|
||||
<span>
|
||||
<strong>{{ row.schedule.host.host }}</strong>
|
||||
<span class="muted">{{ row.schedule.cron_expr }}</span>
|
||||
</span>
|
||||
<span class="schedule-time">
|
||||
{% if row.next_run_at %}
|
||||
{{ row.next_run_at|date:"Y-m-d H:i T" }}
|
||||
<span class="muted">{{ scheduler_timezone }}</span>
|
||||
{% else %}
|
||||
<span class="muted">not due</span>
|
||||
{% endif %}
|
||||
</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">No enabled schedules yet.</p>
|
||||
{% endif %}
|
||||
</article>
|
||||
|
||||
<article class="panel priority-panel dashboard-panel-activity">
|
||||
<h2>Recent Activity <a class="button-link secondary compact" href="{% url 'runs_list' %}">View all</a></h2>
|
||||
{% if recent_runs %}
|
||||
<div class="activity-list">
|
||||
{% for run in recent_runs %}
|
||||
<a class="activity-row" href="{% url 'run_detail' run.id %}">
|
||||
<span class="status {{ run.status }}">{{ run.status }}</span>
|
||||
<span>
|
||||
<strong>Run {{ run.id }}</strong>
|
||||
<span class="muted">{{ run.host.host }} · {{ run.run_type }} · {{ run.started_at|default:run.created_at }}</span>
|
||||
</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">No backup runs recorded yet.</p>
|
||||
{% endif %}
|
||||
</article>
|
||||
|
||||
<article class="panel priority-panel dashboard-panel-storage">
|
||||
<h2>Storage Pressure</h2>
|
||||
{% if stats_summary.runs_sampled %}
|
||||
<div class="storage-priority">
|
||||
<div>
|
||||
<div class="label">Backup root used</div>
|
||||
<div class="value">
|
||||
{% if stats_summary.capacity.used_percent is not None %}
|
||||
{{ stats_summary.capacity.used_percent|floatformat:1 }}%
|
||||
{% else %}
|
||||
unknown
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if stats_summary.capacity.used_percent is not None %}
|
||||
<div class="storage-meter" aria-label="Backup root storage usage">
|
||||
<span style="width: {{ stats_summary.capacity.used_percent|floatformat:0 }}%"></span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="storage-priority-facts">
|
||||
<div>
|
||||
<span class="label">Runway</span>
|
||||
<strong>
|
||||
{% if stats_summary.estimated_days_until_full %}
|
||||
{{ stats_summary.estimated_days_until_full }} days
|
||||
{% elif stats_summary.estimated_runs_until_full %}
|
||||
{{ stats_summary.estimated_runs_until_full }} runs
|
||||
{% else %}
|
||||
unknown
|
||||
{% endif %}
|
||||
</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">New data</span>
|
||||
<strong>{{ stats_summary.avg_daily_literal_data_bytes|filesizeformat }}/day</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Available</span>
|
||||
<strong>{{ stats_summary.capacity.available_bytes|filesizeformat }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="muted">Storage pressure appears after the first completed backup with stats.</p>
|
||||
{% endif %}
|
||||
<div class="storage-priority-facts">
|
||||
<div>
|
||||
<span class="label">Scheduled data</span>
|
||||
<strong>{{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Manual data</span>
|
||||
<strong>{{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Incomplete data</span>
|
||||
<strong>{{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">measured from disk</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Total snapshot data</span>
|
||||
<strong>{{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
@@ -0,0 +1,222 @@
|
||||
<section class="grid" aria-label="Run summary">
|
||||
<div class="metric"><div class="label">Host</div><div class="value">{{ run.host.host }}</div></div>
|
||||
<div class="metric"><div class="label">Status</div><div class="value"><span class="status {{ run.status }}">{{ run.status }}</span></div></div>
|
||||
<div class="metric"><div class="label">Type</div><div class="value">{{ run.run_type }}</div></div>
|
||||
<div class="metric"><div class="label">Rsync</div><div class="value">{{ run.rsync_exit_code|default:"" }}</div></div>
|
||||
</section>
|
||||
|
||||
{% if can_cancel %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Run Control</h2>
|
||||
<p>
|
||||
Cancelling a queued run stops it immediately. Cancelling a running run asks the worker to stop
|
||||
and records the cancellation request on this run.
|
||||
</p>
|
||||
<form method="post" action="{% url 'cancel_run' run.id %}">
|
||||
{% csrf_token %}
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="danger">Cancel run</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if failure %}
|
||||
<section class="panel highlight failed">
|
||||
<h2>Failure</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Category:</strong> {{ failure.category|default:"unknown" }}</div>
|
||||
<div><strong>Summary:</strong> {{ failure_summary }}</div>
|
||||
<div><strong>Hint:</strong> {{ failure.hint|default:"" }}</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if can_manage_control_panel and run.status == "failed" or can_manage_control_panel and run.status == "warning" %}
|
||||
{% if not run.reviewed_at %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Review Required</h2>
|
||||
<p>Mark this run as reviewed after you have checked the failure or warning and no longer need it in the action queue.</p>
|
||||
<form method="post" action="{% url 'resolve_run_review' run.id %}">
|
||||
{% csrf_token %}
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="secondary">Mark reviewed</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% if run.reviewed_at %}
|
||||
<section class="panel highlight success">
|
||||
<h2>Review</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Reviewed:</strong> {{ run.reviewed_at }}</div>
|
||||
<div><strong>Reviewed by:</strong> {{ run.reviewed_by|default:"unknown" }}</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if dry_run_summary %}
|
||||
<section class="panel highlight {{ dry_run_summary.highlight_class }}">
|
||||
<h2>Run Progress</h2>
|
||||
<section class="grid" aria-label="Run progress">
|
||||
<div class="metric"><div class="label">Status</div><div class="value">{{ dry_run_summary.status }}</div></div>
|
||||
<div class="metric">
|
||||
<div class="label">Mode</div>
|
||||
<div class="value">dry run</div>
|
||||
</div>
|
||||
<div class="metric"><div class="label">Files Seen</div><div class="value">{{ dry_run_summary.files_seen|default:"unknown" }}</div></div>
|
||||
<div class="metric"><div class="label">Would Transfer</div><div class="value">{{ dry_run_summary.files_would_transfer|default:"unknown" }}</div></div>
|
||||
<div class="metric"><div class="label">Transfer Estimate</div><div class="value">{{ dry_run_summary.transfer_estimate_bytes|filesizeformat }}</div></div>
|
||||
<div class="metric"><div class="label">Total Size</div><div class="value">{{ dry_run_summary.total_file_size_bytes|filesizeformat }}</div></div>
|
||||
<div class="metric"><div class="label">Link-Dest Saving</div><div class="value">{{ dry_run_summary.link_dest_estimated_savings_bytes|filesizeformat }}</div></div>
|
||||
</section>
|
||||
<div class="stack">
|
||||
{% if dry_run_summary.duration_seconds is not None %}
|
||||
<div><strong>Duration:</strong> {{ dry_run_summary.duration_seconds }}s</div>
|
||||
{% endif %}
|
||||
<div>
|
||||
<strong>Log:</strong>
|
||||
{% if dry_run_summary.log_available and can_manage_control_panel %}
|
||||
<a href="{% url 'run_rsync_log' run.id %}">Open full rsync log</a>
|
||||
{% elif rsync_log_path %}
|
||||
<span class="muted">{{ rsync_log_path }} (missing)</span>
|
||||
{% else %}
|
||||
<span class="muted">not recorded yet</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if dry_run_summary.warnings %}
|
||||
<div><strong>Warnings:</strong></div>
|
||||
<ul>
|
||||
{% for warning in dry_run_summary.warnings %}
|
||||
<li>{{ warning }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% else %}
|
||||
<div><strong>Warnings:</strong> none recorded</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if live_progress %}
|
||||
<section class="panel highlight running">
|
||||
<h2>Run Progress</h2>
|
||||
<section class="grid" aria-label="Run progress">
|
||||
<div class="metric">
|
||||
<div class="label">Status</div>
|
||||
<div class="value">{{ run.status }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Mode</div>
|
||||
<div class="value">backup</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Phase</div>
|
||||
<div class="value">{{ live_progress.phase }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Rsync PID</div>
|
||||
<div class="value">{{ live_progress.rsync_pid|default:"" }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Log Updated</div>
|
||||
<div class="value">
|
||||
{% if live_progress.log.exists %}
|
||||
{{ live_progress.log.seconds_since_modified }}s ago
|
||||
{% else %}
|
||||
missing
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Log Size</div>
|
||||
<div class="value">{{ live_progress.log.size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
{% if live_progress.snapshot.exists %}
|
||||
<div class="metric">
|
||||
<div class="label">Data Files</div>
|
||||
<div class="value">{% if live_progress.snapshot.scan_limited %}at least {% endif %}{{ live_progress.snapshot.files }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Data Size</div>
|
||||
<div class="value">{% if live_progress.snapshot.scan_limited %}at least {% endif %}{{ live_progress.snapshot.apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
<div class="stack">
|
||||
{% if live_progress.snapshot.path %}
|
||||
<div><strong>Snapshot path:</strong> {{ live_progress.snapshot.path }}</div>
|
||||
{% endif %}
|
||||
{% if live_progress.snapshot.scan_limited %}
|
||||
<div class="muted">Progress scan was capped to keep the UI responsive.</div>
|
||||
{% endif %}
|
||||
{% if live_progress.log.path %}
|
||||
<div>
|
||||
<strong>Log:</strong>
|
||||
{% if live_progress.log.exists and can_manage_control_panel %}
|
||||
<a href="{% url 'run_rsync_log' run.id %}">Open full rsync log</a>
|
||||
{% else %}
|
||||
<span class="muted">{{ live_progress.log.path }} (missing)</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div><strong>Log path:</strong> {{ live_progress.log.path }}</div>
|
||||
{% endif %}
|
||||
<div><strong>Warnings:</strong> none recorded</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<div class="two-col">
|
||||
<section class="panel">
|
||||
<h2>Timing</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Created:</strong> {{ run.created_at }}</div>
|
||||
<div><strong>Started:</strong> {{ run.started_at|default:"" }}</div>
|
||||
<div><strong>Ended:</strong> {{ run.ended_at|default:"" }}</div>
|
||||
{% if execution %}
|
||||
<div><strong>Worker:</strong> {{ execution.worker_host|default:"unknown" }}{% if execution.worker_pid %} pid {{ execution.worker_pid }}{% endif %}</div>
|
||||
<div><strong>Worker heartbeat:</strong> {{ execution.heartbeat_at|default:"" }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Snapshot</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Snapshot:</strong> {% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}{{ run.snapshot_path|default:"" }}{% endif %}</div>
|
||||
<div><strong>Base:</strong> {{ run.base_path|default:"" }}</div>
|
||||
<div>
|
||||
<strong>Rsync log:</strong>
|
||||
{% if rsync_log_exists and can_manage_control_panel %}
|
||||
<a href="{% url 'run_rsync_log' run.id %}">{{ rsync_log_path }}</a>
|
||||
{% elif rsync_log_path %}
|
||||
<span class="muted">{{ rsync_log_path }} (missing)</span>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Rsync Log</h2>
|
||||
<div class="stack spaced">
|
||||
{% if rsync_log_exists and can_manage_control_panel %}
|
||||
<div><a href="{% url 'run_rsync_log' run.id %}">Open full rsync log</a></div>
|
||||
<div class="muted">{{ rsync_log_path }}</div>
|
||||
{% elif rsync_log_path %}
|
||||
<div class="muted">{{ rsync_log_path }} (missing)</div>
|
||||
{% else %}
|
||||
<div class="muted">No rsync log path recorded yet.</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if rsync_log_tail %}
|
||||
<pre>{% for line in rsync_log_tail %}{{ line }}{% if not forloop.last %}
|
||||
{% endif %}{% endfor %}</pre>
|
||||
{% else %}
|
||||
<p class="muted">No recent rsync log output recorded yet.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
@@ -0,0 +1,79 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Purged Snapshots | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Retention</div>
|
||||
<h1>Purged Snapshots</h1>
|
||||
<div class="page-subtitle">Audit trail for snapshots removed by retention or manual purge actions.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Purged snapshot actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form method="get" class="filter-form">
|
||||
<div class="field">
|
||||
<label for="host">Host</label>
|
||||
<select id="host" name="host">
|
||||
<option value="">All hosts</option>
|
||||
{% for host in hosts %}
|
||||
<option value="{{ host.host }}" {% if selected_host == host.host %}selected{% endif %}>{{ host.host }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="action">Action</label>
|
||||
<select id="action" name="action">
|
||||
<option value="">All actions</option>
|
||||
{% for value, label in actions %}
|
||||
<option value="{{ value }}" {% if selected_action == value %}selected{% endif %}>{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'purged_snapshots' %}">Clear</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Purged Snapshot History</h2>
|
||||
<p class="muted">Showing up to 200 of {{ total_count }} purged snapshot record(s).</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Purged</th>
|
||||
<th>Host</th>
|
||||
<th>Kind</th>
|
||||
<th>Dirname</th>
|
||||
<th>Action</th>
|
||||
<th>Reason</th>
|
||||
<th>Triggered by</th>
|
||||
<th>Path</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for snapshot in purged_snapshots %}
|
||||
<tr>
|
||||
<td>{{ snapshot.purged_at }}</td>
|
||||
<td>{% if snapshot.host %}<a href="{% url 'host_detail' snapshot.host.host %}">{{ snapshot.host_name }}</a>{% else %}{{ snapshot.host_name }}{% endif %}</td>
|
||||
<td>{{ snapshot.kind }}</td>
|
||||
<td>{{ snapshot.dirname }}</td>
|
||||
<td><span class="status skipped">{{ snapshot.get_action_display }}</span></td>
|
||||
<td>{{ snapshot.reason|default:"" }}</td>
|
||||
<td>{{ snapshot.triggered_by|default:"" }}</td>
|
||||
<td class="muted">{{ snapshot.path }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="8" class="muted">No purged snapshots recorded yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -3,23 +3,55 @@
|
||||
{% block title %}Retention plan | {{ host.host }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Retention Plan: {{ host.host }}</h1>
|
||||
|
||||
<section class="actions" aria-label="Retention filters">
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=scheduled">Scheduled</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=manual">Manual</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=all">All</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind={{ kind }}&protect_bases=1">Protect bases</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Retention</div>
|
||||
<h1>{{ host.host }}</h1>
|
||||
<div class="page-subtitle">Preview which snapshots stay, which would be deleted, and whether incomplete cleanup is needed.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Retention filters">
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=scheduled">Scheduled</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=manual">Manual</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind=all">All</a>
|
||||
<a class="button-link" href="{% url 'host_retention_plan' host.host %}?kind={{ kind }}&protect_bases=1">Protect bases</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="grid" aria-label="Retention plan summary">
|
||||
<div class="metric"><div class="label">Source</div><div class="value">{{ plan.source }}</div></div>
|
||||
<div class="metric"><div class="label">Kind</div><div class="value">{{ plan.kind }}</div></div>
|
||||
<div class="metric"><div class="label">Keep</div><div class="value">{{ plan.keep|length }}</div></div>
|
||||
<div class="metric"><div class="label">Would Delete</div><div class="value">{{ plan.delete|length }}</div></div>
|
||||
<div class="metric"><div class="label">Scheduled Limit</div><div class="value">{{ scheduled_prune_limit|default:"none" }}</div></div>
|
||||
<div class="metric"><div class="label">Incomplete</div><div class="value">{{ plan.incomplete|length }}</div></div>
|
||||
</section>
|
||||
|
||||
{% if scheduled_prune_exceeded %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Scheduled Prune Limit</h2>
|
||||
<p>
|
||||
This plan would delete {{ plan.delete|length }} snapshot(s), which exceeds the scheduled prune limit of
|
||||
{{ scheduled_prune_limit }}. Scheduled pruning will refuse to apply this plan until the limit or retention
|
||||
selection is adjusted.
|
||||
</p>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if plan.incomplete %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Incomplete Snapshots</h2>
|
||||
<p>
|
||||
{{ plan.incomplete|length }} incomplete snapshot(s) exist for this host. Retention does not delete incomplete
|
||||
snapshots automatically because they can indicate an interrupted backup that should be inspected first.
|
||||
</p>
|
||||
<p>
|
||||
{{ incomplete_unreviewed_count }} still need review. After inspection, mark them reviewed and use the dedicated
|
||||
cleanup form below to delete only incomplete snapshot directories and their tracking records. Successful
|
||||
scheduled and manual snapshots are not touched by this cleanup.
|
||||
</p>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Policy</h2>
|
||||
<div class="stack">
|
||||
@@ -28,6 +60,17 @@
|
||||
<div><strong>Monthly:</strong> {{ plan.retention.monthly }}</div>
|
||||
<div><strong>Yearly:</strong> {{ plan.retention.yearly }}</div>
|
||||
<div><strong>Protect bases:</strong> {{ protect_bases|yesno:"yes,no" }}</div>
|
||||
<div class="muted">
|
||||
{% if protect_bases %}
|
||||
Base snapshots referenced by kept snapshots are also kept and marked with a base-of reason.
|
||||
{% else %}
|
||||
Base snapshots are only kept when they match the regular retention policy.
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if schedule %}
|
||||
<div><strong>Schedule pruning:</strong> {{ schedule.prune|yesno:"enabled,disabled" }}</div>
|
||||
<div><strong>Schedule max delete:</strong> {{ schedule.prune_max_delete }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -40,6 +83,7 @@
|
||||
<th>Dirname</th>
|
||||
<th>Started</th>
|
||||
<th>Status</th>
|
||||
<th>Reason</th>
|
||||
<th>Path</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -50,18 +94,23 @@
|
||||
<td>{{ snapshot.dirname }}</td>
|
||||
<td>{{ snapshot.dt }}</td>
|
||||
<td>{{ snapshot.status|default:"" }}</td>
|
||||
<td>{{ snapshot.reason }}</td>
|
||||
<td class="muted">{{ snapshot.path }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">Retention would not delete snapshots for this selection.</td></tr>
|
||||
<tr><td colspan="6" class="muted">Retention would not delete snapshots for this selection.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
{% if plan.delete %}
|
||||
<section class="panel">
|
||||
<section class="panel highlight warning">
|
||||
<h2>Apply Retention</h2>
|
||||
<p class="muted">
|
||||
This permanently deletes the snapshot directories listed in Would Delete. Confirm the host and delete count
|
||||
before applying the plan.
|
||||
</p>
|
||||
<form method="post" action="{% url 'apply_host_retention' host.host %}" class="form-grid">
|
||||
{% csrf_token %}
|
||||
{{ apply_form.non_field_errors }}
|
||||
@@ -71,7 +120,7 @@
|
||||
{{ apply_form.max_delete.errors }}
|
||||
<label for="{{ apply_form.max_delete.id_for_label }}">Max delete</label>
|
||||
{{ apply_form.max_delete }}
|
||||
<div class="helptext">Must be at least the number of snapshots shown in Would Delete.</div>
|
||||
<div class="helptext">Must be at least {{ plan.delete|length }} for the snapshots shown in Would Delete.</div>
|
||||
</div>
|
||||
|
||||
<div class="field">
|
||||
@@ -87,8 +136,16 @@
|
||||
<div class="helptext">{{ apply_form.confirm_host.help_text }}</div>
|
||||
</div>
|
||||
|
||||
<div class="actions">
|
||||
<button type="submit">Apply retention</button>
|
||||
<div class="field">
|
||||
{{ apply_form.confirm_delete_count.errors }}
|
||||
<label for="{{ apply_form.confirm_delete_count.id_for_label }}">Confirm delete count</label>
|
||||
{{ apply_form.confirm_delete_count }}
|
||||
<div class="helptext">{{ apply_form.confirm_delete_count.help_text }}</div>
|
||||
</div>
|
||||
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="danger">Apply retention</button>
|
||||
<a class="button-link secondary" href="{% url 'host_retention_plan' host.host %}?kind={{ kind }}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
@@ -99,20 +156,110 @@
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Kind</th>
|
||||
<th>Dirname</th>
|
||||
<th>Started</th>
|
||||
<th>Status</th>
|
||||
<th>Reasons</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for dirname, reasons in plan.reasons.items %}
|
||||
{% for snapshot in plan.keep_items %}
|
||||
<tr>
|
||||
<td>{{ dirname }}</td>
|
||||
<td>{{ reasons|join:", " }}</td>
|
||||
<td>{{ snapshot.kind }}</td>
|
||||
<td>{{ snapshot.dirname }}</td>
|
||||
<td>{{ snapshot.dt }}</td>
|
||||
<td>{{ snapshot.status|default:"" }}</td>
|
||||
<td>{{ snapshot.reason }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="2" class="muted">No snapshots matched this retention selection.</td></tr>
|
||||
<tr><td colspan="5" class="muted">No snapshots matched this retention selection.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
{% if plan.incomplete %}
|
||||
<section class="panel">
|
||||
<h2>Incomplete Snapshots</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Dirname</th>
|
||||
<th>Started</th>
|
||||
<th>Status</th>
|
||||
<th>Review</th>
|
||||
<th>Reason</th>
|
||||
<th>Path</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for snapshot in plan.incomplete %}
|
||||
<tr>
|
||||
<td>{{ snapshot.dirname }}</td>
|
||||
<td>{{ snapshot.dt }}</td>
|
||||
<td>{{ snapshot.status|default:"" }}</td>
|
||||
<td>
|
||||
{% if snapshot.reviewed %}
|
||||
<span class="status ok">reviewed</span>
|
||||
<span class="muted">{{ snapshot.reviewed_by|default:"unknown" }}</span>
|
||||
{% else %}
|
||||
<span class="status warning">needs review</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ snapshot.reason }}</td>
|
||||
<td class="muted">{{ snapshot.path }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h3>Cleanup Incomplete Snapshots</h3>
|
||||
{% if incomplete_unreviewed_count %}
|
||||
<p class="muted">
|
||||
Cleanup is blocked until all incomplete snapshots are reviewed. This extra step makes it explicit that the
|
||||
interrupted backup was inspected before deletion.
|
||||
</p>
|
||||
<form method="post" action="{% url 'resolve_host_incomplete_reviews' host.host %}" class="actions inline">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Mark incomplete snapshots reviewed</button>
|
||||
</form>
|
||||
{% else %}
|
||||
<p class="muted">
|
||||
This deletes only reviewed incomplete snapshot directories and their tracking records. Successful manual and
|
||||
scheduled snapshots are not touched.
|
||||
</p>
|
||||
<form method="post" action="{% url 'cleanup_host_incomplete_snapshots' host.host %}" class="form-grid">
|
||||
{% csrf_token %}
|
||||
{{ incomplete_cleanup_form.non_field_errors }}
|
||||
|
||||
<div class="field">
|
||||
{{ incomplete_cleanup_form.max_delete.errors }}
|
||||
<label for="{{ incomplete_cleanup_form.max_delete.id_for_label }}">Max delete</label>
|
||||
{{ incomplete_cleanup_form.max_delete }}
|
||||
<div class="helptext">{{ incomplete_cleanup_form.max_delete.help_text }}</div>
|
||||
</div>
|
||||
|
||||
<div class="field">
|
||||
{{ incomplete_cleanup_form.confirm_host.errors }}
|
||||
<label for="{{ incomplete_cleanup_form.confirm_host.id_for_label }}">Confirm host</label>
|
||||
{{ incomplete_cleanup_form.confirm_host }}
|
||||
<div class="helptext">{{ incomplete_cleanup_form.confirm_host.help_text }}</div>
|
||||
</div>
|
||||
|
||||
<div class="field">
|
||||
{{ incomplete_cleanup_form.confirm_delete_count.errors }}
|
||||
<label for="{{ incomplete_cleanup_form.confirm_delete_count.id_for_label }}">Confirm incomplete count</label>
|
||||
{{ incomplete_cleanup_form.confirm_delete_count }}
|
||||
<div class="helptext">{{ incomplete_cleanup_form.confirm_delete_count.help_text }}</div>
|
||||
</div>
|
||||
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="danger">Delete incomplete snapshots</button>
|
||||
<a class="button-link secondary" href="{% url 'host_retention_plan' host.host %}?kind={{ kind }}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
@@ -3,42 +3,36 @@
|
||||
{% block title %}Run {{ run.id }} | {{ run.host.host }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Run {{ run.id }}</h1>
|
||||
|
||||
<section class="actions" aria-label="Run actions">
|
||||
<a class="button-link" href="{% url 'host_detail' run.host.host %}">Back to host</a>
|
||||
{% if can_cancel %}
|
||||
<form method="post" action="{% url 'cancel_run' run.id %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Cancel run</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
<section class="grid" aria-label="Run summary">
|
||||
<div class="metric"><div class="label">Host</div><div class="value">{{ run.host.host }}</div></div>
|
||||
<div class="metric"><div class="label">Status</div><div class="value">{{ run.status }}</div></div>
|
||||
<div class="metric"><div class="label">Type</div><div class="value">{{ run.run_type }}</div></div>
|
||||
<div class="metric"><div class="label">Rsync</div><div class="value">{{ run.rsync_exit_code|default:"" }}</div></div>
|
||||
</section>
|
||||
|
||||
<div class="two-col">
|
||||
<section class="panel">
|
||||
<h2>Timing</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Created:</strong> {{ run.created_at }}</div>
|
||||
<div><strong>Started:</strong> {{ run.started_at|default:"" }}</div>
|
||||
<div><strong>Ended:</strong> {{ run.ended_at|default:"" }}</div>
|
||||
</div>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Backup run</div>
|
||||
<h1>Run {{ run.id }}</h1>
|
||||
<div class="page-subtitle">{{ run.host.host }} · {{ run.run_type }} · {{ run.status }}</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Run actions">
|
||||
<a class="button-link" href="{% url 'host_detail' run.host.host %}">Back to host</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Snapshot</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Snapshot:</strong> {% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}{{ run.snapshot_path|default:"" }}{% endif %}</div>
|
||||
<div><strong>Base:</strong> {{ run.base_path|default:"" }}</div>
|
||||
{% if can_auto_refresh %}
|
||||
<section class="panel refresh-controls" aria-label="Live refresh controls">
|
||||
<div>
|
||||
<h2>Live Updates</h2>
|
||||
<p class="muted">Auto-refresh is <strong data-refresh-state="run-live-region">on</strong> while this run is active.</p>
|
||||
</div>
|
||||
<button type="button" class="secondary" data-refresh-toggle data-refresh-target="run-live-region">Pause refresh</button>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<div
|
||||
id="run-live-region"
|
||||
data-refresh-url="{% url 'run_detail_live' run.id %}"
|
||||
data-refresh-interval="5000"
|
||||
data-refresh-active="{{ can_auto_refresh|yesno:'true,false' }}"
|
||||
data-refresh-paused="false"
|
||||
aria-live="polite"
|
||||
>
|
||||
{% include "pobsync_backend/partials/run_detail_live.html" %}
|
||||
</div>
|
||||
|
||||
{% if requested %}
|
||||
@@ -54,6 +48,20 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Rsync Command</h2>
|
||||
<p class="muted">
|
||||
<strong>Bandwidth limit:</strong>
|
||||
{% if rsync_bwlimit_kbps %}{{ rsync_bwlimit_kbps }} KB/s{% else %}unlimited{% endif %}
|
||||
</p>
|
||||
{% if rsync_command %}
|
||||
<pre>{% for part in rsync_command %}{{ part }}{% if not forloop.last %}
|
||||
{% endif %}{% endfor %}</pre>
|
||||
{% else %}
|
||||
<p class="muted">No rsync command recorded yet.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
{% if stats %}
|
||||
<section class="panel">
|
||||
<h2>Stats</h2>
|
||||
@@ -76,8 +84,56 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if has_prune_result %}
|
||||
<section class="panel highlight {% if prune_result.ok %}success{% else %}warning{% endif %}">
|
||||
<h2>Retention</h2>
|
||||
<div class="stack">
|
||||
<div><strong>Status:</strong> {% if prune_result.ok %}ok{% else %}warning{% endif %}</div>
|
||||
{% if prune_result.kind %}<div><strong>Kind:</strong> {{ prune_result.kind }}</div>{% endif %}
|
||||
{% if prune_result.planned_delete_count is not None %}<div><strong>Planned deletions:</strong> {{ prune_result.planned_delete_count }}</div>{% endif %}
|
||||
{% if prune_result.deleted %}<div><strong>Deleted:</strong> {{ prune_result.deleted|length }}</div>{% endif %}
|
||||
{% if prune_result.max_delete is not None %}<div><strong>Max delete:</strong> {{ prune_result.max_delete }}</div>{% endif %}
|
||||
{% if prune_result.protect_bases is not None %}<div><strong>Protect bases:</strong> {{ prune_result.protect_bases|yesno:"yes,no" }}</div>{% endif %}
|
||||
{% if prune_result.incomplete_ignored_count %}<div><strong>Incomplete ignored:</strong> {{ prune_result.incomplete_ignored_count }}</div>{% endif %}
|
||||
{% if prune_result.actions %}
|
||||
<div><strong>Actions:</strong></div>
|
||||
<ul>
|
||||
{% for action in prune_result.actions %}
|
||||
<li>{{ action }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
{% if prune_result.error %}<div><strong>Error:</strong> {{ prune_result.error }}</div>{% endif %}
|
||||
{% if prune_result.type %}<div><strong>Type:</strong> {{ prune_result.type }}</div>{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if retention_warning.has_warning %}
|
||||
<section class="panel highlight warning">
|
||||
<h2>Retention Warnings</h2>
|
||||
<div class="stack">
|
||||
{% if retention_warning.prune_exceeded %}
|
||||
<div>
|
||||
Scheduled pruning for this host would delete {{ retention_warning.delete_count }} snapshot(s), above max
|
||||
delete {{ retention_warning.max_delete }}.
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if retention_warning.incomplete_count %}
|
||||
<div>
|
||||
{{ retention_warning.incomplete_count }} incomplete snapshot(s) exist for this host and are excluded from
|
||||
retention cleanup.
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if retention_warning.error %}
|
||||
<div>{{ retention_warning.error }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Result</h2>
|
||||
<h2>Raw Result</h2>
|
||||
<pre>{{ result_json }}</pre>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
121
src/pobsync_backend/templates/pobsync_backend/runs_list.html
Normal file
121
src/pobsync_backend/templates/pobsync_backend/runs_list.html
Normal file
@@ -0,0 +1,121 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Runs | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Activity</div>
|
||||
<h1>Runs</h1>
|
||||
<div class="page-subtitle">Review queued, running, completed, warning, failed, and cancelled backup runs.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Run list actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form method="get" class="filter-form">
|
||||
<div class="field">
|
||||
<label for="status">Status</label>
|
||||
<select id="status" name="status">
|
||||
<option value="">All statuses</option>
|
||||
{% for value, label in statuses %}
|
||||
<option value="{{ value }}" {% if selected_status == value %}selected{% endif %}>{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="type">Type</label>
|
||||
<select id="type" name="type">
|
||||
<option value="">All types</option>
|
||||
{% for value, label in run_types %}
|
||||
<option value="{{ value }}" {% if selected_type == value %}selected{% endif %}>{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="host">Host</label>
|
||||
<select id="host" name="host">
|
||||
<option value="">All hosts</option>
|
||||
{% for host in hosts %}
|
||||
<option value="{{ host.host }}" {% if selected_host == host.host %}selected{% endif %}>{{ host.host }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="review">Review</label>
|
||||
<select id="review" name="review">
|
||||
<option value="">All review states</option>
|
||||
<option value="needed" {% if selected_review == "needed" %}selected{% endif %}>Needs review</option>
|
||||
<option value="reviewed" {% if selected_review == "reviewed" %}selected{% endif %}>Reviewed</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'runs_list' %}">Clear</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Backup Runs</h2>
|
||||
<p class="muted">Showing up to 200 of {{ total_count }} run{{ total_count|pluralize }}.</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Run</th>
|
||||
<th>Host</th>
|
||||
<th>Status</th>
|
||||
<th>Type</th>
|
||||
<th>Created</th>
|
||||
<th>Started</th>
|
||||
<th>Ended</th>
|
||||
<th>Snapshot</th>
|
||||
<th>Review</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for run in runs %}
|
||||
<tr>
|
||||
<td><a href="{% url 'run_detail' run.id %}">Run {{ run.id }}</a></td>
|
||||
<td><a href="{% url 'host_detail' run.host.host %}">{{ run.host.host }}</a></td>
|
||||
<td><span class="status {{ run.status }}">{{ run.status }}</span></td>
|
||||
<td>{{ run.run_type }}</td>
|
||||
<td>{{ run.created_at }}</td>
|
||||
<td>{{ run.started_at|default:"" }}</td>
|
||||
<td>{{ run.ended_at|default:"" }}</td>
|
||||
<td>
|
||||
{% if run.snapshot %}
|
||||
<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>
|
||||
{% elif run.snapshot_path %}
|
||||
<span class="muted">{{ run.snapshot_path }}</span>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if run.reviewed_at %}
|
||||
reviewed
|
||||
{% elif run.status == "failed" or run.status == "warning" %}
|
||||
<div class="stack">
|
||||
<span class="status warning">needed</span>
|
||||
<form class="inline-form" method="post" action="{% url 'resolve_run_review' run.id %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<button type="submit" class="secondary compact">Mark reviewed</button>
|
||||
</form>
|
||||
</div>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="9" class="muted">No runs matched the current filter.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -3,11 +3,16 @@
|
||||
{% block title %}Schedule | {{ host.host }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Schedule: {{ host.host }}</h1>
|
||||
|
||||
<section class="actions" aria-label="Schedule actions">
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Schedule</div>
|
||||
<h1>{{ host.host }}</h1>
|
||||
<div class="page-subtitle">Automatic backup timing and scheduled prune behavior for this host.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Schedule actions">
|
||||
<a class="button-link" href="{% url 'host_detail' host.host %}">Back to host</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if schedule %}Edit Schedule{% else %}Create Schedule{% endif %}</h2>
|
||||
@@ -25,8 +30,9 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">Save schedule</button>
|
||||
<a class="button-link secondary" href="{% url 'host_detail' host.host %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Schedules | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Scheduler</div>
|
||||
<h1>Schedules</h1>
|
||||
<div class="page-subtitle">Review configured backup schedules, next run times, prune settings, and recent scheduler state.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Schedule list actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form method="get" class="filter-form">
|
||||
<div class="field">
|
||||
<label for="host">Host</label>
|
||||
<select id="host" name="host">
|
||||
<option value="">All hosts</option>
|
||||
{% for host in hosts %}
|
||||
<option value="{{ host.host }}" {% if selected_host == host.host %}selected{% endif %}>{{ host.host }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="enabled">Enabled</label>
|
||||
<select id="enabled" name="enabled">
|
||||
<option value="">All schedules</option>
|
||||
<option value="yes" {% if selected_enabled == "yes" %}selected{% endif %}>Enabled</option>
|
||||
<option value="no" {% if selected_enabled == "no" %}selected{% endif %}>Disabled</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="prune">Prune</label>
|
||||
<select id="prune" name="prune">
|
||||
<option value="">All prune states</option>
|
||||
<option value="yes" {% if selected_prune == "yes" %}selected{% endif %}>Prune enabled</option>
|
||||
<option value="no" {% if selected_prune == "no" %}selected{% endif %}>Prune disabled</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'schedules_list' %}">Clear</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Configured Schedules</h2>
|
||||
<p class="muted">Showing up to 200 of {{ total_count }} schedule{{ total_count|pluralize }}. Times use {{ scheduler_timezone }}.</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Host</th>
|
||||
<th>Expression</th>
|
||||
<th>Enabled</th>
|
||||
<th>Next Run</th>
|
||||
<th>Prune</th>
|
||||
<th>Last Status</th>
|
||||
<th>Last Started</th>
|
||||
<th>Last Finished</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for row in schedule_rows %}
|
||||
{% with schedule=row.schedule %}
|
||||
<tr>
|
||||
<td><a href="{% url 'host_detail' schedule.host.host %}">{{ schedule.host.host }}</a></td>
|
||||
<td><code>{{ schedule.cron_expr }}</code></td>
|
||||
<td><span class="status {% if schedule.enabled %}ok{% else %}skipped{% endif %}">{{ schedule.enabled|yesno:"enabled,disabled" }}</span></td>
|
||||
<td>
|
||||
{% if row.next_run_at %}
|
||||
{{ row.next_run_at|date:"Y-m-d H:i T" }}
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<span class="status {% if schedule.prune %}ok{% else %}skipped{% endif %}">{{ schedule.prune|yesno:"enabled,disabled" }}</span>
|
||||
{% if schedule.prune %}
|
||||
<div class="muted">max {{ schedule.prune_max_delete }}{% if schedule.prune_protect_bases %}, protects bases{% endif %}</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{% if schedule.last_status %}<span class="status {{ schedule.last_status }}">{{ schedule.last_status }}</span>{% else %}<span class="muted">none</span>{% endif %}</td>
|
||||
<td>{{ schedule.last_started_at|default:"" }}</td>
|
||||
<td>{{ schedule.last_finished_at|default:"" }}</td>
|
||||
<td><a class="button-link secondary" href="{% url 'edit_host_schedule' schedule.host.host %}">Edit</a></td>
|
||||
</tr>
|
||||
{% endwith %}
|
||||
{% empty %}
|
||||
<tr><td colspan="9" class="muted">No schedules matched the current filter.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -3,11 +3,16 @@
|
||||
{% block title %}Self Check | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Self Check</h1>
|
||||
|
||||
<section class="actions" aria-label="Self check actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Operations</div>
|
||||
<h1>Self Check</h1>
|
||||
<div class="page-subtitle">Runtime, filesystem, service, and configuration checks for this pobsync installation.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Self check actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="grid" aria-label="Self check summary">
|
||||
<div class="metric"><div class="label">OK</div><div class="value">{{ summary.ok }}</div></div>
|
||||
|
||||
@@ -3,11 +3,16 @@
|
||||
{% block title %}Snapshot {{ snapshot.dirname }} | {{ snapshot.host.host }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{{ snapshot.dirname }}</h1>
|
||||
|
||||
<section class="actions" aria-label="Snapshot actions">
|
||||
<a class="button-link" href="{% url 'host_detail' snapshot.host.host %}">Back to host</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Snapshot</div>
|
||||
<h1>{{ snapshot.dirname }}</h1>
|
||||
<div class="page-subtitle">{{ snapshot.host.host }} · {{ snapshot.kind }} · {{ snapshot.status }}</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Snapshot actions">
|
||||
<a class="button-link" href="{% url 'host_detail' snapshot.host.host %}">Back to host</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="grid" aria-label="Snapshot summary">
|
||||
<div class="metric"><div class="label">Host</div><div class="value">{{ snapshot.host.host }}</div></div>
|
||||
@@ -60,6 +65,48 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<h2>Restore Guidance</h2>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Snapshot data path:</strong> {{ restore.source_path }}</div>
|
||||
<div><strong>Example staging destination:</strong> {{ restore.destination_path }}</div>
|
||||
<div class="muted">
|
||||
Restore from the snapshot's <code>data/</code> directory. Start with a dry run, restore to a staging path first,
|
||||
and only then copy data back to a live host or service path.
|
||||
</div>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Inspect the snapshot:</strong></div>
|
||||
<pre>{{ restore.inspect_command }}</pre>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Dry-run restore to staging:</strong></div>
|
||||
<pre>{{ restore.dry_run_command }}</pre>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Restore to staging:</strong></div>
|
||||
<pre>{{ restore.local_command }}</pre>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Dry-run a directory restore:</strong></div>
|
||||
<pre>{{ restore.partial_dry_run_command }}</pre>
|
||||
<div class="muted">Replace <code>{{ restore.example_relative_path }}</code> with the path you want to restore.</div>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Dry-run a single file restore:</strong></div>
|
||||
<pre>{{ restore.file_dry_run_command }}</pre>
|
||||
<div class="muted">Replace <code>{{ restore.example_file_relative_path }}</code> with the file you want to restore.</div>
|
||||
</div>
|
||||
<div class="stack spaced">
|
||||
<div><strong>Dry-run restore back to the original host:</strong></div>
|
||||
<pre>{{ restore.remote_dry_run_command }}</pre>
|
||||
</div>
|
||||
<p class="muted">
|
||||
Snapshots can contain hardlinks to files shared with earlier snapshots. Treat snapshot directories as read-only:
|
||||
copy data out with rsync instead of editing files in place.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Backup Runs</h2>
|
||||
<table>
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Snapshots | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Snapshots</div>
|
||||
<h1>Snapshots</h1>
|
||||
<div class="page-subtitle">Browse discovered scheduled, manual, and incomplete snapshots across all hosts.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Snapshot list actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form method="get" class="filter-form">
|
||||
<div class="field">
|
||||
<label for="host">Host</label>
|
||||
<select id="host" name="host">
|
||||
<option value="">All hosts</option>
|
||||
{% for host in hosts %}
|
||||
<option value="{{ host.host }}" {% if selected_host == host.host %}selected{% endif %}>{{ host.host }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="kind">Kind</label>
|
||||
<select id="kind" name="kind">
|
||||
<option value="">All kinds</option>
|
||||
{% for value, label in kinds %}
|
||||
<option value="{{ value }}" {% if selected_kind == value %}selected{% endif %}>{{ label }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="status">Status</label>
|
||||
<select id="status" name="status">
|
||||
<option value="">All statuses</option>
|
||||
{% for value in statuses %}
|
||||
<option value="{{ value }}" {% if selected_status == value %}selected{% endif %}>{{ value }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'snapshots_list' %}">Clear</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Snapshot Records</h2>
|
||||
<p class="muted">Showing up to 200 of {{ total_count }} snapshot{{ total_count|pluralize }}.</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Snapshot</th>
|
||||
<th>Host</th>
|
||||
<th>Kind</th>
|
||||
<th>Status</th>
|
||||
<th>Started</th>
|
||||
<th>Ended</th>
|
||||
<th>Base</th>
|
||||
<th>Path</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for snapshot in snapshots %}
|
||||
<tr>
|
||||
<td><a href="{% url 'snapshot_detail' snapshot.id %}">{{ snapshot.dirname }}</a></td>
|
||||
<td><a href="{% url 'host_detail' snapshot.host.host %}">{{ snapshot.host.host }}</a></td>
|
||||
<td>{{ snapshot.kind }}</td>
|
||||
<td>{% if snapshot.status %}<span class="status {{ snapshot.status }}">{{ snapshot.status }}</span>{% else %}<span class="muted">unknown</span>{% endif %}</td>
|
||||
<td>{{ snapshot.started_at|default:"" }}</td>
|
||||
<td>{{ snapshot.ended_at|default:"" }}</td>
|
||||
<td>
|
||||
{% if snapshot.base %}
|
||||
<a href="{% url 'snapshot_detail' snapshot.base.id %}">{{ snapshot.base.dirname }}</a>
|
||||
{% elif snapshot.base_dirname %}
|
||||
<span class="muted">{{ snapshot.base_dirname }}</span>
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="muted">{{ snapshot.path }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="8" class="muted">No snapshots matched the current filter.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -3,11 +3,16 @@
|
||||
{% block title %}{% if credential %}SSH Key | {{ credential.name }}{% else %}New SSH Key{% endif %} | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>{% if credential %}SSH Key: {{ credential.name }}{% else %}New SSH Key{% endif %}</h1>
|
||||
|
||||
<section class="actions" aria-label="SSH key form actions">
|
||||
<a class="button-link" href="{% url 'ssh_credentials' %}">Back to SSH keys</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Access</div>
|
||||
<h1>{% if credential %}SSH Key: {{ credential.name }}{% else %}New SSH Key{% endif %}</h1>
|
||||
<div class="page-subtitle">{% if credential %}Review key metadata, known hosts, and deletion safety for this credential.{% else %}Register an existing private key for use by pobsync backups.{% endif %}</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="SSH key form actions">
|
||||
<a class="button-link" href="{% url 'ssh_credentials' %}">Back to SSH keys</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if credential %}Edit SSH Credential{% else %}Create SSH Credential{% endif %}</h2>
|
||||
@@ -36,8 +41,9 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">{% if credential %}Save SSH key{% else %}Create SSH key{% endif %}</button>
|
||||
<a class="button-link secondary" href="{% url 'ssh_credentials' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
@@ -45,9 +51,24 @@
|
||||
{% if credential %}
|
||||
<section class="panel">
|
||||
<h2>Delete SSH Key</h2>
|
||||
{% if credential.hosts.exists or credential.global_configs.exists %}
|
||||
<p class="muted">
|
||||
This SSH key is still selected by {{ credential.hosts.count }} host(s) or
|
||||
{{ credential.global_configs.count }} global config(s). Select another key there before deleting it.
|
||||
</p>
|
||||
{% else %}
|
||||
<p class="muted">Type <strong>{{ credential.name }}</strong> to confirm deletion.</p>
|
||||
{% endif %}
|
||||
<form method="post" action="{% url 'delete_ssh_credential' credential.id %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="danger">Delete SSH key</button>
|
||||
<div class="field">
|
||||
<label for="confirm_name">Confirm key name</label>
|
||||
<input id="confirm_name" name="confirm_name" type="text" {% if credential.hosts.exists or credential.global_configs.exists %}disabled{% endif %}>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="danger" {% if credential.hosts.exists or credential.global_configs.exists %}disabled{% endif %}>Delete SSH key</button>
|
||||
<a class="button-link secondary" href="{% url 'ssh_credentials' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
@@ -3,11 +3,16 @@
|
||||
{% block title %}Generate SSH Key | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Generate SSH Key</h1>
|
||||
|
||||
<section class="actions" aria-label="SSH key form actions">
|
||||
<a class="button-link" href="{% url 'ssh_credentials' %}">Back to SSH keys</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Access</div>
|
||||
<h1>Generate SSH Key</h1>
|
||||
<div class="page-subtitle">Create a pobsync-managed SSH key pair for one or more backup targets.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="SSH key form actions">
|
||||
<a class="button-link" href="{% url 'ssh_credentials' %}">Back to SSH keys</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Create Key Pair</h2>
|
||||
@@ -24,8 +29,9 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="actions">
|
||||
<div class="form-actions">
|
||||
<button type="submit">Generate SSH key</button>
|
||||
<a class="button-link secondary" href="{% url 'ssh_credentials' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
@@ -3,13 +3,18 @@
|
||||
{% block title %}SSH Keys | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>SSH Keys</h1>
|
||||
|
||||
<section class="actions" aria-label="SSH key actions">
|
||||
<a class="button-link" href="{% url 'generate_ssh_credential' %}">Generate SSH key</a>
|
||||
<a class="button-link secondary" href="{% url 'create_ssh_credential' %}">Add existing key</a>
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Access</div>
|
||||
<h1>SSH Keys</h1>
|
||||
<div class="page-subtitle">Manage the key pairs pobsync uses to reach backup targets.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="SSH key actions">
|
||||
<a class="button-link" href="{% url 'generate_ssh_credential' %}">Generate SSH key</a>
|
||||
<a class="button-link secondary" href="{% url 'create_ssh_credential' %}">Add existing key</a>
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Credentials</h2>
|
||||
@@ -23,6 +28,7 @@
|
||||
<th>Known hosts</th>
|
||||
<th>Hosts</th>
|
||||
<th>Updated</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -35,9 +41,10 @@
|
||||
<td>{{ credential.known_hosts|yesno:"yes,no" }}</td>
|
||||
<td>{{ credential.hosts.count }}</td>
|
||||
<td>{{ credential.updated_at }}</td>
|
||||
<td><a class="button-link secondary" href="{% url 'edit_ssh_credential' credential.id %}">Edit</a></td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="7" class="muted">No SSH credentials configured yet.</td></tr>
|
||||
<tr><td colspan="8" class="muted">No SSH credentials configured yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
122
src/pobsync_backend/templates/pobsync_backend/updater.html
Normal file
122
src/pobsync_backend/templates/pobsync_backend/updater.html
Normal file
@@ -0,0 +1,122 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Updater | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Operations</div>
|
||||
<h1>Updater</h1>
|
||||
<div class="page-subtitle">Check Gitea releases, pull the installed git checkout, and run the native systemd updater.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Updater actions">
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel-grid">
|
||||
<section class="panel">
|
||||
<h2>Installed App</h2>
|
||||
<dl class="detail-list">
|
||||
<dt>Version</dt>
|
||||
<dd>{{ status.installed_version }}</dd>
|
||||
<dt>Git branch</dt>
|
||||
<dd>{{ status.git.branch|default:"unknown" }}</dd>
|
||||
<dt>Git commit</dt>
|
||||
<dd>{{ status.git.commit|default:"unknown" }}</dd>
|
||||
<dt>Git describe</dt>
|
||||
<dd>{{ status.git.describe|default:"unknown" }}</dd>
|
||||
<dt>App directory</dt>
|
||||
<dd>{{ status.app_dir }}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Release Check</h2>
|
||||
<dl class="detail-list">
|
||||
<dt>Status</dt>
|
||||
<dd>
|
||||
{% if status.update_available == True %}
|
||||
<span class="status warning">update available</span>
|
||||
{% elif status.update_available == False %}
|
||||
<span class="status ok">up to date</span>
|
||||
{% elif status.release_check_configured %}
|
||||
<span class="status skipped">not checked</span>
|
||||
{% else %}
|
||||
<span class="status skipped">not configured</span>
|
||||
{% endif %}
|
||||
</dd>
|
||||
<dt>Latest release</dt>
|
||||
<dd>
|
||||
{% if status.latest_release %}
|
||||
{% if status.latest_release.html_url %}
|
||||
<a href="{{ status.latest_release.html_url }}">
|
||||
{% if status.latest_release.tag_name %}{{ status.latest_release.tag_name }}{% else %}{{ status.latest_release.name }}{% endif %}
|
||||
</a>
|
||||
{% else %}
|
||||
{% if status.latest_release.tag_name %}{{ status.latest_release.tag_name }}{% else %}{{ status.latest_release.name }}{% endif %}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
none
|
||||
{% endif %}
|
||||
</dd>
|
||||
<dt>Release endpoint</dt>
|
||||
<dd>{% if status.release_check_configured %}configured{% else %}set POBSYNC_UPDATE_RELEASES_URL{% endif %}</dd>
|
||||
</dl>
|
||||
{% if status.release_error %}
|
||||
<p class="status failed">{{ status.release_error }}</p>
|
||||
{% endif %}
|
||||
<form method="post" class="actions inline">
|
||||
{% csrf_token %}
|
||||
<button type="submit" name="action" value="check_release">Check releases</button>
|
||||
</form>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Update Actions</h2>
|
||||
<p class="muted">Run these from the installed checkout. The native updater may require a sudoers rule for the pobsync service user.</p>
|
||||
<dl class="detail-list">
|
||||
<dt>Git remote</dt>
|
||||
<dd>{{ status.git_remote }}</dd>
|
||||
<dt>Update command</dt>
|
||||
<dd><code>{{ status.update_command }}</code></dd>
|
||||
</dl>
|
||||
<div class="actions">
|
||||
<form method="post" class="inline-form">
|
||||
{% csrf_token %}
|
||||
<button class="secondary" type="submit" name="action" value="git_fetch">Fetch releases</button>
|
||||
</form>
|
||||
<form method="post" class="inline-form">
|
||||
{% csrf_token %}
|
||||
<button class="secondary" type="submit" name="action" value="git_pull">Pull current branch</button>
|
||||
</form>
|
||||
<form method="post" class="inline-form">
|
||||
{% csrf_token %}
|
||||
<button type="submit" name="action" value="run_update">Run native updater</button>
|
||||
</form>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{% if action_result %}
|
||||
<section class="panel">
|
||||
<h2>Last Action Result</h2>
|
||||
<dl class="detail-list">
|
||||
<dt>Status</dt>
|
||||
<dd><span class="status {% if action_result.ok %}ok{% else %}failed{% endif %}">{% if action_result.ok %}ok{% else %}failed{% endif %}</span></dd>
|
||||
<dt>Exit code</dt>
|
||||
<dd>{{ action_result.exit_code }}</dd>
|
||||
<dt>Command</dt>
|
||||
<dd><code>{{ action_result.command|join:" " }}</code></dd>
|
||||
</dl>
|
||||
{% if action_result.stdout %}
|
||||
<h3>Stdout</h3>
|
||||
<pre>{{ action_result.stdout }}</pre>
|
||||
{% endif %}
|
||||
{% if action_result.stderr %}
|
||||
<h3>Stderr</h3>
|
||||
<pre>{{ action_result.stderr }}</pre>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -5,11 +5,27 @@ from datetime import datetime, timezone
|
||||
from django.contrib.admin.sites import AdminSite
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync_backend.admin import BackupRunAdmin, HostConfigAdmin, SnapshotRecordAdmin
|
||||
from pobsync_backend.models import BackupRun, HostConfig, ScheduleConfig, SnapshotRecord
|
||||
from pobsync_backend.admin import BackupRunAdmin, GlobalConfigAdmin, HostConfigAdmin, SnapshotRecordAdmin
|
||||
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord
|
||||
|
||||
|
||||
class AdminDisplayTests(TestCase):
|
||||
def test_admin_hides_old_global_state_fields_and_labels_host_runtime_state(self) -> None:
|
||||
site = AdminSite()
|
||||
global_admin = GlobalConfigAdmin(GlobalConfig, site)
|
||||
host_admin = HostConfigAdmin(HostConfig, site)
|
||||
|
||||
global_fieldsets = list(global_admin.fieldsets)
|
||||
host_fieldsets = list(host_admin.fieldsets)
|
||||
global_fields = [field for _name, options in global_fieldsets for field in options["fields"]]
|
||||
fieldset_names = [name for name, _options in [*global_fieldsets, *host_fieldsets]]
|
||||
|
||||
self.assertNotIn("pobsync_home", global_fields)
|
||||
self.assertNotIn("data", global_fields)
|
||||
self.assertIn("Runtime state", fieldset_names)
|
||||
self.assertNotIn("Compatibility data", fieldset_names)
|
||||
self.assertNotIn("Legacy JSON", fieldset_names)
|
||||
|
||||
def test_host_admin_links_to_related_snapshots_and_runs(self) -> None:
|
||||
site = AdminSite()
|
||||
admin = HostConfigAdmin(HostConfig, site)
|
||||
|
||||
@@ -18,6 +18,12 @@ class ApiTests(TestCase):
|
||||
is_staff=True,
|
||||
is_superuser=True,
|
||||
)
|
||||
self.readonly_user = user_model.objects.create_user(
|
||||
username="viewer",
|
||||
password="secret",
|
||||
is_staff=False,
|
||||
is_superuser=False,
|
||||
)
|
||||
|
||||
def test_api_requires_staff_login(self) -> None:
|
||||
response = self.client.get("/api/hosts/")
|
||||
@@ -25,6 +31,15 @@ class ApiTests(TestCase):
|
||||
self.assertEqual(response.status_code, 302)
|
||||
self.assertIn("/admin/login/", response["Location"])
|
||||
|
||||
def test_readonly_user_can_access_status_endpoint_only(self) -> None:
|
||||
self.client.force_login(self.readonly_user)
|
||||
|
||||
status_response = self.client.get("/api/status/")
|
||||
hosts_response = self.client.get("/api/hosts/")
|
||||
|
||||
self.assertEqual(status_response.status_code, 200)
|
||||
self.assertEqual(hosts_response.status_code, 403)
|
||||
|
||||
def test_hosts_endpoint_returns_counts_and_schedule(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
@@ -11,7 +11,7 @@ from django.utils import timezone
|
||||
from pobsync.util import write_yaml_atomic
|
||||
from pobsync_backend.backup_runner import queue_backup_run, reconcile_running_runs
|
||||
from pobsync_backend.management.commands.run_pobsync_worker import Command
|
||||
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
||||
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, NotificationDelivery, NotificationTarget, SnapshotRecord
|
||||
|
||||
|
||||
class BackupWorkerTests(TestCase):
|
||||
@@ -39,13 +39,20 @@ class BackupWorkerTests(TestCase):
|
||||
},
|
||||
)
|
||||
|
||||
def test_queue_backup_run_can_request_verbose_output(self) -> None:
|
||||
def test_queue_backup_run_enables_verbose_output_by_default(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
run = queue_backup_run(host=host, verbose_output=True)
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
self.assertTrue(run.result["requested"]["verbose_output"])
|
||||
|
||||
def test_queue_backup_run_can_disable_verbose_output(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
run = queue_backup_run(host=host, verbose_output=False)
|
||||
|
||||
self.assertFalse(run.result["requested"]["verbose_output"])
|
||||
|
||||
def test_worker_executes_next_queued_run(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
@@ -61,6 +68,9 @@ class BackupWorkerTests(TestCase):
|
||||
def fake_run_scheduled(**kwargs):
|
||||
run.refresh_from_db()
|
||||
self.assertIn("execution", run.result)
|
||||
self.assertIn("worker_pid", run.result["execution"])
|
||||
self.assertIn("worker_host", run.result["execution"])
|
||||
self.assertIn("heartbeat_at", run.result["execution"])
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
@@ -82,6 +92,253 @@ class BackupWorkerTests(TestCase):
|
||||
self.assertEqual(SnapshotRecord.objects.count(), 1)
|
||||
self.assertEqual(run.snapshot, SnapshotRecord.objects.get())
|
||||
|
||||
def test_worker_records_warning_status_from_completed_run(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snapshot_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(meta_dir / "meta.yaml", {"status": "warning", "started_at": "2026-05-19T02:15:00Z"})
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
run_scheduled.return_value = {
|
||||
"ok": True,
|
||||
"status": "warning",
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"warning": {"category": "vanished"},
|
||||
"rsync": {"exit_code": 24},
|
||||
}
|
||||
|
||||
count = Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
self.assertEqual(count, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.WARNING)
|
||||
self.assertEqual(run.rsync_exit_code, 24)
|
||||
self.assertEqual(run.result["warning"]["category"], "vanished")
|
||||
|
||||
def test_worker_sends_notification_after_completed_run(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snapshot_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(meta_dir / "meta.yaml", {"status": "success", "started_at": "2026-05-19T02:15:00Z"})
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
with (
|
||||
patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled,
|
||||
patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail,
|
||||
):
|
||||
run_scheduled.return_value = {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.SUCCESS)
|
||||
self.assertEqual(NotificationDelivery.objects.get(run=run).status, NotificationDelivery.Status.SENT)
|
||||
send_mail.assert_called_once()
|
||||
|
||||
def test_worker_refreshes_heartbeat_while_run_is_active(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
def fake_run_scheduled(**kwargs):
|
||||
run.refresh_from_db()
|
||||
old_heartbeat = timezone.now() - timedelta(seconds=120)
|
||||
run.result["execution"]["heartbeat_at"] = old_heartbeat.isoformat()
|
||||
run.save(update_fields=["result"])
|
||||
|
||||
self.assertFalse(kwargs["cancel_check"]())
|
||||
run.refresh_from_db()
|
||||
self.assertGreater(
|
||||
timezone.datetime.fromisoformat(run.result["execution"]["heartbeat_at"]),
|
||||
old_heartbeat,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": "",
|
||||
"base": None,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
|
||||
run_scheduled.side_effect = fake_run_scheduled
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
def test_worker_records_real_run_log_path_while_running(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
snapshot_dir = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH"
|
||||
log_path = snapshot_dir / "meta" / "rsync.log"
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
def fake_run_scheduled(**kwargs):
|
||||
kwargs["state_callback"](
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "rsync",
|
||||
"snapshot": str(snapshot_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": ["rsync"], "exit_code": None, "pid": 1234, "pgid": 1234},
|
||||
}
|
||||
)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.snapshot_path, str(snapshot_dir))
|
||||
self.assertEqual(run.result["execution"]["phase"], "rsync")
|
||||
self.assertEqual(run.result["execution"]["log"], str(log_path))
|
||||
self.assertEqual(run.result["execution"]["snapshot"], str(snapshot_dir))
|
||||
self.assertEqual(run.result["rsync"]["command"], ["rsync"])
|
||||
self.assertEqual(run.result["rsync"]["pid"], 1234)
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": "",
|
||||
"base": None,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
|
||||
run_scheduled.side_effect = fake_run_scheduled
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
def test_worker_reconciles_stale_real_run_after_heartbeat_timeout(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now() - timedelta(seconds=120)
|
||||
run.result["execution"] = {
|
||||
"worker_pid": 123,
|
||||
"worker_host": "backup",
|
||||
"heartbeat_at": (timezone.now() - timedelta(seconds=90)).isoformat(),
|
||||
}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs(stale_worker_seconds=30)
|
||||
|
||||
self.assertEqual(reconciled, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.result["failure"]["category"], "worker")
|
||||
self.assertIn("heartbeat stopped", run.result["failure"]["message"])
|
||||
|
||||
def test_worker_reconciles_real_run_with_terminal_broken_pipe_log(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text(
|
||||
"rsync error: unexplained error (code 255) at rsync.c(716) [generator=3.4.1]\n"
|
||||
"rsync error: received SIGUSR1 (code 19) at main.c(1600) [receiver=3.4.1]\n"
|
||||
"rsync: [generator] write error: Broken pipe (32)\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now()
|
||||
run.result["execution"] = {"log": str(log_path)}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs()
|
||||
|
||||
self.assertEqual(reconciled, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.rsync_exit_code, 255)
|
||||
self.assertEqual(run.result["failure"]["category"], "transport")
|
||||
self.assertIn("Broken pipe", "\n".join(run.result["rsync"]["log_tail"]))
|
||||
|
||||
def test_worker_reconciles_real_run_when_rsync_process_disappears(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text("sending incremental file list\n", encoding="utf-8")
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now() - timedelta(minutes=10)
|
||||
run.result["execution"] = {
|
||||
"phase": "rsync",
|
||||
"log": str(log_path),
|
||||
"heartbeat_at": (timezone.now() - timedelta(minutes=10)).isoformat(),
|
||||
}
|
||||
run.result["rsync"] = {"pid": 999999, "pgid": 999999, "command": ["rsync"]}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs(grace_seconds=300, stale_worker_seconds=24 * 60 * 60)
|
||||
|
||||
self.assertEqual(reconciled, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.result["failure"]["category"], "rsync_process")
|
||||
self.assertEqual(run.rsync_exit_code, 255)
|
||||
|
||||
def test_worker_does_not_reconcile_missing_rsync_process_during_finalizing_phase(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now() - timedelta(minutes=10)
|
||||
run.result["execution"] = {
|
||||
"phase": "finalizing",
|
||||
"heartbeat_at": (timezone.now() - timedelta(minutes=10)).isoformat(),
|
||||
}
|
||||
run.result["rsync"] = {"pid": 999999, "pgid": 999999, "command": ["rsync"], "exit_code": 0}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs(grace_seconds=300, stale_worker_seconds=24 * 60 * 60)
|
||||
|
||||
self.assertEqual(reconciled, 0)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.RUNNING)
|
||||
|
||||
def test_worker_does_not_fail_real_run_for_vanished_file_warning_log(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text(
|
||||
"file has vanished: \"/var/lib/app/session\"\n"
|
||||
"rsync warning: some files vanished before they could be transferred (code 24) at main.c(1338) [sender=3.4.1]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now()
|
||||
run.result["execution"] = {"log": str(log_path)}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs()
|
||||
|
||||
self.assertEqual(reconciled, 0)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.RUNNING)
|
||||
|
||||
def test_worker_records_dry_run_log_path_while_running(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
|
||||
@@ -1,71 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync.config.load import load_global_config, load_host_config
|
||||
from pobsync_backend.config_repository import export_runtime_configs
|
||||
from pobsync_backend.config_repository import ConfigRepositoryError, global_config_data, host_config_data
|
||||
from pobsync_backend.models import GlobalConfig, HostConfig
|
||||
|
||||
|
||||
class ConfigRepositoryTests(TestCase):
|
||||
def test_exports_database_configs_to_engine_yaml(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp)
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home=str(prefix),
|
||||
ssh_user="backup",
|
||||
ssh_port=2222,
|
||||
rsync_args=["--archive"],
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
data={
|
||||
"backup_root": "/ignored",
|
||||
"pobsync_home": "/ignored",
|
||||
"ssh": {"user": "ignored", "port": 22, "options": []},
|
||||
"unknown": "must-not-leak",
|
||||
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
},
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
ssh_user="root",
|
||||
includes=[],
|
||||
excludes_add=["/tmp/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
config={
|
||||
"host": "ignored",
|
||||
"address": "ignored",
|
||||
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
"excludes_add": ["/ignored/***"],
|
||||
"unknown": "must-not-leak",
|
||||
},
|
||||
)
|
||||
def test_builds_runtime_config_from_database_fields(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
ssh_user="backup",
|
||||
ssh_port=2222,
|
||||
rsync_args=["--archive"],
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
ssh_user="root",
|
||||
includes=[],
|
||||
excludes_add=["/tmp/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
config={
|
||||
"host": "ignored",
|
||||
"address": "ignored",
|
||||
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
"excludes_add": ["/ignored/***"],
|
||||
"unknown": "must-not-leak",
|
||||
},
|
||||
)
|
||||
|
||||
written = export_runtime_configs(prefix=prefix, host="web-01")
|
||||
global_cfg = global_config_data()
|
||||
host_cfg = host_config_data("web-01")
|
||||
|
||||
self.assertEqual(len(written), 2)
|
||||
global_cfg = load_global_config(prefix / "config" / "global.yaml")
|
||||
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
|
||||
self.assertEqual(global_cfg["backup_root"], "/backups")
|
||||
self.assertEqual(global_cfg["pobsync_home"], str(prefix))
|
||||
self.assertEqual(global_cfg["ssh"]["user"], "backup")
|
||||
self.assertEqual(global_cfg["ssh"]["port"], 2222)
|
||||
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["host"], "web-01")
|
||||
self.assertEqual(host_cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(host_cfg["retention"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])
|
||||
self.assertNotIn("unknown", global_cfg)
|
||||
self.assertNotIn("unknown", host_cfg)
|
||||
self.assertEqual(global_cfg["backup_root"], "/backups")
|
||||
self.assertEqual(global_cfg["ssh"]["user"], "backup")
|
||||
self.assertEqual(global_cfg["ssh"]["port"], 2222)
|
||||
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["host"], "web-01")
|
||||
self.assertEqual(host_cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(host_cfg["retention"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])
|
||||
self.assertNotIn("unknown", global_cfg)
|
||||
self.assertNotIn("unknown", host_cfg)
|
||||
|
||||
def test_missing_config_errors_use_operator_labels(self) -> None:
|
||||
with self.assertRaisesMessage(ConfigRepositoryError, "Missing global config 'default'"):
|
||||
global_config_data()
|
||||
|
||||
GlobalConfig.objects.create(name="default", backup_root="/backups")
|
||||
|
||||
with self.assertRaisesMessage(ConfigRepositoryError, "Missing enabled host 'web-01'"):
|
||||
host_config_data("web-01")
|
||||
|
||||
@@ -16,7 +16,6 @@ class ConfigureCommandsTests(TestCase):
|
||||
call_command(
|
||||
"configure_pobsync_global",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
retention="daily=3,weekly=2,monthly=1,yearly=0",
|
||||
stdout=out,
|
||||
)
|
||||
@@ -24,7 +23,7 @@ class ConfigureCommandsTests(TestCase):
|
||||
config = GlobalConfig.objects.get(name="default")
|
||||
self.assertEqual(config.backup_root, "/backups")
|
||||
self.assertEqual(config.retention_daily, 3)
|
||||
self.assertIn("Created GlobalConfig", out.getvalue())
|
||||
self.assertIn("Created global config", out.getvalue())
|
||||
|
||||
def test_configure_host_uses_global_retention_defaults(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
@@ -43,6 +42,7 @@ class ConfigureCommandsTests(TestCase):
|
||||
address="web-01.example.test",
|
||||
exclude_add=["/tmp/***"],
|
||||
rsync_extra_arg=["--delete"],
|
||||
rsync_bwlimit_kbps=4096,
|
||||
stdout=out,
|
||||
)
|
||||
|
||||
@@ -50,10 +50,12 @@ class ConfigureCommandsTests(TestCase):
|
||||
self.assertEqual(host.retention_daily, 5)
|
||||
self.assertEqual(host.excludes_add, ["/tmp/***"])
|
||||
self.assertEqual(host.rsync_extra_args, ["--delete"])
|
||||
self.assertEqual(host.rsync_bwlimit_kbps, 4096)
|
||||
|
||||
effective = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
self.assertEqual(effective["retention"]["yearly"], 2)
|
||||
self.assertEqual(effective["excludes_effective"], ["/tmp/***"])
|
||||
self.assertEqual(effective["rsync"]["bwlimit_kbps"], 4096)
|
||||
|
||||
def test_configure_schedule_creates_sql_schedule(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
@@ -62,7 +64,7 @@ class ConfigureCommandsTests(TestCase):
|
||||
call_command(
|
||||
"configure_pobsync_schedule",
|
||||
host.host,
|
||||
cron="15 2 * * *",
|
||||
schedule_expression="15 2 * * *",
|
||||
prune=True,
|
||||
stdout=out,
|
||||
)
|
||||
|
||||
@@ -5,10 +5,19 @@ from unittest.mock import patch
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync import __version__
|
||||
from pobsync.cli import main
|
||||
|
||||
|
||||
class ConsoleEntrypointTests(SimpleTestCase):
|
||||
def test_version_prints_package_version(self) -> None:
|
||||
stdout = StringIO()
|
||||
with patch("sys.stdout", stdout):
|
||||
exit_code = main(["--version"])
|
||||
|
||||
self.assertEqual(exit_code, 0)
|
||||
self.assertEqual(stdout.getvalue().strip(), f"pobsync {__version__}")
|
||||
|
||||
def test_maps_backup_alias_to_django_command(self) -> None:
|
||||
with patch("pobsync.cli.execute_from_command_line") as execute:
|
||||
exit_code = main(["backup", "web-01", "--dry-run"])
|
||||
@@ -31,15 +40,6 @@ class ConsoleEntrypointTests(SimpleTestCase):
|
||||
self.assertEqual(exit_code, 0)
|
||||
execute.assert_called_once_with(["pobsync", "check"])
|
||||
|
||||
def test_maps_schedule_alias_to_django_command(self) -> None:
|
||||
with patch("pobsync.cli.execute_from_command_line") as execute:
|
||||
exit_code = main(["schedule", "web-01", "--cron", "15 2 * * *"])
|
||||
|
||||
self.assertEqual(exit_code, 0)
|
||||
execute.assert_called_once_with(
|
||||
["pobsync", "configure_pobsync_schedule", "web-01", "--cron", "15 2 * * *"]
|
||||
)
|
||||
|
||||
def test_maps_discover_snapshots_alias_to_django_command(self) -> None:
|
||||
with patch("pobsync.cli.execute_from_command_line") as execute:
|
||||
exit_code = main(["discover-snapshots", "--host", "web-01"])
|
||||
@@ -53,3 +53,12 @@ class ConsoleEntrypointTests(SimpleTestCase):
|
||||
|
||||
self.assertEqual(exit_code, 0)
|
||||
execute.assert_called_once_with(["pobsync", "run_pobsync_worker", "--once"])
|
||||
|
||||
def test_configuration_aliases_are_not_public_commands(self) -> None:
|
||||
stderr = StringIO()
|
||||
with patch("sys.stderr", stderr):
|
||||
exit_code = main(["schedule", "web-01", "--cron", "15 2 * * *"])
|
||||
|
||||
self.assertEqual(exit_code, 2)
|
||||
self.assertIn("Unknown pobsync command", stderr.getvalue())
|
||||
self.assertIn("pobsync django <management-command>", stderr.getvalue())
|
||||
|
||||
@@ -15,35 +15,21 @@ class DjangoConfigSourceTests(TestCase):
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
rsync_args=["--archive"],
|
||||
rsync_extra_args=["--numeric-ids"],
|
||||
rsync_bwlimit_kbps=10000,
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
data={
|
||||
"backup_root": "/ignored",
|
||||
"pobsync_home": "/ignored",
|
||||
"ssh": {"user": "root", "port": 22, "options": []},
|
||||
"rsync": {
|
||||
"binary": "rsync",
|
||||
"args": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
"extra_args": ["--numeric-ids"],
|
||||
},
|
||||
"defaults": {"source_root": "/", "destination_subdir": ""},
|
||||
"excludes_default": ["/proc/***"],
|
||||
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
},
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
excludes_add=["/tmp/***"],
|
||||
rsync_extra_args=["--delete"],
|
||||
rsync_bwlimit_kbps=2500,
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
@@ -62,6 +48,24 @@ class DjangoConfigSourceTests(TestCase):
|
||||
self.assertEqual(cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"])
|
||||
self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"])
|
||||
self.assertEqual(cfg["rsync"]["bwlimit_kbps"], 2500)
|
||||
|
||||
def test_host_can_disable_global_rsync_bandwidth_limit(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
rsync_args=["--archive"],
|
||||
rsync_bwlimit_kbps=5000,
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
rsync_bwlimit_kbps=0,
|
||||
)
|
||||
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertEqual(cfg["rsync"]["bwlimit_kbps"], 0)
|
||||
|
||||
def test_materializes_global_ssh_credential_for_runtime_config(self) -> None:
|
||||
credential = SshCredential.objects.create(
|
||||
@@ -72,7 +76,6 @@ class DjangoConfigSourceTests(TestCase):
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
default_ssh_credential=credential,
|
||||
ssh_options=["-oBatchMode=yes"],
|
||||
)
|
||||
@@ -99,7 +102,6 @@ class DjangoConfigSourceTests(TestCase):
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
default_ssh_credential=global_credential,
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
@@ -127,12 +129,12 @@ class DjangoConfigSourceTests(TestCase):
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
default_ssh_credential=credential,
|
||||
)
|
||||
HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
with override_settings(POBSYNC_HOME=str(Path(tmp) / "home")):
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertIn(f"-oIdentityFile={identity_file}", cfg["ssh"]["options"])
|
||||
self.assertEqual(cfg["ssh_credential"]["storage"], "filesystem")
|
||||
@@ -146,7 +148,6 @@ class DjangoConfigSourceTests(TestCase):
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
default_ssh_credential=credential,
|
||||
)
|
||||
HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
125
src/pobsync_backend/tests/test_notifications.py
Normal file
125
src/pobsync_backend/tests/test_notifications.py
Normal file
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget
|
||||
from pobsync_backend.notifications import notify_backup_run_completed
|
||||
|
||||
|
||||
class NotificationTests(TestCase):
|
||||
def test_email_notification_is_sent_for_matching_status(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
statuses=[BackupRun.Status.FAILED],
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
status=BackupRun.Status.FAILED,
|
||||
run_type=BackupRun.RunType.MANUAL,
|
||||
started_at=timezone.now() - timedelta(minutes=5),
|
||||
ended_at=timezone.now(),
|
||||
rsync_exit_code=12,
|
||||
result={"failure": {"message": "rsync failed"}},
|
||||
)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertTrue(results[0].sent)
|
||||
send_mail.assert_called_once()
|
||||
subject, message, _from_email, recipients = send_mail.call_args.args
|
||||
self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}")
|
||||
self.assertIn("Failure: rsync failed", message)
|
||||
self.assertEqual(recipients, ["ops@example.test"])
|
||||
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
||||
self.assertEqual(delivery.status, NotificationDelivery.Status.SENT)
|
||||
target.refresh_from_db()
|
||||
self.assertEqual(target.last_status, NotificationDelivery.Status.SENT)
|
||||
self.assertEqual(target.last_error, "")
|
||||
self.assertIsNotNone(target.last_sent_at)
|
||||
|
||||
def test_webhook_notification_posts_payload(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="discord",
|
||||
channel=NotificationTarget.Channel.WEBHOOK,
|
||||
webhook_url="https://hooks.example.test/pobsync",
|
||||
webhook_headers={"X-Token": "secret"},
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED)
|
||||
response = Mock()
|
||||
response.status = 204
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
|
||||
with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen:
|
||||
notify_backup_run_completed(run)
|
||||
|
||||
request = urlopen.call_args.args[0]
|
||||
self.assertEqual(request.full_url, "https://hooks.example.test/pobsync")
|
||||
self.assertEqual(request.get_method(), "POST")
|
||||
self.assertEqual(request.headers["X-token"], "secret")
|
||||
self.assertIn(f'"id": {run.id}', request.data.decode("utf-8"))
|
||||
self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT)
|
||||
|
||||
def test_notification_filters_statuses(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
NotificationTarget.objects.create(
|
||||
name="failures-only",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
statuses=[BackupRun.Status.FAILED],
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail") as send_mail:
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(results, [])
|
||||
send_mail.assert_not_called()
|
||||
self.assertEqual(NotificationDelivery.objects.count(), 0)
|
||||
|
||||
def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
||||
notify_backup_run_completed(run)
|
||||
notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1)
|
||||
send_mail.assert_called_once()
|
||||
|
||||
def test_failed_delivery_is_recorded_without_raising(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="broken",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")):
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertFalse(results[0].sent)
|
||||
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
||||
self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED)
|
||||
self.assertEqual(delivery.error, "smtp down")
|
||||
target.refresh_from_db()
|
||||
self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED)
|
||||
self.assertEqual(target.last_error, "smtp down")
|
||||
@@ -7,6 +7,7 @@ from tempfile import TemporaryDirectory
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync.commands.retention_plan import run_retention_plan
|
||||
from pobsync.errors import ConfigError
|
||||
from pobsync.util import write_yaml_atomic
|
||||
|
||||
|
||||
@@ -24,6 +25,15 @@ class FakeConfigSource:
|
||||
|
||||
|
||||
class RetentionConfigSourceTests(SimpleTestCase):
|
||||
def test_retention_plan_requires_explicit_config_source(self) -> None:
|
||||
with self.assertRaisesMessage(ConfigError, "A Django config source is required."):
|
||||
run_retention_plan(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
)
|
||||
|
||||
def test_retention_plan_uses_injected_config_source(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
root = Path(tmp) / "backups"
|
||||
|
||||
@@ -39,12 +39,16 @@ class RunBackupRecordsSnapshotTests(TestCase):
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"verbose_output": True,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
call_command("run_pobsync_backup", host.host, prefix=str(Path(tmp) / "home"), stdout=StringIO())
|
||||
|
||||
run_scheduled.assert_called_once()
|
||||
self.assertTrue(run_scheduled.call_args.kwargs["verbose_output"])
|
||||
self.assertEqual(BackupRun.objects.count(), 1)
|
||||
run = BackupRun.objects.get()
|
||||
self.assertTrue(run.result["verbose_output"])
|
||||
self.assertEqual(SnapshotRecord.objects.count(), 1)
|
||||
record = SnapshotRecord.objects.get()
|
||||
self.assertEqual(run.snapshot, record)
|
||||
@@ -52,6 +56,45 @@ class RunBackupRecordsSnapshotTests(TestCase):
|
||||
self.assertEqual(record.kind, "scheduled")
|
||||
self.assertEqual(record.status, "success")
|
||||
|
||||
def test_backup_command_can_skip_default_verbose_rsync_output(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snapshot_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(
|
||||
meta_dir / "meta.yaml",
|
||||
{
|
||||
"status": "success",
|
||||
"started_at": "2026-05-19T02:15:00Z",
|
||||
"ended_at": "2026-05-19T02:16:00Z",
|
||||
},
|
||||
)
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
run_scheduled.return_value = {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"verbose_output": False,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
call_command(
|
||||
"run_pobsync_backup",
|
||||
host.host,
|
||||
prefix=str(Path(tmp) / "home"),
|
||||
quiet_rsync=True,
|
||||
stdout=StringIO(),
|
||||
)
|
||||
|
||||
run_scheduled.assert_called_once()
|
||||
self.assertFalse(run_scheduled.call_args.kwargs["verbose_output"])
|
||||
self.assertFalse(BackupRun.objects.get().result["verbose_output"])
|
||||
|
||||
def test_prune_uses_sql_retention_after_snapshot_record_is_created(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
@@ -96,13 +139,14 @@ class RunBackupRecordsSnapshotTests(TestCase):
|
||||
protect_bases=True,
|
||||
yes=True,
|
||||
max_delete=3,
|
||||
action=BackupRun.RunType.SCHEDULED,
|
||||
acquire_lock=False,
|
||||
)
|
||||
run = BackupRun.objects.get()
|
||||
self.assertEqual(run.status, BackupRun.Status.SUCCESS)
|
||||
self.assertEqual(run.result["prune"], {"ok": True, "source": "sql", "deleted": []})
|
||||
|
||||
def test_prune_failure_is_recorded_on_backup_run(self) -> None:
|
||||
def test_prune_failure_marks_backup_run_as_warning(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
@@ -128,19 +172,20 @@ class RunBackupRecordsSnapshotTests(TestCase):
|
||||
}
|
||||
retention_apply.side_effect = ConfigError("Deletion blocked by --max-delete=0")
|
||||
|
||||
with self.assertRaises(ConfigError):
|
||||
call_command(
|
||||
"run_pobsync_backup",
|
||||
host.host,
|
||||
prefix=str(Path(tmp) / "home"),
|
||||
prune=True,
|
||||
prune_max_delete=0,
|
||||
stdout=StringIO(),
|
||||
)
|
||||
output = StringIO()
|
||||
call_command(
|
||||
"run_pobsync_backup",
|
||||
host.host,
|
||||
prefix=str(Path(tmp) / "home"),
|
||||
prune=True,
|
||||
prune_max_delete=0,
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
run = BackupRun.objects.get()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.status, BackupRun.Status.WARNING)
|
||||
self.assertIsNotNone(run.snapshot)
|
||||
self.assertIn("completed with warnings", output.getvalue())
|
||||
self.assertEqual(run.result["prune"]["ok"], False)
|
||||
self.assertEqual(run.result["prune"]["type"], "ConfigError")
|
||||
self.assertEqual(run.result["prune"]["error"], "Deletion blocked by --max-delete=0")
|
||||
|
||||
@@ -7,12 +7,14 @@ from unittest.mock import patch
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.errors import ConfigError
|
||||
from pobsync.rsync import RsyncResult
|
||||
|
||||
|
||||
class FakeConfigSource:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups", bwlimit_kbps: int = 0) -> None:
|
||||
self.backup_root = backup_root
|
||||
self.bwlimit_kbps = bwlimit_kbps
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict:
|
||||
return {
|
||||
@@ -24,7 +26,7 @@ class FakeConfigSource:
|
||||
"binary": "rsync",
|
||||
"args_effective": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
"bwlimit_kbps": self.bwlimit_kbps,
|
||||
},
|
||||
"source_root": "/",
|
||||
"includes": [],
|
||||
@@ -34,6 +36,10 @@ class FakeConfigSource:
|
||||
|
||||
|
||||
class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
def test_requires_explicit_config_source(self) -> None:
|
||||
with self.assertRaisesMessage(ConfigError, "A Django config source is required."):
|
||||
run_scheduled(prefix=Path("/missing-prefix"), host="web-01", dry_run=True)
|
||||
|
||||
def test_dry_run_uses_injected_config_source(self) -> None:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
|
||||
@@ -49,6 +55,21 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
self.assertEqual(result["host"], "web-01")
|
||||
run_rsync.assert_called_once()
|
||||
|
||||
def test_dry_run_applies_configured_bandwidth_limit(self) -> None:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--bwlimit=4096"])
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
dry_run=True,
|
||||
config_source=FakeConfigSource(bwlimit_kbps=4096),
|
||||
)
|
||||
|
||||
command = run_rsync.call_args.args[0]
|
||||
self.assertIn("--bwlimit=4096", command)
|
||||
self.assertEqual(result["rsync"]["bwlimit_kbps"], 4096)
|
||||
|
||||
def test_failed_dry_run_includes_log_tail(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -181,11 +202,13 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
verbose_output=True,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups"), bwlimit_kbps=2048),
|
||||
)
|
||||
|
||||
command = run_rsync.call_args.args[0]
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertIn("--bwlimit=2048", command)
|
||||
self.assertEqual(result["rsync"]["bwlimit_kbps"], 2048)
|
||||
self.assertIn("--stats", command)
|
||||
self.assertIn("--itemize-changes", command)
|
||||
self.assertIn("--info=flist2,progress2,stats2", command)
|
||||
@@ -242,6 +265,7 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
meta_text = meta_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["log"], str(Path(result["snapshot"]) / "meta" / "rsync.log"))
|
||||
self.assertEqual(result["stats"]["rsync"]["files_total"], 10)
|
||||
self.assertEqual(result["stats"]["rsync"]["files_transferred"], 2)
|
||||
self.assertEqual(result["stats"]["rsync"]["link_dest_estimated_savings_bytes"], 1500)
|
||||
@@ -250,6 +274,71 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
self.assertIn("stats:", meta_text)
|
||||
self.assertIn("files_total: 10", meta_text)
|
||||
|
||||
def test_real_run_reports_running_state_callback_before_rsync_returns(self) -> None:
|
||||
states = []
|
||||
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None, process_started=None):
|
||||
self.assertEqual(len(states), 1)
|
||||
self.assertEqual(states[0]["status"], "running")
|
||||
self.assertEqual(states[0]["phase"], "preparing")
|
||||
self.assertEqual(states[0]["log"], str(log_path))
|
||||
self.assertEqual(states[0]["rsync"]["command"], command)
|
||||
self.assertIsNotNone(process_started)
|
||||
process_started(1234, 1234)
|
||||
self.assertEqual(len(states), 2)
|
||||
self.assertEqual(states[1]["phase"], "rsync")
|
||||
self.assertEqual(states[1]["rsync"]["pid"], 1234)
|
||||
self.assertEqual(states[1]["rsync"]["pgid"], 1234)
|
||||
log_path.write_text("Number of files: 1\n", encoding="utf-8")
|
||||
return RsyncResult(exit_code=0, command=command)
|
||||
|
||||
with TemporaryDirectory() as tmp:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
|
||||
run_scheduled(
|
||||
prefix=Path(tmp) / "home",
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
state_callback=states.append,
|
||||
)
|
||||
|
||||
self.assertEqual(len(states), 3)
|
||||
self.assertIn("/.incomplete/", states[0]["snapshot"])
|
||||
self.assertEqual(states[2]["phase"], "finalizing")
|
||||
self.assertEqual(states[2]["rsync"]["exit_code"], 0)
|
||||
|
||||
def test_real_run_keeps_snapshot_with_warning_for_vanished_files(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
log_path.write_text(
|
||||
"file has vanished: \"/var/lib/app/session\"\n"
|
||||
"rsync warning: some files vanished before they could be transferred (code 24) at main.c(1338) [sender=3.4.1]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
data_dir = log_path.parent.parent / "data"
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
(data_dir / "payload.txt").write_text("payload", encoding="utf-8")
|
||||
return RsyncResult(exit_code=24, command=command)
|
||||
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
|
||||
result = run_scheduled(
|
||||
prefix=Path(tmp) / "home",
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
config_source=FakeConfigSource(backup_root=str(backup_root)),
|
||||
)
|
||||
|
||||
snapshot = Path(result["snapshot"])
|
||||
self.assertTrue((snapshot / "data" / "payload.txt").exists())
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["status"], "warning")
|
||||
self.assertEqual(result["rsync"]["exit_code"], 24)
|
||||
self.assertEqual(result["warning"]["category"], "vanished")
|
||||
self.assertEqual(snapshot.parent.name, "scheduled")
|
||||
self.assertIn("file has vanished", "\n".join(result["rsync"]["log_tail"]))
|
||||
|
||||
def test_dry_run_reports_cancelled_rsync(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
self.assertTrue(cancel_check())
|
||||
|
||||
@@ -8,7 +8,7 @@ from zoneinfo import ZoneInfo
|
||||
from django.test import SimpleTestCase, TestCase
|
||||
|
||||
from pobsync_backend.management.commands.run_pobsync_scheduler import Command
|
||||
from pobsync_backend.models import HostConfig, ScheduleConfig
|
||||
from pobsync_backend.models import BackupRun, HostConfig, ScheduleConfig
|
||||
from pobsync_backend.scheduler import due_key, is_due, next_due_after
|
||||
|
||||
|
||||
@@ -64,3 +64,30 @@ class SchedulerCommandTests(TestCase):
|
||||
self.assertEqual(call.call_count, 1)
|
||||
schedule = ScheduleConfig.objects.get(host=host)
|
||||
self.assertEqual(schedule.last_status, "success")
|
||||
|
||||
def test_run_due_records_warning_status_from_scheduled_backup_run(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
ScheduleConfig.objects.create(host=host, cron_expr="* * * * *", prune=True, prune_max_delete=1)
|
||||
|
||||
def create_warning_run(*args, **kwargs) -> None:
|
||||
BackupRun.objects.create(
|
||||
host=host,
|
||||
run_type=BackupRun.RunType.SCHEDULED,
|
||||
status=BackupRun.Status.WARNING,
|
||||
result={
|
||||
"ok": True,
|
||||
"prune": {
|
||||
"ok": False,
|
||||
"type": "ConfigError",
|
||||
"error": "Refusing to delete 2 snapshots (exceeds --max-delete=1)",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
command = Command()
|
||||
with patch("pobsync_backend.management.commands.run_pobsync_scheduler.call_command", side_effect=create_warning_run):
|
||||
count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=False)
|
||||
|
||||
self.assertEqual(count, 1)
|
||||
schedule = ScheduleConfig.objects.get(host=host)
|
||||
self.assertEqual(schedule.last_status, "warning")
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
from django.test import SimpleTestCase, override_settings
|
||||
|
||||
from pobsync_backend.self_check import _systemd_checks
|
||||
from pobsync_backend.self_check import SelfCheck, _install_checks, _sqlite_database_check, _systemd_checks
|
||||
|
||||
|
||||
class SystemdSelfCheckTests(SimpleTestCase):
|
||||
@@ -40,3 +45,92 @@ class SystemdSelfCheckTests(SimpleTestCase):
|
||||
journal_check = next(check for check in checks if check.name == "Journal access")
|
||||
self.assertEqual(journal_check.status, "failed")
|
||||
self.assertEqual(journal_check.message, "pobsync cannot read service logs.")
|
||||
|
||||
|
||||
class InstallSelfCheckTests(SimpleTestCase):
|
||||
def test_install_checks_skip_native_paths_in_development_runtime(self) -> None:
|
||||
with override_settings(POBSYNC_ENV_FILE="/missing/pobsync.env"), patch(
|
||||
"pobsync_backend.self_check._native_runtime_available",
|
||||
return_value=False,
|
||||
):
|
||||
checks = _install_checks()
|
||||
|
||||
self.assertEqual([check.status for check in checks], ["skipped", "skipped", "skipped"])
|
||||
self.assertEqual(checks[0].name, "Environment file")
|
||||
self.assertEqual(checks[1].name, "Service user")
|
||||
self.assertEqual(checks[2].name, "Backup root owner")
|
||||
|
||||
def test_service_user_warns_when_current_user_differs(self) -> None:
|
||||
with override_settings(
|
||||
POBSYNC_ENV_FILE="/etc/pobsync/pobsync.env",
|
||||
POBSYNC_SERVICE_USER="pobsync",
|
||||
POBSYNC_BACKUP_ROOT="/backups",
|
||||
), patch("pobsync_backend.self_check._native_runtime_available", return_value=True), patch(
|
||||
"pobsync_backend.self_check._env_file_check",
|
||||
return_value=SelfCheck("Environment file", "ok", "/etc/pobsync/pobsync.env"),
|
||||
), patch(
|
||||
"pobsync_backend.self_check._backup_root_owner_check",
|
||||
return_value=SelfCheck("Backup root owner", "ok", "/backups owner=pobsync"),
|
||||
), patch(
|
||||
"pobsync_backend.self_check.os.geteuid",
|
||||
return_value=0,
|
||||
), patch(
|
||||
"pobsync_backend.self_check.pwd.getpwuid",
|
||||
) as getpwuid:
|
||||
getpwuid.return_value.pw_name = "root"
|
||||
checks = _install_checks()
|
||||
|
||||
service_user_check = next(check for check in checks if check.name == "Service user")
|
||||
self.assertEqual(service_user_check.status, "warning")
|
||||
self.assertIn("expected pobsync", service_user_check.message)
|
||||
|
||||
def test_sqlite_database_check_reports_existing_database(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "pobsync.sqlite3"
|
||||
db_path.write_text("", encoding="utf-8")
|
||||
|
||||
check = _sqlite_database_check(db_path)
|
||||
|
||||
self.assertEqual(check.status, "ok")
|
||||
self.assertEqual(check.name, "SQLite database")
|
||||
|
||||
|
||||
class CheckPobsyncInstallCommandTests(SimpleTestCase):
|
||||
def test_command_prints_summary_for_successful_checks(self) -> None:
|
||||
stdout = StringIO()
|
||||
stderr = StringIO()
|
||||
checks = [
|
||||
SelfCheck("Database connection", "ok", "django.db.backends.sqlite3"),
|
||||
SelfCheck("Systemd services", "skipped", "systemd is not available in this runtime."),
|
||||
]
|
||||
|
||||
with patch("pobsync_backend.management.commands.check_pobsync_install.collect_self_checks", return_value=checks):
|
||||
call_command("check_pobsync_install", stdout=stdout, stderr=stderr)
|
||||
|
||||
self.assertIn("[OK] Database connection", stdout.getvalue())
|
||||
self.assertIn("[SKIPPED] Systemd services", stdout.getvalue())
|
||||
self.assertIn("Summary: 1 ok, 0 warning(s), 0 failed, 1 skipped", stdout.getvalue())
|
||||
self.assertEqual(stderr.getvalue(), "")
|
||||
|
||||
def test_command_fails_when_checks_fail(self) -> None:
|
||||
stdout = StringIO()
|
||||
stderr = StringIO()
|
||||
checks = [
|
||||
SelfCheck("POBSYNC_BACKUP_ROOT", "failed", "/backups does not exist."),
|
||||
]
|
||||
|
||||
with patch("pobsync_backend.management.commands.check_pobsync_install.collect_self_checks", return_value=checks):
|
||||
with self.assertRaisesMessage(CommandError, "pobsync install self check failed."):
|
||||
call_command("check_pobsync_install", stdout=stdout, stderr=stderr)
|
||||
|
||||
self.assertIn("[FAILED] POBSYNC_BACKUP_ROOT", stderr.getvalue())
|
||||
self.assertIn("Summary: 0 ok, 0 warning(s), 1 failed, 0 skipped", stdout.getvalue())
|
||||
|
||||
def test_command_can_fail_on_warnings(self) -> None:
|
||||
checks = [
|
||||
SelfCheck("Global config", "warning", "Default global config has not been created yet."),
|
||||
]
|
||||
|
||||
with patch("pobsync_backend.management.commands.check_pobsync_install.collect_self_checks", return_value=checks):
|
||||
with self.assertRaisesMessage(CommandError, "pobsync install self check reported warnings."):
|
||||
call_command("check_pobsync_install", "--fail-on-warning", stdout=StringIO(), stderr=StringIO())
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import stat
|
||||
from datetime import datetime, timezone
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
@@ -10,8 +11,8 @@ from django.core.management import call_command
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync.errors import ConfigError
|
||||
from pobsync_backend.models import HostConfig, SnapshotRecord
|
||||
from pobsync_backend.retention import run_sql_retention_apply, run_sql_retention_plan
|
||||
from pobsync_backend.models import HostConfig, PurgedSnapshot, SnapshotRecord
|
||||
from pobsync_backend.retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan
|
||||
|
||||
|
||||
class SqlRetentionTests(TestCase):
|
||||
@@ -31,7 +32,10 @@ class SqlRetentionTests(TestCase):
|
||||
|
||||
self.assertEqual(plan["source"], "sql")
|
||||
self.assertEqual(plan["keep"], [new.dirname])
|
||||
self.assertEqual([item["dirname"] for item in plan["keep_items"]], [new.dirname])
|
||||
self.assertEqual([item["dirname"] for item in plan["delete"]], [old.dirname])
|
||||
self.assertEqual(plan["delete"][0]["reason"], "outside retention policy")
|
||||
self.assertEqual(plan["incomplete"], [])
|
||||
|
||||
def test_plan_can_protect_base_snapshot_from_sql_relation(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
@@ -83,7 +87,134 @@ class SqlRetentionTests(TestCase):
|
||||
self.assertTrue(new_dir.exists())
|
||||
self.assertTrue(SnapshotRecord.objects.filter(pk=new.pk).exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
|
||||
self.assertEqual(result["deleted"], [{"dirname": old.dirname, "kind": "scheduled", "path": str(old_dir)}])
|
||||
self.assertEqual(
|
||||
result["deleted"],
|
||||
[
|
||||
{
|
||||
"dirname": old.dirname,
|
||||
"kind": "scheduled",
|
||||
"path": str(old_dir),
|
||||
"reason": "outside retention policy",
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertEqual(result["planned_delete_count"], 1)
|
||||
self.assertEqual(result["max_delete"], 1)
|
||||
self.assertEqual(result["incomplete_ignored_count"], 0)
|
||||
purged = PurgedSnapshot.objects.get(dirname=old.dirname)
|
||||
self.assertEqual(purged.host_name, host.host)
|
||||
self.assertEqual(purged.kind, "scheduled")
|
||||
self.assertEqual(purged.path, str(old_dir))
|
||||
self.assertEqual(purged.reason, "outside retention policy")
|
||||
self.assertEqual(purged.action, PurgedSnapshot.Action.MANUAL)
|
||||
|
||||
def test_apply_deletes_snapshot_with_readonly_data_directory(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
retention_daily=0,
|
||||
retention_weekly=0,
|
||||
retention_monthly=0,
|
||||
retention_yearly=0,
|
||||
)
|
||||
old_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260518-021500Z__OLD"
|
||||
old_data = old_dir / "data"
|
||||
old_data.mkdir(parents=True)
|
||||
old_data.joinpath("etc").mkdir()
|
||||
old_data.joinpath("etc", "config").write_text("preserved permissions\n")
|
||||
old_data.chmod(stat.S_IREAD | stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
|
||||
new_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260519-021500Z__NEW"
|
||||
new_dir.mkdir(parents=True)
|
||||
old = self._snapshot(host, old_dir.name, path=str(old_data))
|
||||
self._snapshot(host, new_dir.name, path=str(new_dir))
|
||||
|
||||
result = run_sql_retention_apply(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(old_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
|
||||
self.assertEqual(
|
||||
result["deleted"],
|
||||
[
|
||||
{
|
||||
"dirname": old.dirname,
|
||||
"kind": "scheduled",
|
||||
"path": str(old_dir),
|
||||
"reason": "outside retention policy",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_apply_deletes_snapshot_with_non_traversable_nested_directory(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
retention_daily=0,
|
||||
retention_weekly=0,
|
||||
retention_monthly=0,
|
||||
retention_yearly=0,
|
||||
)
|
||||
old_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260518-021500Z__OLD"
|
||||
restricted_dir = old_dir / "data" / "var" / "lib" / "snapd" / "void"
|
||||
restricted_dir.mkdir(parents=True)
|
||||
restricted_dir.joinpath("state").write_text("preserved permissions\n")
|
||||
restricted_dir.chmod(0)
|
||||
new_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260519-021500Z__NEW"
|
||||
new_dir.mkdir(parents=True)
|
||||
old = self._snapshot(host, old_dir.name, path=str(old_dir))
|
||||
self._snapshot(host, new_dir.name, path=str(new_dir))
|
||||
|
||||
result = run_sql_retention_apply(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(old_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
|
||||
self.assertEqual(result["deleted"][0]["dirname"], old.dirname)
|
||||
|
||||
def test_apply_rejects_scheduled_snapshot_path_outside_host_kind_directory(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
retention_daily=0,
|
||||
retention_weekly=0,
|
||||
retention_monthly=0,
|
||||
retention_yearly=0,
|
||||
)
|
||||
self._snapshot(
|
||||
host,
|
||||
"20260518-021500Z__OLD",
|
||||
path="/backups/web-01/manual/20260518-021500Z__OLD",
|
||||
)
|
||||
self._snapshot(host, "20260519-021500Z__NEW")
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "unexpected snapshot path"):
|
||||
run_sql_retention_apply(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_apply_respects_max_delete(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
@@ -109,6 +240,139 @@ class SqlRetentionTests(TestCase):
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_incomplete_cleanup_deletes_directory_and_record(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
incomplete_dir = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-031500Z__BROKEN01"
|
||||
incomplete_dir.mkdir(parents=True)
|
||||
incomplete_dir.joinpath("partial-file").write_text("interrupted\n")
|
||||
record = SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
result = run_incomplete_cleanup(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(incomplete_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=record.pk).exists())
|
||||
self.assertEqual(
|
||||
result["deleted"],
|
||||
[{"dirname": incomplete_dir.name, "kind": SnapshotRecord.Kind.INCOMPLETE, "path": str(incomplete_dir)}],
|
||||
)
|
||||
self.assertEqual(result["planned_delete_count"], 1)
|
||||
purged = PurgedSnapshot.objects.get(dirname=incomplete_dir.name)
|
||||
self.assertEqual(purged.action, PurgedSnapshot.Action.INCOMPLETE_CLEANUP)
|
||||
self.assertEqual(purged.reason, "manual incomplete cleanup")
|
||||
|
||||
def test_incomplete_cleanup_deletes_non_traversable_nested_directory(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
incomplete_dir = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-031500Z__BROKEN01"
|
||||
restricted_dir = incomplete_dir / "data" / "var" / "lib" / "snapd" / "void"
|
||||
restricted_dir.mkdir(parents=True)
|
||||
restricted_dir.joinpath("state").write_text("interrupted\n")
|
||||
restricted_dir.chmod(0)
|
||||
record = SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
result = run_incomplete_cleanup(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(incomplete_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=record.pk).exists())
|
||||
self.assertEqual(result["deleted"][0]["dirname"], incomplete_dir.name)
|
||||
|
||||
def test_incomplete_cleanup_requires_reviewed_snapshots(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "have not been reviewed"):
|
||||
run_incomplete_cleanup(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_incomplete_cleanup_respects_max_delete(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "blocked by --max-delete=0"):
|
||||
run_incomplete_cleanup(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=0,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_incomplete_cleanup_rejects_unexpected_path(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/scheduled/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "unexpected incomplete snapshot path"):
|
||||
run_incomplete_cleanup(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_management_command_plans_from_sql(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
|
||||
195
src/pobsync_backend/tests/test_stats_summary.py
Normal file
195
src/pobsync_backend/tests/test_stats_summary.py
Normal file
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync.run_stats import tree_usage
|
||||
from pobsync_backend.models import HostConfig, SnapshotRecord
|
||||
from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats
|
||||
|
||||
|
||||
class StatsSummaryTests(TestCase):
|
||||
def test_collect_dashboard_stats_sums_backup_data_across_hosts(self) -> None:
|
||||
web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
db = HostConfig.objects.create(host="db-01", address="db-01.example.test")
|
||||
self._snapshot(web, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
||||
self._snapshot(web, "20260519-031500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=200)
|
||||
self._snapshot(db, "20260519-041500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=300)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_usage = self._incomplete_snapshot_on_disk(
|
||||
db,
|
||||
Path(tmp),
|
||||
"20260519-051500Z__BROKEN1",
|
||||
)
|
||||
|
||||
stats = collect_dashboard_stats(hosts=[web, db], global_config=None)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 400)
|
||||
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 200)
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
||||
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 600 + incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 1200 + incomplete_usage["apparent_size_bytes"])
|
||||
|
||||
def test_collect_host_stats_sums_backup_data_by_snapshot_kind(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
self._snapshot(host, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
||||
self._snapshot(host, "20260519-031500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=200)
|
||||
self._snapshot(host, "20260519-041500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=300)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_usage = self._incomplete_snapshot_on_disk(
|
||||
host,
|
||||
Path(tmp),
|
||||
"20260519-051500Z__BROKEN1",
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 300)
|
||||
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 300)
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
||||
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 600 + incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 1200 + incomplete_usage["apparent_size_bytes"])
|
||||
|
||||
def test_collect_host_stats_falls_back_to_filesystem_usage_for_snapshots_without_metadata(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-051500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
meta_dir = incomplete_dir / "meta"
|
||||
data_dir.mkdir(parents=True)
|
||||
meta_dir.mkdir()
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
meta_dir.joinpath("rsync.log").write_text("not part of the backup data total\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
metadata={},
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["count"], 1)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["apparent_size_bytes"],
|
||||
expected_usage["apparent_size_bytes"],
|
||||
)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["total"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
|
||||
def test_collect_host_stats_measures_incomplete_data_from_disk_even_with_stale_metadata(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-051500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
metadata={
|
||||
"stats": {
|
||||
"storage": {
|
||||
"snapshot": {
|
||||
"apparent_size_bytes": 0,
|
||||
"allocated_size_bytes": 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
self.assertGreater(stats["backup_data"]["incomplete"]["apparent_size_bytes"], 0)
|
||||
|
||||
def test_collect_host_stats_reports_non_hardlinked_snapshot_data(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
self._snapshot_with_sizes(
|
||||
host,
|
||||
"20260519-021500Z__SCHED01",
|
||||
SnapshotRecord.Kind.SCHEDULED,
|
||||
allocated=1_200,
|
||||
apparent=2_000,
|
||||
hardlinked_apparent=1_500,
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["apparent_size_bytes"], 2_000)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["unique_apparent_size_bytes"], 500)
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 500)
|
||||
|
||||
def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord:
|
||||
return self._snapshot_with_sizes(host, dirname, kind, allocated=allocated)
|
||||
|
||||
def _incomplete_snapshot_on_disk(self, host: HostConfig, root: Path, dirname: str) -> dict:
|
||||
incomplete_dir = root / host.host / ".incomplete" / dirname
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=dirname,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
)
|
||||
return usage
|
||||
|
||||
def _snapshot_with_sizes(
|
||||
self,
|
||||
host: HostConfig,
|
||||
dirname: str,
|
||||
kind: str,
|
||||
*,
|
||||
allocated: int,
|
||||
apparent: int | None = None,
|
||||
hardlinked_apparent: int = 0,
|
||||
) -> SnapshotRecord:
|
||||
started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc)
|
||||
apparent_size = apparent if apparent is not None else allocated * 2
|
||||
return SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=kind,
|
||||
dirname=dirname,
|
||||
path=f"/backups/{host.host}/{kind}/{dirname}",
|
||||
status="success",
|
||||
started_at=started_at,
|
||||
metadata={
|
||||
"stats": {
|
||||
"storage": {
|
||||
"snapshot": {
|
||||
"apparent_size_bytes": apparent_size,
|
||||
"allocated_size_bytes": allocated,
|
||||
"hardlinked_apparent_size_bytes": hardlinked_apparent,
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
72
src/pobsync_backend/tests/test_updater.py
Normal file
72
src/pobsync_backend/tests/test_updater.py
Normal file
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from django.test import SimpleTestCase, override_settings
|
||||
|
||||
from pobsync_backend import updater
|
||||
|
||||
|
||||
class UpdaterTests(SimpleTestCase):
|
||||
@override_settings(
|
||||
POBSYNC_UPDATE_RELEASES_URL="https://code.example.test/api/v1/repos/owner/pobsync/releases",
|
||||
POBSYNC_UPDATE_RELEASES_TOKEN="secret",
|
||||
)
|
||||
def test_fetch_latest_release_reads_first_gitea_release(self) -> None:
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = json.dumps(
|
||||
[
|
||||
{
|
||||
"tag_name": "v1.2.0",
|
||||
"name": "1.2.0",
|
||||
"html_url": "https://code.example.test/releases/v1.2.0",
|
||||
}
|
||||
]
|
||||
).encode("utf-8")
|
||||
|
||||
with patch("pobsync_backend.updater.urlopen", return_value=response) as urlopen:
|
||||
release = updater.fetch_latest_release()
|
||||
|
||||
self.assertEqual(release["tag_name"], "v1.2.0")
|
||||
request = urlopen.call_args.args[0]
|
||||
self.assertEqual(request.full_url, "https://code.example.test/api/v1/repos/owner/pobsync/releases")
|
||||
self.assertEqual(request.headers["Authorization"], "token secret")
|
||||
|
||||
@override_settings(POBSYNC_UPDATE_RELEASES_URL="")
|
||||
def test_collect_update_status_reports_unconfigured_release_check(self) -> None:
|
||||
with patch("pobsync_backend.updater._git_status", return_value={"branch": "master"}):
|
||||
status = updater.collect_update_status(check_release=True)
|
||||
|
||||
self.assertFalse(status["release_check_configured"])
|
||||
self.assertEqual(status["release_error"], "POBSYNC_UPDATE_RELEASES_URL is not configured.")
|
||||
self.assertIsNone(status["update_available"])
|
||||
|
||||
@override_settings(POBSYNC_UPDATE_GIT_REMOTE="upstream")
|
||||
def test_run_git_fetch_uses_configured_remote(self) -> None:
|
||||
completed = MagicMock(returncode=0, stdout="ok", stderr="")
|
||||
with patch("pobsync_backend.updater.subprocess.run", return_value=completed) as run:
|
||||
result = updater.run_git_fetch()
|
||||
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(result.command, ["git", "fetch", "--tags", "--prune", "upstream"])
|
||||
run.assert_called_once()
|
||||
|
||||
@override_settings(POBSYNC_UPDATE_GIT_REMOTE="origin")
|
||||
def test_run_git_pull_rejects_detached_checkout(self) -> None:
|
||||
with patch("pobsync_backend.updater._git_current_branch", return_value=""):
|
||||
result = updater.run_git_pull()
|
||||
|
||||
self.assertFalse(result.ok)
|
||||
self.assertEqual(result.exit_code, 2)
|
||||
self.assertIn("not on a branch", result.stderr)
|
||||
|
||||
@override_settings(POBSYNC_UPDATE_COMMAND="sudo -n scripts/update-systemd --verbose")
|
||||
def test_run_native_update_splits_configured_command(self) -> None:
|
||||
completed = MagicMock(returncode=1, stdout="", stderr="sudo failed")
|
||||
with patch("pobsync_backend.updater.subprocess.run", return_value=completed):
|
||||
result = updater.run_native_update()
|
||||
|
||||
self.assertFalse(result.ok)
|
||||
self.assertEqual(result.command, ["sudo", "-n", "scripts/update-systemd", "--verbose"])
|
||||
self.assertEqual(result.stderr, "sudo failed")
|
||||
File diff suppressed because it is too large
Load Diff
149
src/pobsync_backend/updater.py
Normal file
149
src/pobsync_backend/updater.py
Normal file
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from pobsync import __version__
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommandResult:
|
||||
command: list[str]
|
||||
exit_code: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.exit_code == 0
|
||||
|
||||
|
||||
def collect_update_status(*, check_release: bool = False) -> dict[str, Any]:
|
||||
app_dir = Path(settings.BASE_DIR)
|
||||
status: dict[str, Any] = {
|
||||
"app_dir": app_dir,
|
||||
"installed_version": __version__,
|
||||
"release_check_configured": bool(settings.POBSYNC_UPDATE_RELEASES_URL),
|
||||
"update_command": settings.POBSYNC_UPDATE_COMMAND,
|
||||
"git_remote": settings.POBSYNC_UPDATE_GIT_REMOTE,
|
||||
"git": _git_status(app_dir),
|
||||
"latest_release": None,
|
||||
"release_error": "",
|
||||
"update_available": None,
|
||||
}
|
||||
|
||||
if check_release:
|
||||
if not settings.POBSYNC_UPDATE_RELEASES_URL:
|
||||
status["release_error"] = "POBSYNC_UPDATE_RELEASES_URL is not configured."
|
||||
else:
|
||||
try:
|
||||
latest_release = fetch_latest_release()
|
||||
status["latest_release"] = latest_release
|
||||
status["update_available"] = _version_key(latest_release.get("tag_name", "")) != _version_key(__version__)
|
||||
except (HTTPError, URLError, TimeoutError, json.JSONDecodeError, ValueError) as exc:
|
||||
status["release_error"] = str(exc)
|
||||
|
||||
return status
|
||||
|
||||
|
||||
def fetch_latest_release() -> dict[str, Any]:
|
||||
request = Request(settings.POBSYNC_UPDATE_RELEASES_URL, headers={"Accept": "application/json"})
|
||||
if settings.POBSYNC_UPDATE_RELEASES_TOKEN:
|
||||
request.add_header("Authorization", f"token {settings.POBSYNC_UPDATE_RELEASES_TOKEN}")
|
||||
|
||||
with urlopen(request, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
if isinstance(payload, list):
|
||||
if not payload:
|
||||
raise ValueError("No releases were returned.")
|
||||
release = payload[0]
|
||||
elif isinstance(payload, dict):
|
||||
release = payload
|
||||
else:
|
||||
raise ValueError("Release endpoint returned an unexpected payload.")
|
||||
|
||||
if not isinstance(release, dict):
|
||||
raise ValueError("Release endpoint returned an unexpected release entry.")
|
||||
return release
|
||||
|
||||
|
||||
def run_git_fetch() -> CommandResult:
|
||||
remote = settings.POBSYNC_UPDATE_GIT_REMOTE
|
||||
return _run_command(["git", "fetch", "--tags", "--prune", remote])
|
||||
|
||||
|
||||
def run_git_pull() -> CommandResult:
|
||||
remote = settings.POBSYNC_UPDATE_GIT_REMOTE
|
||||
branch = _git_current_branch(Path(settings.BASE_DIR))
|
||||
if not branch:
|
||||
return CommandResult(
|
||||
command=["git", "pull", "--ff-only", remote],
|
||||
exit_code=2,
|
||||
stdout="",
|
||||
stderr="Cannot pull automatically because the installed checkout is not on a branch.",
|
||||
)
|
||||
return _run_command(["git", "pull", "--ff-only", remote, branch])
|
||||
|
||||
|
||||
def run_native_update() -> CommandResult:
|
||||
return _run_command(shlex.split(settings.POBSYNC_UPDATE_COMMAND))
|
||||
|
||||
|
||||
def _run_command(command: list[str]) -> CommandResult:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=settings.BASE_DIR,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
return CommandResult(
|
||||
command=command,
|
||||
exit_code=completed.returncode,
|
||||
stdout=completed.stdout[-6000:],
|
||||
stderr=completed.stderr[-6000:],
|
||||
)
|
||||
|
||||
|
||||
def _git_status(app_dir: Path) -> dict[str, str]:
|
||||
return {
|
||||
"branch": _git_current_branch(app_dir),
|
||||
"commit": _git_output(app_dir, ["git", "rev-parse", "--short", "HEAD"]),
|
||||
"describe": _git_output(app_dir, ["git", "describe", "--tags", "--always", "--dirty"]),
|
||||
}
|
||||
|
||||
|
||||
def _git_current_branch(app_dir: Path) -> str:
|
||||
branch = _git_output(app_dir, ["git", "branch", "--show-current"])
|
||||
return branch or _git_output(app_dir, ["git", "rev-parse", "--abbrev-ref", "HEAD"])
|
||||
|
||||
|
||||
def _git_output(app_dir: Path, command: list[str]) -> str:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=app_dir,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
if completed.returncode != 0:
|
||||
return ""
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def _version_key(value: str) -> str:
|
||||
return value.strip().removeprefix("v")
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user