Compare commits
26 Commits
v1.1.0
...
41ceab5a40
| Author | SHA1 | Date | |
|---|---|---|---|
| 41ceab5a40 | |||
| 2ad119e214 | |||
| eb121453c8 | |||
| 67ffd6101b | |||
| 1f5c4e0756 | |||
| b5e87abad2 | |||
| fc6df89370 | |||
| 3893df4640 | |||
| f86c67aeee | |||
| 7dc4c1df84 | |||
| 10e0293559 | |||
| 9dd690bb3b | |||
| 8740b75841 | |||
| ce1cb9d157 | |||
| 8e83fee7b5 | |||
| a6d6468da8 | |||
| b87203c538 | |||
| 515330c436 | |||
| fdf401a0be | |||
| 3b77f2e5d0 | |||
| df9ec5b04c | |||
| 5788f53854 | |||
| 28da9c4096 | |||
| 6eb1b4add3 | |||
| 8633cbea26 | |||
| 3fb8209aef |
13
README.md
13
README.md
@@ -154,6 +154,19 @@ The UI includes:
|
||||
- `/self-check/` for runtime checks
|
||||
- `/logs/` for filtered pobsync service logs
|
||||
|
||||
## Bandwidth Limits
|
||||
|
||||
Global config can set an rsync bandwidth limit in KB/s. The default `0` means unlimited. Each host can inherit the
|
||||
global value, set `0` to explicitly run unlimited, or set its own limit for slower remote links.
|
||||
|
||||
For VPN-backed or remote backups, start conservatively and adjust after watching normal traffic:
|
||||
|
||||
- `2500` KB/s is roughly 20 Mbit/s
|
||||
- `5000` KB/s is roughly 40 Mbit/s
|
||||
- `10000` KB/s is roughly 80 Mbit/s
|
||||
|
||||
pobsync passes the effective value to rsync as `--bwlimit=<KB/s>` and shows it on the host detail and run detail pages.
|
||||
|
||||
## Restoring Data
|
||||
|
||||
pobsync treats restores as an explicit manual operation. The control panel shows restore guidance on each snapshot
|
||||
|
||||
@@ -23,6 +23,7 @@ from ..util import ensure_dir, realpath_startswith, sanitize_host, write_yaml_at
|
||||
|
||||
|
||||
DEFAULT_DRY_RUN_TIMEOUT_SECONDS = 900
|
||||
RSYNC_PARTIAL_VANISHED_EXIT_CODE = 24
|
||||
|
||||
|
||||
def dry_run_log_path(host: str, run_id: int | None = None) -> Path:
|
||||
@@ -72,6 +73,24 @@ def classify_rsync_failure(exit_code: int | None, log_tail: list[str]) -> dict[s
|
||||
}
|
||||
|
||||
|
||||
def classify_rsync_warning(exit_code: int | None, log_tail: list[str]) -> dict[str, str] | None:
|
||||
joined_tail = "\n".join(log_tail).lower()
|
||||
if exit_code == RSYNC_PARTIAL_VANISHED_EXIT_CODE:
|
||||
return {
|
||||
"category": "vanished",
|
||||
"message": "Some source files vanished during rsync.",
|
||||
"hint": "This is common on live systems. The snapshot was kept, but review the rsync log if this happens often.",
|
||||
}
|
||||
if exit_code in (None, RSYNC_PARTIAL_VANISHED_EXIT_CODE) and (
|
||||
"file has vanished" in joined_tail or "vanished before it could be transferred" in joined_tail
|
||||
):
|
||||
return {
|
||||
"category": "vanished",
|
||||
"message": "Some source files vanished during rsync.",
|
||||
"hint": "This is common on live systems. The snapshot was kept, but review the rsync log if this happens often.",
|
||||
}
|
||||
return None
|
||||
|
||||
def _collect_run_stats(
|
||||
*,
|
||||
log_path: Path,
|
||||
@@ -158,6 +177,7 @@ def run_scheduled(
|
||||
run_id: int | None = None,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
verbose_output: bool = False,
|
||||
state_callback: Callable[[dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
|
||||
host = sanitize_host(host)
|
||||
@@ -258,6 +278,7 @@ def run_scheduled(
|
||||
"exit_code": result.exit_code,
|
||||
"command": result.command,
|
||||
"log_tail": log_tail,
|
||||
"bwlimit_kbps": bwlimit_kbps,
|
||||
},
|
||||
}
|
||||
if result.exit_code != 0:
|
||||
@@ -316,20 +337,65 @@ def run_scheduled(
|
||||
"ended_at": None,
|
||||
"duration_seconds": None,
|
||||
"base": _base_meta_from_path(base_dir, link_dest),
|
||||
"rsync": {"exit_code": None, "command": cmd, "stats": {}},
|
||||
"rsync": {"exit_code": None, "command": cmd, "stats": {}, "bwlimit_kbps": bwlimit_kbps},
|
||||
"overrides": {"includes": [], "excludes": [], "base": None},
|
||||
}
|
||||
|
||||
log_path.touch(exist_ok=True)
|
||||
write_yaml_atomic(meta_path, meta)
|
||||
if state_callback is not None:
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "preparing",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": None, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
result = run_rsync(cmd, log_path=log_path, timeout_seconds=timeout_seconds, cancel_check=cancel_check)
|
||||
def process_started(pid: int, pgid: int) -> None:
|
||||
if state_callback is None:
|
||||
return
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "rsync",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": None, "pid": pid, "pgid": pgid, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
run_rsync_kwargs: dict[str, Any] = {
|
||||
"log_path": log_path,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"cancel_check": cancel_check,
|
||||
}
|
||||
if state_callback is not None:
|
||||
run_rsync_kwargs["process_started"] = process_started
|
||||
result = run_rsync(cmd, **run_rsync_kwargs)
|
||||
log_tail = _read_log_tail(log_path)
|
||||
warning = classify_rsync_warning(result.exit_code, log_tail)
|
||||
successful_or_warning = result.exit_code == 0 or warning is not None
|
||||
if state_callback is not None:
|
||||
state_callback(
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "finalizing",
|
||||
"snapshot": str(incomplete_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": cmd, "exit_code": result.exit_code, "log_tail": log_tail, "bwlimit_kbps": bwlimit_kbps},
|
||||
}
|
||||
)
|
||||
|
||||
end_ts = utc_now()
|
||||
meta["ended_at"] = format_iso_z(end_ts)
|
||||
meta["duration_seconds"] = int((end_ts - ts).total_seconds())
|
||||
meta["rsync"]["exit_code"] = result.exit_code
|
||||
meta["status"] = "cancelled" if result.cancelled else ("success" if result.exit_code == 0 else "failed")
|
||||
meta["status"] = "cancelled" if result.cancelled else ("warning" if warning else ("success" if result.exit_code == 0 else "failed"))
|
||||
if warning is not None:
|
||||
meta["warning"] = warning
|
||||
meta["stats"] = _collect_run_stats(
|
||||
log_path=log_path,
|
||||
backup_root=Path(backup_root),
|
||||
@@ -349,8 +415,7 @@ def run_scheduled(
|
||||
"error": "rsync.log missing after execution",
|
||||
}
|
||||
|
||||
if result.exit_code != 0:
|
||||
log_tail = _read_log_tail(log_path)
|
||||
if not successful_or_warning:
|
||||
return {
|
||||
"ok": False,
|
||||
"dry_run": False,
|
||||
@@ -366,6 +431,7 @@ def run_scheduled(
|
||||
"exit_code": result.exit_code,
|
||||
"command": result.command,
|
||||
"log_tail": log_tail,
|
||||
"bwlimit_kbps": bwlimit_kbps,
|
||||
},
|
||||
"failure": classify_rsync_failure(result.exit_code, log_tail),
|
||||
}
|
||||
@@ -404,7 +470,9 @@ def run_scheduled(
|
||||
"snapshot": str(final_dir),
|
||||
"base": str(base_dir) if base_dir else None,
|
||||
"log": str(final_log_path),
|
||||
"rsync": {"exit_code": result.exit_code},
|
||||
"status": meta["status"],
|
||||
"warning": warning,
|
||||
"rsync": {"exit_code": result.exit_code, "command": result.command, "log_tail": log_tail, "bwlimit_kbps": bwlimit_kbps},
|
||||
"verbose_output": bool(verbose_output),
|
||||
"duration_seconds": meta["duration_seconds"],
|
||||
"stats": meta["stats"],
|
||||
|
||||
@@ -110,6 +110,7 @@ GLOBAL_SCHEMA = Schema(
|
||||
|
||||
HOST_RSYNC_SCHEMA = Schema(
|
||||
fields={
|
||||
"bwlimit_kbps": FieldSpec(int, required=False, min_value=0),
|
||||
"extra_args": FieldSpec(list, required=False, default=[], item=FieldSpec(str)),
|
||||
},
|
||||
allow_unknown=False,
|
||||
|
||||
@@ -82,6 +82,7 @@ def run_rsync(
|
||||
log_path: Path,
|
||||
timeout_seconds: int,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
process_started: Callable[[int, int], None] | None = None,
|
||||
) -> RsyncResult:
|
||||
"""
|
||||
Run rsync and always write stdout/stderr to log_path.
|
||||
@@ -95,6 +96,8 @@ def run_rsync(
|
||||
|
||||
with log_path.open("ab") as f:
|
||||
process = subprocess.Popen(command, stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
if process_started is not None:
|
||||
process_started(process.pid, os.getpgid(process.pid))
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
exit_code = process.poll()
|
||||
|
||||
@@ -6,7 +6,17 @@ from django.urls import reverse
|
||||
from django.utils.html import format_html
|
||||
from django.utils.http import urlencode
|
||||
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential
|
||||
from .models import (
|
||||
BackupRun,
|
||||
GlobalConfig,
|
||||
HostConfig,
|
||||
NotificationDelivery,
|
||||
NotificationTarget,
|
||||
PurgedSnapshot,
|
||||
ScheduleConfig,
|
||||
SnapshotRecord,
|
||||
SshCredential,
|
||||
)
|
||||
|
||||
|
||||
@admin.register(SshCredential)
|
||||
@@ -73,7 +83,7 @@ class HostConfigAdmin(admin.ModelAdmin):
|
||||
(None, {"fields": ("host", "address", "enabled")}),
|
||||
("SSH override", {"fields": ("ssh_credential", "ssh_user", "ssh_port")}),
|
||||
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
|
||||
("Rsync override", {"fields": ("rsync_extra_args",)}),
|
||||
("Rsync override", {"fields": ("rsync_extra_args", "rsync_bwlimit_kbps")}),
|
||||
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
|
||||
("Runtime state", {"fields": ("config",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
@@ -136,6 +146,38 @@ class BackupRunAdmin(admin.ModelAdmin):
|
||||
return format_html('<a href="{}">{}</a>', url, obj.snapshot.dirname)
|
||||
|
||||
|
||||
@admin.register(NotificationTarget)
|
||||
class NotificationTargetAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "channel", "enabled", "last_status", "last_sent_at", "updated_at")
|
||||
list_filter = ("enabled", "channel", "last_status")
|
||||
search_fields = ("name", "email_to", "webhook_url", "notes")
|
||||
readonly_fields = ("created_at", "updated_at", "last_status", "last_error", "last_sent_at")
|
||||
fieldsets = (
|
||||
(None, {"fields": ("name", "enabled", "channel", "statuses")}),
|
||||
("Email", {"fields": ("email_to",)}),
|
||||
("Webhook", {"fields": ("webhook_url", "webhook_headers")}),
|
||||
("State", {"fields": ("last_status", "last_error", "last_sent_at"), "classes": ("collapse",)}),
|
||||
("Notes", {"fields": ("notes",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
|
||||
@admin.register(NotificationDelivery)
|
||||
class NotificationDeliveryAdmin(admin.ModelAdmin):
|
||||
list_display = ("target", "run", "status", "created_at")
|
||||
list_filter = ("status", "target__channel", "created_at")
|
||||
search_fields = ("target__name", "run__host__host", "error")
|
||||
readonly_fields = ("target", "run", "status", "error", "payload", "created_at")
|
||||
list_select_related = ("target", "run", "run__host")
|
||||
date_hierarchy = "created_at"
|
||||
|
||||
def has_add_permission(self, request) -> bool:
|
||||
return False
|
||||
|
||||
def has_change_permission(self, request, obj=None) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@admin.register(SnapshotRecord)
|
||||
class SnapshotRecordAdmin(admin.ModelAdmin):
|
||||
list_display = ("host", "kind", "dirname", "status", "base_link", "backup_run_count_link", "started_at", "discovered_at")
|
||||
|
||||
@@ -8,9 +8,16 @@ from pathlib import Path
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync.commands.run_scheduled import DEFAULT_DRY_RUN_TIMEOUT_SECONDS, classify_rsync_failure, dry_run_log_path, run_scheduled
|
||||
from pobsync.commands.run_scheduled import (
|
||||
DEFAULT_DRY_RUN_TIMEOUT_SECONDS,
|
||||
classify_rsync_failure,
|
||||
classify_rsync_warning,
|
||||
dry_run_log_path,
|
||||
run_scheduled,
|
||||
)
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import BackupRun, HostConfig
|
||||
from pobsync_backend.notifications import notify_backup_run_completed
|
||||
from pobsync_backend.retention import run_sql_retention_apply
|
||||
from pobsync_backend.snapshot_discovery import infer_snapshot_kind, upsert_snapshot_record
|
||||
|
||||
@@ -66,6 +73,7 @@ def execute_backup_run(
|
||||
run_id=run.id,
|
||||
cancel_check=lambda: _run_cancel_requested(run.id),
|
||||
verbose_output=bool(dry_run or verbose_output),
|
||||
state_callback=lambda state: _record_running_state(run.id, state),
|
||||
)
|
||||
except Exception as exc:
|
||||
run.refresh_from_db()
|
||||
@@ -78,11 +86,14 @@ def execute_backup_run(
|
||||
"type": type(exc).__name__,
|
||||
}
|
||||
run.save(update_fields=["status", "ended_at", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
raise
|
||||
|
||||
run.refresh_from_db()
|
||||
if result.get("cancelled") or run.status == BackupRun.Status.CANCELLED:
|
||||
run.status = BackupRun.Status.CANCELLED
|
||||
elif result.get("status") == BackupRun.Status.WARNING:
|
||||
run.status = BackupRun.Status.WARNING
|
||||
else:
|
||||
run.status = BackupRun.Status.SUCCESS if result.get("ok") else BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
@@ -142,6 +153,7 @@ def execute_backup_run(
|
||||
"result",
|
||||
],
|
||||
)
|
||||
notify_backup_run_completed(run)
|
||||
return run
|
||||
|
||||
|
||||
@@ -201,11 +213,100 @@ def _run_cancel_requested(run_id: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _record_running_state(run_id: int, state: dict[str, object]) -> None:
|
||||
try:
|
||||
run = BackupRun.objects.only("id", "status", "result", "snapshot_path", "rsync_exit_code").get(id=run_id)
|
||||
except BackupRun.DoesNotExist:
|
||||
return
|
||||
if run.status != BackupRun.Status.RUNNING:
|
||||
return
|
||||
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
|
||||
incoming_rsync = state.get("rsync") if isinstance(state.get("rsync"), dict) else {}
|
||||
|
||||
log_path = state.get("log")
|
||||
snapshot_path = state.get("snapshot")
|
||||
phase = state.get("phase")
|
||||
if isinstance(phase, str) and phase:
|
||||
execution["phase"] = phase
|
||||
if isinstance(log_path, str) and log_path:
|
||||
execution["log"] = log_path
|
||||
if isinstance(snapshot_path, str) and snapshot_path:
|
||||
execution["snapshot"] = snapshot_path
|
||||
run.snapshot_path = snapshot_path
|
||||
if incoming_rsync:
|
||||
result["rsync"] = {**rsync, **incoming_rsync}
|
||||
exit_code = incoming_rsync.get("exit_code")
|
||||
if isinstance(exit_code, int):
|
||||
run.rsync_exit_code = exit_code
|
||||
result["execution"] = {
|
||||
**execution,
|
||||
"worker_pid": os.getpid(),
|
||||
"worker_host": socket.gethostname(),
|
||||
"heartbeat_at": timezone.now().isoformat(),
|
||||
}
|
||||
run.result = result
|
||||
run.save(update_fields=["snapshot_path", "rsync_exit_code", "result"])
|
||||
|
||||
|
||||
def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_seconds: int) -> bool:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
requested = result.get("requested") if isinstance(result.get("requested"), dict) else {}
|
||||
log_path = _execution_log_path(result)
|
||||
log_tail = _read_log_tail(log_path) if log_path is not None else []
|
||||
terminal_log = _terminal_rsync_log(log_tail)
|
||||
exit_code = _exit_code_from_log(log_tail)
|
||||
stale_worker = _running_worker_timed_out(run=run, stale_worker_seconds=stale_worker_seconds)
|
||||
if not requested.get("dry_run"):
|
||||
if terminal_log:
|
||||
failure = classify_rsync_failure(exit_code or 255, log_tail)
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
"host": run.host.host,
|
||||
"log": str(log_path) if log_path else "",
|
||||
"failure": failure,
|
||||
"rsync": {
|
||||
**(result.get("rsync") if isinstance(result.get("rsync"), dict) else {}),
|
||||
"exit_code": exit_code or 255,
|
||||
"log_tail": log_tail,
|
||||
},
|
||||
}
|
||||
)
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.rsync_exit_code = exit_code or 255
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
if _running_rsync_process_missing(run=run, grace_seconds=grace_seconds):
|
||||
result.update(
|
||||
{
|
||||
"ok": False,
|
||||
"host": run.host.host,
|
||||
"log": str(log_path) if log_path else "",
|
||||
"failure": {
|
||||
"category": "rsync_process",
|
||||
"message": "The rsync process is no longer running while the backup is still marked running.",
|
||||
"hint": "Check the rsync log and pobsync-worker.service logs before retrying the backup.",
|
||||
},
|
||||
"rsync": {
|
||||
**(result.get("rsync") if isinstance(result.get("rsync"), dict) else {}),
|
||||
"exit_code": exit_code or 255,
|
||||
"log_tail": log_tail,
|
||||
},
|
||||
}
|
||||
)
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.rsync_exit_code = exit_code or 255
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
if stale_worker:
|
||||
result.update(
|
||||
{
|
||||
@@ -222,17 +323,15 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
|
||||
run.ended_at = timezone.now()
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
return False
|
||||
|
||||
log_path = _execution_log_path(result)
|
||||
log_tail = _read_log_tail(log_path) if log_path is not None else []
|
||||
terminal_log = _terminal_rsync_log(log_tail)
|
||||
timed_out = _running_dry_run_timed_out(run=run, grace_seconds=grace_seconds)
|
||||
if not terminal_log and not timed_out and not stale_worker:
|
||||
return False
|
||||
|
||||
exit_code = _exit_code_from_log(log_tail) or (124 if timed_out or stale_worker else 255)
|
||||
exit_code = exit_code or (124 if timed_out or stale_worker else 255)
|
||||
failure = classify_rsync_failure(exit_code, log_tail)
|
||||
if stale_worker and not terminal_log:
|
||||
failure = {
|
||||
@@ -260,6 +359,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
|
||||
run.rsync_exit_code = exit_code
|
||||
run.result = result
|
||||
run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
|
||||
notify_backup_run_completed(run)
|
||||
return True
|
||||
|
||||
|
||||
@@ -305,6 +405,9 @@ def _read_log_tail(log_path: Path | None, *, max_lines: int = 40) -> list[str]:
|
||||
|
||||
|
||||
def _terminal_rsync_log(log_tail: list[str]) -> bool:
|
||||
warning = classify_rsync_warning(_exit_code_from_log(log_tail), log_tail)
|
||||
if warning is not None:
|
||||
return False
|
||||
return any(line.startswith("rsync error:") for line in log_tail)
|
||||
|
||||
|
||||
@@ -312,6 +415,8 @@ def _exit_code_from_log(log_tail: list[str]) -> int | None:
|
||||
for line in reversed(log_tail):
|
||||
if "code 255" in line:
|
||||
return 255
|
||||
if "code 24" in line:
|
||||
return 24
|
||||
if "code 124" in line:
|
||||
return 124
|
||||
if "code 12" in line:
|
||||
@@ -342,6 +447,33 @@ def _running_worker_timed_out(*, run: BackupRun, stale_worker_seconds: int) -> b
|
||||
return timezone.now() >= heartbeat_at + timedelta(seconds=stale_worker_seconds)
|
||||
|
||||
|
||||
def _running_rsync_process_missing(*, run: BackupRun, grace_seconds: int) -> bool:
|
||||
if grace_seconds <= 0:
|
||||
return False
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
if execution.get("phase") != "rsync":
|
||||
return False
|
||||
rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
|
||||
pid = rsync.get("pid")
|
||||
if not isinstance(pid, int) or pid <= 0:
|
||||
return False
|
||||
heartbeat_at = _parse_iso_datetime(execution.get("heartbeat_at")) or run.started_at
|
||||
if heartbeat_at is None or timezone.now() < heartbeat_at + timedelta(seconds=grace_seconds):
|
||||
return False
|
||||
return not _process_exists(pid)
|
||||
|
||||
|
||||
def _process_exists(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _parse_iso_datetime(value: object):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
|
||||
@@ -68,8 +68,12 @@ def _host_runtime_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
data["excludes_replace"] = list(host_config.excludes_replace or [])
|
||||
else:
|
||||
data["excludes_add"] = list(host_config.excludes_add or [])
|
||||
if host_config.rsync_extra_args or host_config.rsync_bwlimit_kbps is not None:
|
||||
data["rsync"] = {}
|
||||
if host_config.rsync_extra_args:
|
||||
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
|
||||
data["rsync"]["extra_args"] = list(host_config.rsync_extra_args or [])
|
||||
if host_config.rsync_bwlimit_kbps is not None:
|
||||
data["rsync"]["bwlimit_kbps"] = host_config.rsync_bwlimit_kbps
|
||||
return validate_dict(data, HOST_SCHEMA, path="host")
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
|
||||
from django import forms
|
||||
from django.conf import settings
|
||||
|
||||
from .models import GlobalConfig, HostConfig, ScheduleConfig, SshCredential
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, NotificationTarget, ScheduleConfig, SshCredential
|
||||
from .scheduler import parse_cron_expr
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ class HostConfigForm(forms.ModelForm):
|
||||
"excludes_add",
|
||||
"excludes_replace",
|
||||
"rsync_extra_args",
|
||||
"rsync_bwlimit_kbps",
|
||||
"retention_daily",
|
||||
"retention_weekly",
|
||||
"retention_monthly",
|
||||
@@ -70,6 +71,7 @@ class HostConfigForm(forms.ModelForm):
|
||||
"ssh_user": "Leave empty to use the global SSH user.",
|
||||
"ssh_port": "Leave empty to use the global SSH port.",
|
||||
"source_root": "Leave empty to use the global default source root.",
|
||||
"rsync_bwlimit_kbps": "Leave empty to inherit the global limit. Use 0 for unlimited on this host.",
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +114,7 @@ class GlobalConfigForm(forms.ModelForm):
|
||||
help_texts = {
|
||||
"name": "Usually 'default'. The backup engine currently reads the default config.",
|
||||
"default_ssh_credential": "Optional. Used by hosts without their own SSH credential.",
|
||||
"rsync_bwlimit_kbps": "Rsync bandwidth limit in KB/s. Use 0 for unlimited.",
|
||||
"default_source_root": "Used by hosts without a custom source root.",
|
||||
"default_destination_subdir": "Optional subdirectory below each snapshot.",
|
||||
}
|
||||
@@ -150,6 +153,62 @@ class ManualBackupForm(forms.Form):
|
||||
)
|
||||
|
||||
|
||||
class NotificationTargetForm(forms.ModelForm):
|
||||
TERMINAL_STATUS_CHOICES = (
|
||||
(BackupRun.Status.SUCCESS, BackupRun.Status.SUCCESS.label),
|
||||
(BackupRun.Status.WARNING, BackupRun.Status.WARNING.label),
|
||||
(BackupRun.Status.FAILED, BackupRun.Status.FAILED.label),
|
||||
(BackupRun.Status.CANCELLED, BackupRun.Status.CANCELLED.label),
|
||||
)
|
||||
|
||||
statuses = forms.MultipleChoiceField(
|
||||
choices=TERMINAL_STATUS_CHOICES,
|
||||
widget=forms.CheckboxSelectMultiple,
|
||||
initial=[choice[0] for choice in TERMINAL_STATUS_CHOICES],
|
||||
help_text="Send notifications for these terminal run statuses.",
|
||||
)
|
||||
email_to = forms.CharField(
|
||||
widget=forms.Textarea,
|
||||
required=False,
|
||||
help_text="One recipient per line, or comma-separated.",
|
||||
)
|
||||
webhook_headers = forms.JSONField(
|
||||
required=False,
|
||||
widget=forms.Textarea(attrs={"rows": 4}),
|
||||
help_text='Optional JSON object with extra headers, for example {"Authorization": "Bearer ..."}.',
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = NotificationTarget
|
||||
fields = (
|
||||
"name",
|
||||
"enabled",
|
||||
"channel",
|
||||
"statuses",
|
||||
"email_to",
|
||||
"webhook_url",
|
||||
"webhook_headers",
|
||||
"notes",
|
||||
)
|
||||
widgets = {
|
||||
"notes": forms.Textarea,
|
||||
}
|
||||
|
||||
def clean(self):
|
||||
cleaned_data = super().clean()
|
||||
channel = cleaned_data.get("channel")
|
||||
if channel == NotificationTarget.Channel.EMAIL and not cleaned_data.get("email_to", "").strip():
|
||||
self.add_error("email_to", "Email targets need at least one recipient.")
|
||||
if channel == NotificationTarget.Channel.WEBHOOK and not cleaned_data.get("webhook_url"):
|
||||
self.add_error("webhook_url", "Webhook targets need a URL.")
|
||||
return cleaned_data
|
||||
|
||||
def clean_email_to(self) -> str:
|
||||
value = self.cleaned_data.get("email_to", "")
|
||||
recipients = [line.strip() for line in value.replace(",", "\n").splitlines() if line.strip()]
|
||||
return "\n".join(recipients)
|
||||
|
||||
|
||||
class SshCredentialForm(forms.ModelForm):
|
||||
private_key_file = forms.FileField(
|
||||
required=False,
|
||||
|
||||
@@ -22,6 +22,12 @@ class Command(BaseCommand):
|
||||
parser.add_argument("--exclude-add", action="append", default=[])
|
||||
parser.add_argument("--exclude-replace", action="append", default=None)
|
||||
parser.add_argument("--rsync-extra-arg", action="append", default=[])
|
||||
parser.add_argument(
|
||||
"--rsync-bwlimit-kbps",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Host rsync bandwidth limit in KB/s. Omit to inherit global; set 0 for unlimited.",
|
||||
)
|
||||
parser.add_argument("--retention", default=None)
|
||||
parser.add_argument("--disabled", action="store_true")
|
||||
parser.add_argument("--force", action="store_true", help="Update existing host")
|
||||
@@ -42,6 +48,7 @@ class Command(BaseCommand):
|
||||
"excludes_add": [] if options["exclude_replace"] is not None else list(options["exclude_add"]),
|
||||
"excludes_replace": options["exclude_replace"],
|
||||
"rsync_extra_args": list(options["rsync_extra_arg"]),
|
||||
"rsync_bwlimit_kbps": options["rsync_bwlimit_kbps"],
|
||||
"retention_daily": retention["daily"],
|
||||
"retention_weekly": retention["weekly"],
|
||||
"retention_monthly": retention["monthly"],
|
||||
|
||||
18
src/pobsync_backend/migrations/0014_host_bwlimit_override.py
Normal file
18
src/pobsync_backend/migrations/0014_host_bwlimit_override.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.14 on 2026-05-22 22:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('pobsync_backend', '0013_purgedsnapshot'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='hostconfig',
|
||||
name='rsync_bwlimit_kbps',
|
||||
field=models.PositiveIntegerField(blank=True, null=True),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,54 @@
|
||||
# Generated by Django 5.2.14 on 2026-05-28 19:11
|
||||
|
||||
import django.db.models.deletion
|
||||
import pobsync_backend.models
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('pobsync_backend', '0014_host_bwlimit_override'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='NotificationTarget',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('name', models.CharField(max_length=128, unique=True)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('channel', models.CharField(choices=[('email', 'Email'), ('webhook', 'Webhook')], max_length=16)),
|
||||
('statuses', models.JSONField(blank=True, default=pobsync_backend.models.default_notification_statuses)),
|
||||
('email_to', models.TextField(blank=True)),
|
||||
('webhook_url', models.URLField(blank=True, max_length=1024)),
|
||||
('webhook_headers', models.JSONField(blank=True, default=dict)),
|
||||
('notes', models.TextField(blank=True)),
|
||||
('last_status', models.CharField(blank=True, max_length=16)),
|
||||
('last_error', models.TextField(blank=True)),
|
||||
('last_sent_at', models.DateTimeField(blank=True, null=True)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['name'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='NotificationDelivery',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('status', models.CharField(choices=[('sent', 'Sent'), ('failed', 'Failed'), ('skipped', 'Skipped')], max_length=16)),
|
||||
('error', models.TextField(blank=True)),
|
||||
('payload', models.JSONField(blank=True, default=dict)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notification_deliveries', to='pobsync_backend.backuprun')),
|
||||
('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='pobsync_backend.notificationtarget')),
|
||||
],
|
||||
options={
|
||||
'verbose_name_plural': 'notification deliveries',
|
||||
'ordering': ['-created_at', 'target__name'],
|
||||
'constraints': [models.UniqueConstraint(fields=('target', 'run'), name='unique_notification_delivery_per_target_run')],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -63,6 +63,7 @@ class HostConfig(TimestampedModel):
|
||||
excludes_add = models.JSONField(default=list, blank=True)
|
||||
excludes_replace = models.JSONField(null=True, blank=True)
|
||||
rsync_extra_args = models.JSONField(default=list, blank=True)
|
||||
rsync_bwlimit_kbps = models.PositiveIntegerField(null=True, blank=True)
|
||||
retention_daily = models.PositiveIntegerField(default=14)
|
||||
retention_weekly = models.PositiveIntegerField(default=8)
|
||||
retention_monthly = models.PositiveIntegerField(default=12)
|
||||
@@ -134,6 +135,63 @@ class BackupRun(models.Model):
|
||||
return f"{self.host} {self.run_type} {self.status}"
|
||||
|
||||
|
||||
def default_notification_statuses() -> list[str]:
|
||||
return [
|
||||
BackupRun.Status.SUCCESS,
|
||||
BackupRun.Status.WARNING,
|
||||
BackupRun.Status.FAILED,
|
||||
BackupRun.Status.CANCELLED,
|
||||
]
|
||||
|
||||
|
||||
class NotificationTarget(TimestampedModel):
|
||||
class Channel(models.TextChoices):
|
||||
EMAIL = "email", "Email"
|
||||
WEBHOOK = "webhook", "Webhook"
|
||||
|
||||
name = models.CharField(max_length=128, unique=True)
|
||||
enabled = models.BooleanField(default=True)
|
||||
channel = models.CharField(max_length=16, choices=Channel.choices)
|
||||
statuses = models.JSONField(default=default_notification_statuses, blank=True)
|
||||
email_to = models.TextField(blank=True)
|
||||
webhook_url = models.URLField(max_length=1024, blank=True)
|
||||
webhook_headers = models.JSONField(default=dict, blank=True)
|
||||
notes = models.TextField(blank=True)
|
||||
last_status = models.CharField(max_length=16, blank=True)
|
||||
last_error = models.TextField(blank=True)
|
||||
last_sent_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["name"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class NotificationDelivery(models.Model):
|
||||
class Status(models.TextChoices):
|
||||
SENT = "sent", "Sent"
|
||||
FAILED = "failed", "Failed"
|
||||
SKIPPED = "skipped", "Skipped"
|
||||
|
||||
target = models.ForeignKey(NotificationTarget, on_delete=models.CASCADE, related_name="deliveries")
|
||||
run = models.ForeignKey(BackupRun, on_delete=models.CASCADE, related_name="notification_deliveries")
|
||||
status = models.CharField(max_length=16, choices=Status.choices)
|
||||
error = models.TextField(blank=True)
|
||||
payload = models.JSONField(default=dict, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["target", "run"], name="unique_notification_delivery_per_target_run"),
|
||||
]
|
||||
ordering = ["-created_at", "target__name"]
|
||||
verbose_name_plural = "notification deliveries"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.target} run {self.run_id} {self.status}"
|
||||
|
||||
|
||||
class SnapshotRecord(models.Model):
|
||||
class Kind(models.TextChoices):
|
||||
SCHEDULED = "scheduled", "Scheduled"
|
||||
|
||||
168
src/pobsync_backend/notifications.py
Normal file
168
src/pobsync_backend/notifications.py
Normal file
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.mail import send_mail
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import BackupRun, NotificationDelivery, NotificationTarget
|
||||
|
||||
|
||||
TERMINAL_RUN_STATUSES = {
|
||||
BackupRun.Status.SUCCESS,
|
||||
BackupRun.Status.WARNING,
|
||||
BackupRun.Status.FAILED,
|
||||
BackupRun.Status.CANCELLED,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryResult:
|
||||
target: NotificationTarget
|
||||
delivery: NotificationDelivery
|
||||
sent: bool
|
||||
|
||||
|
||||
def notify_backup_run_completed(run: BackupRun) -> list[DeliveryResult]:
|
||||
if run.status not in TERMINAL_RUN_STATUSES:
|
||||
return []
|
||||
|
||||
targets = [target for target in NotificationTarget.objects.filter(enabled=True) if _target_wants_status(target, run.status)]
|
||||
return [_notify_target(target=target, run=run) for target in targets]
|
||||
|
||||
|
||||
def _target_wants_status(target: NotificationTarget, status: str) -> bool:
|
||||
statuses = target.statuses
|
||||
if not isinstance(statuses, list):
|
||||
return False
|
||||
return status in {str(item) for item in statuses}
|
||||
|
||||
|
||||
def _notify_target(*, target: NotificationTarget, run: BackupRun) -> DeliveryResult:
|
||||
payload = _run_payload(run)
|
||||
delivery, created = NotificationDelivery.objects.get_or_create(
|
||||
target=target,
|
||||
run=run,
|
||||
defaults={
|
||||
"status": NotificationDelivery.Status.SKIPPED,
|
||||
"payload": payload,
|
||||
},
|
||||
)
|
||||
if not created:
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
||||
|
||||
try:
|
||||
if target.channel == NotificationTarget.Channel.EMAIL:
|
||||
_send_email(target=target, run=run, payload=payload)
|
||||
elif target.channel == NotificationTarget.Channel.WEBHOOK:
|
||||
_send_webhook(target=target, payload=payload)
|
||||
else:
|
||||
raise ValueError(f"Unsupported notification channel: {target.channel}")
|
||||
except Exception as exc:
|
||||
delivery.status = NotificationDelivery.Status.FAILED
|
||||
delivery.error = str(exc)
|
||||
delivery.save(update_fields=["status", "error"])
|
||||
target.last_status = NotificationDelivery.Status.FAILED
|
||||
target.last_error = str(exc)
|
||||
target.save(update_fields=["last_status", "last_error", "updated_at"])
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
||||
|
||||
delivery.status = NotificationDelivery.Status.SENT
|
||||
delivery.save(update_fields=["status"])
|
||||
target.last_status = NotificationDelivery.Status.SENT
|
||||
target.last_error = ""
|
||||
target.last_sent_at = timezone.now()
|
||||
target.save(update_fields=["last_status", "last_error", "last_sent_at", "updated_at"])
|
||||
return DeliveryResult(target=target, delivery=delivery, sent=True)
|
||||
|
||||
|
||||
def _send_email(*, target: NotificationTarget, run: BackupRun, payload: dict[str, Any]) -> None:
|
||||
recipients = [line.strip() for line in target.email_to.replace(",", "\n").splitlines() if line.strip()]
|
||||
if not recipients:
|
||||
raise ValueError("Email notification target has no recipients.")
|
||||
|
||||
subject = f"pobsync {run.status}: {run.host.host} run {run.id}"
|
||||
message = _email_message(payload)
|
||||
from_email = getattr(settings, "DEFAULT_FROM_EMAIL", "") or "pobsync@localhost"
|
||||
sent = send_mail(subject, message, from_email, recipients, fail_silently=False)
|
||||
if sent == 0:
|
||||
raise ValueError("Django email backend reported zero sent messages.")
|
||||
|
||||
|
||||
def _send_webhook(*, target: NotificationTarget, payload: dict[str, Any]) -> None:
|
||||
if not target.webhook_url:
|
||||
raise ValueError("Webhook notification target has no URL.")
|
||||
|
||||
headers = {"Content-Type": "application/json", **_string_headers(target.webhook_headers)}
|
||||
request = urllib.request.Request(
|
||||
target.webhook_url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
if response.status >= 400:
|
||||
raise ValueError(f"Webhook returned HTTP {response.status}.")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise ValueError(f"Webhook returned HTTP {exc.code}.") from exc
|
||||
|
||||
|
||||
def _string_headers(headers: object) -> dict[str, str]:
|
||||
if not isinstance(headers, dict):
|
||||
return {}
|
||||
return {str(key): str(value) for key, value in headers.items() if str(key).strip()}
|
||||
|
||||
|
||||
def _run_payload(run: BackupRun) -> dict[str, Any]:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
failure = result.get("failure") if isinstance(result.get("failure"), dict) else {}
|
||||
prune = result.get("prune") if isinstance(result.get("prune"), dict) else {}
|
||||
return {
|
||||
"event": "backup_run.completed",
|
||||
"run": {
|
||||
"id": run.id,
|
||||
"host": run.host.host,
|
||||
"type": run.run_type,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"ended_at": run.ended_at.isoformat() if run.ended_at else None,
|
||||
"snapshot": run.snapshot_path,
|
||||
"rsync_exit_code": run.rsync_exit_code,
|
||||
},
|
||||
"failure": {
|
||||
"category": failure.get("category"),
|
||||
"message": failure.get("message") or result.get("error"),
|
||||
"hint": failure.get("hint"),
|
||||
},
|
||||
"prune": {
|
||||
"ok": prune.get("ok") if prune else None,
|
||||
"error": prune.get("error") if prune else "",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _email_message(payload: dict[str, Any]) -> str:
|
||||
run = payload["run"]
|
||||
lines = [
|
||||
f"Host: {run['host']}",
|
||||
f"Run: {run['id']}",
|
||||
f"Type: {run['type']}",
|
||||
f"Status: {run['status']}",
|
||||
f"Started: {run['started_at'] or '-'}",
|
||||
f"Ended: {run['ended_at'] or '-'}",
|
||||
f"Snapshot: {run['snapshot'] or '-'}",
|
||||
f"Rsync exit code: {run['rsync_exit_code'] if run['rsync_exit_code'] is not None else '-'}",
|
||||
]
|
||||
failure = payload.get("failure") if isinstance(payload.get("failure"), dict) else {}
|
||||
if failure.get("message"):
|
||||
lines.extend(["", f"Failure: {failure['message']}"])
|
||||
prune = payload.get("prune") if isinstance(payload.get("prune"), dict) else {}
|
||||
if prune.get("error"):
|
||||
lines.extend(["", f"Retention: {prune['error']}"])
|
||||
return "\n".join(lines)
|
||||
@@ -97,9 +97,7 @@ def run_remote_preflight(host: HostConfig, *, timeout_seconds: int = 20) -> dict
|
||||
*ssh_cmd,
|
||||
"-oBatchMode=yes",
|
||||
target,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"command -v {shlex.quote(rsync_binary)} >/dev/null",
|
||||
_remote_shell_command(f"command -v {shlex.quote(rsync_binary)} >/dev/null"),
|
||||
],
|
||||
timeout_seconds=timeout_seconds,
|
||||
),
|
||||
@@ -109,9 +107,7 @@ def run_remote_preflight(host: HostConfig, *, timeout_seconds: int = 20) -> dict
|
||||
*ssh_cmd,
|
||||
"-oBatchMode=yes",
|
||||
target,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"test -e {shlex.quote(source_root)} && test -r {shlex.quote(source_root)}",
|
||||
_remote_shell_command(f"test -e {shlex.quote(source_root)} && test -r {shlex.quote(source_root)}"),
|
||||
],
|
||||
timeout_seconds=timeout_seconds,
|
||||
),
|
||||
@@ -129,6 +125,10 @@ def run_remote_preflight(host: HostConfig, *, timeout_seconds: int = 20) -> dict
|
||||
return result
|
||||
|
||||
|
||||
def _remote_shell_command(script: str) -> str:
|
||||
return f"sh -lc {shlex.quote(script)}"
|
||||
|
||||
|
||||
def effective_host_config_preview(host: HostConfig, global_config: GlobalConfig) -> dict[str, Any]:
|
||||
config = build_effective_config(global_config_object_data(global_config), host_config_object_data(host))
|
||||
credential = host.ssh_credential or global_config.default_ssh_credential
|
||||
|
||||
@@ -23,7 +23,7 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
|
||||
host_config = _enabled_host_config(host)
|
||||
retention = _retention_for_host(host_config)
|
||||
snapshots = _snapshots_for_retention(host_config=host_config, kind=kind)
|
||||
incomplete_snapshots = _incomplete_snapshots_for_host(host_config)
|
||||
incomplete_items = _incomplete_snapshot_items_for_host(host_config)
|
||||
|
||||
plan = build_retention_plan(
|
||||
snapshots=snapshots,
|
||||
@@ -49,10 +49,9 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
|
||||
"keep": sorted(keep),
|
||||
"keep_items": [_snapshot_to_item(snapshot, reasons=reasons.get(snapshot.dirname, [])) for snapshot in keep_items],
|
||||
"delete": [_snapshot_to_item(snapshot, reasons=["outside retention policy"]) for snapshot in delete],
|
||||
"incomplete": [
|
||||
_snapshot_to_item(snapshot, reasons=["incomplete snapshot; excluded from retention cleanup"])
|
||||
for snapshot in incomplete_snapshots
|
||||
],
|
||||
"incomplete": incomplete_items,
|
||||
"incomplete_reviewed_count": sum(1 for item in incomplete_items if item["reviewed"]),
|
||||
"incomplete_unreviewed_count": sum(1 for item in incomplete_items if not item["reviewed"]),
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
@@ -103,6 +102,7 @@ def run_sql_retention_apply(
|
||||
raise ConfigError(f"Refusing to delete unsupported snapshot kind: {snap_kind!r}")
|
||||
|
||||
path = _snapshot_delete_path(path=Path(snap_path), dirname=dirname)
|
||||
_validate_snapshot_delete_path(host=host, kind=snap_kind, path=path, dirname=dirname)
|
||||
reason = str(item.get("reason") or "outside retention policy")
|
||||
if not path.exists():
|
||||
actions.append(f"skip missing {snap_kind}/{dirname}")
|
||||
@@ -163,9 +163,15 @@ def run_incomplete_cleanup(
|
||||
|
||||
def _do_cleanup() -> dict[str, Any]:
|
||||
host_config = _enabled_host_config(host)
|
||||
unreviewed_count = _unreviewed_incomplete_count(host_config)
|
||||
if unreviewed_count:
|
||||
raise ConfigError(
|
||||
f"Refusing to delete {unreviewed_count} incomplete snapshot(s) that have not been reviewed."
|
||||
)
|
||||
|
||||
incomplete_list = [
|
||||
_snapshot_to_item(snapshot, reasons=["manual incomplete cleanup"])
|
||||
for snapshot in _incomplete_snapshots_for_host(host_config)
|
||||
for snapshot in _reviewed_incomplete_snapshots_for_host(host_config)
|
||||
]
|
||||
if max_delete == 0 and len(incomplete_list) > 0:
|
||||
raise ConfigError("Incomplete cleanup blocked by --max-delete=0")
|
||||
@@ -252,15 +258,39 @@ def _snapshots_for_retention(*, host_config: HostConfig, kind: str) -> list[Snap
|
||||
return [_snapshot_from_record(record) for record in records]
|
||||
|
||||
|
||||
def _incomplete_snapshots_for_host(host_config: HostConfig) -> list[Snapshot]:
|
||||
def _incomplete_snapshot_items_for_host(host_config: HostConfig) -> list[dict[str, Any]]:
|
||||
records = (
|
||||
SnapshotRecord.objects.filter(host=host_config, kind=SnapshotRecord.Kind.INCOMPLETE)
|
||||
.select_related("base")
|
||||
.order_by("-started_at", "dirname")
|
||||
)
|
||||
return [
|
||||
_snapshot_record_to_item(record, reasons=["incomplete snapshot; excluded from retention cleanup"])
|
||||
for record in records
|
||||
]
|
||||
|
||||
|
||||
def _reviewed_incomplete_snapshots_for_host(host_config: HostConfig) -> list[Snapshot]:
|
||||
records = (
|
||||
SnapshotRecord.objects.filter(
|
||||
host=host_config,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
reviewed_at__isnull=False,
|
||||
)
|
||||
.select_related("base")
|
||||
.order_by("-started_at", "dirname")
|
||||
)
|
||||
return [_snapshot_from_record(record) for record in records]
|
||||
|
||||
|
||||
def _unreviewed_incomplete_count(host_config: HostConfig) -> int:
|
||||
return SnapshotRecord.objects.filter(
|
||||
host=host_config,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
reviewed_at__isnull=True,
|
||||
).count()
|
||||
|
||||
|
||||
def _snapshot_from_record(record: SnapshotRecord) -> Snapshot:
|
||||
return Snapshot(
|
||||
kind=record.kind,
|
||||
@@ -300,6 +330,14 @@ def _snapshot_to_item(snapshot: Snapshot, *, reasons: list[str]) -> dict[str, An
|
||||
}
|
||||
|
||||
|
||||
def _snapshot_record_to_item(record: SnapshotRecord, *, reasons: list[str]) -> dict[str, Any]:
|
||||
item = _snapshot_to_item(_snapshot_from_record(record), reasons=reasons)
|
||||
item["reviewed"] = record.reviewed_at is not None
|
||||
item["reviewed_at"] = record.reviewed_at.isoformat() if record.reviewed_at else ""
|
||||
item["reviewed_by"] = record.reviewed_by
|
||||
return item
|
||||
|
||||
|
||||
def _snapshot_delete_path(*, path: Path, dirname: str) -> Path:
|
||||
if path.name == "data" and path.parent.name == dirname:
|
||||
return path.parent
|
||||
@@ -339,14 +377,55 @@ def _validate_incomplete_delete_path(*, host: str, path: Path, dirname: str) ->
|
||||
raise ConfigError(f"Refusing to delete incomplete snapshot outside host backup root: {path}")
|
||||
|
||||
|
||||
def _validate_snapshot_delete_path(*, host: str, kind: str, path: Path, dirname: str) -> None:
|
||||
if kind not in {SnapshotRecord.Kind.SCHEDULED, SnapshotRecord.Kind.MANUAL}:
|
||||
raise ConfigError(f"Refusing to delete unsupported snapshot kind: {kind!r}")
|
||||
path_parts = path.parts
|
||||
if path.name != dirname or kind not in path_parts or host not in path_parts:
|
||||
raise ConfigError(f"Refusing to delete unexpected snapshot path: {path}")
|
||||
kind_index = path_parts.index(kind)
|
||||
if kind_index == 0 or path_parts[kind_index - 1] != host:
|
||||
raise ConfigError(f"Refusing to delete snapshot outside host backup root: {path}")
|
||||
|
||||
|
||||
def _remove_snapshot_tree(path: Path) -> None:
|
||||
_make_directories_user_writable(path)
|
||||
shutil.rmtree(path)
|
||||
_make_snapshot_tree_user_removable(path)
|
||||
shutil.rmtree(path, onexc=_retry_remove_with_user_permissions)
|
||||
|
||||
|
||||
def _make_directories_user_writable(path: Path) -> None:
|
||||
for directory in [path, *[child for child in path.rglob("*") if child.is_dir() and not child.is_symlink()]]:
|
||||
mode = directory.stat().st_mode
|
||||
if mode & stat.S_IWUSR:
|
||||
def _make_snapshot_tree_user_removable(path: Path) -> None:
|
||||
stack = [path]
|
||||
while stack:
|
||||
directory = stack.pop()
|
||||
if directory.is_symlink():
|
||||
continue
|
||||
directory.chmod(mode | stat.S_IWUSR)
|
||||
_make_path_user_removable(directory)
|
||||
try:
|
||||
children = list(directory.iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for child in children:
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
stack.append(child)
|
||||
|
||||
|
||||
def _retry_remove_with_user_permissions(function: Any, path: str, excinfo: BaseException) -> None:
|
||||
failed_path = Path(path)
|
||||
_make_path_user_removable(failed_path)
|
||||
function(path)
|
||||
|
||||
|
||||
def _make_path_user_removable(path: Path) -> None:
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
except OSError:
|
||||
return
|
||||
wanted = stat.S_IRUSR | stat.S_IWUSR
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
wanted |= stat.S_IXUSR
|
||||
if mode & wanted == wanted:
|
||||
return
|
||||
try:
|
||||
path.chmod(mode | wanted)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
@@ -5,12 +5,13 @@ from typing import Any, Iterable
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync.run_stats import filesystem_capacity
|
||||
from pobsync.run_stats import filesystem_capacity, tree_usage
|
||||
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
||||
|
||||
|
||||
def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: GlobalConfig | None) -> dict[str, Any]:
|
||||
hosts = list(hosts)
|
||||
runs = list(
|
||||
BackupRun.objects.select_related("host", "snapshot")
|
||||
.filter(status__in=_COMPLETED_BACKUP_STATUSES)
|
||||
@@ -21,6 +22,7 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
||||
|
||||
for host in hosts:
|
||||
host.stats_summary = collect_host_stats(host=host)
|
||||
backup_data = _sum_backup_data_by_kind(host.stats_summary["backup_data"] for host in hosts)
|
||||
|
||||
literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in real_runs]
|
||||
literal_values = [value for value in literal_values if value is not None]
|
||||
@@ -51,6 +53,7 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
||||
"estimated_runs_until_full": int(available / avg_literal) if available and avg_literal else None,
|
||||
"estimated_days_until_full": int(available / daily_literal) if available and daily_literal else None,
|
||||
"capacity": capacity,
|
||||
"backup_data": backup_data,
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +64,7 @@ def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]:
|
||||
trend_runs = [run for run in completed_real_runs if run["has_stats"]][:limit]
|
||||
latest_snapshot = host.snapshots.order_by("-started_at", "-discovered_at", "-id").first()
|
||||
latest_snapshot_stats = _snapshot_summary(latest_snapshot) if latest_snapshot else {}
|
||||
backup_data = _backup_data_by_kind(host)
|
||||
|
||||
literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in trend_runs]
|
||||
literal_values = [value for value in literal_values if value is not None]
|
||||
@@ -75,6 +79,7 @@ def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]:
|
||||
"latest_good_run": _first_run_with_status(real_runs, {BackupRun.Status.SUCCESS}),
|
||||
"latest_problem_run": _first_run_with_status(real_runs, {BackupRun.Status.WARNING, BackupRun.Status.FAILED}),
|
||||
"latest_snapshot": latest_snapshot_stats,
|
||||
"backup_data": backup_data,
|
||||
"avg_literal_data_bytes": _average(literal_values),
|
||||
"avg_daily_literal_data_bytes": _average_daily_literal(trend_runs),
|
||||
"total_literal_data_bytes": sum(literal_values),
|
||||
@@ -102,6 +107,65 @@ def _run_summary(run: BackupRun) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _backup_data_by_kind(host: HostConfig) -> dict[str, Any]:
|
||||
rows: dict[str, dict[str, int]] = {
|
||||
SnapshotRecord.Kind.SCHEDULED: _empty_snapshot_data_row(),
|
||||
SnapshotRecord.Kind.MANUAL: _empty_snapshot_data_row(),
|
||||
SnapshotRecord.Kind.INCOMPLETE: _empty_snapshot_data_row(),
|
||||
}
|
||||
total = _empty_snapshot_data_row()
|
||||
|
||||
for snapshot in host.snapshots.all():
|
||||
summary = _snapshot_summary(snapshot)
|
||||
row = rows.setdefault(snapshot.kind, _empty_snapshot_data_row())
|
||||
allocated = summary.get("allocated_size_bytes") or summary.get("apparent_size_bytes") or 0
|
||||
apparent = summary.get("apparent_size_bytes") or 0
|
||||
unique_apparent = summary.get("unique_apparent_size_bytes") or 0
|
||||
row["count"] += 1
|
||||
row["allocated_size_bytes"] += int(allocated)
|
||||
row["apparent_size_bytes"] += int(apparent)
|
||||
row["unique_apparent_size_bytes"] += int(unique_apparent)
|
||||
total["count"] += 1
|
||||
total["allocated_size_bytes"] += int(allocated)
|
||||
total["apparent_size_bytes"] += int(apparent)
|
||||
total["unique_apparent_size_bytes"] += int(unique_apparent)
|
||||
|
||||
return {
|
||||
"scheduled": rows[SnapshotRecord.Kind.SCHEDULED],
|
||||
"manual": rows[SnapshotRecord.Kind.MANUAL],
|
||||
"incomplete": rows[SnapshotRecord.Kind.INCOMPLETE],
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
def _empty_snapshot_data_row() -> dict[str, int]:
|
||||
return {
|
||||
"count": 0,
|
||||
"allocated_size_bytes": 0,
|
||||
"apparent_size_bytes": 0,
|
||||
"unique_apparent_size_bytes": 0,
|
||||
}
|
||||
|
||||
|
||||
def _sum_backup_data_by_kind(rows: Iterable[dict[str, dict[str, int]]]) -> dict[str, dict[str, int]]:
|
||||
total_rows: dict[str, dict[str, int]] = {
|
||||
"scheduled": _empty_snapshot_data_row(),
|
||||
"manual": _empty_snapshot_data_row(),
|
||||
"incomplete": _empty_snapshot_data_row(),
|
||||
"total": _empty_snapshot_data_row(),
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
for kind, values in row.items():
|
||||
total_row = total_rows.setdefault(kind, _empty_snapshot_data_row())
|
||||
total_row["count"] += values.get("count", 0)
|
||||
total_row["allocated_size_bytes"] += values.get("allocated_size_bytes", 0)
|
||||
total_row["apparent_size_bytes"] += values.get("apparent_size_bytes", 0)
|
||||
total_row["unique_apparent_size_bytes"] += values.get("unique_apparent_size_bytes", 0)
|
||||
|
||||
return total_rows
|
||||
|
||||
|
||||
def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
|
||||
if snapshot is None:
|
||||
return {}
|
||||
@@ -109,18 +173,43 @@ def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
|
||||
stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
|
||||
storage = stats.get("storage") if isinstance(stats.get("storage"), dict) else {}
|
||||
snapshot_storage = storage.get("snapshot") if isinstance(storage.get("snapshot"), dict) else {}
|
||||
if snapshot.kind == SnapshotRecord.Kind.INCOMPLETE:
|
||||
snapshot_storage = _snapshot_storage_from_filesystem(snapshot)
|
||||
else:
|
||||
has_recorded_size = (
|
||||
_int_at(snapshot_storage, "allocated_size_bytes") is not None
|
||||
or _int_at(snapshot_storage, "apparent_size_bytes") is not None
|
||||
)
|
||||
if not has_recorded_size:
|
||||
snapshot_storage = _snapshot_storage_from_filesystem(snapshot)
|
||||
apparent_size = _int_at(snapshot_storage, "apparent_size_bytes")
|
||||
hardlinked_apparent = _int_at(snapshot_storage, "hardlinked_apparent_size_bytes") or 0
|
||||
return {
|
||||
"id": snapshot.id,
|
||||
"dirname": snapshot.dirname,
|
||||
"kind": snapshot.kind,
|
||||
"status": snapshot.status,
|
||||
"started_at": snapshot.started_at,
|
||||
"apparent_size_bytes": _int_at(snapshot_storage, "apparent_size_bytes"),
|
||||
"apparent_size_bytes": apparent_size,
|
||||
"allocated_size_bytes": _int_at(snapshot_storage, "allocated_size_bytes"),
|
||||
"hardlinked_files": _int_at(snapshot_storage, "hardlinked_files"),
|
||||
"hardlinked_apparent_size_bytes": hardlinked_apparent,
|
||||
"unique_apparent_size_bytes": max((apparent_size or 0) - hardlinked_apparent, 0),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot_storage_from_filesystem(snapshot: SnapshotRecord) -> dict[str, Any]:
|
||||
if not snapshot.path:
|
||||
return {}
|
||||
snapshot_path = Path(snapshot.path)
|
||||
data_path = snapshot_path / "data"
|
||||
if snapshot_path.name == "data":
|
||||
return tree_usage(snapshot_path)
|
||||
if data_path.exists():
|
||||
return tree_usage(data_path)
|
||||
return tree_usage(snapshot_path)
|
||||
|
||||
|
||||
def _is_real_run(run: BackupRun) -> bool:
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
if result.get("dry_run") is True:
|
||||
|
||||
@@ -559,6 +559,15 @@
|
||||
gap: 8px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
.refresh-controls {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: 14px;
|
||||
justify-content: space-between;
|
||||
}
|
||||
.refresh-controls h2 {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.trend-bars {
|
||||
display: grid;
|
||||
gap: 5px;
|
||||
@@ -743,6 +752,15 @@
|
||||
.host-card-warning > * {
|
||||
min-width: 0;
|
||||
}
|
||||
.host-card-actions {
|
||||
align-items: center;
|
||||
border-top: 1px solid var(--border);
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 14px;
|
||||
padding-top: 12px;
|
||||
}
|
||||
.messages { display: grid; gap: 8px; margin-bottom: 18px; }
|
||||
.message {
|
||||
background: var(--panel);
|
||||
@@ -837,6 +855,10 @@
|
||||
.page-header .actions { justify-content: flex-start; }
|
||||
.two-col,
|
||||
.panel-grid { grid-template-columns: 1fr; }
|
||||
.refresh-controls {
|
||||
align-items: stretch;
|
||||
display: grid;
|
||||
}
|
||||
.dashboard-priority-grid { grid-template-columns: 1fr; }
|
||||
.host-control-grid { grid-template-columns: 1fr; }
|
||||
.schedule-row { grid-template-columns: 1fr; }
|
||||
@@ -896,7 +918,9 @@
|
||||
<strong><a href="{% url 'dashboard' %}">pobsync</a></strong>
|
||||
<span class="nav-primary" aria-label="Primary navigation">
|
||||
<a href="{% url 'dashboard' %}" {% if request.resolver_match.url_name == "dashboard" %}aria-current="page"{% endif %}>Dashboard</a>
|
||||
<a href="{% url 'hosts_list' %}" {% if request.resolver_match.url_name == "hosts_list" or request.resolver_match.url_name == "host_detail" or request.resolver_match.url_name == "create_host_config" or request.resolver_match.url_name == "edit_host_config" or request.resolver_match.url_name == "edit_host_schedule" %}aria-current="page"{% endif %}>Hosts</a>
|
||||
<a href="{% url 'ssh_credentials' %}" {% if request.resolver_match.url_name == "ssh_credentials" or request.resolver_match.url_name == "create_ssh_credential" or request.resolver_match.url_name == "generate_ssh_credential" or request.resolver_match.url_name == "edit_ssh_credential" %}aria-current="page"{% endif %}>SSH Keys</a>
|
||||
<a href="{% url 'notification_targets' %}" {% if request.resolver_match.url_name == "notification_targets" or request.resolver_match.url_name == "create_notification_target" or request.resolver_match.url_name == "edit_notification_target" %}aria-current="page"{% endif %}>Notifications</a>
|
||||
<a href="{% url 'logs' %}" {% if request.resolver_match.url_name == "logs" %}aria-current="page"{% endif %}>Logs</a>
|
||||
<a href="{% url 'purged_snapshots' %}" {% if request.resolver_match.url_name == "purged_snapshots" %}aria-current="page"{% endif %}>Purged</a>
|
||||
</span>
|
||||
@@ -922,8 +946,20 @@
|
||||
</main>
|
||||
<script>
|
||||
(() => {
|
||||
const updateRefreshControls = (region) => {
|
||||
const toggle = document.querySelector(`[data-refresh-toggle][data-refresh-target="${region.id}"]`);
|
||||
const state = document.querySelector(`[data-refresh-state="${region.id}"]`);
|
||||
const paused = region.dataset.refreshPaused === "true";
|
||||
const active = region.dataset.refreshActive === "true";
|
||||
if (state) state.textContent = paused ? "paused" : (active ? "on" : "off");
|
||||
if (toggle) {
|
||||
toggle.textContent = paused ? "Resume refresh" : "Pause refresh";
|
||||
toggle.disabled = !active && !paused;
|
||||
}
|
||||
};
|
||||
|
||||
const refreshRegion = async (region) => {
|
||||
if (region.dataset.refreshActive !== "true" || document.hidden) return;
|
||||
if (region.dataset.refreshActive !== "true" || region.dataset.refreshPaused === "true" || document.hidden) return;
|
||||
try {
|
||||
const response = await fetch(region.dataset.refreshUrl, {
|
||||
credentials: "same-origin",
|
||||
@@ -933,13 +969,26 @@
|
||||
region.innerHTML = await response.text();
|
||||
const refreshActive = response.headers.get("X-Pobsync-Refresh-Active");
|
||||
if (refreshActive) region.dataset.refreshActive = refreshActive;
|
||||
updateRefreshControls(region);
|
||||
} catch (error) {
|
||||
// Keep the current server-rendered content visible if a refresh fails.
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener("click", (event) => {
|
||||
const toggle = event.target.closest("[data-refresh-toggle]");
|
||||
if (!toggle) return;
|
||||
const region = document.getElementById(toggle.dataset.refreshTarget);
|
||||
if (!region) return;
|
||||
const paused = region.dataset.refreshPaused === "true";
|
||||
region.dataset.refreshPaused = paused ? "false" : "true";
|
||||
if (paused && region.dataset.refreshActive === "true") refreshRegion(region);
|
||||
updateRefreshControls(region);
|
||||
});
|
||||
|
||||
document.querySelectorAll("[data-refresh-url]").forEach((region) => {
|
||||
const interval = Number.parseInt(region.dataset.refreshInterval || "5000", 10);
|
||||
updateRefreshControls(region);
|
||||
window.setInterval(() => refreshRegion(region), Number.isFinite(interval) ? interval : 5000);
|
||||
});
|
||||
})();
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
</div>
|
||||
|
||||
<section class="grid dashboard-summary-grid" aria-label="Summary">
|
||||
<a class="metric metric-link" href="#hosts"><div class="label">Hosts</div><div class="value">{{ counts.enabled_hosts }}/{{ counts.hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}"><div class="label">Hosts</div><div class="value">{{ counts.enabled_hosts }}/{{ counts.hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'schedules_list' %}"><div class="label">Schedules</div><div class="value">{{ counts.enabled_schedules }}/{{ counts.schedules }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'snapshots_list' %}"><div class="label">Snapshots</div><div class="value">{{ counts.snapshots }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'runs_list' %}"><div class="label">Runs</div><div class="value">{{ counts.runs }}</div></a>
|
||||
|
||||
@@ -172,6 +172,36 @@
|
||||
<div class="metric"><div class="label">Incomplete</div><div class="value">{{ counts.incomplete_snapshots }}</div></div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Backup Data</h2>
|
||||
<section class="grid" aria-label="Host backup data totals">
|
||||
<div class="metric">
|
||||
<div class="label">Scheduled</div>
|
||||
<div class="value">{{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Manual</div>
|
||||
<div class="value">{{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Incomplete</div>
|
||||
<div class="value">{{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">measured from disk</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Total</div>
|
||||
<div class="value">{{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
</section>
|
||||
<p class="muted">
|
||||
Main totals use allocated snapshot size. Unique values estimate non-hardlinked visible data; incomplete
|
||||
snapshots are measured from disk because their metadata can be stale.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
{% if stats_summary.runs %}
|
||||
<section class="panel">
|
||||
<h2>Backup Trends</h2>
|
||||
@@ -373,7 +403,10 @@
|
||||
<div class="record-fact"><span class="label">Rsync binary:</span><strong>{{ effective_config.rsync.binary }}</strong></div>
|
||||
<div class="record-fact"><span class="label">Rsync args:</span><span>{{ effective_config.rsync.args|join:" " }}</span></div>
|
||||
<div class="record-fact"><span class="label">Timeout:</span><strong>{{ effective_config.rsync.timeout_seconds }}s</strong></div>
|
||||
<div class="record-fact"><span class="label">Bandwidth limit:</span><strong>{{ effective_config.rsync.bwlimit_kbps }} KB/s</strong></div>
|
||||
<div class="record-fact">
|
||||
<span class="label">Bandwidth limit:</span>
|
||||
<strong>{% if effective_config.rsync.bwlimit_kbps %}{{ effective_config.rsync.bwlimit_kbps }} KB/s{% else %}unlimited{% endif %}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article class="record-card">
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Hosts | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Inventory</div>
|
||||
<h1>Hosts</h1>
|
||||
<div class="page-subtitle">Configured backup targets, schedules, retention state, and host-level controls.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Host actions">
|
||||
<a class="button-link" href="{% url 'create_host_config' %}">New host</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="grid dashboard-summary-grid" aria-label="Host summary">
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}"><div class="label">Showing</div><div class="value">{{ counts.hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}?enabled=yes"><div class="label">Enabled</div><div class="value">{{ counts.enabled_hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'hosts_list' %}?enabled=no"><div class="label">Disabled</div><div class="value">{{ counts.disabled_hosts }}</div></a>
|
||||
<a class="metric metric-link" href="{% url 'dashboard' %}"><div class="label">Total</div><div class="value">{{ total_count }}</div></a>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Filters</h2>
|
||||
<form class="filter-form" method="get">
|
||||
<div class="field">
|
||||
<label for="enabled">Host state</label>
|
||||
<select id="enabled" name="enabled">
|
||||
<option value="" {% if selected_enabled == "" %}selected{% endif %}>All hosts</option>
|
||||
<option value="yes" {% if selected_enabled == "yes" %}selected{% endif %}>Enabled only</option>
|
||||
<option value="no" {% if selected_enabled == "no" %}selected{% endif %}>Disabled only</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="submit">Apply filters</button>
|
||||
<a class="button-link secondary" href="{% url 'hosts_list' %}">Reset</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
{% include "pobsync_backend/partials/dashboard_hosts.html" %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,38 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}{{ title }} | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Reports</div>
|
||||
<h1>{{ title }}</h1>
|
||||
<div class="page-subtitle">Choose which completed backup statuses should trigger an email or webhook report.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Notification target form actions">
|
||||
<a class="button-link" href="{% url 'notification_targets' %}">Back to notifications</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>{% if target %}Edit Target{% else %}Create Target{% endif %}</h2>
|
||||
<form method="post" class="form-grid">
|
||||
{% csrf_token %}
|
||||
{{ form.non_field_errors }}
|
||||
|
||||
{% for field in form %}
|
||||
<div class="field">
|
||||
{{ field.errors }}
|
||||
<label for="{{ field.id_for_label }}">{{ field.label }}</label>
|
||||
{{ field }}
|
||||
{% if field.help_text %}<div class="helptext">{{ field.help_text }}</div>{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<div class="form-actions">
|
||||
<button type="submit">{{ submit_label }}</button>
|
||||
<a class="button-link secondary" href="{% url 'notification_targets' %}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,91 @@
|
||||
{% extends "pobsync_backend/base.html" %}
|
||||
|
||||
{% block title %}Notifications | pobsync{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<header class="page-header">
|
||||
<div class="page-title">
|
||||
<div class="page-kicker">Reports</div>
|
||||
<h1>Notifications</h1>
|
||||
<div class="page-subtitle">Send email or webhook reports when backup runs finish.</div>
|
||||
</div>
|
||||
<section class="actions" aria-label="Notification actions">
|
||||
<a class="button-link" href="{% url 'create_notification_target' %}">New target</a>
|
||||
<a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
|
||||
</section>
|
||||
</header>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Targets</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Channel</th>
|
||||
<th>Status</th>
|
||||
<th>Events</th>
|
||||
<th>Destination</th>
|
||||
<th>Last delivery</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for target in targets %}
|
||||
<tr>
|
||||
<td><a href="{% url 'edit_notification_target' target.id %}">{{ target.name }}</a></td>
|
||||
<td>{{ target.get_channel_display }}</td>
|
||||
<td><span class="status {% if target.enabled %}ok{% else %}skipped{% endif %}">{{ target.enabled|yesno:"enabled,disabled" }}</span></td>
|
||||
<td>{{ target.statuses|join:", " }}</td>
|
||||
<td>
|
||||
{% if target.channel == "email" %}
|
||||
{{ target.email_to|linebreaksbr }}
|
||||
{% else %}
|
||||
<code>{{ target.webhook_url|truncatechars:70 }}</code>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if target.last_status %}
|
||||
<span class="status {{ target.last_status }}">{{ target.last_status }}</span>
|
||||
{% if target.last_error %}<div class="muted">{{ target.last_error|truncatechars:90 }}</div>{% endif %}
|
||||
{% if target.last_sent_at %}<div class="muted">{{ target.last_sent_at }}</div>{% endif %}
|
||||
{% else %}
|
||||
<span class="muted">none</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><a class="button-link secondary" href="{% url 'edit_notification_target' target.id %}">Edit</a></td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="7" class="muted">No notification targets configured yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>Recent Deliveries</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Target</th>
|
||||
<th>Run</th>
|
||||
<th>Status</th>
|
||||
<th>Created</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for delivery in deliveries %}
|
||||
<tr>
|
||||
<td>{{ delivery.target.name }}</td>
|
||||
<td><a href="{% url 'run_detail' delivery.run.id %}">Run {{ delivery.run.id }}</a> {{ delivery.run.host.host }}</td>
|
||||
<td><span class="status {{ delivery.status }}">{{ delivery.status }}</span></td>
|
||||
<td>{{ delivery.created_at }}</td>
|
||||
<td class="muted">{{ delivery.error|default:"" }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="5" class="muted">No notification deliveries recorded yet.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -22,6 +22,14 @@
|
||||
{% if host.failed_run_count %}
|
||||
<span class="status failed">failed {{ host.failed_run_count }}</span>
|
||||
{% endif %}
|
||||
{% if show_host_controls %}
|
||||
{% if host.schedule %}
|
||||
<span class="status {% if host.schedule.enabled %}ok{% else %}skipped{% endif %}">schedule {{ host.schedule.enabled|yesno:"on,paused" }}</span>
|
||||
<span class="status {% if host.schedule.prune %}ok{% else %}skipped{% endif %}">retention {{ host.schedule.prune|yesno:"on,paused" }}</span>
|
||||
{% else %}
|
||||
<span class="status skipped">no schedule</span>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="host-card-layout">
|
||||
@@ -94,6 +102,26 @@
|
||||
<div class="label">Retention</div>
|
||||
<div class="value">d{{ host.retention_daily }} w{{ host.retention_weekly }} m{{ host.retention_monthly }} y{{ host.retention_yearly }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Scheduled data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Manual data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Incomplete data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">measured from disk</div>
|
||||
</div>
|
||||
<div class="host-card-stat">
|
||||
<div class="label">Total data</div>
|
||||
<div class="value">{{ host.stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</div>
|
||||
<div class="muted">unique {{ host.stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -115,6 +143,33 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if show_host_controls %}
|
||||
<div class="host-card-actions">
|
||||
<a class="button-link compact secondary" href="{% url 'host_detail' host.host %}">Open</a>
|
||||
<a class="button-link compact secondary" href="{% url 'edit_host_config' host.host %}">Edit config</a>
|
||||
<a class="button-link compact secondary" href="{% url 'edit_host_schedule' host.host %}">{% if host.schedule %}Edit schedule{% else %}Create schedule{% endif %}</a>
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.enabled %}disable_host{% else %}enable_host{% endif %}">
|
||||
<button class="compact {% if host.enabled %}secondary{% endif %}" type="submit">{{ host.enabled|yesno:"Disable host,Enable host" }}</button>
|
||||
</form>
|
||||
{% if host.schedule %}
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.schedule.enabled %}disable_schedule{% else %}enable_schedule{% endif %}">
|
||||
<button class="compact secondary" type="submit">{{ host.schedule.enabled|yesno:"Pause schedule,Resume schedule" }}</button>
|
||||
</form>
|
||||
<form class="inline-form" method="post" action="{% url 'update_host_state' host.host %}">
|
||||
{% csrf_token %}
|
||||
<input type="hidden" name="next" value="{{ request.get_full_path }}">
|
||||
<input type="hidden" name="action" value="{% if host.schedule.prune %}disable_prune{% else %}enable_prune{% endif %}">
|
||||
<button class="compact secondary" type="submit">{{ host.schedule.prune|yesno:"Pause retention,Resume retention" }}</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% empty %}
|
||||
<p class="muted">No hosts configured yet.</p>
|
||||
|
||||
@@ -126,5 +126,27 @@
|
||||
{% else %}
|
||||
<p class="muted">Storage pressure appears after the first completed backup with stats.</p>
|
||||
{% endif %}
|
||||
<div class="storage-priority-facts">
|
||||
<div>
|
||||
<span class="label">Scheduled data</span>
|
||||
<strong>{{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.scheduled.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Manual data</span>
|
||||
<strong>{{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.manual.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Incomplete data</span>
|
||||
<strong>{{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">measured from disk</span>
|
||||
</div>
|
||||
<div>
|
||||
<span class="label">Total snapshot data</span>
|
||||
<strong>{{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}</strong>
|
||||
<span class="muted">unique {{ stats_summary.backup_data.total.unique_apparent_size_bytes|filesizeformat }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
@@ -59,9 +59,13 @@
|
||||
|
||||
{% if dry_run_summary %}
|
||||
<section class="panel highlight {{ dry_run_summary.highlight_class }}">
|
||||
<h2>Dry Run Summary</h2>
|
||||
<section class="grid" aria-label="Dry run summary">
|
||||
<h2>Run Progress</h2>
|
||||
<section class="grid" aria-label="Run progress">
|
||||
<div class="metric"><div class="label">Status</div><div class="value">{{ dry_run_summary.status }}</div></div>
|
||||
<div class="metric">
|
||||
<div class="label">Mode</div>
|
||||
<div class="value">dry run</div>
|
||||
</div>
|
||||
<div class="metric"><div class="label">Files Seen</div><div class="value">{{ dry_run_summary.files_seen|default:"unknown" }}</div></div>
|
||||
<div class="metric"><div class="label">Would Transfer</div><div class="value">{{ dry_run_summary.files_would_transfer|default:"unknown" }}</div></div>
|
||||
<div class="metric"><div class="label">Transfer Estimate</div><div class="value">{{ dry_run_summary.transfer_estimate_bytes|filesizeformat }}</div></div>
|
||||
@@ -96,6 +100,74 @@
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if live_progress %}
|
||||
<section class="panel highlight running">
|
||||
<h2>Run Progress</h2>
|
||||
<section class="grid" aria-label="Run progress">
|
||||
<div class="metric">
|
||||
<div class="label">Status</div>
|
||||
<div class="value">{{ run.status }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Mode</div>
|
||||
<div class="value">backup</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Phase</div>
|
||||
<div class="value">{{ live_progress.phase }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Rsync PID</div>
|
||||
<div class="value">{{ live_progress.rsync_pid|default:"" }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Log Updated</div>
|
||||
<div class="value">
|
||||
{% if live_progress.log.exists %}
|
||||
{{ live_progress.log.seconds_since_modified }}s ago
|
||||
{% else %}
|
||||
missing
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Log Size</div>
|
||||
<div class="value">{{ live_progress.log.size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
{% if live_progress.snapshot.exists %}
|
||||
<div class="metric">
|
||||
<div class="label">Data Files</div>
|
||||
<div class="value">{% if live_progress.snapshot.scan_limited %}at least {% endif %}{{ live_progress.snapshot.files }}</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="label">Data Size</div>
|
||||
<div class="value">{% if live_progress.snapshot.scan_limited %}at least {% endif %}{{ live_progress.snapshot.apparent_size_bytes|filesizeformat }}</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
<div class="stack">
|
||||
{% if live_progress.snapshot.path %}
|
||||
<div><strong>Snapshot path:</strong> {{ live_progress.snapshot.path }}</div>
|
||||
{% endif %}
|
||||
{% if live_progress.snapshot.scan_limited %}
|
||||
<div class="muted">Progress scan was capped to keep the UI responsive.</div>
|
||||
{% endif %}
|
||||
{% if live_progress.log.path %}
|
||||
<div>
|
||||
<strong>Log:</strong>
|
||||
{% if live_progress.log.exists %}
|
||||
<a href="{% url 'run_rsync_log' run.id %}">Open full rsync log</a>
|
||||
{% else %}
|
||||
<span class="muted">{{ live_progress.log.path }} (missing)</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div><strong>Log path:</strong> {{ live_progress.log.path }}</div>
|
||||
{% endif %}
|
||||
<div><strong>Warnings:</strong> none recorded</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<div class="two-col">
|
||||
<section class="panel">
|
||||
<h2>Timing</h2>
|
||||
|
||||
@@ -45,8 +45,9 @@
|
||||
snapshots automatically because they can indicate an interrupted backup that should be inspected first.
|
||||
</p>
|
||||
<p>
|
||||
After inspection, use the dedicated cleanup form below to delete only incomplete snapshot directories and their
|
||||
tracking records. Successful scheduled and manual snapshots are not touched by this cleanup.
|
||||
{{ incomplete_unreviewed_count }} still need review. After inspection, mark them reviewed and use the dedicated
|
||||
cleanup form below to delete only incomplete snapshot directories and their tracking records. Successful
|
||||
scheduled and manual snapshots are not touched by this cleanup.
|
||||
</p>
|
||||
</section>
|
||||
{% endif %}
|
||||
@@ -187,6 +188,7 @@
|
||||
<th>Dirname</th>
|
||||
<th>Started</th>
|
||||
<th>Status</th>
|
||||
<th>Review</th>
|
||||
<th>Reason</th>
|
||||
<th>Path</th>
|
||||
</tr>
|
||||
@@ -197,6 +199,14 @@
|
||||
<td>{{ snapshot.dirname }}</td>
|
||||
<td>{{ snapshot.dt }}</td>
|
||||
<td>{{ snapshot.status|default:"" }}</td>
|
||||
<td>
|
||||
{% if snapshot.reviewed %}
|
||||
<span class="status ok">reviewed</span>
|
||||
<span class="muted">{{ snapshot.reviewed_by|default:"unknown" }}</span>
|
||||
{% else %}
|
||||
<span class="status warning">needs review</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ snapshot.reason }}</td>
|
||||
<td class="muted">{{ snapshot.path }}</td>
|
||||
</tr>
|
||||
@@ -205,9 +215,19 @@
|
||||
</table>
|
||||
|
||||
<h3>Cleanup Incomplete Snapshots</h3>
|
||||
{% if incomplete_unreviewed_count %}
|
||||
<p class="muted">
|
||||
This deletes only incomplete snapshot directories and their tracking records. Successful manual and scheduled
|
||||
snapshots are not touched.
|
||||
Cleanup is blocked until all incomplete snapshots are reviewed. This extra step makes it explicit that the
|
||||
interrupted backup was inspected before deletion.
|
||||
</p>
|
||||
<form method="post" action="{% url 'resolve_host_incomplete_reviews' host.host %}" class="actions inline">
|
||||
{% csrf_token %}
|
||||
<button type="submit" class="secondary">Mark incomplete snapshots reviewed</button>
|
||||
</form>
|
||||
{% else %}
|
||||
<p class="muted">
|
||||
This deletes only reviewed incomplete snapshot directories and their tracking records. Successful manual and
|
||||
scheduled snapshots are not touched.
|
||||
</p>
|
||||
<form method="post" action="{% url 'cleanup_host_incomplete_snapshots' host.host %}" class="form-grid">
|
||||
{% csrf_token %}
|
||||
@@ -239,6 +259,7 @@
|
||||
<a class="button-link secondary" href="{% url 'host_retention_plan' host.host %}?kind={{ kind }}">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
@@ -14,10 +14,22 @@
|
||||
</section>
|
||||
</header>
|
||||
|
||||
{% if can_auto_refresh %}
|
||||
<section class="panel refresh-controls" aria-label="Live refresh controls">
|
||||
<div>
|
||||
<h2>Live Updates</h2>
|
||||
<p class="muted">Auto-refresh is <strong data-refresh-state="run-live-region">on</strong> while this run is active.</p>
|
||||
</div>
|
||||
<button type="button" class="secondary" data-refresh-toggle data-refresh-target="run-live-region">Pause refresh</button>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<div
|
||||
id="run-live-region"
|
||||
data-refresh-url="{% url 'run_detail_live' run.id %}"
|
||||
data-refresh-interval="5000"
|
||||
data-refresh-active="{{ can_auto_refresh|yesno:'true,false' }}"
|
||||
data-refresh-paused="false"
|
||||
aria-live="polite"
|
||||
>
|
||||
{% include "pobsync_backend/partials/run_detail_live.html" %}
|
||||
@@ -38,6 +50,10 @@
|
||||
|
||||
<section class="panel">
|
||||
<h2>Rsync Command</h2>
|
||||
<p class="muted">
|
||||
<strong>Bandwidth limit:</strong>
|
||||
{% if rsync_bwlimit_kbps %}{{ rsync_bwlimit_kbps }} KB/s{% else %}unlimited{% endif %}
|
||||
</p>
|
||||
{% if rsync_command %}
|
||||
<pre>{% for part in rsync_command %}{{ part }}{% if not forloop.last %}
|
||||
{% endif %}{% endfor %}</pre>
|
||||
|
||||
@@ -11,7 +11,7 @@ from django.utils import timezone
|
||||
from pobsync.util import write_yaml_atomic
|
||||
from pobsync_backend.backup_runner import queue_backup_run, reconcile_running_runs
|
||||
from pobsync_backend.management.commands.run_pobsync_worker import Command
|
||||
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
||||
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, NotificationDelivery, NotificationTarget, SnapshotRecord
|
||||
|
||||
|
||||
class BackupWorkerTests(TestCase):
|
||||
@@ -85,6 +85,73 @@ class BackupWorkerTests(TestCase):
|
||||
self.assertEqual(SnapshotRecord.objects.count(), 1)
|
||||
self.assertEqual(run.snapshot, SnapshotRecord.objects.get())
|
||||
|
||||
def test_worker_records_warning_status_from_completed_run(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snapshot_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(meta_dir / "meta.yaml", {"status": "warning", "started_at": "2026-05-19T02:15:00Z"})
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
run_scheduled.return_value = {
|
||||
"ok": True,
|
||||
"status": "warning",
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"warning": {"category": "vanished"},
|
||||
"rsync": {"exit_code": 24},
|
||||
}
|
||||
|
||||
count = Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
self.assertEqual(count, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.WARNING)
|
||||
self.assertEqual(run.rsync_exit_code, 24)
|
||||
self.assertEqual(run.result["warning"]["category"], "vanished")
|
||||
|
||||
def test_worker_sends_notification_after_completed_run(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snapshot_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(meta_dir / "meta.yaml", {"status": "success", "started_at": "2026-05-19T02:15:00Z"})
|
||||
run = queue_backup_run(host=host)
|
||||
|
||||
with (
|
||||
patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled,
|
||||
patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail,
|
||||
):
|
||||
run_scheduled.return_value = {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": str(snapshot_dir),
|
||||
"base": None,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.SUCCESS)
|
||||
self.assertEqual(NotificationDelivery.objects.get(run=run).status, NotificationDelivery.Status.SENT)
|
||||
send_mail.assert_called_once()
|
||||
|
||||
def test_worker_refreshes_heartbeat_while_run_is_active(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
@@ -116,6 +183,44 @@ class BackupWorkerTests(TestCase):
|
||||
run_scheduled.side_effect = fake_run_scheduled
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
def test_worker_records_real_run_log_path_while_running(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
snapshot_dir = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH"
|
||||
log_path = snapshot_dir / "meta" / "rsync.log"
|
||||
|
||||
with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled:
|
||||
def fake_run_scheduled(**kwargs):
|
||||
kwargs["state_callback"](
|
||||
{
|
||||
"status": "running",
|
||||
"phase": "rsync",
|
||||
"snapshot": str(snapshot_dir),
|
||||
"log": str(log_path),
|
||||
"rsync": {"command": ["rsync"], "exit_code": None, "pid": 1234, "pgid": 1234},
|
||||
}
|
||||
)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.snapshot_path, str(snapshot_dir))
|
||||
self.assertEqual(run.result["execution"]["phase"], "rsync")
|
||||
self.assertEqual(run.result["execution"]["log"], str(log_path))
|
||||
self.assertEqual(run.result["execution"]["snapshot"], str(snapshot_dir))
|
||||
self.assertEqual(run.result["rsync"]["command"], ["rsync"])
|
||||
self.assertEqual(run.result["rsync"]["pid"], 1234)
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
"host": host.host,
|
||||
"snapshot": "",
|
||||
"base": None,
|
||||
"rsync": {"exit_code": 0},
|
||||
}
|
||||
|
||||
run_scheduled.side_effect = fake_run_scheduled
|
||||
Command()._run_once(prefix=Path(tmp) / "home")
|
||||
|
||||
def test_worker_reconciles_stale_real_run_after_heartbeat_timeout(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
@@ -136,6 +241,97 @@ class BackupWorkerTests(TestCase):
|
||||
self.assertEqual(run.result["failure"]["category"], "worker")
|
||||
self.assertIn("heartbeat stopped", run.result["failure"]["message"])
|
||||
|
||||
def test_worker_reconciles_real_run_with_terminal_broken_pipe_log(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text(
|
||||
"rsync error: unexplained error (code 255) at rsync.c(716) [generator=3.4.1]\n"
|
||||
"rsync error: received SIGUSR1 (code 19) at main.c(1600) [receiver=3.4.1]\n"
|
||||
"rsync: [generator] write error: Broken pipe (32)\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now()
|
||||
run.result["execution"] = {"log": str(log_path)}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs()
|
||||
|
||||
self.assertEqual(reconciled, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.rsync_exit_code, 255)
|
||||
self.assertEqual(run.result["failure"]["category"], "transport")
|
||||
self.assertIn("Broken pipe", "\n".join(run.result["rsync"]["log_tail"]))
|
||||
|
||||
def test_worker_reconciles_real_run_when_rsync_process_disappears(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text("sending incremental file list\n", encoding="utf-8")
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now() - timedelta(minutes=10)
|
||||
run.result["execution"] = {
|
||||
"phase": "rsync",
|
||||
"log": str(log_path),
|
||||
"heartbeat_at": (timezone.now() - timedelta(minutes=10)).isoformat(),
|
||||
}
|
||||
run.result["rsync"] = {"pid": 999999, "pgid": 999999, "command": ["rsync"]}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs(grace_seconds=300, stale_worker_seconds=24 * 60 * 60)
|
||||
|
||||
self.assertEqual(reconciled, 1)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.FAILED)
|
||||
self.assertEqual(run.result["failure"]["category"], "rsync_process")
|
||||
self.assertEqual(run.rsync_exit_code, 255)
|
||||
|
||||
def test_worker_does_not_reconcile_missing_rsync_process_during_finalizing_phase(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now() - timedelta(minutes=10)
|
||||
run.result["execution"] = {
|
||||
"phase": "finalizing",
|
||||
"heartbeat_at": (timezone.now() - timedelta(minutes=10)).isoformat(),
|
||||
}
|
||||
run.result["rsync"] = {"pid": 999999, "pgid": 999999, "command": ["rsync"], "exit_code": 0}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs(grace_seconds=300, stale_worker_seconds=24 * 60 * 60)
|
||||
|
||||
self.assertEqual(reconciled, 0)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.RUNNING)
|
||||
|
||||
def test_worker_does_not_fail_real_run_for_vanished_file_warning_log(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = queue_backup_run(host=host)
|
||||
log_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-021500Z__ABCDEFGH" / "meta" / "rsync.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.write_text(
|
||||
"file has vanished: \"/var/lib/app/session\"\n"
|
||||
"rsync warning: some files vanished before they could be transferred (code 24) at main.c(1338) [sender=3.4.1]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = timezone.now()
|
||||
run.result["execution"] = {"log": str(log_path)}
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
reconciled = reconcile_running_runs()
|
||||
|
||||
self.assertEqual(reconciled, 0)
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(run.status, BackupRun.Status.RUNNING)
|
||||
|
||||
def test_worker_records_dry_run_log_path_while_running(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))
|
||||
|
||||
@@ -42,6 +42,7 @@ class ConfigureCommandsTests(TestCase):
|
||||
address="web-01.example.test",
|
||||
exclude_add=["/tmp/***"],
|
||||
rsync_extra_arg=["--delete"],
|
||||
rsync_bwlimit_kbps=4096,
|
||||
stdout=out,
|
||||
)
|
||||
|
||||
@@ -49,10 +50,12 @@ class ConfigureCommandsTests(TestCase):
|
||||
self.assertEqual(host.retention_daily, 5)
|
||||
self.assertEqual(host.excludes_add, ["/tmp/***"])
|
||||
self.assertEqual(host.rsync_extra_args, ["--delete"])
|
||||
self.assertEqual(host.rsync_bwlimit_kbps, 4096)
|
||||
|
||||
effective = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
self.assertEqual(effective["retention"]["yearly"], 2)
|
||||
self.assertEqual(effective["excludes_effective"], ["/tmp/***"])
|
||||
self.assertEqual(effective["rsync"]["bwlimit_kbps"], 4096)
|
||||
|
||||
def test_configure_schedule_creates_sql_schedule(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
@@ -17,6 +17,7 @@ class DjangoConfigSourceTests(TestCase):
|
||||
backup_root="/backups",
|
||||
rsync_args=["--archive"],
|
||||
rsync_extra_args=["--numeric-ids"],
|
||||
rsync_bwlimit_kbps=10000,
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
@@ -28,6 +29,7 @@ class DjangoConfigSourceTests(TestCase):
|
||||
address="web-01.example.test",
|
||||
excludes_add=["/tmp/***"],
|
||||
rsync_extra_args=["--delete"],
|
||||
rsync_bwlimit_kbps=2500,
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
@@ -46,6 +48,24 @@ class DjangoConfigSourceTests(TestCase):
|
||||
self.assertEqual(cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"])
|
||||
self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"])
|
||||
self.assertEqual(cfg["rsync"]["bwlimit_kbps"], 2500)
|
||||
|
||||
def test_host_can_disable_global_rsync_bandwidth_limit(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
rsync_args=["--archive"],
|
||||
rsync_bwlimit_kbps=5000,
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
rsync_bwlimit_kbps=0,
|
||||
)
|
||||
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertEqual(cfg["rsync"]["bwlimit_kbps"], 0)
|
||||
|
||||
def test_materializes_global_ssh_credential_for_runtime_config(self) -> None:
|
||||
credential = SshCredential.objects.create(
|
||||
@@ -113,6 +133,7 @@ class DjangoConfigSourceTests(TestCase):
|
||||
)
|
||||
HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
|
||||
with override_settings(POBSYNC_HOME=str(Path(tmp) / "home")):
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertIn(f"-oIdentityFile={identity_file}", cfg["ssh"]["options"])
|
||||
|
||||
125
src/pobsync_backend/tests/test_notifications.py
Normal file
125
src/pobsync_backend/tests/test_notifications.py
Normal file
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget
|
||||
from pobsync_backend.notifications import notify_backup_run_completed
|
||||
|
||||
|
||||
class NotificationTests(TestCase):
|
||||
def test_email_notification_is_sent_for_matching_status(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
statuses=[BackupRun.Status.FAILED],
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
status=BackupRun.Status.FAILED,
|
||||
run_type=BackupRun.RunType.MANUAL,
|
||||
started_at=timezone.now() - timedelta(minutes=5),
|
||||
ended_at=timezone.now(),
|
||||
rsync_exit_code=12,
|
||||
result={"failure": {"message": "rsync failed"}},
|
||||
)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertTrue(results[0].sent)
|
||||
send_mail.assert_called_once()
|
||||
subject, message, _from_email, recipients = send_mail.call_args.args
|
||||
self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}")
|
||||
self.assertIn("Failure: rsync failed", message)
|
||||
self.assertEqual(recipients, ["ops@example.test"])
|
||||
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
||||
self.assertEqual(delivery.status, NotificationDelivery.Status.SENT)
|
||||
target.refresh_from_db()
|
||||
self.assertEqual(target.last_status, NotificationDelivery.Status.SENT)
|
||||
self.assertEqual(target.last_error, "")
|
||||
self.assertIsNotNone(target.last_sent_at)
|
||||
|
||||
def test_webhook_notification_posts_payload(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="discord",
|
||||
channel=NotificationTarget.Channel.WEBHOOK,
|
||||
webhook_url="https://hooks.example.test/pobsync",
|
||||
webhook_headers={"X-Token": "secret"},
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED)
|
||||
response = Mock()
|
||||
response.status = 204
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
|
||||
with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen:
|
||||
notify_backup_run_completed(run)
|
||||
|
||||
request = urlopen.call_args.args[0]
|
||||
self.assertEqual(request.full_url, "https://hooks.example.test/pobsync")
|
||||
self.assertEqual(request.get_method(), "POST")
|
||||
self.assertEqual(request.headers["X-token"], "secret")
|
||||
self.assertIn(f'"id": {run.id}', request.data.decode("utf-8"))
|
||||
self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT)
|
||||
|
||||
def test_notification_filters_statuses(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
NotificationTarget.objects.create(
|
||||
name="failures-only",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
statuses=[BackupRun.Status.FAILED],
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail") as send_mail:
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(results, [])
|
||||
send_mail.assert_not_called()
|
||||
self.assertEqual(NotificationDelivery.objects.count(), 0)
|
||||
|
||||
def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
||||
notify_backup_run_completed(run)
|
||||
notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1)
|
||||
send_mail.assert_called_once()
|
||||
|
||||
def test_failed_delivery_is_recorded_without_raising(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
target = NotificationTarget.objects.create(
|
||||
name="broken",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
)
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED)
|
||||
|
||||
with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")):
|
||||
results = notify_backup_run_completed(run)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertFalse(results[0].sent)
|
||||
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
||||
self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED)
|
||||
self.assertEqual(delivery.error, "smtp down")
|
||||
target.refresh_from_db()
|
||||
self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED)
|
||||
self.assertEqual(target.last_error, "smtp down")
|
||||
@@ -12,8 +12,9 @@ from pobsync.rsync import RsyncResult
|
||||
|
||||
|
||||
class FakeConfigSource:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups", bwlimit_kbps: int = 0) -> None:
|
||||
self.backup_root = backup_root
|
||||
self.bwlimit_kbps = bwlimit_kbps
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict:
|
||||
return {
|
||||
@@ -25,7 +26,7 @@ class FakeConfigSource:
|
||||
"binary": "rsync",
|
||||
"args_effective": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
"bwlimit_kbps": self.bwlimit_kbps,
|
||||
},
|
||||
"source_root": "/",
|
||||
"includes": [],
|
||||
@@ -54,6 +55,21 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
self.assertEqual(result["host"], "web-01")
|
||||
run_rsync.assert_called_once()
|
||||
|
||||
def test_dry_run_applies_configured_bandwidth_limit(self) -> None:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--bwlimit=4096"])
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
dry_run=True,
|
||||
config_source=FakeConfigSource(bwlimit_kbps=4096),
|
||||
)
|
||||
|
||||
command = run_rsync.call_args.args[0]
|
||||
self.assertIn("--bwlimit=4096", command)
|
||||
self.assertEqual(result["rsync"]["bwlimit_kbps"], 4096)
|
||||
|
||||
def test_failed_dry_run_includes_log_tail(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -186,11 +202,13 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
verbose_output=True,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups"), bwlimit_kbps=2048),
|
||||
)
|
||||
|
||||
command = run_rsync.call_args.args[0]
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertIn("--bwlimit=2048", command)
|
||||
self.assertEqual(result["rsync"]["bwlimit_kbps"], 2048)
|
||||
self.assertIn("--stats", command)
|
||||
self.assertIn("--itemize-changes", command)
|
||||
self.assertIn("--info=flist2,progress2,stats2", command)
|
||||
@@ -256,6 +274,71 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
self.assertIn("stats:", meta_text)
|
||||
self.assertIn("files_total: 10", meta_text)
|
||||
|
||||
def test_real_run_reports_running_state_callback_before_rsync_returns(self) -> None:
|
||||
states = []
|
||||
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None, process_started=None):
|
||||
self.assertEqual(len(states), 1)
|
||||
self.assertEqual(states[0]["status"], "running")
|
||||
self.assertEqual(states[0]["phase"], "preparing")
|
||||
self.assertEqual(states[0]["log"], str(log_path))
|
||||
self.assertEqual(states[0]["rsync"]["command"], command)
|
||||
self.assertIsNotNone(process_started)
|
||||
process_started(1234, 1234)
|
||||
self.assertEqual(len(states), 2)
|
||||
self.assertEqual(states[1]["phase"], "rsync")
|
||||
self.assertEqual(states[1]["rsync"]["pid"], 1234)
|
||||
self.assertEqual(states[1]["rsync"]["pgid"], 1234)
|
||||
log_path.write_text("Number of files: 1\n", encoding="utf-8")
|
||||
return RsyncResult(exit_code=0, command=command)
|
||||
|
||||
with TemporaryDirectory() as tmp:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
|
||||
run_scheduled(
|
||||
prefix=Path(tmp) / "home",
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
state_callback=states.append,
|
||||
)
|
||||
|
||||
self.assertEqual(len(states), 3)
|
||||
self.assertIn("/.incomplete/", states[0]["snapshot"])
|
||||
self.assertEqual(states[2]["phase"], "finalizing")
|
||||
self.assertEqual(states[2]["rsync"]["exit_code"], 0)
|
||||
|
||||
def test_real_run_keeps_snapshot_with_warning_for_vanished_files(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
log_path.write_text(
|
||||
"file has vanished: \"/var/lib/app/session\"\n"
|
||||
"rsync warning: some files vanished before they could be transferred (code 24) at main.c(1338) [sender=3.4.1]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
data_dir = log_path.parent.parent / "data"
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
(data_dir / "payload.txt").write_text("payload", encoding="utf-8")
|
||||
return RsyncResult(exit_code=24, command=command)
|
||||
|
||||
with TemporaryDirectory() as tmp:
|
||||
backup_root = Path(tmp) / "backups"
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
|
||||
result = run_scheduled(
|
||||
prefix=Path(tmp) / "home",
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
config_source=FakeConfigSource(backup_root=str(backup_root)),
|
||||
)
|
||||
|
||||
snapshot = Path(result["snapshot"])
|
||||
self.assertTrue((snapshot / "data" / "payload.txt").exists())
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["status"], "warning")
|
||||
self.assertEqual(result["rsync"]["exit_code"], 24)
|
||||
self.assertEqual(result["warning"]["category"], "vanished")
|
||||
self.assertEqual(snapshot.parent.name, "scheduled")
|
||||
self.assertIn("file has vanished", "\n".join(result["rsync"]["log_tail"]))
|
||||
|
||||
def test_dry_run_reports_cancelled_rsync(self) -> None:
|
||||
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
|
||||
self.assertTrue(cancel_check())
|
||||
|
||||
@@ -154,6 +154,68 @@ class SqlRetentionTests(TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_apply_deletes_snapshot_with_non_traversable_nested_directory(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
retention_daily=0,
|
||||
retention_weekly=0,
|
||||
retention_monthly=0,
|
||||
retention_yearly=0,
|
||||
)
|
||||
old_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260518-021500Z__OLD"
|
||||
restricted_dir = old_dir / "data" / "var" / "lib" / "snapd" / "void"
|
||||
restricted_dir.mkdir(parents=True)
|
||||
restricted_dir.joinpath("state").write_text("preserved permissions\n")
|
||||
restricted_dir.chmod(0)
|
||||
new_dir = Path(tmp) / "backups" / host.host / "scheduled" / "20260519-021500Z__NEW"
|
||||
new_dir.mkdir(parents=True)
|
||||
old = self._snapshot(host, old_dir.name, path=str(old_dir))
|
||||
self._snapshot(host, new_dir.name, path=str(new_dir))
|
||||
|
||||
result = run_sql_retention_apply(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(old_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
|
||||
self.assertEqual(result["deleted"][0]["dirname"], old.dirname)
|
||||
|
||||
def test_apply_rejects_scheduled_snapshot_path_outside_host_kind_directory(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
retention_daily=0,
|
||||
retention_weekly=0,
|
||||
retention_monthly=0,
|
||||
retention_yearly=0,
|
||||
)
|
||||
self._snapshot(
|
||||
host,
|
||||
"20260518-021500Z__OLD",
|
||||
path="/backups/web-01/manual/20260518-021500Z__OLD",
|
||||
)
|
||||
self._snapshot(host, "20260519-021500Z__NEW")
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "unexpected snapshot path"):
|
||||
run_sql_retention_apply(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_apply_respects_max_delete(self) -> None:
|
||||
host = HostConfig.objects.create(
|
||||
host="web-01",
|
||||
@@ -192,6 +254,8 @@ class SqlRetentionTests(TestCase):
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
result = run_incomplete_cleanup(
|
||||
@@ -213,6 +277,58 @@ class SqlRetentionTests(TestCase):
|
||||
self.assertEqual(purged.action, PurgedSnapshot.Action.INCOMPLETE_CLEANUP)
|
||||
self.assertEqual(purged.reason, "manual incomplete cleanup")
|
||||
|
||||
def test_incomplete_cleanup_deletes_non_traversable_nested_directory(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
incomplete_dir = Path(tmp) / "backups" / host.host / ".incomplete" / "20260519-031500Z__BROKEN01"
|
||||
restricted_dir = incomplete_dir / "data" / "var" / "lib" / "snapd" / "void"
|
||||
restricted_dir.mkdir(parents=True)
|
||||
restricted_dir.joinpath("state").write_text("interrupted\n")
|
||||
restricted_dir.chmod(0)
|
||||
record = SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
result = run_incomplete_cleanup(
|
||||
prefix=prefix,
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
self.assertFalse(incomplete_dir.exists())
|
||||
self.assertFalse(SnapshotRecord.objects.filter(pk=record.pk).exists())
|
||||
self.assertEqual(result["deleted"][0]["dirname"], incomplete_dir.name)
|
||||
|
||||
def test_incomplete_cleanup_requires_reviewed_snapshots(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "have not been reviewed"):
|
||||
run_incomplete_cleanup(
|
||||
prefix=Path("/tmp/pobsync-test"),
|
||||
host=host.host,
|
||||
yes=True,
|
||||
max_delete=1,
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
def test_incomplete_cleanup_respects_max_delete(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
@@ -222,6 +338,8 @@ class SqlRetentionTests(TestCase):
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "blocked by --max-delete=0"):
|
||||
@@ -242,6 +360,8 @@ class SqlRetentionTests(TestCase):
|
||||
path=f"/backups/{host.host}/scheduled/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ConfigError, "unexpected incomplete snapshot path"):
|
||||
|
||||
195
src/pobsync_backend/tests/test_stats_summary.py
Normal file
195
src/pobsync_backend/tests/test_stats_summary.py
Normal file
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync.run_stats import tree_usage
|
||||
from pobsync_backend.models import HostConfig, SnapshotRecord
|
||||
from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats
|
||||
|
||||
|
||||
class StatsSummaryTests(TestCase):
|
||||
def test_collect_dashboard_stats_sums_backup_data_across_hosts(self) -> None:
|
||||
web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
db = HostConfig.objects.create(host="db-01", address="db-01.example.test")
|
||||
self._snapshot(web, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
||||
self._snapshot(web, "20260519-031500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=200)
|
||||
self._snapshot(db, "20260519-041500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=300)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_usage = self._incomplete_snapshot_on_disk(
|
||||
db,
|
||||
Path(tmp),
|
||||
"20260519-051500Z__BROKEN1",
|
||||
)
|
||||
|
||||
stats = collect_dashboard_stats(hosts=[web, db], global_config=None)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 400)
|
||||
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 200)
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
||||
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 600 + incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 1200 + incomplete_usage["apparent_size_bytes"])
|
||||
|
||||
def test_collect_host_stats_sums_backup_data_by_snapshot_kind(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
self._snapshot(host, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
||||
self._snapshot(host, "20260519-031500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=200)
|
||||
self._snapshot(host, "20260519-041500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=300)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_usage = self._incomplete_snapshot_on_disk(
|
||||
host,
|
||||
Path(tmp),
|
||||
"20260519-051500Z__BROKEN1",
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 300)
|
||||
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 300)
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
||||
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 600 + incomplete_usage["allocated_size_bytes"])
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 1200 + incomplete_usage["apparent_size_bytes"])
|
||||
|
||||
def test_collect_host_stats_falls_back_to_filesystem_usage_for_snapshots_without_metadata(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-051500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
meta_dir = incomplete_dir / "meta"
|
||||
data_dir.mkdir(parents=True)
|
||||
meta_dir.mkdir()
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
meta_dir.joinpath("rsync.log").write_text("not part of the backup data total\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
metadata={},
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["incomplete"]["count"], 1)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["apparent_size_bytes"],
|
||||
expected_usage["apparent_size_bytes"],
|
||||
)
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["total"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
|
||||
def test_collect_host_stats_measures_incomplete_data_from_disk_even_with_stale_metadata(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-051500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
metadata={
|
||||
"stats": {
|
||||
"storage": {
|
||||
"snapshot": {
|
||||
"apparent_size_bytes": 0,
|
||||
"allocated_size_bytes": 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(
|
||||
stats["backup_data"]["incomplete"]["allocated_size_bytes"],
|
||||
expected_usage["allocated_size_bytes"],
|
||||
)
|
||||
self.assertGreater(stats["backup_data"]["incomplete"]["apparent_size_bytes"], 0)
|
||||
|
||||
def test_collect_host_stats_reports_non_hardlinked_snapshot_data(self) -> None:
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
self._snapshot_with_sizes(
|
||||
host,
|
||||
"20260519-021500Z__SCHED01",
|
||||
SnapshotRecord.Kind.SCHEDULED,
|
||||
allocated=1_200,
|
||||
apparent=2_000,
|
||||
hardlinked_apparent=1_500,
|
||||
)
|
||||
|
||||
stats = collect_host_stats(host=host)
|
||||
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["apparent_size_bytes"], 2_000)
|
||||
self.assertEqual(stats["backup_data"]["scheduled"]["unique_apparent_size_bytes"], 500)
|
||||
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 500)
|
||||
|
||||
def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord:
|
||||
return self._snapshot_with_sizes(host, dirname, kind, allocated=allocated)
|
||||
|
||||
def _incomplete_snapshot_on_disk(self, host: HostConfig, root: Path, dirname: str) -> dict:
|
||||
incomplete_dir = root / host.host / ".incomplete" / dirname
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=dirname,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
)
|
||||
return usage
|
||||
|
||||
def _snapshot_with_sizes(
|
||||
self,
|
||||
host: HostConfig,
|
||||
dirname: str,
|
||||
kind: str,
|
||||
*,
|
||||
allocated: int,
|
||||
apparent: int | None = None,
|
||||
hardlinked_apparent: int = 0,
|
||||
) -> SnapshotRecord:
|
||||
started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc)
|
||||
apparent_size = apparent if apparent is not None else allocated * 2
|
||||
return SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=kind,
|
||||
dirname=dirname,
|
||||
path=f"/backups/{host.host}/{kind}/{dirname}",
|
||||
status="success",
|
||||
started_at=started_at,
|
||||
metadata={
|
||||
"stats": {
|
||||
"storage": {
|
||||
"snapshot": {
|
||||
"apparent_size_bytes": apparent_size,
|
||||
"allocated_size_bytes": allocated,
|
||||
"hardlinked_apparent_size_bytes": hardlinked_apparent,
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -8,14 +8,18 @@ from unittest.mock import patch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.template.defaultfilters import filesizeformat
|
||||
from django.test import TestCase, override_settings
|
||||
from django.urls import reverse
|
||||
|
||||
from pobsync.run_stats import tree_usage
|
||||
from pobsync.util import write_yaml_atomic
|
||||
from pobsync_backend.models import (
|
||||
BackupRun,
|
||||
GlobalConfig,
|
||||
HostConfig,
|
||||
NotificationDelivery,
|
||||
NotificationTarget,
|
||||
PurgedSnapshot,
|
||||
ScheduleConfig,
|
||||
SnapshotRecord,
|
||||
@@ -48,7 +52,9 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, 'aria-label="Primary navigation"', html=False)
|
||||
self.assertContains(response, 'aria-label="System navigation"', html=False)
|
||||
self.assertContains(response, reverse("dashboard"))
|
||||
self.assertContains(response, reverse("hosts_list"))
|
||||
self.assertContains(response, reverse("ssh_credentials"))
|
||||
self.assertContains(response, reverse("notification_targets"))
|
||||
self.assertContains(response, reverse("logs"))
|
||||
self.assertContains(response, reverse("purged_snapshots"))
|
||||
self.assertContains(response, reverse("self_check"))
|
||||
@@ -91,6 +97,70 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Django control panel")
|
||||
self.assertContains(response, "Native systemd installer")
|
||||
|
||||
def test_notification_targets_view_renders_targets_and_deliveries(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
|
||||
target = NotificationTarget.objects.create(
|
||||
name="ops",
|
||||
channel=NotificationTarget.Channel.EMAIL,
|
||||
email_to="ops@example.test",
|
||||
last_status=NotificationDelivery.Status.SENT,
|
||||
)
|
||||
NotificationDelivery.objects.create(target=target, run=run, status=NotificationDelivery.Status.SENT)
|
||||
|
||||
response = self.client.get(reverse("notification_targets"))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Notifications")
|
||||
self.assertContains(response, "ops")
|
||||
self.assertContains(response, "ops@example.test")
|
||||
self.assertContains(response, f"Run {run.id}")
|
||||
|
||||
def test_notification_target_form_creates_email_target(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
|
||||
response = self.client.post(
|
||||
reverse("create_notification_target"),
|
||||
{
|
||||
"name": "ops",
|
||||
"enabled": "on",
|
||||
"channel": NotificationTarget.Channel.EMAIL,
|
||||
"statuses": [BackupRun.Status.FAILED, BackupRun.Status.WARNING],
|
||||
"email_to": "ops@example.test, backup@example.test",
|
||||
"webhook_headers": "{}",
|
||||
"notes": "Notify ops",
|
||||
},
|
||||
follow=True,
|
||||
)
|
||||
|
||||
self.assertRedirects(response, reverse("notification_targets"))
|
||||
self.assertContains(response, "Notification target ops created.")
|
||||
target = NotificationTarget.objects.get(name="ops")
|
||||
self.assertEqual(target.channel, NotificationTarget.Channel.EMAIL)
|
||||
self.assertEqual(target.statuses, [BackupRun.Status.FAILED, BackupRun.Status.WARNING])
|
||||
self.assertEqual(target.email_to, "ops@example.test\nbackup@example.test")
|
||||
|
||||
def test_notification_target_form_requires_channel_destination(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
|
||||
response = self.client.post(
|
||||
reverse("create_notification_target"),
|
||||
{
|
||||
"name": "broken",
|
||||
"enabled": "on",
|
||||
"channel": NotificationTarget.Channel.WEBHOOK,
|
||||
"statuses": [BackupRun.Status.FAILED],
|
||||
"email_to": "",
|
||||
"webhook_url": "",
|
||||
"webhook_headers": "{}",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Webhook targets need a URL.")
|
||||
self.assertFalse(NotificationTarget.objects.exists())
|
||||
|
||||
def test_dashboard_renders_hosts_and_latest_runs(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
@@ -189,6 +259,40 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "running")
|
||||
self.assertNotContains(response, "<html", html=False)
|
||||
|
||||
def test_dashboard_priority_live_renders_global_backup_data_totals(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
db = HostConfig.objects.create(host="db-01", address="db-01.example.test")
|
||||
scheduled = self._snapshot(web, "20260519-021500Z__SCHED01", kind=SnapshotRecord.Kind.SCHEDULED)
|
||||
manual = self._snapshot(web, "20260519-031500Z__MANUAL1", kind=SnapshotRecord.Kind.MANUAL)
|
||||
self._set_snapshot_storage(scheduled, allocated=100)
|
||||
self._set_snapshot_storage(manual, allocated=200)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / db.host / ".incomplete" / "20260519-041500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=db,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("dashboard_priority_live"))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Scheduled data")
|
||||
self.assertContains(response, "Manual data")
|
||||
self.assertContains(response, "Incomplete data")
|
||||
self.assertContains(response, "Total snapshot data")
|
||||
self.assertContains(response, "100 bytes", html=True)
|
||||
self.assertContains(response, "200 bytes", html=True)
|
||||
self.assertContains(response, filesizeformat(expected_usage["allocated_size_bytes"]))
|
||||
self.assertContains(response, filesizeformat(300 + expected_usage["allocated_size_bytes"]))
|
||||
|
||||
def test_dashboard_hosts_live_returns_hosts_partial(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
@@ -202,6 +306,129 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Snapshot health")
|
||||
self.assertNotContains(response, "<html", html=False)
|
||||
|
||||
def test_dashboard_host_cards_render_backup_data_totals(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
scheduled = self._snapshot(host, "20260519-021500Z__SCHED01", kind=SnapshotRecord.Kind.SCHEDULED)
|
||||
manual = self._snapshot(host, "20260519-031500Z__MANUAL1", kind=SnapshotRecord.Kind.MANUAL)
|
||||
self._set_snapshot_storage(scheduled, allocated=100)
|
||||
self._set_snapshot_storage(manual, allocated=200)
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-041500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("dashboard_hosts_live"))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Scheduled data")
|
||||
self.assertContains(response, "Manual data")
|
||||
self.assertContains(response, "Incomplete data")
|
||||
self.assertContains(response, "Total data")
|
||||
self.assertContains(response, "100 bytes", html=True)
|
||||
self.assertContains(response, "200 bytes", html=True)
|
||||
self.assertContains(response, filesizeformat(expected_usage["allocated_size_bytes"]))
|
||||
self.assertContains(response, filesizeformat(300 + expected_usage["allocated_size_bytes"]))
|
||||
|
||||
def test_dashboard_host_cards_measure_incomplete_data_without_snapshot_metadata(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-041500Z__BROKEN1"
|
||||
data_dir = incomplete_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
|
||||
expected_usage = tree_usage(data_dir)
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname=incomplete_dir.name,
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
metadata={},
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("dashboard_hosts_live"))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Incomplete data")
|
||||
self.assertContains(response, filesizeformat(expected_usage["allocated_size_bytes"]))
|
||||
|
||||
def test_hosts_list_renders_host_cards_and_controls(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
db = HostConfig.objects.create(host="db-01", address="db-01.example.test", enabled=False)
|
||||
ScheduleConfig.objects.create(host=web, cron_expr="15 2 * * *", enabled=True, prune=True)
|
||||
BackupRun.objects.create(host=web, status=BackupRun.Status.RUNNING)
|
||||
|
||||
response = self.client.get(reverse("hosts_list"))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Inventory")
|
||||
self.assertContains(response, "Configured backup targets")
|
||||
self.assertContains(response, "web-01")
|
||||
self.assertContains(response, "db-01")
|
||||
self.assertContains(response, "running 1")
|
||||
self.assertContains(response, "schedule on")
|
||||
self.assertContains(response, "retention on")
|
||||
self.assertContains(response, "Disable host")
|
||||
self.assertContains(response, "Enable host")
|
||||
self.assertContains(response, "Pause schedule")
|
||||
self.assertContains(response, "Pause retention")
|
||||
self.assertContains(response, reverse("update_host_state", args=[web.host]))
|
||||
self.assertContains(response, reverse("edit_host_config", args=[web.host]))
|
||||
self.assertContains(response, reverse("edit_host_schedule", args=[web.host]))
|
||||
|
||||
def test_hosts_list_filters_by_enabled_state(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
HostConfig.objects.create(host="db-01", address="db-01.example.test", enabled=False)
|
||||
|
||||
response = self.client.get(reverse("hosts_list"), {"enabled": "no"})
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "db-01")
|
||||
self.assertNotContains(response, "web-01")
|
||||
|
||||
def test_update_host_state_toggles_host_schedule_and_retention(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
schedule = ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", enabled=True, prune=True)
|
||||
|
||||
response = self.client.post(
|
||||
reverse("update_host_state", args=[host.host]),
|
||||
{"action": "disable_host", "next": reverse("hosts_list")},
|
||||
follow=True,
|
||||
)
|
||||
self.assertRedirects(response, reverse("hosts_list"))
|
||||
host.refresh_from_db()
|
||||
self.assertFalse(host.enabled)
|
||||
self.assertContains(response, "Disabled host web-01.")
|
||||
|
||||
self.client.post(reverse("update_host_state", args=[host.host]), {"action": "disable_schedule"})
|
||||
self.client.post(reverse("update_host_state", args=[host.host]), {"action": "disable_prune"})
|
||||
schedule.refresh_from_db()
|
||||
self.assertFalse(schedule.enabled)
|
||||
self.assertFalse(schedule.prune)
|
||||
|
||||
self.client.post(reverse("update_host_state", args=[host.host]), {"action": "enable_host"})
|
||||
self.client.post(reverse("update_host_state", args=[host.host]), {"action": "enable_schedule"})
|
||||
self.client.post(reverse("update_host_state", args=[host.host]), {"action": "enable_prune"})
|
||||
host.refresh_from_db()
|
||||
schedule.refresh_from_db()
|
||||
self.assertTrue(host.enabled)
|
||||
self.assertTrue(schedule.enabled)
|
||||
self.assertTrue(schedule.prune)
|
||||
|
||||
def test_dashboard_renders_backup_trend_summary(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
GlobalConfig.objects.create(name="default", backup_root="/missing-backup-root")
|
||||
@@ -921,6 +1148,7 @@ class ViewTests(TestCase):
|
||||
"excludes_add": "*.tmp",
|
||||
"excludes_replace": "",
|
||||
"rsync_extra_args": "--numeric-ids",
|
||||
"rsync_bwlimit_kbps": "4096",
|
||||
"retention_daily": "7",
|
||||
"retention_weekly": "4",
|
||||
"retention_monthly": "2",
|
||||
@@ -938,6 +1166,7 @@ class ViewTests(TestCase):
|
||||
self.assertEqual(host.includes, ["/srv/www", "/srv/db"])
|
||||
self.assertEqual(host.excludes_add, ["*.tmp"])
|
||||
self.assertEqual(host.rsync_extra_args, ["--numeric-ids"])
|
||||
self.assertEqual(host.rsync_bwlimit_kbps, 4096)
|
||||
self.assertEqual(host.retention_weekly, 4)
|
||||
|
||||
def test_create_host_config_uses_global_defaults_and_prepares_directories(self) -> None:
|
||||
@@ -1006,6 +1235,7 @@ class ViewTests(TestCase):
|
||||
(backup_root / host.host / subdir).mkdir(parents=True)
|
||||
ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", prune=True, last_status="success")
|
||||
snapshot = self._snapshot(host, "20260519-021500Z__ABCDEFGH")
|
||||
self._set_snapshot_storage(snapshot, allocated=100)
|
||||
BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, snapshot=snapshot)
|
||||
|
||||
response = self.client.get(reverse("host_detail", args=[host.host]))
|
||||
@@ -1030,6 +1260,8 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, reverse("prepare_host_directories", args=[host.host]))
|
||||
self.assertContains(response, "warning")
|
||||
self.assertContains(response, "Snapshot Storage")
|
||||
self.assertContains(response, "Backup Data")
|
||||
self.assertContains(response, "100 bytes", html=True)
|
||||
self.assertContains(response, reverse("queue_manual_backup", args=[host.host]))
|
||||
self.assertContains(response, reverse("run_detail", args=[BackupRun.objects.get().id]))
|
||||
self.assertContains(response, reverse("snapshot_detail", args=[snapshot.id]))
|
||||
@@ -1077,6 +1309,7 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "default-key")
|
||||
self.assertContains(response, "-oBatchMode=yes")
|
||||
self.assertContains(response, "--archive --numeric-ids --delete --one-file-system")
|
||||
self.assertContains(response, "2048 KB/s")
|
||||
self.assertContains(response, "/srv/www/***")
|
||||
self.assertContains(response, "/srv/www/cache/***")
|
||||
self.assertContains(response, "d14")
|
||||
@@ -1100,6 +1333,10 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Remote rsync")
|
||||
self.assertContains(response, "Remote source root")
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.kwargs["args"] if "args" in call.kwargs else call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual(commands[1][-1], "sh -lc 'command -v rsync >/dev/null'")
|
||||
self.assertEqual(commands[2][-1], "sh -lc 'test -e / && test -r /'")
|
||||
self.assertNotIn("sh", commands[2][commands[2].index("root@web-01.example.test") + 1 : -1])
|
||||
host.refresh_from_db()
|
||||
self.assertTrue(host.config["last_preflight"]["ok"])
|
||||
self.assertEqual(host.config["last_preflight"]["target"], "root@web-01.example.test")
|
||||
@@ -1469,9 +1706,10 @@ class ViewTests(TestCase):
|
||||
"ok": True,
|
||||
"snapshot": snapshot.path,
|
||||
"rsync": {
|
||||
"command": ["rsync", "--archive", "root@web-01:/", snapshot.path],
|
||||
"command": ["rsync", "--archive", "--bwlimit=2048", "root@web-01:/", snapshot.path],
|
||||
"exit_code": 0,
|
||||
"log_tail": ["sending incremental file list", "sent 500 bytes"],
|
||||
"bwlimit_kbps": 2048,
|
||||
},
|
||||
"requested": {
|
||||
"dry_run": True,
|
||||
@@ -1506,10 +1744,13 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Dry run:</strong> yes")
|
||||
self.assertContains(response, "Verbose rsync output:</strong> yes")
|
||||
self.assertContains(response, "Rsync Command")
|
||||
self.assertContains(response, "Bandwidth limit:</strong>")
|
||||
self.assertContains(response, "2048 KB/s")
|
||||
self.assertContains(response, "--archive")
|
||||
self.assertContains(response, "Rsync Log")
|
||||
self.assertContains(response, "sending incremental file list")
|
||||
self.assertContains(response, "Dry Run Summary")
|
||||
self.assertContains(response, "Run Progress")
|
||||
self.assertContains(response, "dry run")
|
||||
self.assertContains(response, "Files Seen")
|
||||
self.assertContains(response, "Would Transfer")
|
||||
self.assertContains(response, "Transfer Estimate")
|
||||
@@ -1559,7 +1800,8 @@ class ViewTests(TestCase):
|
||||
response = self.client.get(reverse("run_detail", args=[run.id]))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Dry Run Summary")
|
||||
self.assertContains(response, "Run Progress")
|
||||
self.assertContains(response, "dry run")
|
||||
self.assertContains(response, "failed")
|
||||
self.assertContains(response, "Files Seen")
|
||||
self.assertContains(response, "25")
|
||||
@@ -1745,6 +1987,8 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, f'data-refresh-url="{reverse("run_detail_live", args=[run.id])}"', html=False)
|
||||
self.assertContains(response, 'data-refresh-interval="5000"', html=False)
|
||||
self.assertContains(response, 'data-refresh-active="true"', html=False)
|
||||
self.assertContains(response, "Live Updates")
|
||||
self.assertContains(response, "Pause refresh")
|
||||
|
||||
def test_run_detail_live_returns_partial_for_active_run(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
@@ -1763,6 +2007,45 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "sending incremental file list")
|
||||
self.assertNotContains(response, "<html", html=False)
|
||||
|
||||
def test_run_detail_live_shows_progress_for_running_real_run(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
with TemporaryDirectory() as tmp:
|
||||
snapshot_path = Path(tmp) / "backups" / host.host / ".incomplete" / "20260523-010000Z__ABCDEFGH"
|
||||
data_path = snapshot_path / "data"
|
||||
log_path = snapshot_path / "meta" / "rsync.log"
|
||||
data_path.mkdir(parents=True)
|
||||
log_path.parent.mkdir(parents=True)
|
||||
(data_path / "payload.txt").write_text("payload", encoding="utf-8")
|
||||
log_path.write_text("sending incremental file list\npayload.txt\n", encoding="utf-8")
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
status=BackupRun.Status.RUNNING,
|
||||
snapshot_path=str(snapshot_path),
|
||||
result={
|
||||
"requested": {"dry_run": False},
|
||||
"execution": {
|
||||
"phase": "rsync",
|
||||
"snapshot": str(snapshot_path),
|
||||
"log": str(log_path),
|
||||
"heartbeat_at": "2026-05-23T01:00:00+02:00",
|
||||
},
|
||||
"rsync": {"pid": 1234, "pgid": 1234, "command": ["rsync"]},
|
||||
},
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("run_detail_live", args=[run.id]))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "Run Progress")
|
||||
self.assertContains(response, "backup")
|
||||
self.assertContains(response, "rsync")
|
||||
self.assertContains(response, "1234")
|
||||
self.assertContains(response, "Data Files")
|
||||
self.assertContains(response, "Open full rsync log")
|
||||
self.assertContains(response, "payload.txt")
|
||||
self.assertContains(response, "sending incremental file list")
|
||||
|
||||
def test_run_detail_live_stops_refresh_for_terminal_run(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
@@ -2032,9 +2315,33 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Incomplete Snapshots")
|
||||
self.assertContains(response, "20260519-031500Z__BROKEN01")
|
||||
self.assertContains(response, "excluded from retention cleanup")
|
||||
self.assertContains(response, "needs review")
|
||||
self.assertContains(response, "Cleanup is blocked until all incomplete snapshots are reviewed.")
|
||||
self.assertContains(response, "Mark incomplete snapshots reviewed")
|
||||
self.assertContains(response, "delete only incomplete snapshot directories")
|
||||
self.assertNotContains(response, "Delete incomplete snapshots")
|
||||
|
||||
def test_retention_plan_offers_incomplete_cleanup_after_review(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
response = self.client.get(reverse("host_retention_plan", args=[host.host]))
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertContains(response, "reviewed")
|
||||
self.assertContains(response, "Delete incomplete snapshots")
|
||||
self.assertContains(response, "Type 1 to confirm the current number of incomplete snapshots.")
|
||||
self.assertContains(response, "This deletes only incomplete snapshot directories")
|
||||
self.assertContains(response, "This deletes only reviewed incomplete snapshot directories")
|
||||
self.assertContains(response, 'class="danger"', html=False)
|
||||
|
||||
def test_incomplete_cleanup_deletes_incomplete_snapshot_after_confirmation(self) -> None:
|
||||
@@ -2052,6 +2359,8 @@ class ViewTests(TestCase):
|
||||
path=str(incomplete_dir),
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
reviewed_at=datetime(2026, 5, 19, 4, 15, tzinfo=timezone.utc),
|
||||
reviewed_by="admin",
|
||||
)
|
||||
|
||||
with override_settings(POBSYNC_HOME=str(home)):
|
||||
@@ -2097,6 +2406,33 @@ class ViewTests(TestCase):
|
||||
self.assertContains(response, "Incomplete cleanup confirmation is invalid.")
|
||||
self.assertEqual(SnapshotRecord.objects.filter(kind=SnapshotRecord.Kind.INCOMPLETE).count(), 1)
|
||||
|
||||
def test_incomplete_cleanup_rejects_unreviewed_snapshots(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
||||
SnapshotRecord.objects.create(
|
||||
host=host,
|
||||
kind=SnapshotRecord.Kind.INCOMPLETE,
|
||||
dirname="20260519-031500Z__BROKEN01",
|
||||
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
|
||||
status="failed",
|
||||
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
with TemporaryDirectory() as tmp, override_settings(POBSYNC_HOME=str(Path(tmp) / "home")):
|
||||
response = self.client.post(
|
||||
reverse("cleanup_host_incomplete_snapshots", args=[host.host]),
|
||||
{
|
||||
"max_delete": "0",
|
||||
"confirm_host": host.host,
|
||||
"confirm_delete_count": "0",
|
||||
},
|
||||
follow=True,
|
||||
)
|
||||
|
||||
self.assertRedirects(response, reverse("host_retention_plan", args=[host.host]))
|
||||
self.assertContains(response, "have not been reviewed")
|
||||
self.assertEqual(SnapshotRecord.objects.filter(kind=SnapshotRecord.Kind.INCOMPLETE).count(), 1)
|
||||
|
||||
def test_host_detail_surfaces_retention_warnings(self) -> None:
|
||||
self.client.force_login(self.staff_user)
|
||||
host = HostConfig.objects.create(
|
||||
@@ -2417,6 +2753,7 @@ class ViewTests(TestCase):
|
||||
"excludes_add": "*.tmp\ncache/",
|
||||
"excludes_replace": "",
|
||||
"rsync_extra_args": "--numeric-ids\n--delete",
|
||||
"rsync_bwlimit_kbps": "8192",
|
||||
"retention_daily": "7",
|
||||
"retention_weekly": "4",
|
||||
"retention_monthly": "2",
|
||||
@@ -2436,6 +2773,7 @@ class ViewTests(TestCase):
|
||||
self.assertEqual(host.excludes_add, ["*.tmp", "cache/"])
|
||||
self.assertIsNone(host.excludes_replace)
|
||||
self.assertEqual(host.rsync_extra_args, ["--numeric-ids", "--delete"])
|
||||
self.assertEqual(host.rsync_bwlimit_kbps, 8192)
|
||||
self.assertEqual(host.retention_daily, 7)
|
||||
self.assertEqual(host.retention_yearly, 1)
|
||||
|
||||
@@ -2484,3 +2822,16 @@ class ViewTests(TestCase):
|
||||
status="success",
|
||||
started_at=started_at,
|
||||
)
|
||||
|
||||
def _set_snapshot_storage(self, snapshot: SnapshotRecord, *, allocated: int) -> None:
|
||||
snapshot.metadata = {
|
||||
"stats": {
|
||||
"storage": {
|
||||
"snapshot": {
|
||||
"apparent_size_bytes": allocated * 2,
|
||||
"allocated_size_bytes": allocated,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshot.save(update_fields=["metadata"])
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import shlex
|
||||
import shutil
|
||||
@@ -16,6 +17,7 @@ from django.db.models import Count, Q
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
from django.utils.http import url_has_allowed_host_and_scheme
|
||||
from django.views.decorators.http import require_POST
|
||||
|
||||
from pobsync import __version__
|
||||
@@ -30,13 +32,24 @@ from .forms import (
|
||||
HostConfigForm,
|
||||
IncompleteCleanupForm,
|
||||
ManualBackupForm,
|
||||
NotificationTargetForm,
|
||||
RetentionApplyForm,
|
||||
SshCredentialGenerateForm,
|
||||
ScheduleConfigForm,
|
||||
SshCredentialForm,
|
||||
)
|
||||
from .host_ops import ensure_host_directories
|
||||
from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential
|
||||
from .models import (
|
||||
BackupRun,
|
||||
GlobalConfig,
|
||||
HostConfig,
|
||||
NotificationDelivery,
|
||||
NotificationTarget,
|
||||
PurgedSnapshot,
|
||||
ScheduleConfig,
|
||||
SnapshotRecord,
|
||||
SshCredential,
|
||||
)
|
||||
from .preflight import collect_backup_gate, effective_host_config_preview, run_remote_preflight
|
||||
from .retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan
|
||||
from .self_check import collect_self_checks, summarize_self_checks
|
||||
@@ -61,8 +74,7 @@ def dashboard_hosts_live(request):
|
||||
return render(request, "pobsync_backend/partials/dashboard_hosts.html", _dashboard_context())
|
||||
|
||||
|
||||
def _dashboard_context() -> dict[str, object]:
|
||||
global_config = GlobalConfig.objects.filter(name="default").first()
|
||||
def _host_cards_context(*, enabled: str = "") -> dict[str, object]:
|
||||
hosts = list(
|
||||
HostConfig.objects.select_related("schedule")
|
||||
.annotate(
|
||||
@@ -83,6 +95,11 @@ def _dashboard_context() -> dict[str, object]:
|
||||
)
|
||||
.order_by("host")
|
||||
)
|
||||
if enabled == "yes":
|
||||
hosts = [host for host in hosts if host.enabled]
|
||||
elif enabled == "no":
|
||||
hosts = [host for host in hosts if not host.enabled]
|
||||
|
||||
for host_config in hosts:
|
||||
host_config.latest_snapshot = (
|
||||
host_config.snapshots.select_related("base")
|
||||
@@ -91,6 +108,22 @@ def _dashboard_context() -> dict[str, object]:
|
||||
)
|
||||
host_config.next_run_at = _next_run_for_host(host_config)
|
||||
host_config.retention_warning = _retention_warning_for_host(host_config, _schedule_for_host(host_config))
|
||||
return {
|
||||
"hosts": hosts,
|
||||
"scheduler_timezone": timezone.get_current_timezone_name(),
|
||||
"selected_enabled": enabled,
|
||||
"counts": {
|
||||
"hosts": len(hosts),
|
||||
"enabled_hosts": sum(1 for host in hosts if host.enabled),
|
||||
"disabled_hosts": sum(1 for host in hosts if not host.enabled),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _dashboard_context() -> dict[str, object]:
|
||||
global_config = GlobalConfig.objects.filter(name="default").first()
|
||||
host_context = _host_cards_context()
|
||||
hosts = host_context["hosts"]
|
||||
action_items = _dashboard_action_items(hosts)
|
||||
next_schedule_rows = _dashboard_next_schedule_rows()
|
||||
recent_runs = BackupRun.objects.select_related("host", "snapshot").order_by("-created_at", "-id")[:6]
|
||||
@@ -99,7 +132,7 @@ def _dashboard_context() -> dict[str, object]:
|
||||
"hosts": hosts,
|
||||
"global_config": global_config,
|
||||
"stats_summary": stats_summary,
|
||||
"scheduler_timezone": timezone.get_current_timezone_name(),
|
||||
"scheduler_timezone": host_context["scheduler_timezone"],
|
||||
"action_items": action_items,
|
||||
"next_schedule_rows": next_schedule_rows,
|
||||
"recent_runs": recent_runs,
|
||||
@@ -126,6 +159,69 @@ def _dashboard_context() -> dict[str, object]:
|
||||
return context
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def hosts_list(request):
|
||||
enabled = request.GET.get("enabled", "").strip()
|
||||
if enabled not in {"", "yes", "no"}:
|
||||
enabled = ""
|
||||
global_config = GlobalConfig.objects.filter(name="default").first()
|
||||
context = _host_cards_context(enabled=enabled)
|
||||
collect_dashboard_stats(hosts=context["hosts"], global_config=global_config)
|
||||
return render(
|
||||
request,
|
||||
"pobsync_backend/hosts_list.html",
|
||||
{
|
||||
**context,
|
||||
"global_config": global_config,
|
||||
"show_host_controls": True,
|
||||
"total_count": HostConfig.objects.count(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
@require_POST
|
||||
def update_host_state(request, host: str):
|
||||
host_config = get_object_or_404(HostConfig, host=host)
|
||||
action = request.POST.get("action", "").strip()
|
||||
next_url = request.POST.get("next") or reverse("hosts_list")
|
||||
if not url_has_allowed_host_and_scheme(next_url, allowed_hosts={request.get_host()}):
|
||||
next_url = reverse("hosts_list")
|
||||
|
||||
if action == "enable_host":
|
||||
host_config.enabled = True
|
||||
host_config.save(update_fields=["enabled", "updated_at"])
|
||||
messages.success(request, f"Enabled host {host_config.host}.")
|
||||
elif action == "disable_host":
|
||||
host_config.enabled = False
|
||||
host_config.save(update_fields=["enabled", "updated_at"])
|
||||
messages.success(request, f"Disabled host {host_config.host}.")
|
||||
elif action in {"enable_schedule", "disable_schedule", "enable_prune", "disable_prune"}:
|
||||
try:
|
||||
schedule = host_config.schedule
|
||||
except ScheduleConfig.DoesNotExist:
|
||||
messages.warning(request, f"{host_config.host} does not have a schedule yet.")
|
||||
else:
|
||||
if action == "enable_schedule":
|
||||
schedule.enabled = True
|
||||
message = f"Enabled backup schedule for {host_config.host}."
|
||||
elif action == "disable_schedule":
|
||||
schedule.enabled = False
|
||||
message = f"Paused backup schedule for {host_config.host}."
|
||||
elif action == "enable_prune":
|
||||
schedule.prune = True
|
||||
message = f"Enabled scheduled retention for {host_config.host}."
|
||||
else:
|
||||
schedule.prune = False
|
||||
message = f"Paused scheduled retention for {host_config.host}."
|
||||
schedule.save(update_fields=["enabled", "prune", "updated_at"])
|
||||
messages.success(request, message)
|
||||
else:
|
||||
messages.error(request, "Unknown host state action.")
|
||||
|
||||
return redirect(next_url)
|
||||
|
||||
|
||||
def _dashboard_action_items(hosts: list[HostConfig]) -> list[dict[str, object]]:
|
||||
action_items: list[dict[str, object]] = []
|
||||
for host_config in hosts:
|
||||
@@ -235,6 +331,65 @@ def logs(request):
|
||||
return render(request, "pobsync_backend/logs.html", context)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def notification_targets(request):
|
||||
targets = NotificationTarget.objects.order_by("name")
|
||||
deliveries = NotificationDelivery.objects.select_related("target", "run", "run__host").order_by("-created_at")[:12]
|
||||
return render(
|
||||
request,
|
||||
"pobsync_backend/notification_targets.html",
|
||||
{
|
||||
"targets": targets,
|
||||
"deliveries": deliveries,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def create_notification_target(request):
|
||||
if request.method == "POST":
|
||||
form = NotificationTargetForm(request.POST)
|
||||
if form.is_valid():
|
||||
target = form.save()
|
||||
messages.success(request, f"Notification target {target.name} created.")
|
||||
return redirect("notification_targets")
|
||||
else:
|
||||
form = NotificationTargetForm()
|
||||
return render(
|
||||
request,
|
||||
"pobsync_backend/notification_target_form.html",
|
||||
{
|
||||
"form": form,
|
||||
"target": None,
|
||||
"title": "New notification target",
|
||||
"submit_label": "Create target",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def edit_notification_target(request, target_id: int):
|
||||
target = get_object_or_404(NotificationTarget, id=target_id)
|
||||
if request.method == "POST":
|
||||
form = NotificationTargetForm(request.POST, instance=target)
|
||||
if form.is_valid():
|
||||
target = form.save()
|
||||
messages.success(request, f"Notification target {target.name} updated.")
|
||||
return redirect("notification_targets")
|
||||
else:
|
||||
form = NotificationTargetForm(instance=target)
|
||||
return render(
|
||||
request,
|
||||
"pobsync_backend/notification_target_form.html",
|
||||
{
|
||||
"form": form,
|
||||
"target": target,
|
||||
"title": f"Edit notification target: {target.name}",
|
||||
"submit_label": "Save target",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def runs_list(request):
|
||||
status = request.GET.get("status", "").strip()
|
||||
@@ -703,6 +858,7 @@ def _run_detail_context(run: BackupRun) -> dict[str, object]:
|
||||
"stats": run_stats if isinstance(run_stats, dict) else {},
|
||||
"rsync": rsync_result,
|
||||
"rsync_command": _run_rsync_command(rsync_result),
|
||||
"rsync_bwlimit_kbps": _run_rsync_bwlimit_kbps(rsync_result),
|
||||
"failure": failure,
|
||||
"failure_summary": failure.get("message") or failure.get("summary") or "",
|
||||
"prune_result": prune_result,
|
||||
@@ -711,6 +867,7 @@ def _run_detail_context(run: BackupRun) -> dict[str, object]:
|
||||
"rsync_log_path": str(rsync_log_path) if rsync_log_path is not None else "",
|
||||
"rsync_log_exists": bool(rsync_log_path and rsync_log_path.exists()),
|
||||
"rsync_log_tail": rsync_log_tail,
|
||||
"live_progress": _run_live_progress(run, rsync_log_path),
|
||||
"dry_run_summary": _dry_run_summary(
|
||||
result=result,
|
||||
requested=requested,
|
||||
@@ -846,6 +1003,8 @@ def host_retention_plan(request, host: str):
|
||||
scheduled_prune_limit = schedule.prune_max_delete if schedule and schedule.prune else None
|
||||
delete_count = len(plan["delete"])
|
||||
incomplete_count = len(plan["incomplete"])
|
||||
incomplete_reviewed_count = int(plan.get("incomplete_reviewed_count") or 0)
|
||||
incomplete_unreviewed_count = int(plan.get("incomplete_unreviewed_count") or 0)
|
||||
context = {
|
||||
"host": host_config,
|
||||
"kind": kind,
|
||||
@@ -854,6 +1013,8 @@ def host_retention_plan(request, host: str):
|
||||
"schedule": schedule,
|
||||
"scheduled_prune_limit": scheduled_prune_limit,
|
||||
"scheduled_prune_exceeded": scheduled_prune_limit is not None and delete_count > scheduled_prune_limit,
|
||||
"incomplete_reviewed_count": incomplete_reviewed_count,
|
||||
"incomplete_unreviewed_count": incomplete_unreviewed_count,
|
||||
"apply_form": RetentionApplyForm(
|
||||
host_name=host_config.host,
|
||||
expected_delete_count=delete_count,
|
||||
@@ -866,10 +1027,10 @@ def host_retention_plan(request, host: str):
|
||||
),
|
||||
"incomplete_cleanup_form": IncompleteCleanupForm(
|
||||
host_name=host_config.host,
|
||||
expected_delete_count=incomplete_count,
|
||||
expected_delete_count=incomplete_reviewed_count,
|
||||
initial={
|
||||
"max_delete": incomplete_count,
|
||||
"confirm_delete_count": incomplete_count,
|
||||
"max_delete": incomplete_reviewed_count,
|
||||
"confirm_delete_count": incomplete_reviewed_count,
|
||||
},
|
||||
),
|
||||
}
|
||||
@@ -940,7 +1101,7 @@ def cleanup_host_incomplete_snapshots(request, host: str):
|
||||
messages.error(request, str(exc))
|
||||
return redirect("host_retention_plan", host=host_config.host)
|
||||
|
||||
incomplete_count = len(plan.get("incomplete") or [])
|
||||
incomplete_count = int(plan.get("incomplete_reviewed_count") or 0)
|
||||
form = IncompleteCleanupForm(
|
||||
request.POST,
|
||||
host_name=host_config.host,
|
||||
@@ -1247,6 +1408,23 @@ def _run_rsync_command(rsync_result: dict) -> list[str]:
|
||||
return [str(part) for part in command]
|
||||
|
||||
|
||||
def _run_rsync_bwlimit_kbps(rsync_result: dict) -> int:
|
||||
stored_limit = rsync_result.get("bwlimit_kbps")
|
||||
if stored_limit is not None:
|
||||
try:
|
||||
return max(0, int(stored_limit))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
for part in _run_rsync_command(rsync_result):
|
||||
if part.startswith("--bwlimit="):
|
||||
try:
|
||||
return max(0, int(part.split("=", 1)[1]))
|
||||
except ValueError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _run_rsync_log_tail(rsync_result: dict, log_path: Path | None, *, max_lines: int = 30) -> list[str]:
|
||||
log_tail = rsync_result.get("log_tail")
|
||||
if isinstance(log_tail, list):
|
||||
@@ -1260,6 +1438,97 @@ def _run_rsync_log_tail(rsync_result: dict, log_path: Path | None, *, max_lines:
|
||||
return []
|
||||
|
||||
|
||||
def _run_live_progress(run: BackupRun, log_path: Path | None) -> dict[str, object]:
|
||||
if run.status not in {BackupRun.Status.QUEUED, BackupRun.Status.RUNNING}:
|
||||
return {}
|
||||
|
||||
result = run.result if isinstance(run.result, dict) else {}
|
||||
requested = result.get("requested") if isinstance(result.get("requested"), dict) else {}
|
||||
if requested.get("dry_run") or result.get("dry_run"):
|
||||
return {}
|
||||
|
||||
execution = result.get("execution") if isinstance(result.get("execution"), dict) else {}
|
||||
rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
|
||||
progress: dict[str, object] = {
|
||||
"phase": execution.get("phase") or ("queued" if run.status == BackupRun.Status.QUEUED else "running"),
|
||||
"worker_pid": execution.get("worker_pid"),
|
||||
"rsync_pid": rsync.get("pid"),
|
||||
"rsync_pgid": rsync.get("pgid"),
|
||||
}
|
||||
|
||||
log_stats = _live_log_stats(log_path)
|
||||
if log_stats:
|
||||
progress["log"] = log_stats
|
||||
|
||||
snapshot_path = _run_progress_snapshot_path(run, execution)
|
||||
if snapshot_path is not None:
|
||||
progress["snapshot"] = {
|
||||
"path": str(snapshot_path),
|
||||
**_scan_snapshot_progress(snapshot_path / "data" if (snapshot_path / "data").exists() else snapshot_path),
|
||||
}
|
||||
return progress
|
||||
|
||||
|
||||
def _run_progress_snapshot_path(run: BackupRun, execution: dict) -> Path | None:
|
||||
snapshot = execution.get("snapshot")
|
||||
if isinstance(snapshot, str) and snapshot:
|
||||
return Path(snapshot)
|
||||
if run.snapshot_path:
|
||||
return Path(run.snapshot_path)
|
||||
return None
|
||||
|
||||
|
||||
def _live_log_stats(log_path: Path | None) -> dict[str, object]:
|
||||
if log_path is None:
|
||||
return {}
|
||||
try:
|
||||
stat = log_path.stat()
|
||||
except OSError:
|
||||
return {"path": str(log_path), "exists": False}
|
||||
modified_at = timezone.datetime.fromtimestamp(stat.st_mtime, tz=timezone.get_current_timezone())
|
||||
return {
|
||||
"path": str(log_path),
|
||||
"exists": True,
|
||||
"size_bytes": stat.st_size,
|
||||
"modified_at": modified_at,
|
||||
"seconds_since_modified": max(0, int((timezone.now() - modified_at).total_seconds())),
|
||||
}
|
||||
|
||||
|
||||
def _scan_snapshot_progress(data_path: Path, *, max_entries: int = 20000) -> dict[str, object]:
|
||||
progress: dict[str, object] = {
|
||||
"data_path": str(data_path),
|
||||
"exists": data_path.exists(),
|
||||
"files": 0,
|
||||
"directories": 0,
|
||||
"apparent_size_bytes": 0,
|
||||
"scan_limited": False,
|
||||
}
|
||||
if not data_path.exists():
|
||||
return progress
|
||||
|
||||
entries_seen = 0
|
||||
for root, dirnames, filenames in os.walk(data_path):
|
||||
progress["directories"] = int(progress["directories"]) + len(dirnames)
|
||||
entries_seen += len(dirnames)
|
||||
for filename in filenames:
|
||||
file_path = Path(root) / filename
|
||||
try:
|
||||
file_stat = file_path.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
progress["files"] = int(progress["files"]) + 1
|
||||
progress["apparent_size_bytes"] = int(progress["apparent_size_bytes"]) + int(file_stat.st_size)
|
||||
entries_seen += 1
|
||||
if entries_seen >= max_entries:
|
||||
progress["scan_limited"] = True
|
||||
return progress
|
||||
if entries_seen >= max_entries:
|
||||
progress["scan_limited"] = True
|
||||
return progress
|
||||
return progress
|
||||
|
||||
|
||||
def _dry_run_summary(
|
||||
*,
|
||||
result: dict,
|
||||
|
||||
@@ -13,6 +13,9 @@ urlpatterns = [
|
||||
path("changelog/", views.changelog, name="changelog"),
|
||||
path("self-check/", views.self_check, name="self_check"),
|
||||
path("logs/", views.logs, name="logs"),
|
||||
path("notifications/", views.notification_targets, name="notification_targets"),
|
||||
path("notifications/new/", views.create_notification_target, name="create_notification_target"),
|
||||
path("notifications/<int:target_id>/", views.edit_notification_target, name="edit_notification_target"),
|
||||
path("purged-snapshots/", views.purged_snapshots, name="purged_snapshots"),
|
||||
path("schedules/", views.schedules_list, name="schedules_list"),
|
||||
path("config/global/", views.edit_global_config, name="edit_global_config"),
|
||||
@@ -21,8 +24,10 @@ urlpatterns = [
|
||||
path("ssh-credentials/generate/", views.generate_ssh_credential, name="generate_ssh_credential"),
|
||||
path("ssh-credentials/<int:credential_id>/", views.edit_ssh_credential, name="edit_ssh_credential"),
|
||||
path("ssh-credentials/<int:credential_id>/delete/", views.delete_ssh_credential, name="delete_ssh_credential"),
|
||||
path("hosts/", views.hosts_list, name="hosts_list"),
|
||||
path("hosts/new/", views.create_host_config, name="create_host_config"),
|
||||
path("hosts/<str:host>/", views.host_detail, name="host_detail"),
|
||||
path("hosts/<str:host>/state/", views.update_host_state, name="update_host_state"),
|
||||
path("hosts/<str:host>/config/", views.edit_host_config, name="edit_host_config"),
|
||||
path("hosts/<str:host>/prepare-directories/", views.prepare_host_directories, name="prepare_host_directories"),
|
||||
path("hosts/<str:host>/scan-known-key/", views.scan_host_known_key, name="scan_host_known_key"),
|
||||
|
||||
Reference in New Issue
Block a user