(feature) Add cancellable backup runs and clearer dry-run logs
Add a cancel action for queued and running backup runs. Queued runs are cancelled immediately, while running runs are marked for cancellation and the worker terminates the active rsync process group. Make dry-run log paths run-specific and add a defensive default dry-run timeout so stuck dry-runs do not remain running indefinitely. Remove rsync exit codes from run overview tables while keeping detailed diagnostics available on the run detail payload.
This commit is contained in:
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.commands.run_scheduled import dry_run_log_path, run_scheduled
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import BackupRun, HostConfig
|
||||
from pobsync_backend.retention import run_sql_retention_apply
|
||||
@@ -47,7 +47,8 @@ def execute_backup_run(
|
||||
) -> BackupRun:
|
||||
run.status = BackupRun.Status.RUNNING
|
||||
run.started_at = run.started_at or timezone.now()
|
||||
run.save(update_fields=["status", "started_at"])
|
||||
run.result = _running_result(run=run, dry_run=bool(dry_run))
|
||||
run.save(update_fields=["status", "started_at", "result"])
|
||||
|
||||
try:
|
||||
result = run_scheduled(
|
||||
@@ -56,15 +57,27 @@ def execute_backup_run(
|
||||
dry_run=bool(dry_run),
|
||||
prune=False,
|
||||
config_source=DjangoConfigSource(),
|
||||
run_id=run.id,
|
||||
cancel_check=lambda: _run_cancel_requested(run.id),
|
||||
)
|
||||
except Exception as exc:
|
||||
run.status = BackupRun.Status.FAILED
|
||||
run.refresh_from_db()
|
||||
run.status = BackupRun.Status.CANCELLED if run.status == BackupRun.Status.CANCELLED else BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.result = {"ok": False, "error": str(exc), "type": type(exc).__name__}
|
||||
run.result = {
|
||||
**(run.result if isinstance(run.result, dict) else {}),
|
||||
"ok": False,
|
||||
"error": str(exc),
|
||||
"type": type(exc).__name__,
|
||||
}
|
||||
run.save(update_fields=["status", "ended_at", "result"])
|
||||
raise
|
||||
|
||||
run.status = BackupRun.Status.SUCCESS if result.get("ok") else BackupRun.Status.FAILED
|
||||
run.refresh_from_db()
|
||||
if result.get("cancelled") or run.status == BackupRun.Status.CANCELLED:
|
||||
run.status = BackupRun.Status.CANCELLED
|
||||
else:
|
||||
run.status = BackupRun.Status.SUCCESS if result.get("ok") else BackupRun.Status.FAILED
|
||||
run.ended_at = timezone.now()
|
||||
run.snapshot_path = str(result.get("snapshot") or "")
|
||||
run.base_path = str(result.get("base") or "")
|
||||
@@ -146,3 +159,18 @@ def requested_options(run: BackupRun) -> dict[str, object]:
|
||||
if not isinstance(requested, dict):
|
||||
return {}
|
||||
return requested
|
||||
|
||||
|
||||
def _running_result(*, run: BackupRun, dry_run: bool) -> dict[str, object]:
|
||||
result = dict(run.result) if isinstance(run.result, dict) else {}
|
||||
execution = {
|
||||
"started_at": (run.started_at or timezone.now()).isoformat(),
|
||||
}
|
||||
if dry_run:
|
||||
execution["log"] = str(dry_run_log_path(run.host.host, run_id=run.id))
|
||||
result["execution"] = execution
|
||||
return result
|
||||
|
||||
|
||||
def _run_cancel_requested(run_id: int) -> bool:
|
||||
return BackupRun.objects.filter(id=run_id, status=BackupRun.Status.CANCELLED).exists()
|
||||
|
||||
Reference in New Issue
Block a user