From 728e5c740af112d67cab40ad06002066a72bea21 Mon Sep 17 00:00:00 2001 From: Peter van Arkel Date: Tue, 19 May 2026 22:13:33 +0200 Subject: [PATCH] (feature) Add optional verbose rsync output for manual backups Expose a verbose rsync output option in the Django manual backup form and store the selected value with the queued run request. Propagate the option through the worker, direct management command, and rsync command builder so real backups can emit itemized changes, file-list progress, and stats when requested. Dry-runs continue to use verbose output by default and report that consistently in requested options. Cover the queue, worker, view, and rsync command behavior with focused tests. --- src/pobsync/commands/run_scheduled.py | 6 +++ src/pobsync/rsync.py | 9 +++-- src/pobsync_backend/backup_runner.py | 4 ++ src/pobsync_backend/forms.py | 5 +++ .../management/commands/run_pobsync_backup.py | 11 ++++++ .../management/commands/run_pobsync_worker.py | 1 + .../pobsync_backend/host_detail.html | 1 + .../templates/pobsync_backend/run_detail.html | 1 + .../tests/test_backup_worker.py | 11 +++++- .../tests/test_run_scheduled_config_source.py | 39 +++++++++++++++++++ src/pobsync_backend/tests/test_views.py | 5 +++ src/pobsync_backend/views.py | 1 + 12 files changed, 89 insertions(+), 5 deletions(-) diff --git a/src/pobsync/commands/run_scheduled.py b/src/pobsync/commands/run_scheduled.py index 435e5ac..8076625 100644 --- a/src/pobsync/commands/run_scheduled.py +++ b/src/pobsync/commands/run_scheduled.py @@ -140,6 +140,7 @@ def run_scheduled( config_source: ConfigSource | None = None, run_id: int | None = None, cancel_check: Callable[[], bool] | None = None, + verbose_output: bool = False, ) -> dict[str, Any]: host = sanitize_host(host) @@ -228,6 +229,7 @@ def run_scheduled( "log": str(dryrun_log), "cancelled": result.cancelled, "timeout_seconds": effective_timeout_seconds, + "verbose_output": True, "ssh_credential": cfg.get("ssh_credential"), "rsync": { "exit_code": result.exit_code, @@ -276,6 +278,7 @@ def run_scheduled( bwlimit_kbps=bwlimit_kbps, extra_excludes=list(excludes), extra_includes=list(includes), + verbose_output=bool(verbose_output), ) meta: dict[str, Any] = { @@ -284,6 +287,7 @@ def run_scheduled( "host": host, "type": "scheduled", "label": None, + "verbose_output": bool(verbose_output), "status": "running", "started_at": format_iso_z(ts), "ended_at": None, @@ -328,6 +332,7 @@ def run_scheduled( "status": meta["status"], "cancelled": result.cancelled, "log": str(log_path), + "verbose_output": bool(verbose_output), "ssh_credential": cfg.get("ssh_credential"), "rsync": { "exit_code": result.exit_code, @@ -362,5 +367,6 @@ def run_scheduled( "snapshot": str(final_dir), "base": str(base_dir) if base_dir else None, "rsync": {"exit_code": result.exit_code}, + "verbose_output": bool(verbose_output), "prune": prune_result, } diff --git a/src/pobsync/rsync.py b/src/pobsync/rsync.py index d3b5990..8ab6c79 100644 --- a/src/pobsync/rsync.py +++ b/src/pobsync/rsync.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Callable, Sequence -DEFAULT_DRY_RUN_OUTPUT_ARGS = ["--itemize-changes", "--info=flist2,progress2,stats2"] +DEFAULT_VERBOSE_OUTPUT_ARGS = ["--itemize-changes", "--info=flist2,progress2,stats2"] @dataclass(frozen=True) @@ -43,12 +43,13 @@ def build_rsync_command( bwlimit_kbps: int, extra_excludes: Sequence[str], extra_includes: Sequence[str], + verbose_output: bool = False, ) -> list[str]: cmd: list[str] = [rsync_binary] cmd.extend(list(rsync_args)) - if dry_run: - _append_default_dry_run_output_args(cmd) + if dry_run or verbose_output: + _append_default_verbose_output_args(cmd) # includes/excludes: keep it simple for now: # - if includes are provided, user is responsible for correct rsync include logic. @@ -123,7 +124,7 @@ def _terminate_process_group(process: subprocess.Popen) -> None: process.wait(timeout=10) -def _append_default_dry_run_output_args(command: list[str]) -> None: +def _append_default_verbose_output_args(command: list[str]) -> None: if not _has_itemize_arg(command): command.append("--itemize-changes") if not any(arg.startswith("--info=") for arg in command): diff --git a/src/pobsync_backend/backup_runner.py b/src/pobsync_backend/backup_runner.py index 8f4876b..26372be 100644 --- a/src/pobsync_backend/backup_runner.py +++ b/src/pobsync_backend/backup_runner.py @@ -18,6 +18,7 @@ def queue_backup_run( host: HostConfig, run_type: str = BackupRun.RunType.MANUAL, dry_run: bool = False, + verbose_output: bool = False, prune: bool = False, prune_max_delete: int = 10, prune_protect_bases: bool = False, @@ -29,6 +30,7 @@ def queue_backup_run( result={ "requested": { "dry_run": bool(dry_run), + "verbose_output": bool(dry_run or verbose_output), "prune": bool(prune), "prune_max_delete": int(prune_max_delete), "prune_protect_bases": bool(prune_protect_bases), @@ -42,6 +44,7 @@ def execute_backup_run( run: BackupRun, prefix: Path, dry_run: bool = False, + verbose_output: bool = False, prune: bool = False, prune_max_delete: int = 10, prune_protect_bases: bool = False, @@ -60,6 +63,7 @@ def execute_backup_run( config_source=DjangoConfigSource(), run_id=run.id, cancel_check=lambda: _run_cancel_requested(run.id), + verbose_output=bool(dry_run or verbose_output), ) except Exception as exc: run.refresh_from_db() diff --git a/src/pobsync_backend/forms.py b/src/pobsync_backend/forms.py index 9a4b3f8..887c082 100644 --- a/src/pobsync_backend/forms.py +++ b/src/pobsync_backend/forms.py @@ -133,6 +133,11 @@ class ManualBackupForm(forms.Form): initial=True, help_text="Queue rsync in dry-run mode without writing a snapshot.", ) + verbose_output = forms.BooleanField( + label="Verbose rsync output", + required=False, + help_text="Write itemized rsync changes, file-list progress, and stats to the run log. Dry-runs always use this.", + ) prune = forms.BooleanField( label="Apply retention after success", required=False, diff --git a/src/pobsync_backend/management/commands/run_pobsync_backup.py b/src/pobsync_backend/management/commands/run_pobsync_backup.py index d54fcee..a0ffcce 100644 --- a/src/pobsync_backend/management/commands/run_pobsync_backup.py +++ b/src/pobsync_backend/management/commands/run_pobsync_backup.py @@ -18,6 +18,7 @@ class Command(BaseCommand): parser.add_argument("host", help="Host to back up") parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory") parser.add_argument("--dry-run", action="store_true", help="Run rsync --dry-run") + parser.add_argument("--verbose-rsync", action="store_true", help="Write itemized rsync output to the run log") parser.add_argument("--prune", action="store_true", help="Apply retention after a successful run") parser.add_argument("--prune-max-delete", type=int, default=10) parser.add_argument("--prune-protect-bases", action="store_true") @@ -35,11 +36,21 @@ class Command(BaseCommand): host=host, run_type=BackupRun.RunType.MANUAL if options["manual"] else BackupRun.RunType.SCHEDULED, status=BackupRun.Status.RUNNING, + result={ + "requested": { + "dry_run": bool(options["dry_run"]), + "verbose_output": bool(options["dry_run"] or options["verbose_rsync"]), + "prune": bool(options["prune"]), + "prune_max_delete": int(options["prune_max_delete"]), + "prune_protect_bases": bool(options["prune_protect_bases"]), + } + }, ) execute_backup_run( run=run, prefix=paths.home, dry_run=bool(options["dry_run"]), + verbose_output=bool(options["dry_run"] or options["verbose_rsync"]), prune=bool(options["prune"]), prune_max_delete=int(options["prune_max_delete"]), prune_protect_bases=bool(options["prune_protect_bases"]), diff --git a/src/pobsync_backend/management/commands/run_pobsync_worker.py b/src/pobsync_backend/management/commands/run_pobsync_worker.py index 303c951..e1288c0 100644 --- a/src/pobsync_backend/management/commands/run_pobsync_worker.py +++ b/src/pobsync_backend/management/commands/run_pobsync_worker.py @@ -44,6 +44,7 @@ class Command(BaseCommand): run=run, prefix=prefix, dry_run=bool(options.get("dry_run", False)), + verbose_output=bool(options.get("verbose_output", False)), prune=bool(options.get("prune", False)), prune_max_delete=int(options.get("prune_max_delete", 10)), prune_protect_bases=bool(options.get("prune_protect_bases", False)), diff --git a/src/pobsync_backend/templates/pobsync_backend/host_detail.html b/src/pobsync_backend/templates/pobsync_backend/host_detail.html index f812772..fc2be55 100644 --- a/src/pobsync_backend/templates/pobsync_backend/host_detail.html +++ b/src/pobsync_backend/templates/pobsync_backend/host_detail.html @@ -127,6 +127,7 @@
{% csrf_token %} +
diff --git a/src/pobsync_backend/templates/pobsync_backend/run_detail.html b/src/pobsync_backend/templates/pobsync_backend/run_detail.html index c8bc9d3..5c9581c 100644 --- a/src/pobsync_backend/templates/pobsync_backend/run_detail.html +++ b/src/pobsync_backend/templates/pobsync_backend/run_detail.html @@ -46,6 +46,7 @@

Requested Options

Dry run: {{ requested.dry_run|yesno:"yes,no" }}
+
Verbose rsync output: {{ requested.verbose_output|yesno:"yes,no" }}
Apply retention: {{ requested.prune|yesno:"yes,no" }}
Retention max delete: {{ requested.prune_max_delete }}
Protect bases: {{ requested.prune_protect_bases|yesno:"yes,no" }}
diff --git a/src/pobsync_backend/tests/test_backup_worker.py b/src/pobsync_backend/tests/test_backup_worker.py index f24d49c..1f7175c 100644 --- a/src/pobsync_backend/tests/test_backup_worker.py +++ b/src/pobsync_backend/tests/test_backup_worker.py @@ -32,12 +32,20 @@ class BackupWorkerTests(TestCase): run.result["requested"], { "dry_run": True, + "verbose_output": True, "prune": True, "prune_max_delete": 3, "prune_protect_bases": True, }, ) + def test_queue_backup_run_can_request_verbose_output(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + + run = queue_backup_run(host=host, verbose_output=True) + + self.assertTrue(run.result["requested"]["verbose_output"]) + def test_worker_executes_next_queued_run(self) -> None: with TemporaryDirectory() as tmp: backup_root = Path(tmp) / "backups" @@ -47,7 +55,7 @@ class BackupWorkerTests(TestCase): meta_dir = snapshot_dir / "meta" meta_dir.mkdir(parents=True) write_yaml_atomic(meta_dir / "meta.yaml", {"status": "success", "started_at": "2026-05-19T02:15:00Z"}) - run = queue_backup_run(host=host) + run = queue_backup_run(host=host, verbose_output=True) with patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled: def fake_run_scheduled(**kwargs): @@ -68,6 +76,7 @@ class BackupWorkerTests(TestCase): self.assertEqual(count, 1) self.assertEqual(run_scheduled.call_args.kwargs["run_id"], run.id) + self.assertTrue(run_scheduled.call_args.kwargs["verbose_output"]) run.refresh_from_db() self.assertEqual(run.status, BackupRun.Status.SUCCESS) self.assertEqual(SnapshotRecord.objects.count(), 1) diff --git a/src/pobsync_backend/tests/test_run_scheduled_config_source.py b/src/pobsync_backend/tests/test_run_scheduled_config_source.py index 0d97e87..c1e6d2d 100644 --- a/src/pobsync_backend/tests/test_run_scheduled_config_source.py +++ b/src/pobsync_backend/tests/test_run_scheduled_config_source.py @@ -160,6 +160,45 @@ class RunScheduledConfigSourceTests(SimpleTestCase): self.assertNotIn("--info=flist2,progress2,stats2", command) self.assertIn("--info=name1,stats2", command) + def test_real_run_can_request_verbose_output_args(self) -> None: + with TemporaryDirectory() as tmp: + prefix = Path(tmp) / "home" + + with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync: + run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"]) + result = run_scheduled( + prefix=prefix, + host="web-01", + dry_run=False, + verbose_output=True, + config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")), + ) + + command = run_rsync.call_args.args[0] + self.assertTrue(result["ok"]) + self.assertIn("--itemize-changes", command) + self.assertIn("--info=flist2,progress2,stats2", command) + self.assertTrue(result["verbose_output"]) + + def test_real_run_keeps_default_output_quiet(self) -> None: + with TemporaryDirectory() as tmp: + prefix = Path(tmp) / "home" + + with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync: + run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"]) + result = run_scheduled( + prefix=prefix, + host="web-01", + dry_run=False, + config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")), + ) + + command = run_rsync.call_args.args[0] + self.assertTrue(result["ok"]) + self.assertNotIn("--itemize-changes", command) + self.assertNotIn("--info=flist2,progress2,stats2", command) + self.assertFalse(result["verbose_output"]) + def test_dry_run_reports_cancelled_rsync(self) -> None: def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None): self.assertTrue(cancel_check()) diff --git a/src/pobsync_backend/tests/test_views.py b/src/pobsync_backend/tests/test_views.py index d5bdb28..0e95529 100644 --- a/src/pobsync_backend/tests/test_views.py +++ b/src/pobsync_backend/tests/test_views.py @@ -655,6 +655,7 @@ class ViewTests(TestCase): reverse("queue_manual_backup", args=[host.host]), { "dry_run": "on", + "verbose_output": "on", "prune": "on", "prune_max_delete": "4", "prune_protect_bases": "on", @@ -671,6 +672,7 @@ class ViewTests(TestCase): run.result["requested"], { "dry_run": True, + "verbose_output": True, "prune": True, "prune_max_delete": 4, "prune_protect_bases": True, @@ -694,6 +696,7 @@ class ViewTests(TestCase): run.result["requested"], { "dry_run": False, + "verbose_output": False, "prune": False, "prune_max_delete": 10, "prune_protect_bases": False, @@ -745,6 +748,7 @@ class ViewTests(TestCase): "snapshot": snapshot.path, "requested": { "dry_run": True, + "verbose_output": True, "prune": False, "prune_max_delete": 10, "prune_protect_bases": False, @@ -761,6 +765,7 @@ class ViewTests(TestCase): self.assertContains(response, "ABCDEFGH") self.assertContains(response, "Requested Options") self.assertContains(response, "Dry run: yes") + self.assertContains(response, "Verbose rsync output: yes") self.assertContains(response, ""ok": true") self.assertContains(response, reverse("snapshot_detail", args=[snapshot.id])) diff --git a/src/pobsync_backend/views.py b/src/pobsync_backend/views.py index 78cc6b9..28b7154 100644 --- a/src/pobsync_backend/views.py +++ b/src/pobsync_backend/views.py @@ -336,6 +336,7 @@ def queue_manual_backup(request, host: str): run = queue_backup_run( host=host_config, dry_run=form.cleaned_data["dry_run"], + verbose_output=form.cleaned_data["verbose_output"], prune=form.cleaned_data["prune"], prune_max_delete=form.cleaned_data["prune_max_delete"], prune_protect_bases=form.cleaned_data["prune_protect_bases"],