hippogrief/pobsync
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

(feature) Capture structured backup statistics

Browse Source 
Parse rsync --stats output into structured run metrics for file counts,
transferred bytes, literal data, matched data, speedup, and estimated
link-dest savings.

Store collected stats on backup run results and successful snapshot metadata,
including snapshot data usage and backup-root capacity details for future
dashboard graphs and disk-full projections.

Render the collected metrics on run and snapshot detail pages, with tests
covering parsing, metadata persistence, and UI output.
This commit is contained in:
Peter van Arkel
2026-05-19 22:25:04 +02:00
parent 728e5c740a
commit 6940dc55b7
9 changed files with 484 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/pobsync_backend/tests/test_run_scheduled_config_source.py
View File

@@ -119,8 +119,16 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
self.assertEqual(timeout_seconds, 900)
self.assertIn("--itemize-changes", command)
self.assertIn("--info=flist2,progress2,stats2", command)
self.assertIn("--stats", command)
log_path.parent.mkdir(parents=True, exist_ok=True)
log_path.write_text("run 42\n", encoding="utf-8")
log_path.write_text(
"Number of files: 42\n"
"Number of regular files transferred: 3\n"
"Total file size: 1,000 bytes\n"
"Literal data: 100 bytes\n"
"Matched data: 900 bytes\n",
encoding="utf-8",
)
return RsyncResult(exit_code=0, command=command)
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
@@ -135,6 +143,8 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
self.assertTrue(result["ok"])
self.assertEqual(result["log"], "/tmp/pobsync-dryrun/web-01/run-42/rsync.log")
self.assertEqual(result["timeout_seconds"], 900)
self.assertEqual(result["stats"]["rsync"]["files_total"], 42)
self.assertEqual(result["stats"]["rsync"]["link_dest_estimated_savings_ratio"], 0.9)
def test_dry_run_does_not_duplicate_custom_output_args(self) -> None:
config_source = FakeConfigSource()
@@ -176,6 +186,7 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
command = run_rsync.call_args.args[0]
self.assertTrue(result["ok"])
self.assertIn("--stats", command)
self.assertIn("--itemize-changes", command)
self.assertIn("--info=flist2,progress2,stats2", command)
self.assertTrue(result["verbose_output"])
@@ -195,10 +206,50 @@ class RunScheduledConfigSourceTests(SimpleTestCase):
command = run_rsync.call_args.args[0]
self.assertTrue(result["ok"])
self.assertIn("--stats", command)
self.assertNotIn("--itemize-changes", command)
self.assertNotIn("--info=flist2,progress2,stats2", command)
self.assertFalse(result["verbose_output"])
def test_successful_real_run_records_stats_in_result_and_metadata(self) -> None:
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
log_path.parent.mkdir(parents=True, exist_ok=True)
log_path.write_text(
"Number of files: 10\n"
"Number of regular files transferred: 2\n"
"Total file size: 2,000 bytes\n"
"Total transferred file size: 500 bytes\n"
"Literal data: 500 bytes\n"
"Matched data: 1,500 bytes\n",
encoding="utf-8",
)
data_dir = log_path.parent.parent / "data"
data_dir.mkdir(parents=True, exist_ok=True)
(data_dir / "payload.txt").write_text("payload", encoding="utf-8")
return RsyncResult(exit_code=0, command=command)
with TemporaryDirectory() as tmp:
backup_root = Path(tmp) / "backups"
with patch("pobsync.commands.run_scheduled.run_rsync", side_effect=fake_run_rsync):
result = run_scheduled(
prefix=Path(tmp) / "home",
host="web-01",
dry_run=False,
config_source=FakeConfigSource(backup_root=str(backup_root)),
)
meta_path = Path(result["snapshot"]) / "meta" / "meta.yaml"
meta_text = meta_path.read_text(encoding="utf-8")
self.assertTrue(result["ok"])
self.assertEqual(result["stats"]["rsync"]["files_total"], 10)
self.assertEqual(result["stats"]["rsync"]["files_transferred"], 2)
self.assertEqual(result["stats"]["rsync"]["link_dest_estimated_savings_bytes"], 1500)
self.assertIn("snapshot", result["stats"]["storage"])
self.assertIn("capacity", result["stats"]["storage"])
self.assertIn("stats:", meta_text)
self.assertIn("files_total: 10", meta_text)
def test_dry_run_reports_cancelled_rsync(self) -> None:
def fake_run_rsync(command, log_path, timeout_seconds, cancel_check=None):
self.assertTrue(cancel_check())