From 9dd690bb3be568f609f9ff162340d07f0a6eb7cc Mon Sep 17 00:00:00 2001 From: Peter van Arkel Date: Sat, 23 May 2026 01:27:51 +0200 Subject: [PATCH] (feature) Show backup data totals by snapshot kind Aggregate snapshot storage metadata by snapshot kind so operators can see scheduled, manual, incomplete, and total backup data. Surface the totals per host and across all hosts on the dashboard, using allocated snapshot size from recorded backup metadata without rescanning backup storage. --- src/pobsync_backend/stats_summary.py | 59 +++++++++++++++++ .../pobsync_backend/host_detail.html | 23 +++++++ .../partials/dashboard_hosts.html | 16 +++++ .../partials/dashboard_priority.html | 18 ++++++ .../tests/test_stats_summary.py | 64 +++++++++++++++++++ src/pobsync_backend/tests/test_views.py | 61 ++++++++++++++++++ 6 files changed, 241 insertions(+) create mode 100644 src/pobsync_backend/tests/test_stats_summary.py diff --git a/src/pobsync_backend/stats_summary.py b/src/pobsync_backend/stats_summary.py index 62d5134..a63e8cd 100644 --- a/src/pobsync_backend/stats_summary.py +++ b/src/pobsync_backend/stats_summary.py @@ -11,6 +11,7 @@ from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: GlobalConfig | None) -> dict[str, Any]: + hosts = list(hosts) runs = list( BackupRun.objects.select_related("host", "snapshot") .filter(status__in=_COMPLETED_BACKUP_STATUSES) @@ -21,6 +22,7 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa for host in hosts: host.stats_summary = collect_host_stats(host=host) + backup_data = _sum_backup_data_by_kind(host.stats_summary["backup_data"] for host in hosts) literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in real_runs] literal_values = [value for value in literal_values if value is not None] @@ -51,6 +53,7 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa "estimated_runs_until_full": int(available / avg_literal) if available and avg_literal else None, "estimated_days_until_full": int(available / daily_literal) if available and daily_literal else None, "capacity": capacity, + "backup_data": backup_data, } @@ -61,6 +64,7 @@ def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]: trend_runs = [run for run in completed_real_runs if run["has_stats"]][:limit] latest_snapshot = host.snapshots.order_by("-started_at", "-discovered_at", "-id").first() latest_snapshot_stats = _snapshot_summary(latest_snapshot) if latest_snapshot else {} + backup_data = _backup_data_by_kind(host) literal_values = [_int_at(run, "rsync", "literal_data_bytes") for run in trend_runs] literal_values = [value for value in literal_values if value is not None] @@ -75,6 +79,7 @@ def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]: "latest_good_run": _first_run_with_status(real_runs, {BackupRun.Status.SUCCESS}), "latest_problem_run": _first_run_with_status(real_runs, {BackupRun.Status.WARNING, BackupRun.Status.FAILED}), "latest_snapshot": latest_snapshot_stats, + "backup_data": backup_data, "avg_literal_data_bytes": _average(literal_values), "avg_daily_literal_data_bytes": _average_daily_literal(trend_runs), "total_literal_data_bytes": sum(literal_values), @@ -102,6 +107,60 @@ def _run_summary(run: BackupRun) -> dict[str, Any]: } +def _backup_data_by_kind(host: HostConfig) -> dict[str, Any]: + rows: dict[str, dict[str, int]] = { + SnapshotRecord.Kind.SCHEDULED: _empty_snapshot_data_row(), + SnapshotRecord.Kind.MANUAL: _empty_snapshot_data_row(), + SnapshotRecord.Kind.INCOMPLETE: _empty_snapshot_data_row(), + } + total = _empty_snapshot_data_row() + + for snapshot in host.snapshots.all(): + summary = _snapshot_summary(snapshot) + row = rows.setdefault(snapshot.kind, _empty_snapshot_data_row()) + allocated = summary.get("allocated_size_bytes") or summary.get("apparent_size_bytes") or 0 + apparent = summary.get("apparent_size_bytes") or 0 + row["count"] += 1 + row["allocated_size_bytes"] += int(allocated) + row["apparent_size_bytes"] += int(apparent) + total["count"] += 1 + total["allocated_size_bytes"] += int(allocated) + total["apparent_size_bytes"] += int(apparent) + + return { + "scheduled": rows[SnapshotRecord.Kind.SCHEDULED], + "manual": rows[SnapshotRecord.Kind.MANUAL], + "incomplete": rows[SnapshotRecord.Kind.INCOMPLETE], + "total": total, + } + + +def _empty_snapshot_data_row() -> dict[str, int]: + return { + "count": 0, + "allocated_size_bytes": 0, + "apparent_size_bytes": 0, + } + + +def _sum_backup_data_by_kind(rows: Iterable[dict[str, dict[str, int]]]) -> dict[str, dict[str, int]]: + total_rows: dict[str, dict[str, int]] = { + "scheduled": _empty_snapshot_data_row(), + "manual": _empty_snapshot_data_row(), + "incomplete": _empty_snapshot_data_row(), + "total": _empty_snapshot_data_row(), + } + + for row in rows: + for kind, values in row.items(): + total_row = total_rows.setdefault(kind, _empty_snapshot_data_row()) + total_row["count"] += values.get("count", 0) + total_row["allocated_size_bytes"] += values.get("allocated_size_bytes", 0) + total_row["apparent_size_bytes"] += values.get("apparent_size_bytes", 0) + + return total_rows + + def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]: if snapshot is None: return {} diff --git a/src/pobsync_backend/templates/pobsync_backend/host_detail.html b/src/pobsync_backend/templates/pobsync_backend/host_detail.html index b69d6a2..6ff8faa 100644 --- a/src/pobsync_backend/templates/pobsync_backend/host_detail.html +++ b/src/pobsync_backend/templates/pobsync_backend/host_detail.html @@ -172,6 +172,29 @@
Incomplete
{{ counts.incomplete_snapshots }}
+
+

Backup Data

+
+
+
Scheduled
+
{{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}
+
+
+
Manual
+
{{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}
+
+
+
Incomplete
+
{{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}
+
+
+
Total
+
{{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}
+
+
+

Totals use the allocated snapshot size recorded in backup metadata, grouped by snapshot kind.

+
+ {% if stats_summary.runs %}

Backup Trends

diff --git a/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_hosts.html b/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_hosts.html index a693a56..35b5047 100644 --- a/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_hosts.html +++ b/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_hosts.html @@ -102,6 +102,22 @@
Retention
d{{ host.retention_daily }} w{{ host.retention_weekly }} m{{ host.retention_monthly }} y{{ host.retention_yearly }}
+
+
Scheduled data
+
{{ host.stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }}
+
+
+
Manual data
+
{{ host.stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }}
+
+
+
Incomplete data
+
{{ host.stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }}
+
+
+
Total data
+
{{ host.stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }}
+
diff --git a/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_priority.html b/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_priority.html index d17d01e..c15001a 100644 --- a/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_priority.html +++ b/src/pobsync_backend/templates/pobsync_backend/partials/dashboard_priority.html @@ -126,5 +126,23 @@ {% else %}

Storage pressure appears after the first completed backup with stats.

{% endif %} +
+
+ Scheduled data + {{ stats_summary.backup_data.scheduled.allocated_size_bytes|filesizeformat }} +
+
+ Manual data + {{ stats_summary.backup_data.manual.allocated_size_bytes|filesizeformat }} +
+
+ Incomplete data + {{ stats_summary.backup_data.incomplete.allocated_size_bytes|filesizeformat }} +
+
+ Total snapshot data + {{ stats_summary.backup_data.total.allocated_size_bytes|filesizeformat }} +
+
diff --git a/src/pobsync_backend/tests/test_stats_summary.py b/src/pobsync_backend/tests/test_stats_summary.py new file mode 100644 index 0000000..e043097 --- /dev/null +++ b/src/pobsync_backend/tests/test_stats_summary.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from django.test import TestCase + +from pobsync_backend.models import HostConfig, SnapshotRecord +from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats + + +class StatsSummaryTests(TestCase): + def test_collect_dashboard_stats_sums_backup_data_across_hosts(self) -> None: + web = HostConfig.objects.create(host="web-01", address="web-01.example.test") + db = HostConfig.objects.create(host="db-01", address="db-01.example.test") + self._snapshot(web, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100) + self._snapshot(web, "20260519-031500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=200) + self._snapshot(db, "20260519-041500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=300) + self._snapshot(db, "20260519-051500Z__BROKEN1", SnapshotRecord.Kind.INCOMPLETE, allocated=400) + + stats = collect_dashboard_stats(hosts=[web, db], global_config=None) + + self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2) + self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 400) + self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 200) + self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 400) + self.assertEqual(stats["backup_data"]["total"]["count"], 4) + self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000) + + def test_collect_host_stats_sums_backup_data_by_snapshot_kind(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + self._snapshot(host, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100) + self._snapshot(host, "20260519-031500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=200) + self._snapshot(host, "20260519-041500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=300) + self._snapshot(host, "20260519-051500Z__BROKEN1", SnapshotRecord.Kind.INCOMPLETE, allocated=400) + + stats = collect_host_stats(host=host) + + self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2) + self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 300) + self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 300) + self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 400) + self.assertEqual(stats["backup_data"]["total"]["count"], 4) + self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000) + + def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord: + started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc) + return SnapshotRecord.objects.create( + host=host, + kind=kind, + dirname=dirname, + path=f"/backups/{host.host}/{kind}/{dirname}", + status="success", + started_at=started_at, + metadata={ + "stats": { + "storage": { + "snapshot": { + "apparent_size_bytes": allocated * 2, + "allocated_size_bytes": allocated, + } + } + } + }, + ) diff --git a/src/pobsync_backend/tests/test_views.py b/src/pobsync_backend/tests/test_views.py index 7de778c..5e066ef 100644 --- a/src/pobsync_backend/tests/test_views.py +++ b/src/pobsync_backend/tests/test_views.py @@ -190,6 +190,29 @@ class ViewTests(TestCase): self.assertContains(response, "running") self.assertNotContains(response, " None: + self.client.force_login(self.staff_user) + web = HostConfig.objects.create(host="web-01", address="web-01.example.test") + db = HostConfig.objects.create(host="db-01", address="db-01.example.test") + scheduled = self._snapshot(web, "20260519-021500Z__SCHED01", kind=SnapshotRecord.Kind.SCHEDULED) + manual = self._snapshot(web, "20260519-031500Z__MANUAL1", kind=SnapshotRecord.Kind.MANUAL) + incomplete = self._snapshot(db, "20260519-041500Z__BROKEN1", kind=SnapshotRecord.Kind.INCOMPLETE) + self._set_snapshot_storage(scheduled, allocated=100) + self._set_snapshot_storage(manual, allocated=200) + self._set_snapshot_storage(incomplete, allocated=300) + + response = self.client.get(reverse("dashboard_priority_live")) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Scheduled data") + self.assertContains(response, "Manual data") + self.assertContains(response, "Incomplete data") + self.assertContains(response, "Total snapshot data") + self.assertContains(response, "100 bytes", html=True) + self.assertContains(response, "200 bytes", html=True) + self.assertContains(response, "300 bytes", html=True) + self.assertContains(response, "600 bytes", html=True) + def test_dashboard_hosts_live_returns_hosts_partial(self) -> None: self.client.force_login(self.staff_user) host = HostConfig.objects.create(host="web-01", address="web-01.example.test") @@ -203,6 +226,28 @@ class ViewTests(TestCase): self.assertContains(response, "Snapshot health") self.assertNotContains(response, " None: + self.client.force_login(self.staff_user) + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + scheduled = self._snapshot(host, "20260519-021500Z__SCHED01", kind=SnapshotRecord.Kind.SCHEDULED) + manual = self._snapshot(host, "20260519-031500Z__MANUAL1", kind=SnapshotRecord.Kind.MANUAL) + incomplete = self._snapshot(host, "20260519-041500Z__BROKEN1", kind=SnapshotRecord.Kind.INCOMPLETE) + self._set_snapshot_storage(scheduled, allocated=100) + self._set_snapshot_storage(manual, allocated=200) + self._set_snapshot_storage(incomplete, allocated=300) + + response = self.client.get(reverse("dashboard_hosts_live")) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Scheduled data") + self.assertContains(response, "Manual data") + self.assertContains(response, "Incomplete data") + self.assertContains(response, "Total data") + self.assertContains(response, "100 bytes", html=True) + self.assertContains(response, "200 bytes", html=True) + self.assertContains(response, "300 bytes", html=True) + self.assertContains(response, "600 bytes", html=True) + def test_hosts_list_renders_host_cards_and_controls(self) -> None: self.client.force_login(self.staff_user) web = HostConfig.objects.create(host="web-01", address="web-01.example.test") @@ -1075,6 +1120,7 @@ class ViewTests(TestCase): (backup_root / host.host / subdir).mkdir(parents=True) ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", prune=True, last_status="success") snapshot = self._snapshot(host, "20260519-021500Z__ABCDEFGH") + self._set_snapshot_storage(snapshot, allocated=100) BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, snapshot=snapshot) response = self.client.get(reverse("host_detail", args=[host.host])) @@ -1099,6 +1145,8 @@ class ViewTests(TestCase): self.assertContains(response, reverse("prepare_host_directories", args=[host.host])) self.assertContains(response, "warning") self.assertContains(response, "Snapshot Storage") + self.assertContains(response, "Backup Data") + self.assertContains(response, "100 bytes", html=True) self.assertContains(response, reverse("queue_manual_backup", args=[host.host])) self.assertContains(response, reverse("run_detail", args=[BackupRun.objects.get().id])) self.assertContains(response, reverse("snapshot_detail", args=[snapshot.id])) @@ -2606,3 +2654,16 @@ class ViewTests(TestCase): status="success", started_at=started_at, ) + + def _set_snapshot_storage(self, snapshot: SnapshotRecord, *, allocated: int) -> None: + snapshot.metadata = { + "stats": { + "storage": { + "snapshot": { + "apparent_size_bytes": allocated * 2, + "allocated_size_bytes": allocated, + } + } + } + } + snapshot.save(update_fields=["metadata"])