Peter van Arkel
f886a7b620
Use stored snapshot storage metadata for dashboard and host backup data summaries instead of walking snapshot directories during request rendering. Snapshots without recorded storage metadata are counted as not measured so large backup targets cannot trigger unbounded filesystem scans from live-refresh views. Closes #97
174 lines
8.5 KiB
Python
174 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
|
|
from django.test import TestCase
|
|
|
|
from pobsync_backend.models import HostConfig, SnapshotRecord
|
|
from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats
|
|
|
|
|
|
class StatsSummaryTests(TestCase):
|
|
def test_collect_dashboard_stats_sums_backup_data_across_hosts(self) -> None:
|
|
web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
db = HostConfig.objects.create(host="db-01", address="db-01.example.test")
|
|
self._snapshot(web, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
|
self._snapshot(web, "20260519-031500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=200)
|
|
self._snapshot(db, "20260519-041500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=300)
|
|
self._snapshot(db, "20260519-051500Z__BROKEN1", SnapshotRecord.Kind.INCOMPLETE, allocated=400)
|
|
|
|
stats = collect_dashboard_stats(hosts=[web, db], global_config=None)
|
|
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 400)
|
|
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 200)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 400)
|
|
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
|
self.assertEqual(stats["backup_data"]["total"]["measured_count"], 4)
|
|
self.assertEqual(stats["backup_data"]["total"]["unknown_count"], 0)
|
|
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000)
|
|
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 2000)
|
|
|
|
def test_collect_host_stats_sums_backup_data_by_snapshot_kind(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
self._snapshot(host, "20260519-021500Z__SCHED01", SnapshotRecord.Kind.SCHEDULED, allocated=100)
|
|
self._snapshot(host, "20260519-031500Z__SCHED02", SnapshotRecord.Kind.SCHEDULED, allocated=200)
|
|
self._snapshot(host, "20260519-041500Z__MANUAL1", SnapshotRecord.Kind.MANUAL, allocated=300)
|
|
self._snapshot(host, "20260519-051500Z__BROKEN1", SnapshotRecord.Kind.INCOMPLETE, allocated=400)
|
|
|
|
stats = collect_host_stats(host=host)
|
|
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 2)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 300)
|
|
self.assertEqual(stats["backup_data"]["manual"]["allocated_size_bytes"], 300)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 400)
|
|
self.assertEqual(stats["backup_data"]["total"]["count"], 4)
|
|
self.assertEqual(stats["backup_data"]["total"]["measured_count"], 4)
|
|
self.assertEqual(stats["backup_data"]["total"]["unknown_count"], 0)
|
|
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000)
|
|
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 2000)
|
|
|
|
def test_collect_host_stats_marks_snapshots_without_storage_metadata_unknown(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
SnapshotRecord.objects.create(
|
|
host=host,
|
|
kind=SnapshotRecord.Kind.INCOMPLETE,
|
|
dirname="20260519-051500Z__BROKEN1",
|
|
path="/backups/web-01/.incomplete/20260519-051500Z__BROKEN1",
|
|
status="failed",
|
|
metadata={},
|
|
)
|
|
|
|
with patch("pobsync_backend.stats_summary.tree_usage", create=True) as tree_usage:
|
|
stats = collect_host_stats(host=host)
|
|
|
|
tree_usage.assert_not_called()
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["count"], 1)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["measured_count"], 0)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["unknown_count"], 1)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 0)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["apparent_size_bytes"], 0)
|
|
self.assertEqual(stats["backup_data"]["total"]["unknown_count"], 1)
|
|
|
|
def test_collect_host_stats_uses_recorded_zero_storage_without_rescanning(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
SnapshotRecord.objects.create(
|
|
host=host,
|
|
kind=SnapshotRecord.Kind.INCOMPLETE,
|
|
dirname="20260519-051500Z__BROKEN1",
|
|
path="/backups/web-01/.incomplete/20260519-051500Z__BROKEN1",
|
|
status="failed",
|
|
metadata={
|
|
"stats": {
|
|
"storage": {
|
|
"snapshot": {
|
|
"apparent_size_bytes": 0,
|
|
"allocated_size_bytes": 0,
|
|
}
|
|
}
|
|
}
|
|
},
|
|
)
|
|
|
|
with patch("pobsync_backend.stats_summary.tree_usage", create=True) as tree_usage:
|
|
stats = collect_host_stats(host=host)
|
|
|
|
tree_usage.assert_not_called()
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["measured_count"], 1)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["unknown_count"], 0)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["allocated_size_bytes"], 0)
|
|
self.assertEqual(stats["backup_data"]["incomplete"]["apparent_size_bytes"], 0)
|
|
|
|
def test_collect_dashboard_stats_does_not_scan_filesystem_for_missing_snapshot_metadata(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
SnapshotRecord.objects.create(
|
|
host=host,
|
|
kind=SnapshotRecord.Kind.SCHEDULED,
|
|
dirname="20260519-051500Z__SCHED01",
|
|
path="/backups/web-01/scheduled/20260519-051500Z__SCHED01",
|
|
status="success",
|
|
metadata={},
|
|
)
|
|
|
|
with patch("pobsync_backend.stats_summary.tree_usage", create=True) as tree_usage:
|
|
stats = collect_dashboard_stats(hosts=[host], global_config=None)
|
|
|
|
tree_usage.assert_not_called()
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["count"], 1)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["measured_count"], 0)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["unknown_count"], 1)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["allocated_size_bytes"], 0)
|
|
|
|
def test_collect_host_stats_reports_non_hardlinked_snapshot_data(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
self._snapshot_with_sizes(
|
|
host,
|
|
"20260519-021500Z__SCHED01",
|
|
SnapshotRecord.Kind.SCHEDULED,
|
|
allocated=1_200,
|
|
apparent=2_000,
|
|
hardlinked_apparent=1_500,
|
|
)
|
|
|
|
stats = collect_host_stats(host=host)
|
|
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["apparent_size_bytes"], 2_000)
|
|
self.assertEqual(stats["backup_data"]["scheduled"]["unique_apparent_size_bytes"], 500)
|
|
self.assertEqual(stats["backup_data"]["total"]["unique_apparent_size_bytes"], 500)
|
|
|
|
def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord:
|
|
return self._snapshot_with_sizes(host, dirname, kind, allocated=allocated)
|
|
|
|
def _snapshot_with_sizes(
|
|
self,
|
|
host: HostConfig,
|
|
dirname: str,
|
|
kind: str,
|
|
*,
|
|
allocated: int,
|
|
apparent: int | None = None,
|
|
hardlinked_apparent: int = 0,
|
|
) -> SnapshotRecord:
|
|
started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc)
|
|
apparent_size = apparent if apparent is not None else allocated * 2
|
|
return SnapshotRecord.objects.create(
|
|
host=host,
|
|
kind=kind,
|
|
dirname=dirname,
|
|
path=f"/backups/{host.host}/{kind}/{dirname}",
|
|
status="success",
|
|
started_at=started_at,
|
|
metadata={
|
|
"stats": {
|
|
"storage": {
|
|
"snapshot": {
|
|
"apparent_size_bytes": apparent_size,
|
|
"allocated_size_bytes": allocated,
|
|
"hardlinked_apparent_size_bytes": hardlinked_apparent,
|
|
}
|
|
}
|
|
}
|
|
},
|
|
)
|