## Summary

- Fall back to filesystem measurement when snapshot storage metadata is missing.
- Prefer `data/` inside snapshot directories so incomplete snapshot metadata/log files are not counted as backup data.
- Add stats and dashboard rendering coverage for incomplete snapshots without recorded storage metadata.

## Tests
- .venv/bin/python manage.py test src.pobsync_backend.tests.test_stats_summary src.pobsync_backend.tests.test_views.ViewTests.test_dashboard_host_cards_measure_incomplete_data_without_snapshot_metadata --verbosity 2
- .venv/bin/python manage.py check
- git diff --check
- .venv/bin/python manage.py test src.pobsync_backend --verbosity 2

Closes #62
This commit is contained in:
2026-05-23 01:35:38 +02:00
parent 10e0293559
commit 7dc4c1df84
3 changed files with 84 additions and 1 deletions

View File

@@ -5,7 +5,7 @@ from typing import Any, Iterable
from django.utils import timezone from django.utils import timezone
from pobsync.run_stats import filesystem_capacity from pobsync.run_stats import filesystem_capacity, tree_usage
from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
@@ -168,6 +168,12 @@ def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {} stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
storage = stats.get("storage") if isinstance(stats.get("storage"), dict) else {} storage = stats.get("storage") if isinstance(stats.get("storage"), dict) else {}
snapshot_storage = storage.get("snapshot") if isinstance(storage.get("snapshot"), dict) else {} snapshot_storage = storage.get("snapshot") if isinstance(storage.get("snapshot"), dict) else {}
has_recorded_size = (
_int_at(snapshot_storage, "allocated_size_bytes") is not None
or _int_at(snapshot_storage, "apparent_size_bytes") is not None
)
if not has_recorded_size:
snapshot_storage = _snapshot_storage_from_filesystem(snapshot)
return { return {
"id": snapshot.id, "id": snapshot.id,
"dirname": snapshot.dirname, "dirname": snapshot.dirname,
@@ -180,6 +186,18 @@ def _snapshot_summary(snapshot: SnapshotRecord | None) -> dict[str, Any]:
} }
def _snapshot_storage_from_filesystem(snapshot: SnapshotRecord) -> dict[str, Any]:
if not snapshot.path:
return {}
snapshot_path = Path(snapshot.path)
data_path = snapshot_path / "data"
if snapshot_path.name == "data":
return tree_usage(snapshot_path)
if data_path.exists():
return tree_usage(data_path)
return tree_usage(snapshot_path)
def _is_real_run(run: BackupRun) -> bool: def _is_real_run(run: BackupRun) -> bool:
result = run.result if isinstance(run.result, dict) else {} result = run.result if isinstance(run.result, dict) else {}
if result.get("dry_run") is True: if result.get("dry_run") is True:

View File

@@ -1,9 +1,12 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path
from tempfile import TemporaryDirectory
from django.test import TestCase from django.test import TestCase
from pobsync.run_stats import tree_usage
from pobsync_backend.models import HostConfig, SnapshotRecord from pobsync_backend.models import HostConfig, SnapshotRecord
from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats from pobsync_backend.stats_summary import collect_dashboard_stats, collect_host_stats
@@ -42,6 +45,42 @@ class StatsSummaryTests(TestCase):
self.assertEqual(stats["backup_data"]["total"]["count"], 4) self.assertEqual(stats["backup_data"]["total"]["count"], 4)
self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000) self.assertEqual(stats["backup_data"]["total"]["allocated_size_bytes"], 1000)
def test_collect_host_stats_falls_back_to_filesystem_usage_for_snapshots_without_metadata(self) -> None:
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
with TemporaryDirectory() as tmp:
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-051500Z__BROKEN1"
data_dir = incomplete_dir / "data"
meta_dir = incomplete_dir / "meta"
data_dir.mkdir(parents=True)
meta_dir.mkdir()
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
meta_dir.joinpath("rsync.log").write_text("not part of the backup data total\n", encoding="utf-8")
expected_usage = tree_usage(data_dir)
SnapshotRecord.objects.create(
host=host,
kind=SnapshotRecord.Kind.INCOMPLETE,
dirname=incomplete_dir.name,
path=str(incomplete_dir),
status="failed",
metadata={},
)
stats = collect_host_stats(host=host)
self.assertEqual(stats["backup_data"]["incomplete"]["count"], 1)
self.assertEqual(
stats["backup_data"]["incomplete"]["allocated_size_bytes"],
expected_usage["allocated_size_bytes"],
)
self.assertEqual(
stats["backup_data"]["incomplete"]["apparent_size_bytes"],
expected_usage["apparent_size_bytes"],
)
self.assertEqual(
stats["backup_data"]["total"]["allocated_size_bytes"],
expected_usage["allocated_size_bytes"],
)
def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord: def _snapshot(self, host: HostConfig, dirname: str, kind: str, *, allocated: int) -> SnapshotRecord:
started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc) started_at = datetime.strptime(dirname.split("__", 1)[0], "%Y%m%d-%H%M%SZ").replace(tzinfo=timezone.utc)
return SnapshotRecord.objects.create( return SnapshotRecord.objects.create(

View File

@@ -8,9 +8,11 @@ from unittest.mock import patch
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.template.defaultfilters import filesizeformat
from django.test import TestCase, override_settings from django.test import TestCase, override_settings
from django.urls import reverse from django.urls import reverse
from pobsync.run_stats import tree_usage
from pobsync.util import write_yaml_atomic from pobsync.util import write_yaml_atomic
from pobsync_backend.models import ( from pobsync_backend.models import (
BackupRun, BackupRun,
@@ -248,6 +250,30 @@ class ViewTests(TestCase):
self.assertContains(response, "300 bytes", html=True) self.assertContains(response, "300 bytes", html=True)
self.assertContains(response, "600 bytes", html=True) self.assertContains(response, "600 bytes", html=True)
def test_dashboard_host_cards_measure_incomplete_data_without_snapshot_metadata(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
with TemporaryDirectory() as tmp:
incomplete_dir = Path(tmp) / host.host / ".incomplete" / "20260519-041500Z__BROKEN1"
data_dir = incomplete_dir / "data"
data_dir.mkdir(parents=True)
data_dir.joinpath("partial-file").write_text("interrupted backup data\n", encoding="utf-8")
expected_usage = tree_usage(data_dir)
SnapshotRecord.objects.create(
host=host,
kind=SnapshotRecord.Kind.INCOMPLETE,
dirname=incomplete_dir.name,
path=str(incomplete_dir),
status="failed",
metadata={},
)
response = self.client.get(reverse("dashboard_hosts_live"))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Incomplete data")
self.assertContains(response, filesizeformat(expected_usage["allocated_size_bytes"]))
def test_hosts_list_renders_host_cards_and_controls(self) -> None: def test_hosts_list_renders_host_cards_and_controls(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
web = HostConfig.objects.create(host="web-01", address="web-01.example.test") web = HostConfig.objects.create(host="web-01", address="web-01.example.test")