(feature) Add backup trend forecasting to the Django UI
Extend derived backup statistics with average daily new data and an estimated days-until-full forecast based on recent successful real runs. Show the new forecast metrics on the dashboard and add compact per-run trend bars on the host detail page so new data and matched link-dest data are easier to compare at a glance. Keep the implementation migration-free by deriving everything from the existing BackupRun result stats payload.
This commit is contained in:
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
from pobsync.run_stats import filesystem_capacity
|
from pobsync.run_stats import filesystem_capacity
|
||||||
|
|
||||||
from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
from .models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
|
||||||
@@ -33,15 +35,18 @@ def collect_dashboard_stats(*, hosts: Iterable[HostConfig], global_config: Globa
|
|||||||
savings_basis = total_literal + total_matched
|
savings_basis = total_literal + total_matched
|
||||||
capacity = _capacity_from_system(global_config) or _latest_capacity_from_runs(real_runs) or {}
|
capacity = _capacity_from_system(global_config) or _latest_capacity_from_runs(real_runs) or {}
|
||||||
available = _int_at(capacity, "available_bytes")
|
available = _int_at(capacity, "available_bytes")
|
||||||
|
daily_literal = _average_daily_literal(real_runs)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"runs_sampled": len(real_runs),
|
"runs_sampled": len(real_runs),
|
||||||
"avg_duration_seconds": _average(duration_values),
|
"avg_duration_seconds": _average(duration_values),
|
||||||
|
"avg_daily_literal_data_bytes": daily_literal,
|
||||||
"avg_literal_data_bytes": avg_literal,
|
"avg_literal_data_bytes": avg_literal,
|
||||||
"total_literal_data_bytes": total_literal,
|
"total_literal_data_bytes": total_literal,
|
||||||
"total_matched_data_bytes": total_matched,
|
"total_matched_data_bytes": total_matched,
|
||||||
"link_dest_savings_ratio": round(total_matched / savings_basis, 4) if savings_basis else None,
|
"link_dest_savings_ratio": round(total_matched / savings_basis, 4) if savings_basis else None,
|
||||||
"estimated_runs_until_full": int(available / avg_literal) if available and avg_literal else None,
|
"estimated_runs_until_full": int(available / avg_literal) if available and avg_literal else None,
|
||||||
|
"estimated_days_until_full": int(available / daily_literal) if available and daily_literal else None,
|
||||||
"capacity": capacity,
|
"capacity": capacity,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,12 +62,15 @@ def collect_host_stats(*, host: HostConfig, limit: int = 8) -> dict[str, Any]:
|
|||||||
literal_values = [value for value in literal_values if value is not None]
|
literal_values = [value for value in literal_values if value is not None]
|
||||||
matched_values = [_int_at(run, "rsync", "matched_data_bytes") for run in real_runs]
|
matched_values = [_int_at(run, "rsync", "matched_data_bytes") for run in real_runs]
|
||||||
matched_values = [value for value in matched_values if value is not None]
|
matched_values = [value for value in matched_values if value is not None]
|
||||||
|
max_literal = max(literal_values) if literal_values else 0
|
||||||
|
max_matched = max(matched_values) if matched_values else 0
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"runs": real_runs,
|
"runs": [_with_bar_percentages(run, max_literal=max_literal, max_matched=max_matched) for run in real_runs],
|
||||||
"latest_run": real_runs[0] if real_runs else {},
|
"latest_run": real_runs[0] if real_runs else {},
|
||||||
"latest_snapshot": latest_snapshot_stats,
|
"latest_snapshot": latest_snapshot_stats,
|
||||||
"avg_literal_data_bytes": _average(literal_values),
|
"avg_literal_data_bytes": _average(literal_values),
|
||||||
|
"avg_daily_literal_data_bytes": _average_daily_literal(real_runs),
|
||||||
"total_literal_data_bytes": sum(literal_values),
|
"total_literal_data_bytes": sum(literal_values),
|
||||||
"total_matched_data_bytes": sum(matched_values),
|
"total_matched_data_bytes": sum(matched_values),
|
||||||
}
|
}
|
||||||
@@ -132,6 +140,41 @@ def _average(values: list[int]) -> int | None:
|
|||||||
return int(sum(values) / len(values))
|
return int(sum(values) / len(values))
|
||||||
|
|
||||||
|
|
||||||
|
def _average_daily_literal(runs: list[dict[str, Any]]) -> int | None:
|
||||||
|
values = [_int_at(run, "rsync", "literal_data_bytes") for run in runs]
|
||||||
|
values = [value for value in values if value is not None]
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
|
||||||
|
timestamps = [run["started_at"] for run in runs if run.get("started_at") is not None]
|
||||||
|
if len(timestamps) < 2:
|
||||||
|
return _average(values)
|
||||||
|
|
||||||
|
oldest = min(timestamps)
|
||||||
|
newest = max(timestamps)
|
||||||
|
if timezone.is_naive(oldest):
|
||||||
|
oldest = timezone.make_aware(oldest)
|
||||||
|
if timezone.is_naive(newest):
|
||||||
|
newest = timezone.make_aware(newest)
|
||||||
|
span_days = max((newest - oldest).total_seconds() / 86400, 1)
|
||||||
|
return int(sum(values) / span_days)
|
||||||
|
|
||||||
|
|
||||||
|
def _with_bar_percentages(run: dict[str, Any], *, max_literal: int, max_matched: int) -> dict[str, Any]:
|
||||||
|
run = dict(run)
|
||||||
|
literal = _int_at(run, "rsync", "literal_data_bytes") or 0
|
||||||
|
matched = _int_at(run, "rsync", "matched_data_bytes") or 0
|
||||||
|
run["literal_percent"] = _percentage(literal, max_literal)
|
||||||
|
run["matched_percent"] = _percentage(matched, max_matched)
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def _percentage(value: int, maximum: int) -> int:
|
||||||
|
if maximum <= 0 or value <= 0:
|
||||||
|
return 0
|
||||||
|
return max(1, min(100, int(value / maximum * 100)))
|
||||||
|
|
||||||
|
|
||||||
def _dict_at(data: dict[str, Any], *keys: str) -> dict[str, Any]:
|
def _dict_at(data: dict[str, Any], *keys: str) -> dict[str, Any]:
|
||||||
value: Any = data
|
value: Any = data
|
||||||
for key in keys:
|
for key in keys:
|
||||||
|
|||||||
@@ -120,6 +120,30 @@
|
|||||||
gap: 8px;
|
gap: 8px;
|
||||||
margin-bottom: 14px;
|
margin-bottom: 14px;
|
||||||
}
|
}
|
||||||
|
.trend-bars {
|
||||||
|
display: grid;
|
||||||
|
gap: 5px;
|
||||||
|
min-width: 150px;
|
||||||
|
}
|
||||||
|
.trend-bar {
|
||||||
|
background: #edf2f7;
|
||||||
|
border-radius: 4px;
|
||||||
|
height: 8px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.trend-bar span {
|
||||||
|
background: var(--link);
|
||||||
|
display: block;
|
||||||
|
height: 100%;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.trend-bar.matched span { background: var(--success); }
|
||||||
|
.trend-legend {
|
||||||
|
color: var(--muted);
|
||||||
|
display: flex;
|
||||||
|
font-size: 12px;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
.messages { display: grid; gap: 8px; margin-bottom: 18px; }
|
.messages { display: grid; gap: 8px; margin-bottom: 18px; }
|
||||||
.message {
|
.message {
|
||||||
background: var(--panel);
|
background: var(--panel);
|
||||||
|
|||||||
@@ -42,9 +42,11 @@
|
|||||||
<div class="metric"><div class="label">Backup Root Used</div><div class="value">{{ stats_summary.capacity.used_percent|default:"" }}{% if stats_summary.capacity.used_percent is not None %}%{% endif %}</div></div>
|
<div class="metric"><div class="label">Backup Root Used</div><div class="value">{{ stats_summary.capacity.used_percent|default:"" }}{% if stats_summary.capacity.used_percent is not None %}%{% endif %}</div></div>
|
||||||
<div class="metric"><div class="label">Available</div><div class="value">{{ stats_summary.capacity.available_bytes|filesizeformat }}</div></div>
|
<div class="metric"><div class="label">Available</div><div class="value">{{ stats_summary.capacity.available_bytes|filesizeformat }}</div></div>
|
||||||
<div class="metric"><div class="label">Avg New Data</div><div class="value">{{ stats_summary.avg_literal_data_bytes|filesizeformat }}</div></div>
|
<div class="metric"><div class="label">Avg New Data</div><div class="value">{{ stats_summary.avg_literal_data_bytes|filesizeformat }}</div></div>
|
||||||
|
<div class="metric"><div class="label">Avg Daily New</div><div class="value">{{ stats_summary.avg_daily_literal_data_bytes|filesizeformat }}</div></div>
|
||||||
<div class="metric"><div class="label">Avg Duration</div><div class="value">{{ stats_summary.avg_duration_seconds|default:"" }}{% if stats_summary.avg_duration_seconds is not None %}s{% endif %}</div></div>
|
<div class="metric"><div class="label">Avg Duration</div><div class="value">{{ stats_summary.avg_duration_seconds|default:"" }}{% if stats_summary.avg_duration_seconds is not None %}s{% endif %}</div></div>
|
||||||
<div class="metric"><div class="label">Link-Dest Savings</div><div class="value">{{ stats_summary.link_dest_savings_ratio|default:"" }}</div></div>
|
<div class="metric"><div class="label">Link-Dest Savings</div><div class="value">{{ stats_summary.link_dest_savings_ratio|default:"" }}</div></div>
|
||||||
<div class="metric"><div class="label">Runs Until Full</div><div class="value">{{ stats_summary.estimated_runs_until_full|default:"" }}</div></div>
|
<div class="metric"><div class="label">Runs Until Full</div><div class="value">{{ stats_summary.estimated_runs_until_full|default:"" }}</div></div>
|
||||||
|
<div class="metric"><div class="label">Days Until Full</div><div class="value">{{ stats_summary.estimated_days_until_full|default:"" }}</div></div>
|
||||||
</section>
|
</section>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
|||||||
@@ -83,6 +83,7 @@
|
|||||||
<h2>Backup Trends</h2>
|
<h2>Backup Trends</h2>
|
||||||
<section class="grid" aria-label="Host backup trend summary">
|
<section class="grid" aria-label="Host backup trend summary">
|
||||||
<div class="metric"><div class="label">Avg New Data</div><div class="value">{{ stats_summary.avg_literal_data_bytes|filesizeformat }}</div></div>
|
<div class="metric"><div class="label">Avg New Data</div><div class="value">{{ stats_summary.avg_literal_data_bytes|filesizeformat }}</div></div>
|
||||||
|
<div class="metric"><div class="label">Avg Daily New</div><div class="value">{{ stats_summary.avg_daily_literal_data_bytes|filesizeformat }}</div></div>
|
||||||
<div class="metric"><div class="label">Total New Data</div><div class="value">{{ stats_summary.total_literal_data_bytes|filesizeformat }}</div></div>
|
<div class="metric"><div class="label">Total New Data</div><div class="value">{{ stats_summary.total_literal_data_bytes|filesizeformat }}</div></div>
|
||||||
<div class="metric"><div class="label">Matched Data</div><div class="value">{{ stats_summary.total_matched_data_bytes|filesizeformat }}</div></div>
|
<div class="metric"><div class="label">Matched Data</div><div class="value">{{ stats_summary.total_matched_data_bytes|filesizeformat }}</div></div>
|
||||||
<div class="metric"><div class="label">Latest Duration</div><div class="value">{{ stats_summary.latest_run.duration_seconds|default:"" }}{% if stats_summary.latest_run.duration_seconds is not None %}s{% endif %}</div></div>
|
<div class="metric"><div class="label">Latest Duration</div><div class="value">{{ stats_summary.latest_run.duration_seconds|default:"" }}{% if stats_summary.latest_run.duration_seconds is not None %}s{% endif %}</div></div>
|
||||||
@@ -96,6 +97,7 @@
|
|||||||
<th>Files</th>
|
<th>Files</th>
|
||||||
<th>New Data</th>
|
<th>New Data</th>
|
||||||
<th>Matched</th>
|
<th>Matched</th>
|
||||||
|
<th>Trend</th>
|
||||||
<th>Snapshot</th>
|
<th>Snapshot</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -108,6 +110,13 @@
|
|||||||
<td>{{ run.rsync.files_total|default:"" }}</td>
|
<td>{{ run.rsync.files_total|default:"" }}</td>
|
||||||
<td>{{ run.rsync.literal_data_bytes|filesizeformat }}</td>
|
<td>{{ run.rsync.literal_data_bytes|filesizeformat }}</td>
|
||||||
<td>{{ run.rsync.matched_data_bytes|filesizeformat }}</td>
|
<td>{{ run.rsync.matched_data_bytes|filesizeformat }}</td>
|
||||||
|
<td>
|
||||||
|
<div class="trend-bars" aria-label="Run data trend">
|
||||||
|
<div class="trend-bar" title="New data"><span style="width: {{ run.literal_percent }}%"></span></div>
|
||||||
|
<div class="trend-bar matched" title="Matched data"><span style="width: {{ run.matched_percent }}%"></span></div>
|
||||||
|
<div class="trend-legend"><span>new</span><span>matched</span></div>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
<td>{% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}<span class="muted">{{ run.snapshot_path }}</span>{% endif %}</td>
|
<td>{% if run.snapshot %}<a href="{% url 'snapshot_detail' run.snapshot.id %}">{{ run.snapshot.dirname }}</a>{% else %}<span class="muted">{{ run.snapshot_path }}</span>{% endif %}</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|||||||
@@ -85,6 +85,8 @@ class ViewTests(TestCase):
|
|||||||
self.assertEqual(response.status_code, 200)
|
self.assertEqual(response.status_code, 200)
|
||||||
self.assertContains(response, "Backup Root Used")
|
self.assertContains(response, "Backup Root Used")
|
||||||
self.assertContains(response, "Runs Until Full")
|
self.assertContains(response, "Runs Until Full")
|
||||||
|
self.assertContains(response, "Avg Daily New")
|
||||||
|
self.assertContains(response, "Days Until Full")
|
||||||
self.assertContains(response, "10")
|
self.assertContains(response, "10")
|
||||||
self.assertContains(response, f"Run {run.id}")
|
self.assertContains(response, f"Run {run.id}")
|
||||||
self.assertContains(response, "1000")
|
self.assertContains(response, "1000")
|
||||||
@@ -582,16 +584,38 @@ class ViewTests(TestCase):
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
BackupRun.objects.create(
|
||||||
|
host=host,
|
||||||
|
status=BackupRun.Status.SUCCESS,
|
||||||
|
snapshot=snapshot,
|
||||||
|
started_at=datetime(2026, 5, 18, 2, 15, tzinfo=timezone.utc),
|
||||||
|
result={
|
||||||
|
"ok": True,
|
||||||
|
"dry_run": False,
|
||||||
|
"stats": {
|
||||||
|
"duration_seconds": 35,
|
||||||
|
"rsync": {
|
||||||
|
"files_total": 150,
|
||||||
|
"literal_data_bytes": 1024,
|
||||||
|
"matched_data_bytes": 4096,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
response = self.client.get(reverse("host_detail", args=[host.host]))
|
response = self.client.get(reverse("host_detail", args=[host.host]))
|
||||||
|
|
||||||
self.assertEqual(response.status_code, 200)
|
self.assertEqual(response.status_code, 200)
|
||||||
self.assertContains(response, "Backup Trends")
|
self.assertContains(response, "Backup Trends")
|
||||||
self.assertContains(response, "Avg New Data")
|
self.assertContains(response, "Avg New Data")
|
||||||
|
self.assertContains(response, "Avg Daily New")
|
||||||
self.assertContains(response, "45s")
|
self.assertContains(response, "45s")
|
||||||
self.assertContains(response, "250")
|
self.assertContains(response, "250")
|
||||||
self.assertContains(response, "2.0")
|
self.assertContains(response, "2.0")
|
||||||
self.assertContains(response, "KB")
|
self.assertContains(response, "KB")
|
||||||
|
self.assertContains(response, "Run data trend")
|
||||||
|
self.assertContains(response, "width: 100%")
|
||||||
|
self.assertContains(response, "width: 50%")
|
||||||
|
|
||||||
def test_prepare_host_directories_action_creates_missing_directories(self) -> None:
|
def test_prepare_host_directories_action_creates_missing_directories(self) -> None:
|
||||||
self.client.force_login(self.staff_user)
|
self.client.force_login(self.staff_user)
|
||||||
|
|||||||
Reference in New Issue
Block a user