6 Commits

Author SHA1 Message Date
17215fd191 Merge pull request '(feature) Improve retention UX and prune safety' (#14) from issue-4-retention-ux-and-safety into master
Reviewed-on: #14
2026-05-21 01:27:19 +02:00
97753c3d3c (ui) Show retention apply details on run detail
Record planned delete counts, max-delete settings, base protection, and
ignored incomplete snapshots in retention apply results.

Surface those details on run detail pages so scheduled and manual prune
outcomes are understandable without reading the raw JSON payload.
2026-05-21 01:25:40 +02:00
994f7f66c4 (bugfix) Preserve scheduled backup warning status
Update the scheduler to reflect the actual scheduled BackupRun status after
a run completes, so prune warnings are shown as schedule warnings instead
of being reported as successful schedule executions.
2026-05-21 01:22:06 +02:00
f76b6cad14 (feature) Require delete count confirmation for retention apply
Make manual retention application more explicit by requiring operators to
confirm both the host name and the current number of planned deletions.

This reduces the risk of applying a stale or misunderstood retention plan
when the delete set changes between review and confirmation.
2026-05-21 01:19:08 +02:00
90e293facd (ui) Surface retention warnings on run detail
Show host-level retention warnings on run detail pages so successful or
warning runs still expose scheduled prune limit issues and incomplete
snapshots that need operator attention.
2026-05-21 01:13:44 +02:00
50eb7cf2f3 (ui) Make retention planning warnings explicit
Show keep/delete reasons in the retention plan, surface scheduled prune
limit warnings, and explain base snapshot protection before retention is
applied.

Also surface incomplete snapshots from the retention views without deleting
them automatically, so interrupted backups are visible on the dashboard,
host detail, and retention plan.
2026-05-21 01:10:45 +02:00
12 changed files with 506 additions and 18 deletions

View File

@@ -249,12 +249,18 @@ class RetentionApplyForm(forms.Form):
kind = forms.ChoiceField(choices=(("scheduled", "Scheduled"), ("manual", "Manual"), ("all", "All"))) kind = forms.ChoiceField(choices=(("scheduled", "Scheduled"), ("manual", "Manual"), ("all", "All")))
protect_bases = forms.BooleanField(required=False) protect_bases = forms.BooleanField(required=False)
max_delete = forms.IntegerField(min_value=0, initial=10) max_delete = forms.IntegerField(min_value=0, initial=10)
confirm_delete_count = forms.IntegerField(min_value=0)
confirm_host = forms.CharField() confirm_host = forms.CharField()
def __init__(self, *args, host_name: str, **kwargs) -> None: def __init__(self, *args, host_name: str, expected_delete_count: int | None = None, **kwargs) -> None:
self.host_name = host_name self.host_name = host_name
self.expected_delete_count = expected_delete_count
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.fields["confirm_host"].help_text = f"Type {host_name} to confirm deletion." self.fields["confirm_host"].help_text = f"Type {host_name} to confirm deletion."
if expected_delete_count is not None:
self.fields["confirm_delete_count"].help_text = (
f"Type {expected_delete_count} to confirm the current number of planned deletions."
)
def clean_confirm_host(self) -> str: def clean_confirm_host(self) -> str:
value = self.cleaned_data["confirm_host"].strip() value = self.cleaned_data["confirm_host"].strip()
@@ -262,6 +268,12 @@ class RetentionApplyForm(forms.Form):
raise forms.ValidationError(f"Type {self.host_name} to confirm.") raise forms.ValidationError(f"Type {self.host_name} to confirm.")
return value return value
def clean_confirm_delete_count(self) -> int:
value = self.cleaned_data["confirm_delete_count"]
if self.expected_delete_count is not None and value != self.expected_delete_count:
raise forms.ValidationError(f"Type {self.expected_delete_count} to confirm the delete count.")
return value
class ScheduleConfigForm(forms.ModelForm): class ScheduleConfigForm(forms.ModelForm):
cron_expr = forms.CharField( cron_expr = forms.CharField(

View File

@@ -10,7 +10,7 @@ from django.core.management.base import BaseCommand
from django.db import transaction from django.db import transaction
from django.utils import timezone from django.utils import timezone
from pobsync_backend.models import ScheduleConfig from pobsync_backend.models import BackupRun, ScheduleConfig
from pobsync_backend.scheduler import due_key, is_due from pobsync_backend.scheduler import due_key, is_due
@@ -52,12 +52,13 @@ class Command(BaseCommand):
if not is_due(schedule.cron_expr, now): if not is_due(schedule.cron_expr, now):
continue continue
schedule_started_at = timezone.now()
with transaction.atomic(): with transaction.atomic():
locked = ScheduleConfig.objects.select_for_update().get(pk=schedule.pk) locked = ScheduleConfig.objects.select_for_update().get(pk=schedule.pk)
if locked.last_due_key == current_due_key: if locked.last_due_key == current_due_key:
continue continue
locked.last_due_key = current_due_key locked.last_due_key = current_due_key
locked.last_started_at = timezone.now() locked.last_started_at = schedule_started_at
locked.last_status = "running" locked.last_status = "running"
locked.save(update_fields=["last_due_key", "last_started_at", "last_status", "updated_at"]) locked.save(update_fields=["last_due_key", "last_started_at", "last_status", "updated_at"])
@@ -72,6 +73,7 @@ class Command(BaseCommand):
prune_max_delete=schedule.prune_max_delete, prune_max_delete=schedule.prune_max_delete,
prune_protect_bases=schedule.prune_protect_bases, prune_protect_bases=schedule.prune_protect_bases,
) )
status = _latest_scheduled_run_status(host_id=schedule.host_id, started_at=schedule_started_at) or status
except Exception as exc: except Exception as exc:
status = "failed" status = "failed"
self.stderr.write(f"{schedule.host.host}: {type(exc).__name__}: {exc}") self.stderr.write(f"{schedule.host.host}: {type(exc).__name__}: {exc}")
@@ -83,3 +85,16 @@ class Command(BaseCommand):
ran += 1 ran += 1
return ran return ran
def _latest_scheduled_run_status(*, host_id: int, started_at) -> str | None:
run = (
BackupRun.objects.filter(
host_id=host_id,
run_type=BackupRun.RunType.SCHEDULED,
created_at__gte=started_at,
)
.order_by("-created_at", "-id")
.first()
)
return run.status if run is not None else None

View File

@@ -23,6 +23,7 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
host_config = _enabled_host_config(host) host_config = _enabled_host_config(host)
retention = _retention_for_host(host_config) retention = _retention_for_host(host_config)
snapshots = _snapshots_for_retention(host_config=host_config, kind=kind) snapshots = _snapshots_for_retention(host_config=host_config, kind=kind)
incomplete_snapshots = _incomplete_snapshots_for_host(host_config)
plan = build_retention_plan( plan = build_retention_plan(
snapshots=snapshots, snapshots=snapshots,
@@ -36,6 +37,7 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
keep, reasons = apply_base_protection(snapshots=snapshots, keep=keep, reasons=reasons) keep, reasons = apply_base_protection(snapshots=snapshots, keep=keep, reasons=reasons)
delete = [snapshot for snapshot in snapshots if snapshot.dirname not in keep] delete = [snapshot for snapshot in snapshots if snapshot.dirname not in keep]
keep_items = [snapshot for snapshot in snapshots if snapshot.dirname in keep]
return { return {
"ok": True, "ok": True,
@@ -45,7 +47,12 @@ def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict
"retention": retention, "retention": retention,
"source": "sql", "source": "sql",
"keep": sorted(keep), "keep": sorted(keep),
"delete": [_snapshot_to_delete_item(snapshot) for snapshot in delete], "keep_items": [_snapshot_to_item(snapshot, reasons=reasons.get(snapshot.dirname, [])) for snapshot in keep_items],
"delete": [_snapshot_to_item(snapshot, reasons=["outside retention policy"]) for snapshot in delete],
"incomplete": [
_snapshot_to_item(snapshot, reasons=["incomplete snapshot; excluded from retention cleanup"])
for snapshot in incomplete_snapshots
],
"reasons": reasons, "reasons": reasons,
} }
@@ -71,8 +78,11 @@ def run_sql_retention_apply(
def _do_apply() -> dict[str, Any]: def _do_apply() -> dict[str, Any]:
plan = run_sql_retention_plan(host=host, kind=kind, protect_bases=bool(protect_bases)) plan = run_sql_retention_plan(host=host, kind=kind, protect_bases=bool(protect_bases))
delete_list = plan.get("delete") or [] delete_list = plan.get("delete") or []
incomplete_list = plan.get("incomplete") or []
if not isinstance(delete_list, list): if not isinstance(delete_list, list):
raise ConfigError("Invalid retention plan output: delete is not a list") raise ConfigError("Invalid retention plan output: delete is not a list")
if not isinstance(incomplete_list, list):
raise ConfigError("Invalid retention plan output: incomplete is not a list")
if max_delete == 0 and len(delete_list) > 0: if max_delete == 0 and len(delete_list) > 0:
raise ConfigError("Deletion blocked by --max-delete=0") raise ConfigError("Deletion blocked by --max-delete=0")
if len(delete_list) > max_delete: if len(delete_list) > max_delete:
@@ -109,6 +119,8 @@ def run_sql_retention_apply(
"protect_bases": bool(protect_bases), "protect_bases": bool(protect_bases),
"max_delete": max_delete, "max_delete": max_delete,
"source": "sql", "source": "sql",
"planned_delete_count": len(delete_list),
"incomplete_ignored_count": len(incomplete_list),
"deleted": deleted, "deleted": deleted,
"actions": actions, "actions": actions,
} }
@@ -146,6 +158,15 @@ def _snapshots_for_retention(*, host_config: HostConfig, kind: str) -> list[Snap
return [_snapshot_from_record(record) for record in records] return [_snapshot_from_record(record) for record in records]
def _incomplete_snapshots_for_host(host_config: HostConfig) -> list[Snapshot]:
records = (
SnapshotRecord.objects.filter(host=host_config, kind=SnapshotRecord.Kind.INCOMPLETE)
.select_related("base")
.order_by("-started_at", "dirname")
)
return [_snapshot_from_record(record) for record in records]
def _snapshot_from_record(record: SnapshotRecord) -> Snapshot: def _snapshot_from_record(record: SnapshotRecord) -> Snapshot:
return Snapshot( return Snapshot(
kind=record.kind, kind=record.kind,
@@ -173,13 +194,15 @@ def _base_meta_from_record(record: SnapshotRecord) -> dict[str, str] | None:
return None return None
def _snapshot_to_delete_item(snapshot: Snapshot) -> dict[str, Any]: def _snapshot_to_item(snapshot: Snapshot, *, reasons: list[str]) -> dict[str, Any]:
return { return {
"dirname": snapshot.dirname, "dirname": snapshot.dirname,
"kind": snapshot.kind, "kind": snapshot.kind,
"path": snapshot.path, "path": snapshot.path,
"dt": snapshot.dt.isoformat(), "dt": snapshot.dt.isoformat(),
"status": snapshot.status, "status": snapshot.status,
"reasons": reasons,
"reason": ", ".join(reasons),
} }

View File

@@ -231,6 +231,17 @@
.host-card-stat.wide { .host-card-stat.wide {
grid-column: 1 / -1; grid-column: 1 / -1;
} }
.host-card-warning {
background: #fffaf0;
border: 1px solid #e7cf8a;
border-radius: 6px;
color: var(--running);
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-top: 14px;
padding: 10px;
}
.messages { display: grid; gap: 8px; margin-bottom: 18px; } .messages { display: grid; gap: 8px; margin-bottom: 18px; }
.message { .message {
background: var(--panel); background: var(--panel);

View File

@@ -118,6 +118,20 @@
</div> </div>
</div> </div>
</div> </div>
{% if host.retention_warning.has_warning %}
<div class="host-card-warning">
<span class="status warning">retention</span>
{% if host.retention_warning.prune_exceeded %}
Scheduled prune would delete {{ host.retention_warning.delete_count }} snapshot(s), above max {{ host.retention_warning.max_delete }}.
{% endif %}
{% if host.retention_warning.incomplete_count %}
{{ host.retention_warning.incomplete_count }} incomplete snapshot(s) need review.
{% endif %}
{% if host.retention_warning.error %}
{{ host.retention_warning.error }}
{% endif %}
</div>
{% endif %}
</article> </article>
{% empty %} {% empty %}
<p class="muted">No hosts configured yet.</p> <p class="muted">No hosts configured yet.</p>

View File

@@ -68,6 +68,30 @@
</section> </section>
</div> </div>
{% if retention_warning.has_warning %}
<section class="panel highlight warning">
<h2>Retention Warnings</h2>
<div class="stack">
{% if retention_warning.prune_exceeded %}
<div>
Scheduled pruning would delete {{ retention_warning.delete_count }} snapshot(s), above max delete
{{ retention_warning.max_delete }}. Scheduled pruning will refuse this plan until the limit or retention
selection is adjusted.
</div>
{% endif %}
{% if retention_warning.incomplete_count %}
<div>
{{ retention_warning.incomplete_count }} incomplete snapshot(s) exist. Retention does not delete incomplete
snapshots automatically; inspect them before cleanup.
</div>
{% endif %}
{% if retention_warning.error %}
<div>{{ retention_warning.error }}</div>
{% endif %}
</div>
</section>
{% endif %}
{% if effective_config %} {% if effective_config %}
<section class="panel"> <section class="panel">
<h2>Effective Config</h2> <h2>Effective Config</h2>

View File

@@ -18,8 +18,31 @@
<div class="metric"><div class="label">Kind</div><div class="value">{{ plan.kind }}</div></div> <div class="metric"><div class="label">Kind</div><div class="value">{{ plan.kind }}</div></div>
<div class="metric"><div class="label">Keep</div><div class="value">{{ plan.keep|length }}</div></div> <div class="metric"><div class="label">Keep</div><div class="value">{{ plan.keep|length }}</div></div>
<div class="metric"><div class="label">Would Delete</div><div class="value">{{ plan.delete|length }}</div></div> <div class="metric"><div class="label">Would Delete</div><div class="value">{{ plan.delete|length }}</div></div>
<div class="metric"><div class="label">Scheduled Limit</div><div class="value">{{ scheduled_prune_limit|default:"none" }}</div></div>
<div class="metric"><div class="label">Incomplete</div><div class="value">{{ plan.incomplete|length }}</div></div>
</section> </section>
{% if scheduled_prune_exceeded %}
<section class="panel highlight warning">
<h2>Scheduled Prune Limit</h2>
<p>
This plan would delete {{ plan.delete|length }} snapshot(s), which exceeds the scheduled prune limit of
{{ scheduled_prune_limit }}. Scheduled pruning will refuse to apply this plan until the limit or retention
selection is adjusted.
</p>
</section>
{% endif %}
{% if plan.incomplete %}
<section class="panel highlight warning">
<h2>Incomplete Snapshots</h2>
<p>
{{ plan.incomplete|length }} incomplete snapshot(s) exist for this host. Retention does not delete incomplete
snapshots automatically because they can indicate an interrupted backup that should be inspected first.
</p>
</section>
{% endif %}
<section class="panel"> <section class="panel">
<h2>Policy</h2> <h2>Policy</h2>
<div class="stack"> <div class="stack">
@@ -28,6 +51,17 @@
<div><strong>Monthly:</strong> {{ plan.retention.monthly }}</div> <div><strong>Monthly:</strong> {{ plan.retention.monthly }}</div>
<div><strong>Yearly:</strong> {{ plan.retention.yearly }}</div> <div><strong>Yearly:</strong> {{ plan.retention.yearly }}</div>
<div><strong>Protect bases:</strong> {{ protect_bases|yesno:"yes,no" }}</div> <div><strong>Protect bases:</strong> {{ protect_bases|yesno:"yes,no" }}</div>
<div class="muted">
{% if protect_bases %}
Base snapshots referenced by kept snapshots are also kept and marked with a base-of reason.
{% else %}
Base snapshots are only kept when they match the regular retention policy.
{% endif %}
</div>
{% if schedule %}
<div><strong>Schedule pruning:</strong> {{ schedule.prune|yesno:"enabled,disabled" }}</div>
<div><strong>Schedule max delete:</strong> {{ schedule.prune_max_delete }}</div>
{% endif %}
</div> </div>
</section> </section>
@@ -40,6 +74,7 @@
<th>Dirname</th> <th>Dirname</th>
<th>Started</th> <th>Started</th>
<th>Status</th> <th>Status</th>
<th>Reason</th>
<th>Path</th> <th>Path</th>
</tr> </tr>
</thead> </thead>
@@ -50,10 +85,11 @@
<td>{{ snapshot.dirname }}</td> <td>{{ snapshot.dirname }}</td>
<td>{{ snapshot.dt }}</td> <td>{{ snapshot.dt }}</td>
<td>{{ snapshot.status|default:"" }}</td> <td>{{ snapshot.status|default:"" }}</td>
<td>{{ snapshot.reason }}</td>
<td class="muted">{{ snapshot.path }}</td> <td class="muted">{{ snapshot.path }}</td>
</tr> </tr>
{% empty %} {% empty %}
<tr><td colspan="5" class="muted">Retention would not delete snapshots for this selection.</td></tr> <tr><td colspan="6" class="muted">Retention would not delete snapshots for this selection.</td></tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
@@ -71,7 +107,7 @@
{{ apply_form.max_delete.errors }} {{ apply_form.max_delete.errors }}
<label for="{{ apply_form.max_delete.id_for_label }}">Max delete</label> <label for="{{ apply_form.max_delete.id_for_label }}">Max delete</label>
{{ apply_form.max_delete }} {{ apply_form.max_delete }}
<div class="helptext">Must be at least the number of snapshots shown in Would Delete.</div> <div class="helptext">Must be at least {{ plan.delete|length }} for the snapshots shown in Would Delete.</div>
</div> </div>
<div class="field"> <div class="field">
@@ -87,6 +123,13 @@
<div class="helptext">{{ apply_form.confirm_host.help_text }}</div> <div class="helptext">{{ apply_form.confirm_host.help_text }}</div>
</div> </div>
<div class="field">
{{ apply_form.confirm_delete_count.errors }}
<label for="{{ apply_form.confirm_delete_count.id_for_label }}">Confirm delete count</label>
{{ apply_form.confirm_delete_count }}
<div class="helptext">{{ apply_form.confirm_delete_count.help_text }}</div>
</div>
<div class="actions"> <div class="actions">
<button type="submit">Apply retention</button> <button type="submit">Apply retention</button>
</div> </div>
@@ -99,20 +142,54 @@
<table> <table>
<thead> <thead>
<tr> <tr>
<th>Kind</th>
<th>Dirname</th> <th>Dirname</th>
<th>Started</th>
<th>Status</th>
<th>Reasons</th> <th>Reasons</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for dirname, reasons in plan.reasons.items %} {% for snapshot in plan.keep_items %}
<tr> <tr>
<td>{{ dirname }}</td> <td>{{ snapshot.kind }}</td>
<td>{{ reasons|join:", " }}</td> <td>{{ snapshot.dirname }}</td>
<td>{{ snapshot.dt }}</td>
<td>{{ snapshot.status|default:"" }}</td>
<td>{{ snapshot.reason }}</td>
</tr> </tr>
{% empty %} {% empty %}
<tr><td colspan="2" class="muted">No snapshots matched this retention selection.</td></tr> <tr><td colspan="5" class="muted">No snapshots matched this retention selection.</td></tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
</section> </section>
{% if plan.incomplete %}
<section class="panel">
<h2>Incomplete Snapshots</h2>
<table>
<thead>
<tr>
<th>Dirname</th>
<th>Started</th>
<th>Status</th>
<th>Reason</th>
<th>Path</th>
</tr>
</thead>
<tbody>
{% for snapshot in plan.incomplete %}
<tr>
<td>{{ snapshot.dirname }}</td>
<td>{{ snapshot.dt }}</td>
<td>{{ snapshot.status|default:"" }}</td>
<td>{{ snapshot.reason }}</td>
<td class="muted">{{ snapshot.path }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endif %}
{% endblock %} {% endblock %}

View File

@@ -172,13 +172,49 @@
<div class="stack"> <div class="stack">
<div><strong>Status:</strong> {% if prune_result.ok %}ok{% else %}warning{% endif %}</div> <div><strong>Status:</strong> {% if prune_result.ok %}ok{% else %}warning{% endif %}</div>
{% if prune_result.source %}<div><strong>Source:</strong> {{ prune_result.source }}</div>{% endif %} {% if prune_result.source %}<div><strong>Source:</strong> {{ prune_result.source }}</div>{% endif %}
{% if prune_result.kind %}<div><strong>Kind:</strong> {{ prune_result.kind }}</div>{% endif %}
{% if prune_result.planned_delete_count is not None %}<div><strong>Planned deletions:</strong> {{ prune_result.planned_delete_count }}</div>{% endif %}
{% if prune_result.deleted %}<div><strong>Deleted:</strong> {{ prune_result.deleted|length }}</div>{% endif %} {% if prune_result.deleted %}<div><strong>Deleted:</strong> {{ prune_result.deleted|length }}</div>{% endif %}
{% if prune_result.max_delete is not None %}<div><strong>Max delete:</strong> {{ prune_result.max_delete }}</div>{% endif %}
{% if prune_result.protect_bases is not None %}<div><strong>Protect bases:</strong> {{ prune_result.protect_bases|yesno:"yes,no" }}</div>{% endif %}
{% if prune_result.incomplete_ignored_count %}<div><strong>Incomplete ignored:</strong> {{ prune_result.incomplete_ignored_count }}</div>{% endif %}
{% if prune_result.actions %}
<div><strong>Actions:</strong></div>
<ul>
{% for action in prune_result.actions %}
<li>{{ action }}</li>
{% endfor %}
</ul>
{% endif %}
{% if prune_result.error %}<div><strong>Error:</strong> {{ prune_result.error }}</div>{% endif %} {% if prune_result.error %}<div><strong>Error:</strong> {{ prune_result.error }}</div>{% endif %}
{% if prune_result.type %}<div><strong>Type:</strong> {{ prune_result.type }}</div>{% endif %} {% if prune_result.type %}<div><strong>Type:</strong> {{ prune_result.type }}</div>{% endif %}
</div> </div>
</section> </section>
{% endif %} {% endif %}
{% if retention_warning.has_warning %}
<section class="panel highlight warning">
<h2>Retention Warnings</h2>
<div class="stack">
{% if retention_warning.prune_exceeded %}
<div>
Scheduled pruning for this host would delete {{ retention_warning.delete_count }} snapshot(s), above max
delete {{ retention_warning.max_delete }}.
</div>
{% endif %}
{% if retention_warning.incomplete_count %}
<div>
{{ retention_warning.incomplete_count }} incomplete snapshot(s) exist for this host and are excluded from
retention cleanup.
</div>
{% endif %}
{% if retention_warning.error %}
<div>{{ retention_warning.error }}</div>
{% endif %}
</div>
</section>
{% endif %}
<section class="panel"> <section class="panel">
<h2>Raw Result</h2> <h2>Raw Result</h2>
<pre>{{ result_json }}</pre> <pre>{{ result_json }}</pre>

View File

@@ -8,7 +8,7 @@ from zoneinfo import ZoneInfo
from django.test import SimpleTestCase, TestCase from django.test import SimpleTestCase, TestCase
from pobsync_backend.management.commands.run_pobsync_scheduler import Command from pobsync_backend.management.commands.run_pobsync_scheduler import Command
from pobsync_backend.models import HostConfig, ScheduleConfig from pobsync_backend.models import BackupRun, HostConfig, ScheduleConfig
from pobsync_backend.scheduler import due_key, is_due, next_due_after from pobsync_backend.scheduler import due_key, is_due, next_due_after
@@ -64,3 +64,30 @@ class SchedulerCommandTests(TestCase):
self.assertEqual(call.call_count, 1) self.assertEqual(call.call_count, 1)
schedule = ScheduleConfig.objects.get(host=host) schedule = ScheduleConfig.objects.get(host=host)
self.assertEqual(schedule.last_status, "success") self.assertEqual(schedule.last_status, "success")
def test_run_due_records_warning_status_from_scheduled_backup_run(self) -> None:
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
ScheduleConfig.objects.create(host=host, cron_expr="* * * * *", prune=True, prune_max_delete=1)
def create_warning_run(*args, **kwargs) -> None:
BackupRun.objects.create(
host=host,
run_type=BackupRun.RunType.SCHEDULED,
status=BackupRun.Status.WARNING,
result={
"ok": True,
"prune": {
"ok": False,
"type": "ConfigError",
"error": "Refusing to delete 2 snapshots (exceeds --max-delete=1)",
},
},
)
command = Command()
with patch("pobsync_backend.management.commands.run_pobsync_scheduler.call_command", side_effect=create_warning_run):
count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=False)
self.assertEqual(count, 1)
schedule = ScheduleConfig.objects.get(host=host)
self.assertEqual(schedule.last_status, "warning")

View File

@@ -32,7 +32,10 @@ class SqlRetentionTests(TestCase):
self.assertEqual(plan["source"], "sql") self.assertEqual(plan["source"], "sql")
self.assertEqual(plan["keep"], [new.dirname]) self.assertEqual(plan["keep"], [new.dirname])
self.assertEqual([item["dirname"] for item in plan["keep_items"]], [new.dirname])
self.assertEqual([item["dirname"] for item in plan["delete"]], [old.dirname]) self.assertEqual([item["dirname"] for item in plan["delete"]], [old.dirname])
self.assertEqual(plan["delete"][0]["reason"], "outside retention policy")
self.assertEqual(plan["incomplete"], [])
def test_plan_can_protect_base_snapshot_from_sql_relation(self) -> None: def test_plan_can_protect_base_snapshot_from_sql_relation(self) -> None:
host = HostConfig.objects.create( host = HostConfig.objects.create(
@@ -85,6 +88,9 @@ class SqlRetentionTests(TestCase):
self.assertTrue(SnapshotRecord.objects.filter(pk=new.pk).exists()) self.assertTrue(SnapshotRecord.objects.filter(pk=new.pk).exists())
self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists()) self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
self.assertEqual(result["deleted"], [{"dirname": old.dirname, "kind": "scheduled", "path": str(old_dir)}]) self.assertEqual(result["deleted"], [{"dirname": old.dirname, "kind": "scheduled", "path": str(old_dir)}])
self.assertEqual(result["planned_delete_count"], 1)
self.assertEqual(result["max_delete"], 1)
self.assertEqual(result["incomplete_ignored_count"], 0)
def test_apply_deletes_snapshot_with_readonly_data_directory(self) -> None: def test_apply_deletes_snapshot_with_readonly_data_directory(self) -> None:
with TemporaryDirectory() as tmp: with TemporaryDirectory() as tmp:

View File

@@ -99,6 +99,35 @@ class ViewTests(TestCase):
self.assertContains(response, "manual") self.assertContains(response, "manual")
self.assertContains(response, "1000") self.assertContains(response, "1000")
def test_dashboard_surfaces_retention_warnings(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", enabled=True, prune=True, prune_max_delete=1)
self._snapshot(host, "20260517-021500Z__OLDSNP1")
self._snapshot(host, "20260518-021500Z__OLDSNP2")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
SnapshotRecord.objects.create(
host=host,
kind=SnapshotRecord.Kind.INCOMPLETE,
dirname="20260519-031500Z__BROKEN01",
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
status="failed",
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
)
response = self.client.get(reverse("dashboard"))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Scheduled prune would delete 2 snapshot(s), above max 1.")
self.assertContains(response, "1 incomplete snapshot(s) need review.")
def test_dashboard_links_latest_snapshot_for_each_host(self) -> None: def test_dashboard_links_latest_snapshot_for_each_host(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
host = HostConfig.objects.create(host="web-01", address="web-01.example.test") host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
@@ -1168,9 +1197,15 @@ class ViewTests(TestCase):
"hint": "Check network connectivity.", "hint": "Check network connectivity.",
}, },
"prune": { "prune": {
"ok": False, "ok": True,
"type": "ConfigError", "source": "sql",
"error": "Deletion blocked by --max-delete=0", "kind": "scheduled",
"planned_delete_count": 1,
"max_delete": 1,
"protect_bases": True,
"incomplete_ignored_count": 1,
"deleted": [{"dirname": "20260518-021500Z__OLD"}],
"actions": ["deleted scheduled 20260518-021500Z__OLD"],
}, },
}, },
) )
@@ -1182,7 +1217,42 @@ class ViewTests(TestCase):
self.assertContains(response, "transport") self.assertContains(response, "transport")
self.assertContains(response, "Check network connectivity.") self.assertContains(response, "Check network connectivity.")
self.assertContains(response, "Retention") self.assertContains(response, "Retention")
self.assertContains(response, "Deletion blocked by --max-delete=0") self.assertContains(response, "Planned deletions")
self.assertContains(response, "Max delete")
self.assertContains(response, "Protect bases")
self.assertContains(response, "Incomplete ignored")
self.assertContains(response, "deleted scheduled 20260518-021500Z__OLD")
def test_run_detail_surfaces_host_retention_warnings(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", enabled=True, prune=True, prune_max_delete=1)
self._snapshot(host, "20260517-021500Z__OLDSNP1")
self._snapshot(host, "20260518-021500Z__OLDSNP2")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
SnapshotRecord.objects.create(
host=host,
kind=SnapshotRecord.Kind.INCOMPLETE,
dirname="20260519-031500Z__BROKEN01",
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
status="failed",
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
)
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, result={"ok": True})
response = self.client.get(reverse("run_detail", args=[run.id]))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Retention Warnings")
self.assertContains(response, "Scheduled pruning for this host would delete 2 snapshot(s)")
self.assertContains(response, "1 incomplete snapshot(s) exist for this host")
def test_run_detail_infers_rsync_log_from_snapshot_path(self) -> None: def test_run_detail_infers_rsync_log_from_snapshot_path(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
@@ -1351,6 +1421,32 @@ class ViewTests(TestCase):
self.assertContains(response, new_snapshot.dirname) self.assertContains(response, new_snapshot.dirname)
self.assertContains(response, "newest") self.assertContains(response, "newest")
self.assertContains(response, "Would Delete") self.assertContains(response, "Would Delete")
self.assertContains(response, "outside retention policy")
self.assertContains(response, "Confirm delete count")
self.assertContains(response, "Type 1 to confirm the current number of planned deletions.")
def test_retention_plan_warns_when_scheduled_prune_limit_is_exceeded(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", enabled=True, prune=True, prune_max_delete=1)
self._snapshot(host, "20260517-021500Z__OLDSNP1")
self._snapshot(host, "20260518-021500Z__OLDSNP2")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
response = self.client.get(reverse("host_retention_plan", args=[host.host]))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Scheduled Prune Limit")
self.assertContains(response, "would delete 2 snapshot(s)")
self.assertContains(response, "scheduled prune limit of")
self.assertContains(response, "Schedule max delete:</strong> 1")
def test_retention_plan_can_enable_base_protection(self) -> None: def test_retention_plan_can_enable_base_protection(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
@@ -1371,8 +1467,58 @@ class ViewTests(TestCase):
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertContains(response, "Protect bases:</strong> yes") self.assertContains(response, "Protect bases:</strong> yes")
self.assertContains(response, "Base snapshots referenced by kept snapshots")
self.assertContains(response, f"base-of:{child.dirname}") self.assertContains(response, f"base-of:{child.dirname}")
def test_retention_plan_surfaces_incomplete_snapshots_without_deleting_them(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
self._snapshot(host, "20260518-021500Z__OLDSNAP")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
SnapshotRecord.objects.create(
host=host,
kind=SnapshotRecord.Kind.INCOMPLETE,
dirname="20260519-031500Z__BROKEN01",
path=f"/backups/{host.host}/.incomplete/20260519-031500Z__BROKEN01",
status="failed",
started_at=datetime(2026, 5, 19, 3, 15, tzinfo=timezone.utc),
)
response = self.client.get(reverse("host_retention_plan", args=[host.host]))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Incomplete Snapshots")
self.assertContains(response, "20260519-031500Z__BROKEN01")
self.assertContains(response, "excluded from retention cleanup")
def test_host_detail_surfaces_retention_warnings(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
ScheduleConfig.objects.create(host=host, cron_expr="15 2 * * *", enabled=True, prune=True, prune_max_delete=1)
self._snapshot(host, "20260517-021500Z__OLDSNP1")
self._snapshot(host, "20260518-021500Z__OLDSNP2")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
response = self.client.get(reverse("host_detail", args=[host.host]))
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Retention Warnings")
self.assertContains(response, "Scheduled pruning would delete 2 snapshot(s), above max delete")
def test_retention_plan_rejects_invalid_kind(self) -> None: def test_retention_plan_rejects_invalid_kind(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
host = HostConfig.objects.create(host="web-01", address="web-01.example.test") host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
@@ -1412,6 +1558,7 @@ class ViewTests(TestCase):
"kind": "scheduled", "kind": "scheduled",
"max_delete": "1", "max_delete": "1",
"confirm_host": host.host, "confirm_host": host.host,
"confirm_delete_count": "1",
}, },
follow=True, follow=True,
) )
@@ -1435,6 +1582,7 @@ class ViewTests(TestCase):
"kind": "scheduled", "kind": "scheduled",
"max_delete": "1", "max_delete": "1",
"confirm_host": "wrong", "confirm_host": "wrong",
"confirm_delete_count": "1",
}, },
follow=True, follow=True,
) )
@@ -1443,6 +1591,34 @@ class ViewTests(TestCase):
self.assertContains(response, "Retention apply confirmation is invalid.") self.assertContains(response, "Retention apply confirmation is invalid.")
self.assertEqual(SnapshotRecord.objects.count(), 1) self.assertEqual(SnapshotRecord.objects.count(), 1)
def test_retention_apply_rejects_mismatched_delete_count_confirmation(self) -> None:
self.client.force_login(self.staff_user)
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
retention_daily=0,
retention_weekly=0,
retention_monthly=0,
retention_yearly=0,
)
self._snapshot(host, "20260518-021500Z__OLDSNAP")
self._snapshot(host, "20260519-021500Z__NEWSNAP")
response = self.client.post(
reverse("apply_host_retention", args=[host.host]),
{
"kind": "scheduled",
"max_delete": "1",
"confirm_host": host.host,
"confirm_delete_count": "0",
},
follow=True,
)
self.assertRedirects(response, reverse("host_retention_plan", args=[host.host]))
self.assertContains(response, "Retention apply confirmation is invalid.")
self.assertEqual(SnapshotRecord.objects.count(), 2)
def test_retention_apply_requires_post(self) -> None: def test_retention_apply_requires_post(self) -> None:
self.client.force_login(self.staff_user) self.client.force_login(self.staff_user)
host = HostConfig.objects.create(host="web-01", address="web-01.example.test") host = HostConfig.objects.create(host="web-01", address="web-01.example.test")

View File

@@ -55,6 +55,7 @@ def dashboard(request):
.first() .first()
) )
host_config.next_run_at = _next_run_for_host(host_config) host_config.next_run_at = _next_run_for_host(host_config)
host_config.retention_warning = _retention_warning_for_host(host_config, _schedule_for_host(host_config))
stats_summary = collect_dashboard_stats(hosts=hosts, global_config=global_config) stats_summary = collect_dashboard_stats(hosts=hosts, global_config=global_config)
context = { context = {
"hosts": hosts, "hosts": hosts,
@@ -274,6 +275,7 @@ def host_detail(request, host: str):
context = { context = {
"host": host_config, "host": host_config,
"schedule": schedule, "schedule": schedule,
"retention_warning": _retention_warning_for_host(host_config, schedule),
"next_run_at": _next_run_for_schedule(schedule, host_config), "next_run_at": _next_run_for_schedule(schedule, host_config),
"scheduler_timezone": timezone.get_current_timezone_name(), "scheduler_timezone": timezone.get_current_timezone_name(),
"discovery": inspect_snapshot_discovery(host=host_config), "discovery": inspect_snapshot_discovery(host=host_config),
@@ -427,6 +429,7 @@ def run_detail(request, run_id: int):
"failure_summary": failure.get("message") or failure.get("summary") or "", "failure_summary": failure.get("message") or failure.get("summary") or "",
"prune_result": prune_result, "prune_result": prune_result,
"has_prune_result": bool(prune_result), "has_prune_result": bool(prune_result),
"retention_warning": _retention_warning_for_host(run.host, _schedule_for_host(run.host)),
"rsync_log_path": str(rsync_log_path) if rsync_log_path is not None else "", "rsync_log_path": str(rsync_log_path) if rsync_log_path is not None else "",
"rsync_log_exists": bool(rsync_log_path and rsync_log_path.exists()), "rsync_log_exists": bool(rsync_log_path and rsync_log_path.exists()),
"rsync_log_tail": rsync_log_tail, "rsync_log_tail": rsync_log_tail,
@@ -526,17 +529,25 @@ def host_retention_plan(request, host: str):
except PobsyncError as exc: except PobsyncError as exc:
messages.error(request, str(exc)) messages.error(request, str(exc))
return redirect("host_detail", host=host_config.host) return redirect("host_detail", host=host_config.host)
schedule = _schedule_for_host(host_config)
scheduled_prune_limit = schedule.prune_max_delete if schedule and schedule.prune else None
delete_count = len(plan["delete"])
context = { context = {
"host": host_config, "host": host_config,
"kind": kind, "kind": kind,
"protect_bases": protect_bases, "protect_bases": protect_bases,
"plan": plan, "plan": plan,
"schedule": schedule,
"scheduled_prune_limit": scheduled_prune_limit,
"scheduled_prune_exceeded": scheduled_prune_limit is not None and delete_count > scheduled_prune_limit,
"apply_form": RetentionApplyForm( "apply_form": RetentionApplyForm(
host_name=host_config.host, host_name=host_config.host,
expected_delete_count=delete_count,
initial={ initial={
"kind": kind, "kind": kind,
"protect_bases": protect_bases, "protect_bases": protect_bases,
"max_delete": len(plan["delete"]), "max_delete": delete_count,
"confirm_delete_count": delete_count,
}, },
), ),
} }
@@ -547,7 +558,26 @@ def host_retention_plan(request, host: str):
@require_POST @require_POST
def apply_host_retention(request, host: str): def apply_host_retention(request, host: str):
host_config = get_object_or_404(HostConfig, host=host) host_config = get_object_or_404(HostConfig, host=host)
form = RetentionApplyForm(request.POST, host_name=host_config.host) raw_kind = request.POST.get("kind", "scheduled")
raw_protect_bases = request.POST.get("protect_bases") in {"1", "true", "on", "yes"}
expected_delete_count = None
if raw_kind in {"scheduled", "manual", "all"}:
try:
plan = run_sql_retention_plan(
host=host_config.host,
kind=raw_kind,
protect_bases=raw_protect_bases,
)
except PobsyncError as exc:
messages.error(request, str(exc))
return redirect("host_retention_plan", host=host_config.host)
expected_delete_count = len(plan.get("delete") or [])
form = RetentionApplyForm(
request.POST,
host_name=host_config.host,
expected_delete_count=expected_delete_count,
)
if not form.is_valid(): if not form.is_valid():
messages.error(request, "Retention apply confirmation is invalid.") messages.error(request, "Retention apply confirmation is invalid.")
return redirect("host_retention_plan", host=host_config.host) return redirect("host_retention_plan", host=host_config.host)
@@ -652,6 +682,43 @@ def _next_run_for_schedule(schedule: ScheduleConfig | None, host_config: HostCon
return None return None
def _retention_warning_for_host(host_config: HostConfig, schedule: ScheduleConfig | None) -> dict[str, object]:
incomplete_count = host_config.snapshots.filter(kind=SnapshotRecord.Kind.INCOMPLETE).count()
warning: dict[str, object] = {
"has_warning": incomplete_count > 0,
"incomplete_count": incomplete_count,
}
if schedule is None or not schedule.prune or not host_config.enabled:
return warning
try:
plan = run_sql_retention_plan(
host=host_config.host,
kind="scheduled",
protect_bases=bool(schedule.prune_protect_bases),
)
except PobsyncError as exc:
warning.update(
{
"has_warning": True,
"error": str(exc),
}
)
return warning
delete_count = len(plan.get("delete") or [])
warning.update(
{
"delete_count": delete_count,
"max_delete": schedule.prune_max_delete,
"protect_bases": bool(schedule.prune_protect_bases),
"prune_exceeded": delete_count > schedule.prune_max_delete,
}
)
if warning["prune_exceeded"]:
warning["has_warning"] = True
return warning
def _default_schedule_initial() -> dict[str, object]: def _default_schedule_initial() -> dict[str, object]:
return { return {
"cron_expr": "15 2 * * *", "cron_expr": "15 2 * * *",