(release) Add purged snapshot audit overview

Record snapshot purge history whenever retention or incomplete cleanup removes snapshot directories and SQL records. Store the purge reason, original kind, path, action source, and triggering operator so manual, scheduled, CLI, and incomplete cleanup actions remain auditable after the original snapshot record is deleted. Add a staff-only Purged Snapshots page with host/action filters and register the audit model in Django admin. Refs #16 Refs #8
2026-05-21 03:46:38 +02:00
parent 5b5a5bc637
commit ea9e3e41e3
13 changed files with 340 additions and 8 deletions
--- a/src/pobsync_backend/admin.py
+++ b/src/pobsync_backend/admin.py
@@ -6,7 +6,7 @@ from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.http import urlencode

-from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord, SshCredential
+from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential


@admin.register(SshCredential)
@@ -173,6 +173,16 @@ class SnapshotRecordAdmin(admin.ModelAdmin):
        return format_html('<a href="{}">{}</a>', url, count)


+@admin.register(PurgedSnapshot)
+class PurgedSnapshotAdmin(admin.ModelAdmin):
+    list_display = ("host_name", "kind", "dirname", "action", "reason", "triggered_by", "purged_at")
+    list_filter = ("action", "kind", "purged_at")
+    search_fields = ("host_name", "dirname", "path", "reason", "triggered_by")
+    list_select_related = ("host",)
+    readonly_fields = ("purged_at",)
+    date_hierarchy = "purged_at"
+
+
@admin.register(ScheduleConfig)
 class ScheduleConfigAdmin(admin.ModelAdmin):
    list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at")
--- a/src/pobsync_backend/backup_runner.py
+++ b/src/pobsync_backend/backup_runner.py
@@ -109,6 +109,7 @@ def execute_backup_run(
                protect_bases=bool(prune_protect_bases),
                yes=True,
                max_delete=int(prune_max_delete),
+                action=run.run_type,
                acquire_lock=False,
            )
        except Exception as exc:
--- a/src/pobsync_backend/management/commands/run_pobsync_retention.py
+++ b/src/pobsync_backend/management/commands/run_pobsync_retention.py
@@ -36,6 +36,7 @@ class Command(BaseCommand):
                    protect_bases=bool(options["protect_bases"]),
                    yes=True,
                    max_delete=int(options["max_delete"]),
+                    action="cli",
                )
            else:
                result = run_sql_retention_plan(
--- a/src/pobsync_backend/migrations/0013_purgedsnapshot.py
+++ b/src/pobsync_backend/migrations/0013_purgedsnapshot.py
@@ -0,0 +1,50 @@
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("pobsync_backend", "0012_review_state"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PurgedSnapshot",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("host_name", models.CharField(max_length=255)),
+                ("kind", models.CharField(max_length=16)),
+                ("dirname", models.CharField(max_length=255)),
+                ("path", models.CharField(max_length=1024)),
+                ("reason", models.CharField(blank=True, max_length=512)),
+                (
+                    "action",
+                    models.CharField(
+                        choices=[
+                            ("manual", "Manual"),
+                            ("scheduled", "Scheduled"),
+                            ("cli", "CLI"),
+                            ("incomplete_cleanup", "Incomplete cleanup"),
+                        ],
+                        max_length=32,
+                    ),
+                ),
+                ("triggered_by", models.CharField(blank=True, max_length=150)),
+                ("metadata", models.JSONField(blank=True, default=dict)),
+                ("purged_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "host",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="purged_snapshots",
+                        to="pobsync_backend.hostconfig",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["-purged_at", "host_name", "dirname"],
+            },
+        ),
+    ]
--- a/src/pobsync_backend/models.py
+++ b/src/pobsync_backend/models.py
@@ -173,6 +173,31 @@ class SnapshotRecord(models.Model):
        return f"{self.host}/{self.kind}/{self.dirname}"


+class PurgedSnapshot(models.Model):
+    class Action(models.TextChoices):
+        MANUAL = "manual", "Manual"
+        SCHEDULED = "scheduled", "Scheduled"
+        CLI = "cli", "CLI"
+        INCOMPLETE_CLEANUP = "incomplete_cleanup", "Incomplete cleanup"
+
+    host = models.ForeignKey(HostConfig, on_delete=models.SET_NULL, null=True, blank=True, related_name="purged_snapshots")
+    host_name = models.CharField(max_length=255)
+    kind = models.CharField(max_length=16)
+    dirname = models.CharField(max_length=255)
+    path = models.CharField(max_length=1024)
+    reason = models.CharField(max_length=512, blank=True)
+    action = models.CharField(max_length=32, choices=Action.choices)
+    triggered_by = models.CharField(max_length=150, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+    purged_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["-purged_at", "host_name", "dirname"]
+
+    def __str__(self) -> str:
+        return f"{self.host_name}/{self.kind}/{self.dirname}"
+
+
 class ScheduleConfig(TimestampedModel):
    host = models.OneToOneField(HostConfig, on_delete=models.CASCADE, related_name="schedule")
    cron_expr = models.CharField(max_length=128)
--- a/src/pobsync_backend/retention.py
+++ b/src/pobsync_backend/retention.py
@@ -12,7 +12,7 @@ from pobsync.paths import PobsyncPaths
 from pobsync.retention import Snapshot, apply_base_protection, build_retention_plan
 from pobsync.util import sanitize_host

-from .models import HostConfig, SnapshotRecord
+from .models import HostConfig, PurgedSnapshot, SnapshotRecord


 def run_sql_retention_plan(*, host: str, kind: str, protect_bases: bool) -> dict[str, Any]:
@@ -65,6 +65,8 @@ def run_sql_retention_apply(
    protect_bases: bool,
    yes: bool,
    max_delete: int,
+    action: str = PurgedSnapshot.Action.MANUAL,
+    triggered_by: str = "",
    acquire_lock: bool = True,
 ) -> dict[str, Any]:
    host = sanitize_host(host)
@@ -101,6 +103,7 @@ def run_sql_retention_apply(
                raise ConfigError(f"Refusing to delete unsupported snapshot kind: {snap_kind!r}")

            path = _snapshot_delete_path(path=Path(snap_path), dirname=dirname)
+            reason = str(item.get("reason") or "outside retention policy")
            if not path.exists():
                actions.append(f"skip missing {snap_kind}/{dirname}")
                continue
@@ -108,9 +111,19 @@ def run_sql_retention_apply(
                raise ConfigError(f"Refusing to delete non-directory path: {path}")

            _remove_snapshot_tree(path)
+            _record_purged_snapshot(
+                host_config=_enabled_host_config(host),
+                kind=snap_kind,
+                dirname=dirname,
+                path=path,
+                reason=reason,
+                action=action,
+                triggered_by=triggered_by,
+                metadata={"source": "retention", "protect_bases": bool(protect_bases), "retention_kind": kind},
+            )
            SnapshotRecord.objects.filter(host__host=host, kind=snap_kind, dirname=dirname).delete()
            actions.append(f"deleted {snap_kind} {dirname}")
-            deleted.append({"dirname": dirname, "kind": snap_kind, "path": str(path)})
+            deleted.append({"dirname": dirname, "kind": snap_kind, "path": str(path), "reason": reason})

        return {
            "ok": True,
@@ -137,6 +150,7 @@ def run_incomplete_cleanup(
    host: str,
    yes: bool,
    max_delete: int,
+    triggered_by: str = "",
    acquire_lock: bool = True,
 ) -> dict[str, Any]:
    host = sanitize_host(host)
@@ -177,6 +191,16 @@ def run_incomplete_cleanup(
                _remove_snapshot_tree(path)
                actions.append(f"deleted incomplete {dirname}")

+            _record_purged_snapshot(
+                host_config=host_config,
+                kind=SnapshotRecord.Kind.INCOMPLETE,
+                dirname=dirname,
+                path=path,
+                reason="manual incomplete cleanup",
+                action=PurgedSnapshot.Action.INCOMPLETE_CLEANUP,
+                triggered_by=triggered_by,
+                metadata={"source": "incomplete_cleanup"},
+            )
            SnapshotRecord.objects.filter(
                host__host=host,
                kind=SnapshotRecord.Kind.INCOMPLETE,
@@ -282,6 +306,30 @@ def _snapshot_delete_path(*, path: Path, dirname: str) -> Path:
    return path


+def _record_purged_snapshot(
+    *,
+    host_config: HostConfig,
+    kind: str,
+    dirname: str,
+    path: Path,
+    reason: str,
+    action: str,
+    triggered_by: str,
+    metadata: dict[str, Any],
+) -> None:
+    PurgedSnapshot.objects.create(
+        host=host_config,
+        host_name=host_config.host,
+        kind=kind,
+        dirname=dirname,
+        path=str(path),
+        reason=reason,
+        action=action,
+        triggered_by=triggered_by,
+        metadata=metadata,
+    )
+
+
 def _validate_incomplete_delete_path(*, host: str, path: Path, dirname: str) -> None:
    path_parts = path.parts
    if path.name != dirname or ".incomplete" not in path_parts or host not in path_parts:
--- a/src/pobsync_backend/templates/pobsync_backend/base.html
+++ b/src/pobsync_backend/templates/pobsync_backend/base.html
@@ -375,6 +375,7 @@
      <a href="{% url 'ssh_credentials' %}">SSH Keys</a>
      <a href="{% url 'self_check' %}">Self Check</a>
      <a href="{% url 'logs' %}">Logs</a>
+      <a href="{% url 'purged_snapshots' %}">Purged</a>
      <a href="{% url 'changelog' %}">Changelog</a>
      <a href="/api/status/">Status API</a>
      <span class="spacer"></span>
--- a/src/pobsync_backend/templates/pobsync_backend/purged_snapshots.html
+++ b/src/pobsync_backend/templates/pobsync_backend/purged_snapshots.html
@@ -0,0 +1,74 @@
+{% extends "pobsync_backend/base.html" %}
+
+{% block title %}Purged Snapshots | pobsync{% endblock %}
+
+{% block content %}
+  <h1>Purged Snapshots</h1>
+
+  <section class="actions" aria-label="Purged snapshot actions">
+    <a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
+  </section>
+
+  <section class="panel">
+    <h2>Filters</h2>
+    <form method="get" class="form-grid">
+      <div class="field">
+        <label for="host">Host</label>
+        <select id="host" name="host">
+          <option value="">All hosts</option>
+          {% for host in hosts %}
+            <option value="{{ host.host }}" {% if selected_host == host.host %}selected{% endif %}>{{ host.host }}</option>
+          {% endfor %}
+        </select>
+      </div>
+      <div class="field">
+        <label for="action">Action</label>
+        <select id="action" name="action">
+          <option value="">All actions</option>
+          {% for value, label in actions %}
+            <option value="{{ value }}" {% if selected_action == value %}selected{% endif %}>{{ label }}</option>
+          {% endfor %}
+        </select>
+      </div>
+      <div class="actions">
+        <button type="submit">Apply filters</button>
+        <a class="button-link secondary" href="{% url 'purged_snapshots' %}">Clear</a>
+      </div>
+    </form>
+  </section>
+
+  <section class="panel">
+    <h2>Purged Snapshot History</h2>
+    <p class="muted">Showing up to 200 of {{ total_count }} purged snapshot record(s).</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Purged</th>
+          <th>Host</th>
+          <th>Kind</th>
+          <th>Dirname</th>
+          <th>Action</th>
+          <th>Reason</th>
+          <th>Triggered by</th>
+          <th>Path</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for snapshot in purged_snapshots %}
+          <tr>
+            <td>{{ snapshot.purged_at }}</td>
+            <td>{% if snapshot.host %}<a href="{% url 'host_detail' snapshot.host.host %}">{{ snapshot.host_name }}</a>{% else %}{{ snapshot.host_name }}{% endif %}</td>
+            <td>{{ snapshot.kind }}</td>
+            <td>{{ snapshot.dirname }}</td>
+            <td><span class="status skipped">{{ snapshot.get_action_display }}</span></td>
+            <td>{{ snapshot.reason|default:"" }}</td>
+            <td>{{ snapshot.triggered_by|default:"" }}</td>
+            <td class="muted">{{ snapshot.path }}</td>
+          </tr>
+        {% empty %}
+          <tr><td colspan="8" class="muted">No purged snapshots recorded yet.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </section>
+{% endblock %}
--- a/src/pobsync_backend/tests/test_run_backup_records_snapshot.py
+++ b/src/pobsync_backend/tests/test_run_backup_records_snapshot.py
@@ -96,6 +96,7 @@ class RunBackupRecordsSnapshotTests(TestCase):
            protect_bases=True,
            yes=True,
            max_delete=3,
+            action=BackupRun.RunType.SCHEDULED,
            acquire_lock=False,
        )
        run = BackupRun.objects.get()
--- a/src/pobsync_backend/tests/test_sql_retention.py
+++ b/src/pobsync_backend/tests/test_sql_retention.py
@@ -11,7 +11,7 @@ from django.core.management import call_command
 from django.test import TestCase

 from pobsync.errors import ConfigError
-from pobsync_backend.models import HostConfig, SnapshotRecord
+from pobsync_backend.models import HostConfig, PurgedSnapshot, SnapshotRecord
 from pobsync_backend.retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan


@@ -87,10 +87,26 @@ class SqlRetentionTests(TestCase):
            self.assertTrue(new_dir.exists())
            self.assertTrue(SnapshotRecord.objects.filter(pk=new.pk).exists())
            self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
-            self.assertEqual(result["deleted"], [{"dirname": old.dirname, "kind": "scheduled", "path": str(old_dir)}])
+            self.assertEqual(
+                result["deleted"],
+                [
+                    {
+                        "dirname": old.dirname,
+                        "kind": "scheduled",
+                        "path": str(old_dir),
+                        "reason": "outside retention policy",
+                    }
+                ],
+            )
            self.assertEqual(result["planned_delete_count"], 1)
            self.assertEqual(result["max_delete"], 1)
            self.assertEqual(result["incomplete_ignored_count"], 0)
+            purged = PurgedSnapshot.objects.get(dirname=old.dirname)
+            self.assertEqual(purged.host_name, host.host)
+            self.assertEqual(purged.kind, "scheduled")
+            self.assertEqual(purged.path, str(old_dir))
+            self.assertEqual(purged.reason, "outside retention policy")
+            self.assertEqual(purged.action, PurgedSnapshot.Action.MANUAL)

    def test_apply_deletes_snapshot_with_readonly_data_directory(self) -> None:
        with TemporaryDirectory() as tmp:
@@ -126,7 +142,17 @@ class SqlRetentionTests(TestCase):

            self.assertFalse(old_dir.exists())
            self.assertFalse(SnapshotRecord.objects.filter(pk=old.pk).exists())
-            self.assertEqual(result["deleted"], [{"dirname": old.dirname, "kind": "scheduled", "path": str(old_dir)}])
+            self.assertEqual(
+                result["deleted"],
+                [
+                    {
+                        "dirname": old.dirname,
+                        "kind": "scheduled",
+                        "path": str(old_dir),
+                        "reason": "outside retention policy",
+                    }
+                ],
+            )

    def test_apply_respects_max_delete(self) -> None:
        host = HostConfig.objects.create(
@@ -183,6 +209,9 @@ class SqlRetentionTests(TestCase):
                [{"dirname": incomplete_dir.name, "kind": SnapshotRecord.Kind.INCOMPLETE, "path": str(incomplete_dir)}],
            )
            self.assertEqual(result["planned_delete_count"], 1)
+            purged = PurgedSnapshot.objects.get(dirname=incomplete_dir.name)
+            self.assertEqual(purged.action, PurgedSnapshot.Action.INCOMPLETE_CLEANUP)
+            self.assertEqual(purged.reason, "manual incomplete cleanup")

    def test_incomplete_cleanup_respects_max_delete(self) -> None:
        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
--- a/src/pobsync_backend/tests/test_views.py
+++ b/src/pobsync_backend/tests/test_views.py
@@ -12,7 +12,15 @@ from django.test import TestCase, override_settings
 from django.urls import reverse

 from pobsync.util import write_yaml_atomic
-from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord, SshCredential
+from pobsync_backend.models import (
+    BackupRun,
+    GlobalConfig,
+    HostConfig,
+    PurgedSnapshot,
+    ScheduleConfig,
+    SnapshotRecord,
+    SshCredential,
+)


 class ViewTests(TestCase):
@@ -328,6 +336,61 @@ class ViewTests(TestCase):
        self.assertIn("--since", command)
        self.assertIn("6 hours ago", command)

+    def test_purged_snapshots_view_renders_history(self) -> None:
+        self.client.force_login(self.staff_user)
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        PurgedSnapshot.objects.create(
+            host=host,
+            host_name=host.host,
+            kind=SnapshotRecord.Kind.SCHEDULED,
+            dirname="20260518-021500Z__OLDSNAP",
+            path=f"/backups/{host.host}/scheduled/20260518-021500Z__OLDSNAP",
+            reason="outside retention policy",
+            action=PurgedSnapshot.Action.SCHEDULED,
+            triggered_by="pobsync-scheduler",
+        )
+
+        response = self.client.get(reverse("purged_snapshots"))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Purged Snapshots")
+        self.assertContains(response, "20260518-021500Z__OLDSNAP")
+        self.assertContains(response, "outside retention policy")
+        self.assertContains(response, "Scheduled")
+        self.assertContains(response, "pobsync-scheduler")
+
+    def test_purged_snapshots_view_filters_by_host_and_action(self) -> None:
+        self.client.force_login(self.staff_user)
+        web = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        db = HostConfig.objects.create(host="db-01", address="db-01.example.test")
+        PurgedSnapshot.objects.create(
+            host=web,
+            host_name=web.host,
+            kind=SnapshotRecord.Kind.SCHEDULED,
+            dirname="20260518-021500Z__WEBOLD",
+            path=f"/backups/{web.host}/scheduled/20260518-021500Z__WEBOLD",
+            reason="outside retention policy",
+            action=PurgedSnapshot.Action.MANUAL,
+        )
+        PurgedSnapshot.objects.create(
+            host=db,
+            host_name=db.host,
+            kind=SnapshotRecord.Kind.INCOMPLETE,
+            dirname="20260518-021500Z__DBBROKEN",
+            path=f"/backups/{db.host}/.incomplete/20260518-021500Z__DBBROKEN",
+            reason="manual incomplete cleanup",
+            action=PurgedSnapshot.Action.INCOMPLETE_CLEANUP,
+        )
+
+        response = self.client.get(
+            reverse("purged_snapshots"),
+            {"host": db.host, "action": PurgedSnapshot.Action.INCOMPLETE_CLEANUP},
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "20260518-021500Z__DBBROKEN")
+        self.assertNotContains(response, "20260518-021500Z__WEBOLD")
+
    def test_ssh_credentials_view_creates_key(self) -> None:
        self.client.force_login(self.staff_user)

@@ -1881,6 +1944,10 @@ class ViewTests(TestCase):
        self.assertContains(response, "Retention deleted 1 snapshot(s) for web-01.")
        self.assertFalse(SnapshotRecord.objects.filter(pk=old_snapshot.pk).exists())
        self.assertTrue(SnapshotRecord.objects.filter(pk=new_snapshot.pk).exists())
+        purged = PurgedSnapshot.objects.get(dirname=old_snapshot.dirname)
+        self.assertEqual(purged.action, PurgedSnapshot.Action.MANUAL)
+        self.assertEqual(purged.triggered_by, self.staff_user.username)
+        self.assertEqual(purged.reason, "outside retention policy")

    def test_retention_apply_rejects_bad_confirmation(self) -> None:
        self.client.force_login(self.staff_user)
--- a/src/pobsync_backend/views.py
+++ b/src/pobsync_backend/views.py
@@ -33,7 +33,7 @@ from .forms import (
    SshCredentialForm,
 )
 from .host_ops import ensure_host_directories
-from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord, SshCredential
+from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential
 from .preflight import collect_backup_gate, effective_host_config_preview, run_remote_preflight
 from .retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan
 from .self_check import collect_self_checks, summarize_self_checks
@@ -145,6 +145,27 @@ def logs(request):
    return render(request, "pobsync_backend/logs.html", context)


+@staff_member_required
+def purged_snapshots(request):
+    host = request.GET.get("host", "").strip()
+    action = request.GET.get("action", "").strip()
+    purged = PurgedSnapshot.objects.select_related("host").order_by("-purged_at", "host_name", "dirname")
+    if host:
+        purged = purged.filter(host_name=host)
+    if action:
+        purged = purged.filter(action=action)
+
+    context = {
+        "purged_snapshots": purged[:200],
+        "hosts": HostConfig.objects.order_by("host"),
+        "actions": PurgedSnapshot.Action.choices,
+        "selected_host": host,
+        "selected_action": action,
+        "total_count": purged.count(),
+    }
+    return render(request, "pobsync_backend/purged_snapshots.html", context)
+
+
@staff_member_required
 def ssh_credentials(request):
    context = {
@@ -693,6 +714,8 @@ def apply_host_retention(request, host: str):
            protect_bases=protect_bases,
            yes=True,
            max_delete=form.cleaned_data["max_delete"],
+            action=PurgedSnapshot.Action.MANUAL,
+            triggered_by=request.user.get_username(),
        )
    except PobsyncError as exc:
        messages.error(request, str(exc))
@@ -733,6 +756,7 @@ def cleanup_host_incomplete_snapshots(request, host: str):
            host=host_config.host,
            yes=True,
            max_delete=form.cleaned_data["max_delete"],
+            triggered_by=request.user.get_username(),
        )
    except PobsyncError as exc:
        messages.error(request, str(exc))