(feature) Add run completion notifications

Add email and webhook notification targets with delivery tracking, and send notifications when backup runs reach a terminal status. Expose notification target management in the Django UI and keep delivery failures recorded without failing the backup worker.
2026-05-28 21:20:38 +02:00
parent 1f5c4e0756
commit 67ffd6101b
14 changed files with 819 additions and 4 deletions
--- a/src/pobsync_backend/tests/test_notifications.py
+++ b/src/pobsync_backend/tests/test_notifications.py
@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from django.utils import timezone
+
+from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget
+from pobsync_backend.notifications import notify_backup_run_completed
+
+
+class NotificationTests(TestCase):
+    def test_email_notification_is_sent_for_matching_status(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="ops",
+            channel=NotificationTarget.Channel.EMAIL,
+            statuses=[BackupRun.Status.FAILED],
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(
+            host=host,
+            status=BackupRun.Status.FAILED,
+            run_type=BackupRun.RunType.MANUAL,
+            started_at=timezone.now() - timedelta(minutes=5),
+            ended_at=timezone.now(),
+            rsync_exit_code=12,
+            result={"failure": {"message": "rsync failed"}},
+        )
+
+        with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(len(results), 1)
+        self.assertTrue(results[0].sent)
+        send_mail.assert_called_once()
+        subject, message, _from_email, recipients = send_mail.call_args.args
+        self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}")
+        self.assertIn("Failure: rsync failed", message)
+        self.assertEqual(recipients, ["ops@example.test"])
+        delivery = NotificationDelivery.objects.get(target=target, run=run)
+        self.assertEqual(delivery.status, NotificationDelivery.Status.SENT)
+        target.refresh_from_db()
+        self.assertEqual(target.last_status, NotificationDelivery.Status.SENT)
+        self.assertEqual(target.last_error, "")
+        self.assertIsNotNone(target.last_sent_at)
+
+    def test_webhook_notification_posts_payload(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="discord",
+            channel=NotificationTarget.Channel.WEBHOOK,
+            webhook_url="https://hooks.example.test/pobsync",
+            webhook_headers={"X-Token": "secret"},
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED)
+        response = Mock()
+        response.status = 204
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+
+        with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen:
+            notify_backup_run_completed(run)
+
+        request = urlopen.call_args.args[0]
+        self.assertEqual(request.full_url, "https://hooks.example.test/pobsync")
+        self.assertEqual(request.get_method(), "POST")
+        self.assertEqual(request.headers["X-token"], "secret")
+        self.assertIn(f'"id": {run.id}', request.data.decode("utf-8"))
+        self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT)
+
+    def test_notification_filters_statuses(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        NotificationTarget.objects.create(
+            name="failures-only",
+            channel=NotificationTarget.Channel.EMAIL,
+            statuses=[BackupRun.Status.FAILED],
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
+
+        with patch("pobsync_backend.notifications.send_mail") as send_mail:
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(results, [])
+        send_mail.assert_not_called()
+        self.assertEqual(NotificationDelivery.objects.count(), 0)
+
+    def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="ops",
+            channel=NotificationTarget.Channel.EMAIL,
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING)
+
+        with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
+            notify_backup_run_completed(run)
+            notify_backup_run_completed(run)
+
+        self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1)
+        send_mail.assert_called_once()
+
+    def test_failed_delivery_is_recorded_without_raising(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="broken",
+            channel=NotificationTarget.Channel.EMAIL,
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED)
+
+        with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")):
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(len(results), 1)
+        self.assertFalse(results[0].sent)
+        delivery = NotificationDelivery.objects.get(target=target, run=run)
+        self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED)
+        self.assertEqual(delivery.error, "smtp down")
+        target.refresh_from_db()
+        self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED)
+        self.assertEqual(target.last_error, "smtp down")