Add email and webhook notification targets with delivery tracking, and send notifications when backup runs reach a terminal status. Expose notification target management in the Django UI and keep delivery failures recorded without failing the backup worker.
126 lines
5.7 KiB
Python
126 lines
5.7 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from unittest.mock import Mock, patch
|
|
|
|
from django.test import TestCase
|
|
from django.utils import timezone
|
|
|
|
from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget
|
|
from pobsync_backend.notifications import notify_backup_run_completed
|
|
|
|
|
|
class NotificationTests(TestCase):
|
|
def test_email_notification_is_sent_for_matching_status(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
target = NotificationTarget.objects.create(
|
|
name="ops",
|
|
channel=NotificationTarget.Channel.EMAIL,
|
|
statuses=[BackupRun.Status.FAILED],
|
|
email_to="ops@example.test",
|
|
)
|
|
run = BackupRun.objects.create(
|
|
host=host,
|
|
status=BackupRun.Status.FAILED,
|
|
run_type=BackupRun.RunType.MANUAL,
|
|
started_at=timezone.now() - timedelta(minutes=5),
|
|
ended_at=timezone.now(),
|
|
rsync_exit_code=12,
|
|
result={"failure": {"message": "rsync failed"}},
|
|
)
|
|
|
|
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
|
results = notify_backup_run_completed(run)
|
|
|
|
self.assertEqual(len(results), 1)
|
|
self.assertTrue(results[0].sent)
|
|
send_mail.assert_called_once()
|
|
subject, message, _from_email, recipients = send_mail.call_args.args
|
|
self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}")
|
|
self.assertIn("Failure: rsync failed", message)
|
|
self.assertEqual(recipients, ["ops@example.test"])
|
|
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
|
self.assertEqual(delivery.status, NotificationDelivery.Status.SENT)
|
|
target.refresh_from_db()
|
|
self.assertEqual(target.last_status, NotificationDelivery.Status.SENT)
|
|
self.assertEqual(target.last_error, "")
|
|
self.assertIsNotNone(target.last_sent_at)
|
|
|
|
def test_webhook_notification_posts_payload(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
target = NotificationTarget.objects.create(
|
|
name="discord",
|
|
channel=NotificationTarget.Channel.WEBHOOK,
|
|
webhook_url="https://hooks.example.test/pobsync",
|
|
webhook_headers={"X-Token": "secret"},
|
|
)
|
|
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED)
|
|
response = Mock()
|
|
response.status = 204
|
|
response.__enter__ = Mock(return_value=response)
|
|
response.__exit__ = Mock(return_value=False)
|
|
|
|
with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen:
|
|
notify_backup_run_completed(run)
|
|
|
|
request = urlopen.call_args.args[0]
|
|
self.assertEqual(request.full_url, "https://hooks.example.test/pobsync")
|
|
self.assertEqual(request.get_method(), "POST")
|
|
self.assertEqual(request.headers["X-token"], "secret")
|
|
self.assertIn(f'"id": {run.id}', request.data.decode("utf-8"))
|
|
self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT)
|
|
|
|
def test_notification_filters_statuses(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
NotificationTarget.objects.create(
|
|
name="failures-only",
|
|
channel=NotificationTarget.Channel.EMAIL,
|
|
statuses=[BackupRun.Status.FAILED],
|
|
email_to="ops@example.test",
|
|
)
|
|
run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
|
|
|
|
with patch("pobsync_backend.notifications.send_mail") as send_mail:
|
|
results = notify_backup_run_completed(run)
|
|
|
|
self.assertEqual(results, [])
|
|
send_mail.assert_not_called()
|
|
self.assertEqual(NotificationDelivery.objects.count(), 0)
|
|
|
|
def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
target = NotificationTarget.objects.create(
|
|
name="ops",
|
|
channel=NotificationTarget.Channel.EMAIL,
|
|
email_to="ops@example.test",
|
|
)
|
|
run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING)
|
|
|
|
with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
|
|
notify_backup_run_completed(run)
|
|
notify_backup_run_completed(run)
|
|
|
|
self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1)
|
|
send_mail.assert_called_once()
|
|
|
|
def test_failed_delivery_is_recorded_without_raising(self) -> None:
|
|
host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
|
|
target = NotificationTarget.objects.create(
|
|
name="broken",
|
|
channel=NotificationTarget.Channel.EMAIL,
|
|
email_to="ops@example.test",
|
|
)
|
|
run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED)
|
|
|
|
with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")):
|
|
results = notify_backup_run_completed(run)
|
|
|
|
self.assertEqual(len(results), 1)
|
|
self.assertFalse(results[0].sent)
|
|
delivery = NotificationDelivery.objects.get(target=target, run=run)
|
|
self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED)
|
|
self.assertEqual(delivery.error, "smtp down")
|
|
target.refresh_from_db()
|
|
self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED)
|
|
self.assertEqual(target.last_error, "smtp down")
|