diff --git a/src/pobsync_backend/admin.py b/src/pobsync_backend/admin.py index 694e76d..870f127 100644 --- a/src/pobsync_backend/admin.py +++ b/src/pobsync_backend/admin.py @@ -6,7 +6,17 @@ from django.urls import reverse from django.utils.html import format_html from django.utils.http import urlencode -from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential +from .models import ( + BackupRun, + GlobalConfig, + HostConfig, + NotificationDelivery, + NotificationTarget, + PurgedSnapshot, + ScheduleConfig, + SnapshotRecord, + SshCredential, +) @admin.register(SshCredential) @@ -136,6 +146,38 @@ class BackupRunAdmin(admin.ModelAdmin): return format_html('{}', url, obj.snapshot.dirname) +@admin.register(NotificationTarget) +class NotificationTargetAdmin(admin.ModelAdmin): + list_display = ("name", "channel", "enabled", "last_status", "last_sent_at", "updated_at") + list_filter = ("enabled", "channel", "last_status") + search_fields = ("name", "email_to", "webhook_url", "notes") + readonly_fields = ("created_at", "updated_at", "last_status", "last_error", "last_sent_at") + fieldsets = ( + (None, {"fields": ("name", "enabled", "channel", "statuses")}), + ("Email", {"fields": ("email_to",)}), + ("Webhook", {"fields": ("webhook_url", "webhook_headers")}), + ("State", {"fields": ("last_status", "last_error", "last_sent_at"), "classes": ("collapse",)}), + ("Notes", {"fields": ("notes",), "classes": ("collapse",)}), + ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), + ) + + +@admin.register(NotificationDelivery) +class NotificationDeliveryAdmin(admin.ModelAdmin): + list_display = ("target", "run", "status", "created_at") + list_filter = ("status", "target__channel", "created_at") + search_fields = ("target__name", "run__host__host", "error") + readonly_fields = ("target", "run", "status", "error", "payload", "created_at") + list_select_related = ("target", "run", "run__host") + date_hierarchy = "created_at" + + def has_add_permission(self, request) -> bool: + return False + + def has_change_permission(self, request, obj=None) -> bool: + return False + + @admin.register(SnapshotRecord) class SnapshotRecordAdmin(admin.ModelAdmin): list_display = ("host", "kind", "dirname", "status", "base_link", "backup_run_count_link", "started_at", "discovered_at") diff --git a/src/pobsync_backend/backup_runner.py b/src/pobsync_backend/backup_runner.py index 8b7f9ec..0c2560c 100644 --- a/src/pobsync_backend/backup_runner.py +++ b/src/pobsync_backend/backup_runner.py @@ -17,6 +17,7 @@ from pobsync.commands.run_scheduled import ( ) from pobsync_backend.config_source import DjangoConfigSource from pobsync_backend.models import BackupRun, HostConfig +from pobsync_backend.notifications import notify_backup_run_completed from pobsync_backend.retention import run_sql_retention_apply from pobsync_backend.snapshot_discovery import infer_snapshot_kind, upsert_snapshot_record @@ -85,6 +86,7 @@ def execute_backup_run( "type": type(exc).__name__, } run.save(update_fields=["status", "ended_at", "result"]) + notify_backup_run_completed(run) raise run.refresh_from_db() @@ -151,6 +153,7 @@ def execute_backup_run( "result", ], ) + notify_backup_run_completed(run) return run @@ -277,6 +280,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s run.rsync_exit_code = exit_code or 255 run.result = result run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"]) + notify_backup_run_completed(run) return True if _running_rsync_process_missing(run=run, grace_seconds=grace_seconds): result.update( @@ -301,6 +305,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s run.rsync_exit_code = exit_code or 255 run.result = result run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"]) + notify_backup_run_completed(run) return True if stale_worker: result.update( @@ -318,6 +323,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s run.ended_at = timezone.now() run.result = result run.save(update_fields=["status", "ended_at", "result"]) + notify_backup_run_completed(run) return True return False @@ -353,6 +359,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s run.rsync_exit_code = exit_code run.result = result run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"]) + notify_backup_run_completed(run) return True diff --git a/src/pobsync_backend/forms.py b/src/pobsync_backend/forms.py index 9647e06..b1c8037 100644 --- a/src/pobsync_backend/forms.py +++ b/src/pobsync_backend/forms.py @@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory from django import forms from django.conf import settings -from .models import GlobalConfig, HostConfig, ScheduleConfig, SshCredential +from .models import BackupRun, GlobalConfig, HostConfig, NotificationTarget, ScheduleConfig, SshCredential from .scheduler import parse_cron_expr @@ -153,6 +153,62 @@ class ManualBackupForm(forms.Form): ) +class NotificationTargetForm(forms.ModelForm): + TERMINAL_STATUS_CHOICES = ( + (BackupRun.Status.SUCCESS, BackupRun.Status.SUCCESS.label), + (BackupRun.Status.WARNING, BackupRun.Status.WARNING.label), + (BackupRun.Status.FAILED, BackupRun.Status.FAILED.label), + (BackupRun.Status.CANCELLED, BackupRun.Status.CANCELLED.label), + ) + + statuses = forms.MultipleChoiceField( + choices=TERMINAL_STATUS_CHOICES, + widget=forms.CheckboxSelectMultiple, + initial=[choice[0] for choice in TERMINAL_STATUS_CHOICES], + help_text="Send notifications for these terminal run statuses.", + ) + email_to = forms.CharField( + widget=forms.Textarea, + required=False, + help_text="One recipient per line, or comma-separated.", + ) + webhook_headers = forms.JSONField( + required=False, + widget=forms.Textarea(attrs={"rows": 4}), + help_text='Optional JSON object with extra headers, for example {"Authorization": "Bearer ..."}.', + ) + + class Meta: + model = NotificationTarget + fields = ( + "name", + "enabled", + "channel", + "statuses", + "email_to", + "webhook_url", + "webhook_headers", + "notes", + ) + widgets = { + "notes": forms.Textarea, + } + + def clean(self): + cleaned_data = super().clean() + channel = cleaned_data.get("channel") + if channel == NotificationTarget.Channel.EMAIL and not cleaned_data.get("email_to", "").strip(): + self.add_error("email_to", "Email targets need at least one recipient.") + if channel == NotificationTarget.Channel.WEBHOOK and not cleaned_data.get("webhook_url"): + self.add_error("webhook_url", "Webhook targets need a URL.") + return cleaned_data + + def clean_email_to(self) -> str: + value = self.cleaned_data.get("email_to", "") + recipients = [line.strip() for line in value.replace(",", "\n").splitlines() if line.strip()] + return "\n".join(recipients) + + class SshCredentialForm(forms.ModelForm): private_key_file = forms.FileField( required=False, diff --git a/src/pobsync_backend/migrations/0015_notificationtarget_notificationdelivery.py b/src/pobsync_backend/migrations/0015_notificationtarget_notificationdelivery.py new file mode 100644 index 0000000..9c94aa1 --- /dev/null +++ b/src/pobsync_backend/migrations/0015_notificationtarget_notificationdelivery.py @@ -0,0 +1,54 @@ +# Generated by Django 5.2.14 on 2026-05-28 19:11 + +import django.db.models.deletion +import pobsync_backend.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pobsync_backend', '0014_host_bwlimit_override'), + ] + + operations = [ + migrations.CreateModel( + name='NotificationTarget', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=128, unique=True)), + ('enabled', models.BooleanField(default=True)), + ('channel', models.CharField(choices=[('email', 'Email'), ('webhook', 'Webhook')], max_length=16)), + ('statuses', models.JSONField(blank=True, default=pobsync_backend.models.default_notification_statuses)), + ('email_to', models.TextField(blank=True)), + ('webhook_url', models.URLField(blank=True, max_length=1024)), + ('webhook_headers', models.JSONField(blank=True, default=dict)), + ('notes', models.TextField(blank=True)), + ('last_status', models.CharField(blank=True, max_length=16)), + ('last_error', models.TextField(blank=True)), + ('last_sent_at', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='NotificationDelivery', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('sent', 'Sent'), ('failed', 'Failed'), ('skipped', 'Skipped')], max_length=16)), + ('error', models.TextField(blank=True)), + ('payload', models.JSONField(blank=True, default=dict)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notification_deliveries', to='pobsync_backend.backuprun')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='pobsync_backend.notificationtarget')), + ], + options={ + 'verbose_name_plural': 'notification deliveries', + 'ordering': ['-created_at', 'target__name'], + 'constraints': [models.UniqueConstraint(fields=('target', 'run'), name='unique_notification_delivery_per_target_run')], + }, + ), + ] diff --git a/src/pobsync_backend/models.py b/src/pobsync_backend/models.py index 4f89927..cd107a2 100644 --- a/src/pobsync_backend/models.py +++ b/src/pobsync_backend/models.py @@ -135,6 +135,63 @@ class BackupRun(models.Model): return f"{self.host} {self.run_type} {self.status}" +def default_notification_statuses() -> list[str]: + return [ + BackupRun.Status.SUCCESS, + BackupRun.Status.WARNING, + BackupRun.Status.FAILED, + BackupRun.Status.CANCELLED, + ] + + +class NotificationTarget(TimestampedModel): + class Channel(models.TextChoices): + EMAIL = "email", "Email" + WEBHOOK = "webhook", "Webhook" + + name = models.CharField(max_length=128, unique=True) + enabled = models.BooleanField(default=True) + channel = models.CharField(max_length=16, choices=Channel.choices) + statuses = models.JSONField(default=default_notification_statuses, blank=True) + email_to = models.TextField(blank=True) + webhook_url = models.URLField(max_length=1024, blank=True) + webhook_headers = models.JSONField(default=dict, blank=True) + notes = models.TextField(blank=True) + last_status = models.CharField(max_length=16, blank=True) + last_error = models.TextField(blank=True) + last_sent_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["name"] + + def __str__(self) -> str: + return self.name + + +class NotificationDelivery(models.Model): + class Status(models.TextChoices): + SENT = "sent", "Sent" + FAILED = "failed", "Failed" + SKIPPED = "skipped", "Skipped" + + target = models.ForeignKey(NotificationTarget, on_delete=models.CASCADE, related_name="deliveries") + run = models.ForeignKey(BackupRun, on_delete=models.CASCADE, related_name="notification_deliveries") + status = models.CharField(max_length=16, choices=Status.choices) + error = models.TextField(blank=True) + payload = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["target", "run"], name="unique_notification_delivery_per_target_run"), + ] + ordering = ["-created_at", "target__name"] + verbose_name_plural = "notification deliveries" + + def __str__(self) -> str: + return f"{self.target} run {self.run_id} {self.status}" + + class SnapshotRecord(models.Model): class Kind(models.TextChoices): SCHEDULED = "scheduled", "Scheduled" diff --git a/src/pobsync_backend/notifications.py b/src/pobsync_backend/notifications.py new file mode 100644 index 0000000..70c0197 --- /dev/null +++ b/src/pobsync_backend/notifications.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + +from django.conf import settings +from django.core.mail import send_mail +from django.utils import timezone + +from .models import BackupRun, NotificationDelivery, NotificationTarget + + +TERMINAL_RUN_STATUSES = { + BackupRun.Status.SUCCESS, + BackupRun.Status.WARNING, + BackupRun.Status.FAILED, + BackupRun.Status.CANCELLED, +} + + +@dataclass(frozen=True) +class DeliveryResult: + target: NotificationTarget + delivery: NotificationDelivery + sent: bool + + +def notify_backup_run_completed(run: BackupRun) -> list[DeliveryResult]: + if run.status not in TERMINAL_RUN_STATUSES: + return [] + + targets = [target for target in NotificationTarget.objects.filter(enabled=True) if _target_wants_status(target, run.status)] + return [_notify_target(target=target, run=run) for target in targets] + + +def _target_wants_status(target: NotificationTarget, status: str) -> bool: + statuses = target.statuses + if not isinstance(statuses, list): + return False + return status in {str(item) for item in statuses} + + +def _notify_target(*, target: NotificationTarget, run: BackupRun) -> DeliveryResult: + payload = _run_payload(run) + delivery, created = NotificationDelivery.objects.get_or_create( + target=target, + run=run, + defaults={ + "status": NotificationDelivery.Status.SKIPPED, + "payload": payload, + }, + ) + if not created: + return DeliveryResult(target=target, delivery=delivery, sent=False) + + try: + if target.channel == NotificationTarget.Channel.EMAIL: + _send_email(target=target, run=run, payload=payload) + elif target.channel == NotificationTarget.Channel.WEBHOOK: + _send_webhook(target=target, payload=payload) + else: + raise ValueError(f"Unsupported notification channel: {target.channel}") + except Exception as exc: + delivery.status = NotificationDelivery.Status.FAILED + delivery.error = str(exc) + delivery.save(update_fields=["status", "error"]) + target.last_status = NotificationDelivery.Status.FAILED + target.last_error = str(exc) + target.save(update_fields=["last_status", "last_error", "updated_at"]) + return DeliveryResult(target=target, delivery=delivery, sent=False) + + delivery.status = NotificationDelivery.Status.SENT + delivery.save(update_fields=["status"]) + target.last_status = NotificationDelivery.Status.SENT + target.last_error = "" + target.last_sent_at = timezone.now() + target.save(update_fields=["last_status", "last_error", "last_sent_at", "updated_at"]) + return DeliveryResult(target=target, delivery=delivery, sent=True) + + +def _send_email(*, target: NotificationTarget, run: BackupRun, payload: dict[str, Any]) -> None: + recipients = [line.strip() for line in target.email_to.replace(",", "\n").splitlines() if line.strip()] + if not recipients: + raise ValueError("Email notification target has no recipients.") + + subject = f"pobsync {run.status}: {run.host.host} run {run.id}" + message = _email_message(payload) + from_email = getattr(settings, "DEFAULT_FROM_EMAIL", "") or "pobsync@localhost" + sent = send_mail(subject, message, from_email, recipients, fail_silently=False) + if sent == 0: + raise ValueError("Django email backend reported zero sent messages.") + + +def _send_webhook(*, target: NotificationTarget, payload: dict[str, Any]) -> None: + if not target.webhook_url: + raise ValueError("Webhook notification target has no URL.") + + headers = {"Content-Type": "application/json", **_string_headers(target.webhook_headers)} + request = urllib.request.Request( + target.webhook_url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + if response.status >= 400: + raise ValueError(f"Webhook returned HTTP {response.status}.") + except urllib.error.HTTPError as exc: + raise ValueError(f"Webhook returned HTTP {exc.code}.") from exc + + +def _string_headers(headers: object) -> dict[str, str]: + if not isinstance(headers, dict): + return {} + return {str(key): str(value) for key, value in headers.items() if str(key).strip()} + + +def _run_payload(run: BackupRun) -> dict[str, Any]: + result = run.result if isinstance(run.result, dict) else {} + failure = result.get("failure") if isinstance(result.get("failure"), dict) else {} + prune = result.get("prune") if isinstance(result.get("prune"), dict) else {} + return { + "event": "backup_run.completed", + "run": { + "id": run.id, + "host": run.host.host, + "type": run.run_type, + "status": run.status, + "started_at": run.started_at.isoformat() if run.started_at else None, + "ended_at": run.ended_at.isoformat() if run.ended_at else None, + "snapshot": run.snapshot_path, + "rsync_exit_code": run.rsync_exit_code, + }, + "failure": { + "category": failure.get("category"), + "message": failure.get("message") or result.get("error"), + "hint": failure.get("hint"), + }, + "prune": { + "ok": prune.get("ok") if prune else None, + "error": prune.get("error") if prune else "", + }, + } + + +def _email_message(payload: dict[str, Any]) -> str: + run = payload["run"] + lines = [ + f"Host: {run['host']}", + f"Run: {run['id']}", + f"Type: {run['type']}", + f"Status: {run['status']}", + f"Started: {run['started_at'] or '-'}", + f"Ended: {run['ended_at'] or '-'}", + f"Snapshot: {run['snapshot'] or '-'}", + f"Rsync exit code: {run['rsync_exit_code'] if run['rsync_exit_code'] is not None else '-'}", + ] + failure = payload.get("failure") if isinstance(payload.get("failure"), dict) else {} + if failure.get("message"): + lines.extend(["", f"Failure: {failure['message']}"]) + prune = payload.get("prune") if isinstance(payload.get("prune"), dict) else {} + if prune.get("error"): + lines.extend(["", f"Retention: {prune['error']}"]) + return "\n".join(lines) diff --git a/src/pobsync_backend/templates/pobsync_backend/base.html b/src/pobsync_backend/templates/pobsync_backend/base.html index dc0f733..54919a9 100644 --- a/src/pobsync_backend/templates/pobsync_backend/base.html +++ b/src/pobsync_backend/templates/pobsync_backend/base.html @@ -920,6 +920,7 @@ Dashboard Hosts SSH Keys + Notifications Logs Purged diff --git a/src/pobsync_backend/templates/pobsync_backend/notification_target_form.html b/src/pobsync_backend/templates/pobsync_backend/notification_target_form.html new file mode 100644 index 0000000..75375b2 --- /dev/null +++ b/src/pobsync_backend/templates/pobsync_backend/notification_target_form.html @@ -0,0 +1,38 @@ +{% extends "pobsync_backend/base.html" %} + +{% block title %}{{ title }} | pobsync{% endblock %} + +{% block content %} + + +
+

{% if target %}Edit Target{% else %}Create Target{% endif %}

+
+ {% csrf_token %} + {{ form.non_field_errors }} + + {% for field in form %} +
+ {{ field.errors }} + + {{ field }} + {% if field.help_text %}
{{ field.help_text }}
{% endif %} +
+ {% endfor %} + +
+ + Cancel +
+
+
+{% endblock %} diff --git a/src/pobsync_backend/templates/pobsync_backend/notification_targets.html b/src/pobsync_backend/templates/pobsync_backend/notification_targets.html new file mode 100644 index 0000000..40ecfca --- /dev/null +++ b/src/pobsync_backend/templates/pobsync_backend/notification_targets.html @@ -0,0 +1,91 @@ +{% extends "pobsync_backend/base.html" %} + +{% block title %}Notifications | pobsync{% endblock %} + +{% block content %} + + +
+

Targets

+ + + + + + + + + + + + + + {% for target in targets %} + + + + + + + + + + {% empty %} + + {% endfor %} + +
NameChannelStatusEventsDestinationLast deliveryActions
{{ target.name }}{{ target.get_channel_display }}{{ target.enabled|yesno:"enabled,disabled" }}{{ target.statuses|join:", " }} + {% if target.channel == "email" %} + {{ target.email_to|linebreaksbr }} + {% else %} + {{ target.webhook_url|truncatechars:70 }} + {% endif %} + + {% if target.last_status %} + {{ target.last_status }} + {% if target.last_error %}
{{ target.last_error|truncatechars:90 }}
{% endif %} + {% if target.last_sent_at %}
{{ target.last_sent_at }}
{% endif %} + {% else %} + none + {% endif %} +
Edit
No notification targets configured yet.
+
+ +
+

Recent Deliveries

+ + + + + + + + + + + + {% for delivery in deliveries %} + + + + + + + + {% empty %} + + {% endfor %} + +
TargetRunStatusCreatedError
{{ delivery.target.name }}Run {{ delivery.run.id }} {{ delivery.run.host.host }}{{ delivery.status }}{{ delivery.created_at }}{{ delivery.error|default:"" }}
No notification deliveries recorded yet.
+
+{% endblock %} diff --git a/src/pobsync_backend/tests/test_backup_worker.py b/src/pobsync_backend/tests/test_backup_worker.py index 084fc53..d09600e 100644 --- a/src/pobsync_backend/tests/test_backup_worker.py +++ b/src/pobsync_backend/tests/test_backup_worker.py @@ -11,7 +11,7 @@ from django.utils import timezone from pobsync.util import write_yaml_atomic from pobsync_backend.backup_runner import queue_backup_run, reconcile_running_runs from pobsync_backend.management.commands.run_pobsync_worker import Command -from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord +from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, NotificationDelivery, NotificationTarget, SnapshotRecord class BackupWorkerTests(TestCase): @@ -116,6 +116,42 @@ class BackupWorkerTests(TestCase): self.assertEqual(run.rsync_exit_code, 24) self.assertEqual(run.result["warning"]["category"], "vanished") + def test_worker_sends_notification_after_completed_run(self) -> None: + with TemporaryDirectory() as tmp: + backup_root = Path(tmp) / "backups" + GlobalConfig.objects.create(name="default", backup_root=str(backup_root)) + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + NotificationTarget.objects.create( + name="ops", + channel=NotificationTarget.Channel.EMAIL, + email_to="ops@example.test", + ) + snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH" + meta_dir = snapshot_dir / "meta" + meta_dir.mkdir(parents=True) + write_yaml_atomic(meta_dir / "meta.yaml", {"status": "success", "started_at": "2026-05-19T02:15:00Z"}) + run = queue_backup_run(host=host) + + with ( + patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled, + patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail, + ): + run_scheduled.return_value = { + "ok": True, + "dry_run": False, + "host": host.host, + "snapshot": str(snapshot_dir), + "base": None, + "rsync": {"exit_code": 0}, + } + + Command()._run_once(prefix=Path(tmp) / "home") + + run.refresh_from_db() + self.assertEqual(run.status, BackupRun.Status.SUCCESS) + self.assertEqual(NotificationDelivery.objects.get(run=run).status, NotificationDelivery.Status.SENT) + send_mail.assert_called_once() + def test_worker_refreshes_heartbeat_while_run_is_active(self) -> None: with TemporaryDirectory() as tmp: GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups")) diff --git a/src/pobsync_backend/tests/test_notifications.py b/src/pobsync_backend/tests/test_notifications.py new file mode 100644 index 0000000..bc69e59 --- /dev/null +++ b/src/pobsync_backend/tests/test_notifications.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from datetime import timedelta +from unittest.mock import Mock, patch + +from django.test import TestCase +from django.utils import timezone + +from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget +from pobsync_backend.notifications import notify_backup_run_completed + + +class NotificationTests(TestCase): + def test_email_notification_is_sent_for_matching_status(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + target = NotificationTarget.objects.create( + name="ops", + channel=NotificationTarget.Channel.EMAIL, + statuses=[BackupRun.Status.FAILED], + email_to="ops@example.test", + ) + run = BackupRun.objects.create( + host=host, + status=BackupRun.Status.FAILED, + run_type=BackupRun.RunType.MANUAL, + started_at=timezone.now() - timedelta(minutes=5), + ended_at=timezone.now(), + rsync_exit_code=12, + result={"failure": {"message": "rsync failed"}}, + ) + + with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail: + results = notify_backup_run_completed(run) + + self.assertEqual(len(results), 1) + self.assertTrue(results[0].sent) + send_mail.assert_called_once() + subject, message, _from_email, recipients = send_mail.call_args.args + self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}") + self.assertIn("Failure: rsync failed", message) + self.assertEqual(recipients, ["ops@example.test"]) + delivery = NotificationDelivery.objects.get(target=target, run=run) + self.assertEqual(delivery.status, NotificationDelivery.Status.SENT) + target.refresh_from_db() + self.assertEqual(target.last_status, NotificationDelivery.Status.SENT) + self.assertEqual(target.last_error, "") + self.assertIsNotNone(target.last_sent_at) + + def test_webhook_notification_posts_payload(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + target = NotificationTarget.objects.create( + name="discord", + channel=NotificationTarget.Channel.WEBHOOK, + webhook_url="https://hooks.example.test/pobsync", + webhook_headers={"X-Token": "secret"}, + ) + run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED) + response = Mock() + response.status = 204 + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + + with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen: + notify_backup_run_completed(run) + + request = urlopen.call_args.args[0] + self.assertEqual(request.full_url, "https://hooks.example.test/pobsync") + self.assertEqual(request.get_method(), "POST") + self.assertEqual(request.headers["X-token"], "secret") + self.assertIn(f'"id": {run.id}', request.data.decode("utf-8")) + self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT) + + def test_notification_filters_statuses(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + NotificationTarget.objects.create( + name="failures-only", + channel=NotificationTarget.Channel.EMAIL, + statuses=[BackupRun.Status.FAILED], + email_to="ops@example.test", + ) + run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS) + + with patch("pobsync_backend.notifications.send_mail") as send_mail: + results = notify_backup_run_completed(run) + + self.assertEqual(results, []) + send_mail.assert_not_called() + self.assertEqual(NotificationDelivery.objects.count(), 0) + + def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + target = NotificationTarget.objects.create( + name="ops", + channel=NotificationTarget.Channel.EMAIL, + email_to="ops@example.test", + ) + run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING) + + with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail: + notify_backup_run_completed(run) + notify_backup_run_completed(run) + + self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1) + send_mail.assert_called_once() + + def test_failed_delivery_is_recorded_without_raising(self) -> None: + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + target = NotificationTarget.objects.create( + name="broken", + channel=NotificationTarget.Channel.EMAIL, + email_to="ops@example.test", + ) + run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED) + + with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")): + results = notify_backup_run_completed(run) + + self.assertEqual(len(results), 1) + self.assertFalse(results[0].sent) + delivery = NotificationDelivery.objects.get(target=target, run=run) + self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED) + self.assertEqual(delivery.error, "smtp down") + target.refresh_from_db() + self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED) + self.assertEqual(target.last_error, "smtp down") diff --git a/src/pobsync_backend/tests/test_views.py b/src/pobsync_backend/tests/test_views.py index 324cf96..45b4cc5 100644 --- a/src/pobsync_backend/tests/test_views.py +++ b/src/pobsync_backend/tests/test_views.py @@ -18,6 +18,8 @@ from pobsync_backend.models import ( BackupRun, GlobalConfig, HostConfig, + NotificationDelivery, + NotificationTarget, PurgedSnapshot, ScheduleConfig, SnapshotRecord, @@ -52,6 +54,7 @@ class ViewTests(TestCase): self.assertContains(response, reverse("dashboard")) self.assertContains(response, reverse("hosts_list")) self.assertContains(response, reverse("ssh_credentials")) + self.assertContains(response, reverse("notification_targets")) self.assertContains(response, reverse("logs")) self.assertContains(response, reverse("purged_snapshots")) self.assertContains(response, reverse("self_check")) @@ -94,6 +97,70 @@ class ViewTests(TestCase): self.assertContains(response, "Django control panel") self.assertContains(response, "Native systemd installer") + def test_notification_targets_view_renders_targets_and_deliveries(self) -> None: + self.client.force_login(self.staff_user) + host = HostConfig.objects.create(host="web-01", address="web-01.example.test") + run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS) + target = NotificationTarget.objects.create( + name="ops", + channel=NotificationTarget.Channel.EMAIL, + email_to="ops@example.test", + last_status=NotificationDelivery.Status.SENT, + ) + NotificationDelivery.objects.create(target=target, run=run, status=NotificationDelivery.Status.SENT) + + response = self.client.get(reverse("notification_targets")) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Notifications") + self.assertContains(response, "ops") + self.assertContains(response, "ops@example.test") + self.assertContains(response, f"Run {run.id}") + + def test_notification_target_form_creates_email_target(self) -> None: + self.client.force_login(self.staff_user) + + response = self.client.post( + reverse("create_notification_target"), + { + "name": "ops", + "enabled": "on", + "channel": NotificationTarget.Channel.EMAIL, + "statuses": [BackupRun.Status.FAILED, BackupRun.Status.WARNING], + "email_to": "ops@example.test, backup@example.test", + "webhook_headers": "{}", + "notes": "Notify ops", + }, + follow=True, + ) + + self.assertRedirects(response, reverse("notification_targets")) + self.assertContains(response, "Notification target ops created.") + target = NotificationTarget.objects.get(name="ops") + self.assertEqual(target.channel, NotificationTarget.Channel.EMAIL) + self.assertEqual(target.statuses, [BackupRun.Status.FAILED, BackupRun.Status.WARNING]) + self.assertEqual(target.email_to, "ops@example.test\nbackup@example.test") + + def test_notification_target_form_requires_channel_destination(self) -> None: + self.client.force_login(self.staff_user) + + response = self.client.post( + reverse("create_notification_target"), + { + "name": "broken", + "enabled": "on", + "channel": NotificationTarget.Channel.WEBHOOK, + "statuses": [BackupRun.Status.FAILED], + "email_to": "", + "webhook_url": "", + "webhook_headers": "{}", + }, + ) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Webhook targets need a URL.") + self.assertFalse(NotificationTarget.objects.exists()) + def test_dashboard_renders_hosts_and_latest_runs(self) -> None: self.client.force_login(self.staff_user) host = HostConfig.objects.create(host="web-01", address="web-01.example.test") diff --git a/src/pobsync_backend/views.py b/src/pobsync_backend/views.py index 8bee6ba..7e4a0f0 100644 --- a/src/pobsync_backend/views.py +++ b/src/pobsync_backend/views.py @@ -32,13 +32,24 @@ from .forms import ( HostConfigForm, IncompleteCleanupForm, ManualBackupForm, + NotificationTargetForm, RetentionApplyForm, SshCredentialGenerateForm, ScheduleConfigForm, SshCredentialForm, ) from .host_ops import ensure_host_directories -from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential +from .models import ( + BackupRun, + GlobalConfig, + HostConfig, + NotificationDelivery, + NotificationTarget, + PurgedSnapshot, + ScheduleConfig, + SnapshotRecord, + SshCredential, +) from .preflight import collect_backup_gate, effective_host_config_preview, run_remote_preflight from .retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan from .self_check import collect_self_checks, summarize_self_checks @@ -320,6 +331,65 @@ def logs(request): return render(request, "pobsync_backend/logs.html", context) +@staff_member_required +def notification_targets(request): + targets = NotificationTarget.objects.order_by("name") + deliveries = NotificationDelivery.objects.select_related("target", "run", "run__host").order_by("-created_at")[:12] + return render( + request, + "pobsync_backend/notification_targets.html", + { + "targets": targets, + "deliveries": deliveries, + }, + ) + + +@staff_member_required +def create_notification_target(request): + if request.method == "POST": + form = NotificationTargetForm(request.POST) + if form.is_valid(): + target = form.save() + messages.success(request, f"Notification target {target.name} created.") + return redirect("notification_targets") + else: + form = NotificationTargetForm() + return render( + request, + "pobsync_backend/notification_target_form.html", + { + "form": form, + "target": None, + "title": "New notification target", + "submit_label": "Create target", + }, + ) + + +@staff_member_required +def edit_notification_target(request, target_id: int): + target = get_object_or_404(NotificationTarget, id=target_id) + if request.method == "POST": + form = NotificationTargetForm(request.POST, instance=target) + if form.is_valid(): + target = form.save() + messages.success(request, f"Notification target {target.name} updated.") + return redirect("notification_targets") + else: + form = NotificationTargetForm(instance=target) + return render( + request, + "pobsync_backend/notification_target_form.html", + { + "form": form, + "target": target, + "title": f"Edit notification target: {target.name}", + "submit_label": "Save target", + }, + ) + + @staff_member_required def runs_list(request): status = request.GET.get("status", "").strip() diff --git a/src/pobsync_server/urls.py b/src/pobsync_server/urls.py index 5e4b04b..7f2a561 100644 --- a/src/pobsync_server/urls.py +++ b/src/pobsync_server/urls.py @@ -13,6 +13,9 @@ urlpatterns = [ path("changelog/", views.changelog, name="changelog"), path("self-check/", views.self_check, name="self_check"), path("logs/", views.logs, name="logs"), + path("notifications/", views.notification_targets, name="notification_targets"), + path("notifications/new/", views.create_notification_target, name="create_notification_target"), + path("notifications//", views.edit_notification_target, name="edit_notification_target"), path("purged-snapshots/", views.purged_snapshots, name="purged_snapshots"), path("schedules/", views.schedules_list, name="schedules_list"), path("config/global/", views.edit_global_config, name="edit_global_config"),