Merge pull request '(feature) Add run completion notifications' (#68) from issue-50-run-notifications into master

Reviewed-on: #68

src/pobsync_backend/admin.py

@@ -6,7 +6,17 @@ from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.http import urlencode
 
-from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential
+from .models import (
+    BackupRun,
+    GlobalConfig,
+    HostConfig,
+    NotificationDelivery,
+    NotificationTarget,
+    PurgedSnapshot,
+    ScheduleConfig,
+    SnapshotRecord,
+    SshCredential,
+)
 
 
 @admin.register(SshCredential)
@@ -136,6 +146,38 @@ class BackupRunAdmin(admin.ModelAdmin):
         return format_html('<a href="{}">{}</a>', url, obj.snapshot.dirname)
 
 
+@admin.register(NotificationTarget)
+class NotificationTargetAdmin(admin.ModelAdmin):
+    list_display = ("name", "channel", "enabled", "last_status", "last_sent_at", "updated_at")
+    list_filter = ("enabled", "channel", "last_status")
+    search_fields = ("name", "email_to", "webhook_url", "notes")
+    readonly_fields = ("created_at", "updated_at", "last_status", "last_error", "last_sent_at")
+    fieldsets = (
+        (None, {"fields": ("name", "enabled", "channel", "statuses")}),
+        ("Email", {"fields": ("email_to",)}),
+        ("Webhook", {"fields": ("webhook_url", "webhook_headers")}),
+        ("State", {"fields": ("last_status", "last_error", "last_sent_at"), "classes": ("collapse",)}),
+        ("Notes", {"fields": ("notes",), "classes": ("collapse",)}),
+        ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
+    )
+
+
+@admin.register(NotificationDelivery)
+class NotificationDeliveryAdmin(admin.ModelAdmin):
+    list_display = ("target", "run", "status", "created_at")
+    list_filter = ("status", "target__channel", "created_at")
+    search_fields = ("target__name", "run__host__host", "error")
+    readonly_fields = ("target", "run", "status", "error", "payload", "created_at")
+    list_select_related = ("target", "run", "run__host")
+    date_hierarchy = "created_at"
+
+    def has_add_permission(self, request) -> bool:
+        return False
+
+    def has_change_permission(self, request, obj=None) -> bool:
+        return False
+
+
 @admin.register(SnapshotRecord)
 class SnapshotRecordAdmin(admin.ModelAdmin):
     list_display = ("host", "kind", "dirname", "status", "base_link", "backup_run_count_link", "started_at", "discovered_at")

src/pobsync_backend/backup_runner.py

@@ -17,6 +17,7 @@ from pobsync.commands.run_scheduled import (
 )
 from pobsync_backend.config_source import DjangoConfigSource
 from pobsync_backend.models import BackupRun, HostConfig
+from pobsync_backend.notifications import notify_backup_run_completed
 from pobsync_backend.retention import run_sql_retention_apply
 from pobsync_backend.snapshot_discovery import infer_snapshot_kind, upsert_snapshot_record
 
@@ -85,6 +86,7 @@ def execute_backup_run(
             "type": type(exc).__name__,
         }
         run.save(update_fields=["status", "ended_at", "result"])
+        notify_backup_run_completed(run)
         raise
 
     run.refresh_from_db()
@@ -151,6 +153,7 @@ def execute_backup_run(
             "result",
         ],
     )
+    notify_backup_run_completed(run)
     return run
 
 
@@ -277,6 +280,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
             run.rsync_exit_code = exit_code or 255
             run.result = result
             run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
+            notify_backup_run_completed(run)
             return True
         if _running_rsync_process_missing(run=run, grace_seconds=grace_seconds):
             result.update(
@@ -301,6 +305,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
             run.rsync_exit_code = exit_code or 255
             run.result = result
             run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
+            notify_backup_run_completed(run)
             return True
         if stale_worker:
             result.update(
@@ -318,6 +323,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
             run.ended_at = timezone.now()
             run.result = result
             run.save(update_fields=["status", "ended_at", "result"])
+            notify_backup_run_completed(run)
             return True
         return False
 
@@ -353,6 +359,7 @@ def _reconcile_running_run(*, run: BackupRun, grace_seconds: int, stale_worker_s
     run.rsync_exit_code = exit_code
     run.result = result
     run.save(update_fields=["status", "ended_at", "rsync_exit_code", "result"])
+    notify_backup_run_completed(run)
     return True
 
 

src/pobsync_backend/forms.py

@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
 from django import forms
 from django.conf import settings
 
-from .models import GlobalConfig, HostConfig, ScheduleConfig, SshCredential
+from .models import BackupRun, GlobalConfig, HostConfig, NotificationTarget, ScheduleConfig, SshCredential
 from .scheduler import parse_cron_expr
 
 
@@ -153,6 +153,62 @@ class ManualBackupForm(forms.Form):
     )
 
 
+class NotificationTargetForm(forms.ModelForm):
+    TERMINAL_STATUS_CHOICES = (
+        (BackupRun.Status.SUCCESS, BackupRun.Status.SUCCESS.label),
+        (BackupRun.Status.WARNING, BackupRun.Status.WARNING.label),
+        (BackupRun.Status.FAILED, BackupRun.Status.FAILED.label),
+        (BackupRun.Status.CANCELLED, BackupRun.Status.CANCELLED.label),
+    )
+
+    statuses = forms.MultipleChoiceField(
+        choices=TERMINAL_STATUS_CHOICES,
+        widget=forms.CheckboxSelectMultiple,
+        initial=[choice[0] for choice in TERMINAL_STATUS_CHOICES],
+        help_text="Send notifications for these terminal run statuses.",
+    )
+    email_to = forms.CharField(
+        widget=forms.Textarea,
+        required=False,
+        help_text="One recipient per line, or comma-separated.",
+    )
+    webhook_headers = forms.JSONField(
+        required=False,
+        widget=forms.Textarea(attrs={"rows": 4}),
+        help_text='Optional JSON object with extra headers, for example {"Authorization": "Bearer ..."}.',
+    )
+
+    class Meta:
+        model = NotificationTarget
+        fields = (
+            "name",
+            "enabled",
+            "channel",
+            "statuses",
+            "email_to",
+            "webhook_url",
+            "webhook_headers",
+            "notes",
+        )
+        widgets = {
+            "notes": forms.Textarea,
+        }
+
+    def clean(self):
+        cleaned_data = super().clean()
+        channel = cleaned_data.get("channel")
+        if channel == NotificationTarget.Channel.EMAIL and not cleaned_data.get("email_to", "").strip():
+            self.add_error("email_to", "Email targets need at least one recipient.")
+        if channel == NotificationTarget.Channel.WEBHOOK and not cleaned_data.get("webhook_url"):
+            self.add_error("webhook_url", "Webhook targets need a URL.")
+        return cleaned_data
+
+    def clean_email_to(self) -> str:
+        value = self.cleaned_data.get("email_to", "")
+        recipients = [line.strip() for line in value.replace(",", "\n").splitlines() if line.strip()]
+        return "\n".join(recipients)
+
+
 class SshCredentialForm(forms.ModelForm):
     private_key_file = forms.FileField(
         required=False,

src/pobsync_backend/migrations/0015_notificationtarget_notificationdelivery.py

@@ -0,0 +1,54 @@
+# Generated by Django 5.2.14 on 2026-05-28 19:11
+
+import django.db.models.deletion
+import pobsync_backend.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pobsync_backend', '0014_host_bwlimit_override'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NotificationTarget',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=128, unique=True)),
+                ('enabled', models.BooleanField(default=True)),
+                ('channel', models.CharField(choices=[('email', 'Email'), ('webhook', 'Webhook')], max_length=16)),
+                ('statuses', models.JSONField(blank=True, default=pobsync_backend.models.default_notification_statuses)),
+                ('email_to', models.TextField(blank=True)),
+                ('webhook_url', models.URLField(blank=True, max_length=1024)),
+                ('webhook_headers', models.JSONField(blank=True, default=dict)),
+                ('notes', models.TextField(blank=True)),
+                ('last_status', models.CharField(blank=True, max_length=16)),
+                ('last_error', models.TextField(blank=True)),
+                ('last_sent_at', models.DateTimeField(blank=True, null=True)),
+            ],
+            options={
+                'ordering': ['name'],
+            },
+        ),
+        migrations.CreateModel(
+            name='NotificationDelivery',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('sent', 'Sent'), ('failed', 'Failed'), ('skipped', 'Skipped')], max_length=16)),
+                ('error', models.TextField(blank=True)),
+                ('payload', models.JSONField(blank=True, default=dict)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='notification_deliveries', to='pobsync_backend.backuprun')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='pobsync_backend.notificationtarget')),
+            ],
+            options={
+                'verbose_name_plural': 'notification deliveries',
+                'ordering': ['-created_at', 'target__name'],
+                'constraints': [models.UniqueConstraint(fields=('target', 'run'), name='unique_notification_delivery_per_target_run')],
+            },
+        ),
+    ]

src/pobsync_backend/models.py

@@ -135,6 +135,63 @@ class BackupRun(models.Model):
         return f"{self.host} {self.run_type} {self.status}"
 
 
+def default_notification_statuses() -> list[str]:
+    return [
+        BackupRun.Status.SUCCESS,
+        BackupRun.Status.WARNING,
+        BackupRun.Status.FAILED,
+        BackupRun.Status.CANCELLED,
+    ]
+
+
+class NotificationTarget(TimestampedModel):
+    class Channel(models.TextChoices):
+        EMAIL = "email", "Email"
+        WEBHOOK = "webhook", "Webhook"
+
+    name = models.CharField(max_length=128, unique=True)
+    enabled = models.BooleanField(default=True)
+    channel = models.CharField(max_length=16, choices=Channel.choices)
+    statuses = models.JSONField(default=default_notification_statuses, blank=True)
+    email_to = models.TextField(blank=True)
+    webhook_url = models.URLField(max_length=1024, blank=True)
+    webhook_headers = models.JSONField(default=dict, blank=True)
+    notes = models.TextField(blank=True)
+    last_status = models.CharField(max_length=16, blank=True)
+    last_error = models.TextField(blank=True)
+    last_sent_at = models.DateTimeField(null=True, blank=True)
+
+    class Meta:
+        ordering = ["name"]
+
+    def __str__(self) -> str:
+        return self.name
+
+
+class NotificationDelivery(models.Model):
+    class Status(models.TextChoices):
+        SENT = "sent", "Sent"
+        FAILED = "failed", "Failed"
+        SKIPPED = "skipped", "Skipped"
+
+    target = models.ForeignKey(NotificationTarget, on_delete=models.CASCADE, related_name="deliveries")
+    run = models.ForeignKey(BackupRun, on_delete=models.CASCADE, related_name="notification_deliveries")
+    status = models.CharField(max_length=16, choices=Status.choices)
+    error = models.TextField(blank=True)
+    payload = models.JSONField(default=dict, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=["target", "run"], name="unique_notification_delivery_per_target_run"),
+        ]
+        ordering = ["-created_at", "target__name"]
+        verbose_name_plural = "notification deliveries"
+
+    def __str__(self) -> str:
+        return f"{self.target} run {self.run_id} {self.status}"
+
+
 class SnapshotRecord(models.Model):
     class Kind(models.TextChoices):
         SCHEDULED = "scheduled", "Scheduled"

src/pobsync_backend/notifications.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Any
+
+from django.conf import settings
+from django.core.mail import send_mail
+from django.utils import timezone
+
+from .models import BackupRun, NotificationDelivery, NotificationTarget
+
+
+TERMINAL_RUN_STATUSES = {
+    BackupRun.Status.SUCCESS,
+    BackupRun.Status.WARNING,
+    BackupRun.Status.FAILED,
+    BackupRun.Status.CANCELLED,
+}
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+    target: NotificationTarget
+    delivery: NotificationDelivery
+    sent: bool
+
+
+def notify_backup_run_completed(run: BackupRun) -> list[DeliveryResult]:
+    if run.status not in TERMINAL_RUN_STATUSES:
+        return []
+
+    targets = [target for target in NotificationTarget.objects.filter(enabled=True) if _target_wants_status(target, run.status)]
+    return [_notify_target(target=target, run=run) for target in targets]
+
+
+def _target_wants_status(target: NotificationTarget, status: str) -> bool:
+    statuses = target.statuses
+    if not isinstance(statuses, list):
+        return False
+    return status in {str(item) for item in statuses}
+
+
+def _notify_target(*, target: NotificationTarget, run: BackupRun) -> DeliveryResult:
+    payload = _run_payload(run)
+    delivery, created = NotificationDelivery.objects.get_or_create(
+        target=target,
+        run=run,
+        defaults={
+            "status": NotificationDelivery.Status.SKIPPED,
+            "payload": payload,
+        },
+    )
+    if not created:
+        return DeliveryResult(target=target, delivery=delivery, sent=False)
+
+    try:
+        if target.channel == NotificationTarget.Channel.EMAIL:
+            _send_email(target=target, run=run, payload=payload)
+        elif target.channel == NotificationTarget.Channel.WEBHOOK:
+            _send_webhook(target=target, payload=payload)
+        else:
+            raise ValueError(f"Unsupported notification channel: {target.channel}")
+    except Exception as exc:
+        delivery.status = NotificationDelivery.Status.FAILED
+        delivery.error = str(exc)
+        delivery.save(update_fields=["status", "error"])
+        target.last_status = NotificationDelivery.Status.FAILED
+        target.last_error = str(exc)
+        target.save(update_fields=["last_status", "last_error", "updated_at"])
+        return DeliveryResult(target=target, delivery=delivery, sent=False)
+
+    delivery.status = NotificationDelivery.Status.SENT
+    delivery.save(update_fields=["status"])
+    target.last_status = NotificationDelivery.Status.SENT
+    target.last_error = ""
+    target.last_sent_at = timezone.now()
+    target.save(update_fields=["last_status", "last_error", "last_sent_at", "updated_at"])
+    return DeliveryResult(target=target, delivery=delivery, sent=True)
+
+
+def _send_email(*, target: NotificationTarget, run: BackupRun, payload: dict[str, Any]) -> None:
+    recipients = [line.strip() for line in target.email_to.replace(",", "\n").splitlines() if line.strip()]
+    if not recipients:
+        raise ValueError("Email notification target has no recipients.")
+
+    subject = f"pobsync {run.status}: {run.host.host} run {run.id}"
+    message = _email_message(payload)
+    from_email = getattr(settings, "DEFAULT_FROM_EMAIL", "") or "pobsync@localhost"
+    sent = send_mail(subject, message, from_email, recipients, fail_silently=False)
+    if sent == 0:
+        raise ValueError("Django email backend reported zero sent messages.")
+
+
+def _send_webhook(*, target: NotificationTarget, payload: dict[str, Any]) -> None:
+    if not target.webhook_url:
+        raise ValueError("Webhook notification target has no URL.")
+
+    headers = {"Content-Type": "application/json", **_string_headers(target.webhook_headers)}
+    request = urllib.request.Request(
+        target.webhook_url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=headers,
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=10) as response:
+            if response.status >= 400:
+                raise ValueError(f"Webhook returned HTTP {response.status}.")
+    except urllib.error.HTTPError as exc:
+        raise ValueError(f"Webhook returned HTTP {exc.code}.") from exc
+
+
+def _string_headers(headers: object) -> dict[str, str]:
+    if not isinstance(headers, dict):
+        return {}
+    return {str(key): str(value) for key, value in headers.items() if str(key).strip()}
+
+
+def _run_payload(run: BackupRun) -> dict[str, Any]:
+    result = run.result if isinstance(run.result, dict) else {}
+    failure = result.get("failure") if isinstance(result.get("failure"), dict) else {}
+    prune = result.get("prune") if isinstance(result.get("prune"), dict) else {}
+    return {
+        "event": "backup_run.completed",
+        "run": {
+            "id": run.id,
+            "host": run.host.host,
+            "type": run.run_type,
+            "status": run.status,
+            "started_at": run.started_at.isoformat() if run.started_at else None,
+            "ended_at": run.ended_at.isoformat() if run.ended_at else None,
+            "snapshot": run.snapshot_path,
+            "rsync_exit_code": run.rsync_exit_code,
+        },
+        "failure": {
+            "category": failure.get("category"),
+            "message": failure.get("message") or result.get("error"),
+            "hint": failure.get("hint"),
+        },
+        "prune": {
+            "ok": prune.get("ok") if prune else None,
+            "error": prune.get("error") if prune else "",
+        },
+    }
+
+
+def _email_message(payload: dict[str, Any]) -> str:
+    run = payload["run"]
+    lines = [
+        f"Host: {run['host']}",
+        f"Run: {run['id']}",
+        f"Type: {run['type']}",
+        f"Status: {run['status']}",
+        f"Started: {run['started_at'] or '-'}",
+        f"Ended: {run['ended_at'] or '-'}",
+        f"Snapshot: {run['snapshot'] or '-'}",
+        f"Rsync exit code: {run['rsync_exit_code'] if run['rsync_exit_code'] is not None else '-'}",
+    ]
+    failure = payload.get("failure") if isinstance(payload.get("failure"), dict) else {}
+    if failure.get("message"):
+        lines.extend(["", f"Failure: {failure['message']}"])
+    prune = payload.get("prune") if isinstance(payload.get("prune"), dict) else {}
+    if prune.get("error"):
+        lines.extend(["", f"Retention: {prune['error']}"])
+    return "\n".join(lines)

src/pobsync_backend/templates/pobsync_backend/base.html

@@ -920,6 +920,7 @@
         <a href="{% url 'dashboard' %}" {% if request.resolver_match.url_name == "dashboard" %}aria-current="page"{% endif %}>Dashboard</a>
         <a href="{% url 'hosts_list' %}" {% if request.resolver_match.url_name == "hosts_list" or request.resolver_match.url_name == "host_detail" or request.resolver_match.url_name == "create_host_config" or request.resolver_match.url_name == "edit_host_config" or request.resolver_match.url_name == "edit_host_schedule" %}aria-current="page"{% endif %}>Hosts</a>
         <a href="{% url 'ssh_credentials' %}" {% if request.resolver_match.url_name == "ssh_credentials" or request.resolver_match.url_name == "create_ssh_credential" or request.resolver_match.url_name == "generate_ssh_credential" or request.resolver_match.url_name == "edit_ssh_credential" %}aria-current="page"{% endif %}>SSH Keys</a>
+        <a href="{% url 'notification_targets' %}" {% if request.resolver_match.url_name == "notification_targets" or request.resolver_match.url_name == "create_notification_target" or request.resolver_match.url_name == "edit_notification_target" %}aria-current="page"{% endif %}>Notifications</a>
         <a href="{% url 'logs' %}" {% if request.resolver_match.url_name == "logs" %}aria-current="page"{% endif %}>Logs</a>
         <a href="{% url 'purged_snapshots' %}" {% if request.resolver_match.url_name == "purged_snapshots" %}aria-current="page"{% endif %}>Purged</a>
       </span>

src/pobsync_backend/templates/pobsync_backend/notification_target_form.html

@@ -0,0 +1,38 @@
+{% extends "pobsync_backend/base.html" %}
+
+{% block title %}{{ title }} | pobsync{% endblock %}
+
+{% block content %}
+  <header class="page-header">
+    <div class="page-title">
+      <div class="page-kicker">Reports</div>
+      <h1>{{ title }}</h1>
+      <div class="page-subtitle">Choose which completed backup statuses should trigger an email or webhook report.</div>
+    </div>
+    <section class="actions" aria-label="Notification target form actions">
+      <a class="button-link" href="{% url 'notification_targets' %}">Back to notifications</a>
+    </section>
+  </header>
+
+  <section class="panel">
+    <h2>{% if target %}Edit Target{% else %}Create Target{% endif %}</h2>
+    <form method="post" class="form-grid">
+      {% csrf_token %}
+      {{ form.non_field_errors }}
+
+      {% for field in form %}
+        <div class="field">
+          {{ field.errors }}
+          <label for="{{ field.id_for_label }}">{{ field.label }}</label>
+          {{ field }}
+          {% if field.help_text %}<div class="helptext">{{ field.help_text }}</div>{% endif %}
+        </div>
+      {% endfor %}
+
+      <div class="form-actions">
+        <button type="submit">{{ submit_label }}</button>
+        <a class="button-link secondary" href="{% url 'notification_targets' %}">Cancel</a>
+      </div>
+    </form>
+  </section>
+{% endblock %}

src/pobsync_backend/templates/pobsync_backend/notification_targets.html

@@ -0,0 +1,91 @@
+{% extends "pobsync_backend/base.html" %}
+
+{% block title %}Notifications | pobsync{% endblock %}
+
+{% block content %}
+  <header class="page-header">
+    <div class="page-title">
+      <div class="page-kicker">Reports</div>
+      <h1>Notifications</h1>
+      <div class="page-subtitle">Send email or webhook reports when backup runs finish.</div>
+    </div>
+    <section class="actions" aria-label="Notification actions">
+      <a class="button-link" href="{% url 'create_notification_target' %}">New target</a>
+      <a class="button-link secondary" href="{% url 'dashboard' %}">Back to dashboard</a>
+    </section>
+  </header>
+
+  <section class="panel">
+    <h2>Targets</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Channel</th>
+          <th>Status</th>
+          <th>Events</th>
+          <th>Destination</th>
+          <th>Last delivery</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for target in targets %}
+          <tr>
+            <td><a href="{% url 'edit_notification_target' target.id %}">{{ target.name }}</a></td>
+            <td>{{ target.get_channel_display }}</td>
+            <td><span class="status {% if target.enabled %}ok{% else %}skipped{% endif %}">{{ target.enabled|yesno:"enabled,disabled" }}</span></td>
+            <td>{{ target.statuses|join:", " }}</td>
+            <td>
+              {% if target.channel == "email" %}
+                {{ target.email_to|linebreaksbr }}
+              {% else %}
+                <code>{{ target.webhook_url|truncatechars:70 }}</code>
+              {% endif %}
+            </td>
+            <td>
+              {% if target.last_status %}
+                <span class="status {{ target.last_status }}">{{ target.last_status }}</span>
+                {% if target.last_error %}<div class="muted">{{ target.last_error|truncatechars:90 }}</div>{% endif %}
+                {% if target.last_sent_at %}<div class="muted">{{ target.last_sent_at }}</div>{% endif %}
+              {% else %}
+                <span class="muted">none</span>
+              {% endif %}
+            </td>
+            <td><a class="button-link secondary" href="{% url 'edit_notification_target' target.id %}">Edit</a></td>
+          </tr>
+        {% empty %}
+          <tr><td colspan="7" class="muted">No notification targets configured yet.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </section>
+
+  <section class="panel">
+    <h2>Recent Deliveries</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Target</th>
+          <th>Run</th>
+          <th>Status</th>
+          <th>Created</th>
+          <th>Error</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for delivery in deliveries %}
+          <tr>
+            <td>{{ delivery.target.name }}</td>
+            <td><a href="{% url 'run_detail' delivery.run.id %}">Run {{ delivery.run.id }}</a> {{ delivery.run.host.host }}</td>
+            <td><span class="status {{ delivery.status }}">{{ delivery.status }}</span></td>
+            <td>{{ delivery.created_at }}</td>
+            <td class="muted">{{ delivery.error|default:"" }}</td>
+          </tr>
+        {% empty %}
+          <tr><td colspan="5" class="muted">No notification deliveries recorded yet.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </section>
+{% endblock %}

src/pobsync_backend/tests/test_backup_worker.py

@@ -11,7 +11,7 @@ from django.utils import timezone
 from pobsync.util import write_yaml_atomic
 from pobsync_backend.backup_runner import queue_backup_run, reconcile_running_runs
 from pobsync_backend.management.commands.run_pobsync_worker import Command
-from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, SnapshotRecord
+from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, NotificationDelivery, NotificationTarget, SnapshotRecord
 
 
 class BackupWorkerTests(TestCase):
@@ -116,6 +116,42 @@ class BackupWorkerTests(TestCase):
         self.assertEqual(run.rsync_exit_code, 24)
         self.assertEqual(run.result["warning"]["category"], "vanished")
 
+    def test_worker_sends_notification_after_completed_run(self) -> None:
+        with TemporaryDirectory() as tmp:
+            backup_root = Path(tmp) / "backups"
+            GlobalConfig.objects.create(name="default", backup_root=str(backup_root))
+            host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+            NotificationTarget.objects.create(
+                name="ops",
+                channel=NotificationTarget.Channel.EMAIL,
+                email_to="ops@example.test",
+            )
+            snapshot_dir = backup_root / host.host / "scheduled" / "20260519-021500Z__ABCDEFGH"
+            meta_dir = snapshot_dir / "meta"
+            meta_dir.mkdir(parents=True)
+            write_yaml_atomic(meta_dir / "meta.yaml", {"status": "success", "started_at": "2026-05-19T02:15:00Z"})
+            run = queue_backup_run(host=host)
+
+            with (
+                patch("pobsync_backend.backup_runner.run_scheduled") as run_scheduled,
+                patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail,
+            ):
+                run_scheduled.return_value = {
+                    "ok": True,
+                    "dry_run": False,
+                    "host": host.host,
+                    "snapshot": str(snapshot_dir),
+                    "base": None,
+                    "rsync": {"exit_code": 0},
+                }
+
+                Command()._run_once(prefix=Path(tmp) / "home")
+
+        run.refresh_from_db()
+        self.assertEqual(run.status, BackupRun.Status.SUCCESS)
+        self.assertEqual(NotificationDelivery.objects.get(run=run).status, NotificationDelivery.Status.SENT)
+        send_mail.assert_called_once()
+
     def test_worker_refreshes_heartbeat_while_run_is_active(self) -> None:
         with TemporaryDirectory() as tmp:
             GlobalConfig.objects.create(name="default", backup_root=str(Path(tmp) / "backups"))

src/pobsync_backend/tests/test_notifications.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from django.utils import timezone
+
+from pobsync_backend.models import BackupRun, HostConfig, NotificationDelivery, NotificationTarget
+from pobsync_backend.notifications import notify_backup_run_completed
+
+
+class NotificationTests(TestCase):
+    def test_email_notification_is_sent_for_matching_status(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="ops",
+            channel=NotificationTarget.Channel.EMAIL,
+            statuses=[BackupRun.Status.FAILED],
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(
+            host=host,
+            status=BackupRun.Status.FAILED,
+            run_type=BackupRun.RunType.MANUAL,
+            started_at=timezone.now() - timedelta(minutes=5),
+            ended_at=timezone.now(),
+            rsync_exit_code=12,
+            result={"failure": {"message": "rsync failed"}},
+        )
+
+        with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(len(results), 1)
+        self.assertTrue(results[0].sent)
+        send_mail.assert_called_once()
+        subject, message, _from_email, recipients = send_mail.call_args.args
+        self.assertEqual(subject, f"pobsync failed: web-01 run {run.id}")
+        self.assertIn("Failure: rsync failed", message)
+        self.assertEqual(recipients, ["ops@example.test"])
+        delivery = NotificationDelivery.objects.get(target=target, run=run)
+        self.assertEqual(delivery.status, NotificationDelivery.Status.SENT)
+        target.refresh_from_db()
+        self.assertEqual(target.last_status, NotificationDelivery.Status.SENT)
+        self.assertEqual(target.last_error, "")
+        self.assertIsNotNone(target.last_sent_at)
+
+    def test_webhook_notification_posts_payload(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="discord",
+            channel=NotificationTarget.Channel.WEBHOOK,
+            webhook_url="https://hooks.example.test/pobsync",
+            webhook_headers={"X-Token": "secret"},
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS, run_type=BackupRun.RunType.SCHEDULED)
+        response = Mock()
+        response.status = 204
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+
+        with patch("pobsync_backend.notifications.urllib.request.urlopen", return_value=response) as urlopen:
+            notify_backup_run_completed(run)
+
+        request = urlopen.call_args.args[0]
+        self.assertEqual(request.full_url, "https://hooks.example.test/pobsync")
+        self.assertEqual(request.get_method(), "POST")
+        self.assertEqual(request.headers["X-token"], "secret")
+        self.assertIn(f'"id": {run.id}', request.data.decode("utf-8"))
+        self.assertEqual(NotificationDelivery.objects.get(target=target, run=run).status, NotificationDelivery.Status.SENT)
+
+    def test_notification_filters_statuses(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        NotificationTarget.objects.create(
+            name="failures-only",
+            channel=NotificationTarget.Channel.EMAIL,
+            statuses=[BackupRun.Status.FAILED],
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
+
+        with patch("pobsync_backend.notifications.send_mail") as send_mail:
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(results, [])
+        send_mail.assert_not_called()
+        self.assertEqual(NotificationDelivery.objects.count(), 0)
+
+    def test_notification_delivery_is_idempotent_per_run_and_target(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="ops",
+            channel=NotificationTarget.Channel.EMAIL,
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.WARNING)
+
+        with patch("pobsync_backend.notifications.send_mail", return_value=1) as send_mail:
+            notify_backup_run_completed(run)
+            notify_backup_run_completed(run)
+
+        self.assertEqual(NotificationDelivery.objects.filter(target=target, run=run).count(), 1)
+        send_mail.assert_called_once()
+
+    def test_failed_delivery_is_recorded_without_raising(self) -> None:
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        target = NotificationTarget.objects.create(
+            name="broken",
+            channel=NotificationTarget.Channel.EMAIL,
+            email_to="ops@example.test",
+        )
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.FAILED)
+
+        with patch("pobsync_backend.notifications.send_mail", side_effect=RuntimeError("smtp down")):
+            results = notify_backup_run_completed(run)
+
+        self.assertEqual(len(results), 1)
+        self.assertFalse(results[0].sent)
+        delivery = NotificationDelivery.objects.get(target=target, run=run)
+        self.assertEqual(delivery.status, NotificationDelivery.Status.FAILED)
+        self.assertEqual(delivery.error, "smtp down")
+        target.refresh_from_db()
+        self.assertEqual(target.last_status, NotificationDelivery.Status.FAILED)
+        self.assertEqual(target.last_error, "smtp down")

src/pobsync_backend/tests/test_views.py

@@ -18,6 +18,8 @@ from pobsync_backend.models import (
     BackupRun,
     GlobalConfig,
     HostConfig,
+    NotificationDelivery,
+    NotificationTarget,
     PurgedSnapshot,
     ScheduleConfig,
     SnapshotRecord,
@@ -52,6 +54,7 @@ class ViewTests(TestCase):
         self.assertContains(response, reverse("dashboard"))
         self.assertContains(response, reverse("hosts_list"))
         self.assertContains(response, reverse("ssh_credentials"))
+        self.assertContains(response, reverse("notification_targets"))
         self.assertContains(response, reverse("logs"))
         self.assertContains(response, reverse("purged_snapshots"))
         self.assertContains(response, reverse("self_check"))
@@ -94,6 +97,70 @@ class ViewTests(TestCase):
         self.assertContains(response, "Django control panel")
         self.assertContains(response, "Native systemd installer")
 
+    def test_notification_targets_view_renders_targets_and_deliveries(self) -> None:
+        self.client.force_login(self.staff_user)
+        host = HostConfig.objects.create(host="web-01", address="web-01.example.test")
+        run = BackupRun.objects.create(host=host, status=BackupRun.Status.SUCCESS)
+        target = NotificationTarget.objects.create(
+            name="ops",
+            channel=NotificationTarget.Channel.EMAIL,
+            email_to="ops@example.test",
+            last_status=NotificationDelivery.Status.SENT,
+        )
+        NotificationDelivery.objects.create(target=target, run=run, status=NotificationDelivery.Status.SENT)
+
+        response = self.client.get(reverse("notification_targets"))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Notifications")
+        self.assertContains(response, "ops")
+        self.assertContains(response, "ops@example.test")
+        self.assertContains(response, f"Run {run.id}")
+
+    def test_notification_target_form_creates_email_target(self) -> None:
+        self.client.force_login(self.staff_user)
+
+        response = self.client.post(
+            reverse("create_notification_target"),
+            {
+                "name": "ops",
+                "enabled": "on",
+                "channel": NotificationTarget.Channel.EMAIL,
+                "statuses": [BackupRun.Status.FAILED, BackupRun.Status.WARNING],
+                "email_to": "ops@example.test, backup@example.test",
+                "webhook_headers": "{}",
+                "notes": "Notify ops",
+            },
+            follow=True,
+        )
+
+        self.assertRedirects(response, reverse("notification_targets"))
+        self.assertContains(response, "Notification target ops created.")
+        target = NotificationTarget.objects.get(name="ops")
+        self.assertEqual(target.channel, NotificationTarget.Channel.EMAIL)
+        self.assertEqual(target.statuses, [BackupRun.Status.FAILED, BackupRun.Status.WARNING])
+        self.assertEqual(target.email_to, "ops@example.test\nbackup@example.test")
+
+    def test_notification_target_form_requires_channel_destination(self) -> None:
+        self.client.force_login(self.staff_user)
+
+        response = self.client.post(
+            reverse("create_notification_target"),
+            {
+                "name": "broken",
+                "enabled": "on",
+                "channel": NotificationTarget.Channel.WEBHOOK,
+                "statuses": [BackupRun.Status.FAILED],
+                "email_to": "",
+                "webhook_url": "",
+                "webhook_headers": "{}",
+            },
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Webhook targets need a URL.")
+        self.assertFalse(NotificationTarget.objects.exists())
+
     def test_dashboard_renders_hosts_and_latest_runs(self) -> None:
         self.client.force_login(self.staff_user)
         host = HostConfig.objects.create(host="web-01", address="web-01.example.test")

src/pobsync_backend/views.py

@@ -32,13 +32,24 @@ from .forms import (
     HostConfigForm,
     IncompleteCleanupForm,
     ManualBackupForm,
+    NotificationTargetForm,
     RetentionApplyForm,
     SshCredentialGenerateForm,
     ScheduleConfigForm,
     SshCredentialForm,
 )
 from .host_ops import ensure_host_directories
-from .models import BackupRun, GlobalConfig, HostConfig, PurgedSnapshot, ScheduleConfig, SnapshotRecord, SshCredential
+from .models import (
+    BackupRun,
+    GlobalConfig,
+    HostConfig,
+    NotificationDelivery,
+    NotificationTarget,
+    PurgedSnapshot,
+    ScheduleConfig,
+    SnapshotRecord,
+    SshCredential,
+)
 from .preflight import collect_backup_gate, effective_host_config_preview, run_remote_preflight
 from .retention import run_incomplete_cleanup, run_sql_retention_apply, run_sql_retention_plan
 from .self_check import collect_self_checks, summarize_self_checks
@@ -320,6 +331,65 @@ def logs(request):
     return render(request, "pobsync_backend/logs.html", context)
 
 
+@staff_member_required
+def notification_targets(request):
+    targets = NotificationTarget.objects.order_by("name")
+    deliveries = NotificationDelivery.objects.select_related("target", "run", "run__host").order_by("-created_at")[:12]
+    return render(
+        request,
+        "pobsync_backend/notification_targets.html",
+        {
+            "targets": targets,
+            "deliveries": deliveries,
+        },
+    )
+
+
+@staff_member_required
+def create_notification_target(request):
+    if request.method == "POST":
+        form = NotificationTargetForm(request.POST)
+        if form.is_valid():
+            target = form.save()
+            messages.success(request, f"Notification target {target.name} created.")
+            return redirect("notification_targets")
+    else:
+        form = NotificationTargetForm()
+    return render(
+        request,
+        "pobsync_backend/notification_target_form.html",
+        {
+            "form": form,
+            "target": None,
+            "title": "New notification target",
+            "submit_label": "Create target",
+        },
+    )
+
+
+@staff_member_required
+def edit_notification_target(request, target_id: int):
+    target = get_object_or_404(NotificationTarget, id=target_id)
+    if request.method == "POST":
+        form = NotificationTargetForm(request.POST, instance=target)
+        if form.is_valid():
+            target = form.save()
+            messages.success(request, f"Notification target {target.name} updated.")
+            return redirect("notification_targets")
+    else:
+        form = NotificationTargetForm(instance=target)
+    return render(
+        request,
+        "pobsync_backend/notification_target_form.html",
+        {
+            "form": form,
+            "target": target,
+            "title": f"Edit notification target: {target.name}",
+            "submit_label": "Save target",
+        },
+    )
+
+
 @staff_member_required
 def runs_list(request):
     status = request.GET.get("status", "").strip()

src/pobsync_server/urls.py

@@ -13,6 +13,9 @@ urlpatterns = [
     path("changelog/", views.changelog, name="changelog"),
     path("self-check/", views.self_check, name="self_check"),
     path("logs/", views.logs, name="logs"),
+    path("notifications/", views.notification_targets, name="notification_targets"),
+    path("notifications/new/", views.create_notification_target, name="create_notification_target"),
+    path("notifications/<int:target_id>/", views.edit_notification_target, name="edit_notification_target"),
     path("purged-snapshots/", views.purged_snapshots, name="purged_snapshots"),
     path("schedules/", views.schedules_list, name="schedules_list"),
     path("config/global/", views.edit_global_config, name="edit_global_config"),