Add email and webhook notification targets with delivery tracking, and send notifications when backup runs reach a terminal status. Expose notification target management in the Django UI and keep delivery failures recorded without failing the backup worker.
169 lines
6.2 KiB
Python
169 lines
6.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from django.conf import settings
|
|
from django.core.mail import send_mail
|
|
from django.utils import timezone
|
|
|
|
from .models import BackupRun, NotificationDelivery, NotificationTarget
|
|
|
|
|
|
TERMINAL_RUN_STATUSES = {
|
|
BackupRun.Status.SUCCESS,
|
|
BackupRun.Status.WARNING,
|
|
BackupRun.Status.FAILED,
|
|
BackupRun.Status.CANCELLED,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeliveryResult:
|
|
target: NotificationTarget
|
|
delivery: NotificationDelivery
|
|
sent: bool
|
|
|
|
|
|
def notify_backup_run_completed(run: BackupRun) -> list[DeliveryResult]:
|
|
if run.status not in TERMINAL_RUN_STATUSES:
|
|
return []
|
|
|
|
targets = [target for target in NotificationTarget.objects.filter(enabled=True) if _target_wants_status(target, run.status)]
|
|
return [_notify_target(target=target, run=run) for target in targets]
|
|
|
|
|
|
def _target_wants_status(target: NotificationTarget, status: str) -> bool:
|
|
statuses = target.statuses
|
|
if not isinstance(statuses, list):
|
|
return False
|
|
return status in {str(item) for item in statuses}
|
|
|
|
|
|
def _notify_target(*, target: NotificationTarget, run: BackupRun) -> DeliveryResult:
|
|
payload = _run_payload(run)
|
|
delivery, created = NotificationDelivery.objects.get_or_create(
|
|
target=target,
|
|
run=run,
|
|
defaults={
|
|
"status": NotificationDelivery.Status.SKIPPED,
|
|
"payload": payload,
|
|
},
|
|
)
|
|
if not created:
|
|
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
|
|
|
try:
|
|
if target.channel == NotificationTarget.Channel.EMAIL:
|
|
_send_email(target=target, run=run, payload=payload)
|
|
elif target.channel == NotificationTarget.Channel.WEBHOOK:
|
|
_send_webhook(target=target, payload=payload)
|
|
else:
|
|
raise ValueError(f"Unsupported notification channel: {target.channel}")
|
|
except Exception as exc:
|
|
delivery.status = NotificationDelivery.Status.FAILED
|
|
delivery.error = str(exc)
|
|
delivery.save(update_fields=["status", "error"])
|
|
target.last_status = NotificationDelivery.Status.FAILED
|
|
target.last_error = str(exc)
|
|
target.save(update_fields=["last_status", "last_error", "updated_at"])
|
|
return DeliveryResult(target=target, delivery=delivery, sent=False)
|
|
|
|
delivery.status = NotificationDelivery.Status.SENT
|
|
delivery.save(update_fields=["status"])
|
|
target.last_status = NotificationDelivery.Status.SENT
|
|
target.last_error = ""
|
|
target.last_sent_at = timezone.now()
|
|
target.save(update_fields=["last_status", "last_error", "last_sent_at", "updated_at"])
|
|
return DeliveryResult(target=target, delivery=delivery, sent=True)
|
|
|
|
|
|
def _send_email(*, target: NotificationTarget, run: BackupRun, payload: dict[str, Any]) -> None:
|
|
recipients = [line.strip() for line in target.email_to.replace(",", "\n").splitlines() if line.strip()]
|
|
if not recipients:
|
|
raise ValueError("Email notification target has no recipients.")
|
|
|
|
subject = f"pobsync {run.status}: {run.host.host} run {run.id}"
|
|
message = _email_message(payload)
|
|
from_email = getattr(settings, "DEFAULT_FROM_EMAIL", "") or "pobsync@localhost"
|
|
sent = send_mail(subject, message, from_email, recipients, fail_silently=False)
|
|
if sent == 0:
|
|
raise ValueError("Django email backend reported zero sent messages.")
|
|
|
|
|
|
def _send_webhook(*, target: NotificationTarget, payload: dict[str, Any]) -> None:
|
|
if not target.webhook_url:
|
|
raise ValueError("Webhook notification target has no URL.")
|
|
|
|
headers = {"Content-Type": "application/json", **_string_headers(target.webhook_headers)}
|
|
request = urllib.request.Request(
|
|
target.webhook_url,
|
|
data=json.dumps(payload).encode("utf-8"),
|
|
headers=headers,
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
if response.status >= 400:
|
|
raise ValueError(f"Webhook returned HTTP {response.status}.")
|
|
except urllib.error.HTTPError as exc:
|
|
raise ValueError(f"Webhook returned HTTP {exc.code}.") from exc
|
|
|
|
|
|
def _string_headers(headers: object) -> dict[str, str]:
|
|
if not isinstance(headers, dict):
|
|
return {}
|
|
return {str(key): str(value) for key, value in headers.items() if str(key).strip()}
|
|
|
|
|
|
def _run_payload(run: BackupRun) -> dict[str, Any]:
|
|
result = run.result if isinstance(run.result, dict) else {}
|
|
failure = result.get("failure") if isinstance(result.get("failure"), dict) else {}
|
|
prune = result.get("prune") if isinstance(result.get("prune"), dict) else {}
|
|
return {
|
|
"event": "backup_run.completed",
|
|
"run": {
|
|
"id": run.id,
|
|
"host": run.host.host,
|
|
"type": run.run_type,
|
|
"status": run.status,
|
|
"started_at": run.started_at.isoformat() if run.started_at else None,
|
|
"ended_at": run.ended_at.isoformat() if run.ended_at else None,
|
|
"snapshot": run.snapshot_path,
|
|
"rsync_exit_code": run.rsync_exit_code,
|
|
},
|
|
"failure": {
|
|
"category": failure.get("category"),
|
|
"message": failure.get("message") or result.get("error"),
|
|
"hint": failure.get("hint"),
|
|
},
|
|
"prune": {
|
|
"ok": prune.get("ok") if prune else None,
|
|
"error": prune.get("error") if prune else "",
|
|
},
|
|
}
|
|
|
|
|
|
def _email_message(payload: dict[str, Any]) -> str:
|
|
run = payload["run"]
|
|
lines = [
|
|
f"Host: {run['host']}",
|
|
f"Run: {run['id']}",
|
|
f"Type: {run['type']}",
|
|
f"Status: {run['status']}",
|
|
f"Started: {run['started_at'] or '-'}",
|
|
f"Ended: {run['ended_at'] or '-'}",
|
|
f"Snapshot: {run['snapshot'] or '-'}",
|
|
f"Rsync exit code: {run['rsync_exit_code'] if run['rsync_exit_code'] is not None else '-'}",
|
|
]
|
|
failure = payload.get("failure") if isinstance(payload.get("failure"), dict) else {}
|
|
if failure.get("message"):
|
|
lines.extend(["", f"Failure: {failure['message']}"])
|
|
prune = payload.get("prune") if isinstance(payload.get("prune"), dict) else {}
|
|
if prune.get("error"):
|
|
lines.extend(["", f"Retention: {prune['error']}"])
|
|
return "\n".join(lines)
|