(feature) add queued backup worker foundation

Move backup execution out of the management command into a reusable
backup runner service that can execute an existing BackupRun record.

Add queue primitives and a run_pobsync_worker command so manual backup
requests can be recorded as queued SQL state and processed outside the
web request path.

Add a worker Docker service and pobsync worker CLI alias, with tests for
queued run creation, worker execution, manual run typing, and command
mapping.

src/pobsync_backend/management/commands/run_pobsync_backup.py

@@ -5,18 +5,14 @@ from typing import Any
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from django.utils import timezone
 
-from pobsync.commands.run_scheduled import run_scheduled
 from pobsync.paths import PobsyncPaths
-from pobsync_backend.config_source import DjangoConfigSource
+from pobsync_backend.backup_runner import execute_backup_run
 from pobsync_backend.models import BackupRun, HostConfig
-from pobsync_backend.retention import run_sql_retention_apply
-from pobsync_backend.snapshot_discovery import infer_snapshot_kind, upsert_snapshot_record
 
 
 class Command(BaseCommand):
-    help = "Run a scheduled pobsync backup and record the result in Django."
+    help = "Run a pobsync backup and record the result in Django."
 
     def add_arguments(self, parser) -> None:
         parser.add_argument("host", help="Host to back up")
@@ -25,6 +21,7 @@ class Command(BaseCommand):
         parser.add_argument("--prune", action="store_true", help="Apply retention after a successful run")
         parser.add_argument("--prune-max-delete", type=int, default=10)
         parser.add_argument("--prune-protect-bases", action="store_true")
+        parser.add_argument("--manual", action="store_true", help="Record the run as manual instead of scheduled")
 
     def handle(self, *args: Any, **options: Any) -> None:
         host_name = options["host"]
@@ -36,86 +33,20 @@ class Command(BaseCommand):
 
         run = BackupRun.objects.create(
             host=host,
-            run_type=BackupRun.RunType.SCHEDULED,
+            run_type=BackupRun.RunType.MANUAL if options["manual"] else BackupRun.RunType.SCHEDULED,
             status=BackupRun.Status.RUNNING,
-            started_at=timezone.now(),
         )
-
-        try:
-            result = run_scheduled(
-                prefix=paths.home,
-                host=host.host,
-                dry_run=bool(options["dry_run"]),
-                prune=False,
-                config_source=DjangoConfigSource(),
-            )
-        except Exception as exc:
-            run.status = BackupRun.Status.FAILED
-            run.ended_at = timezone.now()
-            run.result = {"ok": False, "error": str(exc), "type": type(exc).__name__}
-            run.save(update_fields=["status", "ended_at", "result"])
-            raise
-
-        run.status = BackupRun.Status.SUCCESS if result.get("ok") else BackupRun.Status.FAILED
-        run.ended_at = timezone.now()
-        run.snapshot_path = str(result.get("snapshot") or "")
-        run.base_path = str(result.get("base") or "")
-        rsync = result.get("rsync") if isinstance(result.get("rsync"), dict) else {}
-        run.rsync_exit_code = rsync.get("exit_code")
-        run.result = result
-        snapshot_record = None
-        if run.snapshot_path:
-            snapshot_path = Path(run.snapshot_path)
-            try:
-                kind = infer_snapshot_kind(snapshot_path)
-                snapshot_record, _created = upsert_snapshot_record(host=host, kind=kind, snapshot_dir=snapshot_path)
-            except ValueError:
-                snapshot_record = None
-
-        if result.get("ok") and not result.get("dry_run") and options["prune"]:
-            try:
-                result["prune"] = run_sql_retention_apply(
-                    prefix=paths.home,
-                    host=host.host,
-                    kind="scheduled",
-                    protect_bases=bool(options["prune_protect_bases"]),
-                    yes=True,
-                    max_delete=int(options["prune_max_delete"]),
-                    acquire_lock=False,
-                )
-            except Exception as exc:
-                result["prune"] = {"ok": False, "error": str(exc), "type": type(exc).__name__}
-                run.status = BackupRun.Status.FAILED
-                run.result = result
-                run.snapshot = snapshot_record
-                run.save(
-                    update_fields=[
-                        "status",
-                        "ended_at",
-                        "snapshot_path",
-                        "snapshot",
-                        "base_path",
-                        "rsync_exit_code",
-                        "result",
-                    ],
-                )
-                raise
-
-        run.snapshot = snapshot_record
-        run.result = result
-        run.save(
-            update_fields=[
-                "status",
-                "ended_at",
-                "snapshot_path",
-                "snapshot",
-                "base_path",
-                "rsync_exit_code",
-                "result",
-            ],
+        execute_backup_run(
+            run=run,
+            prefix=paths.home,
+            dry_run=bool(options["dry_run"]),
+            prune=bool(options["prune"]),
+            prune_max_delete=int(options["prune_max_delete"]),
+            prune_protect_bases=bool(options["prune_protect_bases"]),
         )
+        run.refresh_from_db()
 
-        if result.get("ok"):
+        if run.status == BackupRun.Status.SUCCESS:
             self.stdout.write(self.style.SUCCESS(f"Backup completed for {host.host}."))
             return
 

src/pobsync_backend/management/commands/run_pobsync_worker.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from pobsync.paths import PobsyncPaths
+from pobsync_backend.backup_runner import claim_next_queued_run, execute_backup_run, requested_options
+
+
+class Command(BaseCommand):
+    help = "Run queued pobsync backup jobs from the Django database."
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
+        parser.add_argument("--once", action="store_true", help="Process one queued run and exit")
+        parser.add_argument("--loop", action="store_true", help="Keep checking for queued runs")
+        parser.add_argument("--interval", type=int, default=15, help="Loop interval in seconds")
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not options["once"] and not options["loop"]:
+            options["once"] = True
+
+        paths = PobsyncPaths(home=Path(options["prefix"]))
+        while True:
+            count = self._run_once(prefix=paths.home)
+            self.stdout.write(f"Ran {count} queued backup run(s).")
+            if options["once"]:
+                return
+            time.sleep(max(1, int(options["interval"])))
+
+    def _run_once(self, *, prefix: Path) -> int:
+        run = claim_next_queued_run()
+        if run is None:
+            return 0
+
+        options = requested_options(run)
+        try:
+            execute_backup_run(
+                run=run,
+                prefix=prefix,
+                dry_run=bool(options.get("dry_run", False)),
+                prune=bool(options.get("prune", False)),
+                prune_max_delete=int(options.get("prune_max_delete", 10)),
+                prune_protect_bases=bool(options.get("prune_protect_bases", False)),
+            )
+        except Exception as exc:
+            self.stderr.write(f"{run.host.host}: {type(exc).__name__}: {exc}")
+        return 1