feat: make Django configs drive backups and scheduling

Treat SQL-backed Django models as the source of truth for pobsync
configuration, exporting runtime YAML only as a compatibility layer for
the existing engine. Add a database-driven scheduler command, Docker
scheduler services, schedule run-state fields, and tests for scheduler,
config export, and retention behavior.
This commit is contained in:
2026-05-19 04:53:47 +02:00
parent 1a51c3e448
commit 18082496e4
13 changed files with 493 additions and 13 deletions

View File

@@ -0,0 +1,64 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.core.exceptions import ObjectDoesNotExist
from pobsync.config.schemas import GLOBAL_SCHEMA, HOST_SCHEMA
from pobsync.paths import PobsyncPaths
from pobsync.util import write_yaml_atomic
from pobsync.validate import validate_dict
from .models import GlobalConfig, HostConfig
class ConfigRepositoryError(RuntimeError):
pass
def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
data = dict(global_config.data or {})
data["backup_root"] = global_config.backup_root
data["pobsync_home"] = global_config.pobsync_home
return validate_dict(data, GLOBAL_SCHEMA, path="global")
def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
data = dict(host_config.config or {})
data["host"] = host_config.host
data["address"] = host_config.address
return validate_dict(data, HOST_SCHEMA, path="host")
def export_global_config(prefix: Path, name: str = "default") -> Path:
try:
global_config = GlobalConfig.objects.get(name=name)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
paths = PobsyncPaths(home=prefix)
write_yaml_atomic(paths.global_config_path, _global_yaml_data(global_config))
return paths.global_config_path
def export_host_config(prefix: Path, host: str) -> Path:
try:
host_config = HostConfig.objects.get(host=host, enabled=True)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
paths = PobsyncPaths(home=prefix)
target = paths.hosts_dir / f"{host_config.host}.yaml"
write_yaml_atomic(target, _host_yaml_data(host_config))
return target
def export_runtime_configs(prefix: Path, host: str | None = None) -> list[Path]:
written = [export_global_config(prefix)]
hosts = HostConfig.objects.filter(enabled=True).order_by("host")
if host is not None:
hosts = hosts.filter(host=host)
for host_config in hosts:
written.append(export_host_config(prefix, host_config.host))
return written

View File

@@ -0,0 +1,23 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from pobsync_backend.config_repository import export_runtime_configs
class Command(BaseCommand):
help = "Export Django database configs to pobsync runtime YAML files."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--host", default=None, help="Export only one enabled host")
def handle(self, *args: Any, **options: Any) -> None:
written = export_runtime_configs(prefix=Path(options["prefix"]), host=options["host"])
for path in written:
self.stdout.write(str(path))
self.stdout.write(self.style.SUCCESS(f"Exported {len(written)} config file(s)."))

View File

@@ -8,8 +8,8 @@ from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from pobsync.commands.run_scheduled import run_scheduled
from pobsync.config.load import load_host_config
from pobsync.paths import PobsyncPaths
from pobsync_backend.config_repository import export_runtime_configs
from pobsync_backend.models import BackupRun, HostConfig
@@ -27,19 +27,12 @@ class Command(BaseCommand):
def handle(self, *args: Any, **options: Any) -> None:
host_name = options["host"]
paths = PobsyncPaths(home=Path(options["prefix"]))
host_path = paths.hosts_dir / f"{host_name}.yaml"
if not host_path.exists():
raise CommandError(f"Missing host config: {host_path}")
try:
host = HostConfig.objects.get(host=host_name, enabled=True)
except HostConfig.DoesNotExist as exc:
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
host_cfg = load_host_config(host_path)
host, _created = HostConfig.objects.update_or_create(
host=host_cfg["host"],
defaults={
"address": host_cfg["address"],
"config": host_cfg,
"enabled": True,
},
)
export_runtime_configs(prefix=paths.home, host=host.host)
run = BackupRun.objects.create(
host=host,

View File

@@ -0,0 +1,85 @@
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils import timezone
from pobsync_backend.models import ScheduleConfig
from pobsync_backend.scheduler import due_key, is_due
class Command(BaseCommand):
help = "Run due pobsync schedules from the Django database."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--once", action="store_true", help="Check once and exit")
parser.add_argument("--loop", action="store_true", help="Keep checking schedules")
parser.add_argument("--interval", type=int, default=60, help="Loop interval in seconds")
parser.add_argument("--dry-run", action="store_true", help="Pass --dry-run to backup runs")
def handle(self, *args: Any, **options: Any) -> None:
if not options["once"] and not options["loop"]:
options["once"] = True
prefix = Path(options["prefix"])
while True:
count = self._run_due(prefix=prefix, dry_run=bool(options["dry_run"]))
self.stdout.write(f"Ran {count} due schedule(s).")
if options["once"]:
return
time.sleep(max(1, int(options["interval"])))
def _run_due(self, *, prefix: Path, dry_run: bool) -> int:
now = timezone.localtime(timezone.now())
current_due_key = due_key(now)
ran = 0
schedules = (
ScheduleConfig.objects.select_related("host")
.filter(enabled=True, host__enabled=True)
.order_by("host__host")
)
for schedule in schedules:
if schedule.last_due_key == current_due_key:
continue
if not is_due(schedule.cron_expr, now):
continue
with transaction.atomic():
locked = ScheduleConfig.objects.select_for_update().get(pk=schedule.pk)
if locked.last_due_key == current_due_key:
continue
locked.last_due_key = current_due_key
locked.last_started_at = timezone.now()
locked.last_status = "running"
locked.save(update_fields=["last_due_key", "last_started_at", "last_status", "updated_at"])
status = "success"
try:
call_command(
"run_pobsync_backup",
schedule.host.host,
prefix=str(prefix),
dry_run=dry_run,
prune=schedule.prune,
prune_max_delete=schedule.prune_max_delete,
prune_protect_bases=schedule.prune_protect_bases,
)
except Exception as exc:
status = "failed"
self.stderr.write(f"{schedule.host.host}: {type(exc).__name__}: {exc}")
finally:
ScheduleConfig.objects.filter(pk=schedule.pk).update(
last_finished_at=timezone.now(),
last_status=status,
)
ran += 1
return ran

View File

@@ -0,0 +1,32 @@
from __future__ import annotations
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("pobsync_backend", "0001_initial"),
]
operations = [
migrations.AddField(
model_name="scheduleconfig",
name="last_due_key",
field=models.CharField(blank=True, max_length=32),
),
migrations.AddField(
model_name="scheduleconfig",
name="last_finished_at",
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name="scheduleconfig",
name="last_started_at",
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name="scheduleconfig",
name="last_status",
field=models.CharField(blank=True, max_length=16),
),
]

View File

@@ -102,6 +102,10 @@ class ScheduleConfig(TimestampedModel):
prune = models.BooleanField(default=False)
prune_max_delete = models.PositiveIntegerField(default=10)
prune_protect_bases = models.BooleanField(default=False)
last_due_key = models.CharField(max_length=32, blank=True)
last_started_at = models.DateTimeField(null=True, blank=True)
last_finished_at = models.DateTimeField(null=True, blank=True)
last_status = models.CharField(max_length=16, blank=True)
class Meta:
ordering = ["host__host"]

View File

@@ -0,0 +1,88 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
@dataclass(frozen=True)
class CronSchedule:
minute: str
hour: str
day_of_month: str
month: str
day_of_week: str
def parse_cron_expr(expr: str) -> CronSchedule:
parts = expr.strip().split()
if len(parts) != 5:
raise ValueError("cron expression must have exactly 5 fields")
return CronSchedule(*parts)
def due_key(moment: datetime) -> str:
return moment.strftime("%Y%m%d%H%M")
def is_due(expr: str, moment: datetime) -> bool:
schedule = parse_cron_expr(expr)
cron_dow = (moment.weekday() + 1) % 7
return (
_field_matches(schedule.minute, moment.minute, 0, 59)
and _field_matches(schedule.hour, moment.hour, 0, 23)
and _field_matches(schedule.day_of_month, moment.day, 1, 31)
and _field_matches(schedule.month, moment.month, 1, 12)
and _field_matches(schedule.day_of_week, cron_dow, 0, 7, sunday_alias=True)
)
def _field_matches(field: str, value: int, min_value: int, max_value: int, sunday_alias: bool = False) -> bool:
for part in field.split(","):
if _part_matches(part.strip(), value, min_value, max_value, sunday_alias=sunday_alias):
return True
return False
def _part_matches(part: str, value: int, min_value: int, max_value: int, sunday_alias: bool) -> bool:
if not part:
return False
step = 1
base = part
if "/" in part:
base, step_s = part.split("/", 1)
if not step_s.isdigit():
return False
step = int(step_s)
if step <= 0:
return False
if base == "*":
start = min_value
end = max_value
elif "-" in base:
start_s, end_s = base.split("-", 1)
if not start_s.isdigit() or not end_s.isdigit():
return False
start = int(start_s)
end = int(end_s)
elif base.isdigit():
start = end = int(base)
else:
return False
values = _normalize_values(range(start, end + 1), sunday_alias=sunday_alias)
normalized_value = 0 if sunday_alias and value == 7 else value
if normalized_value not in values:
return False
return (normalized_value - min(values)) % step == 0
def _normalize_values(values: range, sunday_alias: bool) -> set[int]:
out: set[int] = set()
for value in values:
if sunday_alias and value == 7:
out.add(0)
else:
out.add(value)
return out

View File

@@ -0,0 +1 @@

View File

@@ -0,0 +1,47 @@
from __future__ import annotations
import tempfile
from pathlib import Path
from django.test import TestCase
from pobsync.config.load import load_global_config, load_host_config
from pobsync_backend.config_repository import export_runtime_configs
from pobsync_backend.models import GlobalConfig, HostConfig
class ConfigRepositoryTests(TestCase):
def test_exports_database_configs_to_engine_yaml(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
prefix = Path(tmp)
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home=str(prefix),
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
},
)
HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
config={
"host": "ignored",
"address": "ignored",
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
"includes": [],
"excludes_add": ["/tmp/***"],
},
)
written = export_runtime_configs(prefix=prefix, host="web-01")
self.assertEqual(len(written), 2)
global_cfg = load_global_config(prefix / "config" / "global.yaml")
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
self.assertEqual(global_cfg["backup_root"], "/backups")
self.assertEqual(global_cfg["pobsync_home"], str(prefix))
self.assertEqual(host_cfg["host"], "web-01")
self.assertEqual(host_cfg["address"], "web-01.example.test")

View File

@@ -0,0 +1,24 @@
from __future__ import annotations
from datetime import datetime, timezone
from django.test import SimpleTestCase
from pobsync.retention import Snapshot, build_retention_plan
class RetentionTests(SimpleTestCase):
def test_always_keeps_newest_snapshot(self) -> None:
snapshots = [
Snapshot("scheduled", "old", "/x/old", datetime(2026, 5, 18, tzinfo=timezone.utc), "success", None),
Snapshot("scheduled", "new", "/x/new", datetime(2026, 5, 19, tzinfo=timezone.utc), "failed", None),
]
plan = build_retention_plan(
snapshots,
retention={"daily": 0, "weekly": 0, "monthly": 0, "yearly": 0},
now=datetime(2026, 5, 19, tzinfo=timezone.utc),
)
self.assertEqual(plan.keep, {"new"})
self.assertEqual(plan.reasons["new"], ["newest"])

View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from datetime import datetime
from pathlib import Path
from unittest.mock import patch
from zoneinfo import ZoneInfo
from django.test import SimpleTestCase, TestCase
from pobsync_backend.management.commands.run_pobsync_scheduler import Command
from pobsync_backend.models import HostConfig, ScheduleConfig
from pobsync_backend.scheduler import due_key, is_due
class SchedulerTests(SimpleTestCase):
def test_daily_time_is_due_only_on_matching_minute(self) -> None:
moment = datetime(2026, 5, 19, 2, 15, tzinfo=ZoneInfo("UTC"))
self.assertTrue(is_due("15 2 * * *", moment))
self.assertFalse(is_due("16 2 * * *", moment))
def test_step_values_are_supported(self) -> None:
moment = datetime(2026, 5, 19, 2, 30, tzinfo=ZoneInfo("UTC"))
self.assertTrue(is_due("*/15 * * * *", moment))
self.assertFalse(is_due("*/20 * * * *", moment))
def test_sunday_allows_zero_and_seven(self) -> None:
sunday = datetime(2026, 5, 24, 2, 0, tzinfo=ZoneInfo("UTC"))
self.assertTrue(is_due("0 2 * * 0", sunday))
self.assertTrue(is_due("0 2 * * 7", sunday))
def test_due_key_has_minute_granularity(self) -> None:
moment = datetime(2026, 5, 19, 2, 15, 45, tzinfo=ZoneInfo("UTC"))
self.assertEqual(due_key(moment), "202605190215")
class SchedulerCommandTests(TestCase):
def test_run_due_executes_schedule_once_per_minute(self) -> None:
host = HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
config={
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
},
)
ScheduleConfig.objects.create(host=host, cron_expr="* * * * *")
command = Command()
with patch("pobsync_backend.management.commands.run_pobsync_scheduler.call_command") as call:
first_count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=True)
second_count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=True)
self.assertEqual(first_count, 1)
self.assertEqual(second_count, 0)
self.assertEqual(call.call_count, 1)
schedule = ScheduleConfig.objects.get(host=host)
self.assertEqual(schedule.last_status, "success")