diff --git a/README.md b/README.md index 7587ac1..aefea24 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,18 @@ Run a backup through Django while still using the existing pobsync engine: python3 manage.py run_pobsync_backup --prefix /opt/pobsync --prune ``` +Export database configs to the runtime YAML files consumed by the current engine: + +``` +python3 manage.py export_pobsync_configs --prefix /opt/pobsync +``` + +Run due schedules from the database: + +``` +python3 manage.py run_pobsync_scheduler --loop --interval 60 +``` + ### Docker with SQLite ``` @@ -157,12 +169,24 @@ This starts Django on: The container persists `/opt/pobsync` and the SQLite database in Docker volumes. +Run the Django scheduler alongside the web admin: + +``` +docker compose up --build web scheduler +``` + ### Docker with MariaDB ``` docker compose --profile mariadb up --build web-mariadb ``` +With the scheduler: + +``` +docker compose --profile mariadb up --build web-mariadb scheduler-mariadb +``` + The MariaDB profile is optional. SQLite remains the default because it is enough for a single backup server and keeps deployment simple. ### Refactor direction @@ -171,5 +195,7 @@ Recommended next steps: - Move config reading/writing behind a repository interface that can use YAML or Django models. - Record `run-scheduled` results into `BackupRun`. +- Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine. +- Run schedules from Django/Docker instead of writing host cron files. - Add a snapshot discovery command that syncs existing snapshot metadata into `SnapshotRecord`. - Add tests around retention, scheduling, and config merge before deeper internal reshaping. diff --git a/docker-compose.yml b/docker-compose.yml index dd764af..546e599 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,19 @@ services: - pobsync_state:/opt/pobsync - pobsync_db:/var/lib/pobsync + scheduler: + build: . + command: python manage.py run_pobsync_scheduler --loop --interval 60 + environment: + POBSYNC_DJANGO_DEBUG: "1" + POBSYNC_DJANGO_SECRET_KEY: "dev-only-change-me" + POBSYNC_DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,0.0.0.0" + POBSYNC_HOME: "/opt/pobsync" + POBSYNC_SQLITE_PATH: "/var/lib/pobsync/pobsync.sqlite3" + volumes: + - pobsync_state:/opt/pobsync + - pobsync_db:/var/lib/pobsync + web-mariadb: profiles: ["mariadb"] build: . @@ -36,6 +49,26 @@ services: volumes: - pobsync_state:/opt/pobsync + scheduler-mariadb: + profiles: ["mariadb"] + build: . + command: python manage.py run_pobsync_scheduler --loop --interval 60 + environment: + POBSYNC_DJANGO_DEBUG: "1" + POBSYNC_DJANGO_SECRET_KEY: "dev-only-change-me" + POBSYNC_DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,0.0.0.0" + POBSYNC_HOME: "/opt/pobsync" + POBSYNC_DB_ENGINE: "mariadb" + POBSYNC_DB_HOST: "db" + POBSYNC_DB_NAME: "pobsync" + POBSYNC_DB_USER: "pobsync" + POBSYNC_DB_PASSWORD: "pobsync" + depends_on: + db: + condition: service_healthy + volumes: + - pobsync_state:/opt/pobsync + db: profiles: ["mariadb"] image: mariadb:11 diff --git a/src/pobsync_backend/config_repository.py b/src/pobsync_backend/config_repository.py new file mode 100644 index 0000000..626cce4 --- /dev/null +++ b/src/pobsync_backend/config_repository.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from django.core.exceptions import ObjectDoesNotExist + +from pobsync.config.schemas import GLOBAL_SCHEMA, HOST_SCHEMA +from pobsync.paths import PobsyncPaths +from pobsync.util import write_yaml_atomic +from pobsync.validate import validate_dict + +from .models import GlobalConfig, HostConfig + + +class ConfigRepositoryError(RuntimeError): + pass + + +def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]: + data = dict(global_config.data or {}) + data["backup_root"] = global_config.backup_root + data["pobsync_home"] = global_config.pobsync_home + return validate_dict(data, GLOBAL_SCHEMA, path="global") + + +def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]: + data = dict(host_config.config or {}) + data["host"] = host_config.host + data["address"] = host_config.address + return validate_dict(data, HOST_SCHEMA, path="host") + + +def export_global_config(prefix: Path, name: str = "default") -> Path: + try: + global_config = GlobalConfig.objects.get(name=name) + except ObjectDoesNotExist as exc: + raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc + + paths = PobsyncPaths(home=prefix) + write_yaml_atomic(paths.global_config_path, _global_yaml_data(global_config)) + return paths.global_config_path + + +def export_host_config(prefix: Path, host: str) -> Path: + try: + host_config = HostConfig.objects.get(host=host, enabled=True) + except ObjectDoesNotExist as exc: + raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc + + paths = PobsyncPaths(home=prefix) + target = paths.hosts_dir / f"{host_config.host}.yaml" + write_yaml_atomic(target, _host_yaml_data(host_config)) + return target + + +def export_runtime_configs(prefix: Path, host: str | None = None) -> list[Path]: + written = [export_global_config(prefix)] + hosts = HostConfig.objects.filter(enabled=True).order_by("host") + if host is not None: + hosts = hosts.filter(host=host) + for host_config in hosts: + written.append(export_host_config(prefix, host_config.host)) + return written diff --git a/src/pobsync_backend/management/commands/export_pobsync_configs.py b/src/pobsync_backend/management/commands/export_pobsync_configs.py new file mode 100644 index 0000000..76317b2 --- /dev/null +++ b/src/pobsync_backend/management/commands/export_pobsync_configs.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from django.conf import settings +from django.core.management.base import BaseCommand + +from pobsync_backend.config_repository import export_runtime_configs + + +class Command(BaseCommand): + help = "Export Django database configs to pobsync runtime YAML files." + + def add_arguments(self, parser) -> None: + parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory") + parser.add_argument("--host", default=None, help="Export only one enabled host") + + def handle(self, *args: Any, **options: Any) -> None: + written = export_runtime_configs(prefix=Path(options["prefix"]), host=options["host"]) + for path in written: + self.stdout.write(str(path)) + self.stdout.write(self.style.SUCCESS(f"Exported {len(written)} config file(s).")) diff --git a/src/pobsync_backend/management/commands/run_pobsync_backup.py b/src/pobsync_backend/management/commands/run_pobsync_backup.py index 4ce19d9..4234458 100644 --- a/src/pobsync_backend/management/commands/run_pobsync_backup.py +++ b/src/pobsync_backend/management/commands/run_pobsync_backup.py @@ -8,8 +8,8 @@ from django.core.management.base import BaseCommand, CommandError from django.utils import timezone from pobsync.commands.run_scheduled import run_scheduled -from pobsync.config.load import load_host_config from pobsync.paths import PobsyncPaths +from pobsync_backend.config_repository import export_runtime_configs from pobsync_backend.models import BackupRun, HostConfig @@ -27,19 +27,12 @@ class Command(BaseCommand): def handle(self, *args: Any, **options: Any) -> None: host_name = options["host"] paths = PobsyncPaths(home=Path(options["prefix"])) - host_path = paths.hosts_dir / f"{host_name}.yaml" - if not host_path.exists(): - raise CommandError(f"Missing host config: {host_path}") + try: + host = HostConfig.objects.get(host=host_name, enabled=True) + except HostConfig.DoesNotExist as exc: + raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc - host_cfg = load_host_config(host_path) - host, _created = HostConfig.objects.update_or_create( - host=host_cfg["host"], - defaults={ - "address": host_cfg["address"], - "config": host_cfg, - "enabled": True, - }, - ) + export_runtime_configs(prefix=paths.home, host=host.host) run = BackupRun.objects.create( host=host, diff --git a/src/pobsync_backend/management/commands/run_pobsync_scheduler.py b/src/pobsync_backend/management/commands/run_pobsync_scheduler.py new file mode 100644 index 0000000..7443ad7 --- /dev/null +++ b/src/pobsync_backend/management/commands/run_pobsync_scheduler.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.db import transaction +from django.utils import timezone + +from pobsync_backend.models import ScheduleConfig +from pobsync_backend.scheduler import due_key, is_due + + +class Command(BaseCommand): + help = "Run due pobsync schedules from the Django database." + + def add_arguments(self, parser) -> None: + parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory") + parser.add_argument("--once", action="store_true", help="Check once and exit") + parser.add_argument("--loop", action="store_true", help="Keep checking schedules") + parser.add_argument("--interval", type=int, default=60, help="Loop interval in seconds") + parser.add_argument("--dry-run", action="store_true", help="Pass --dry-run to backup runs") + + def handle(self, *args: Any, **options: Any) -> None: + if not options["once"] and not options["loop"]: + options["once"] = True + + prefix = Path(options["prefix"]) + while True: + count = self._run_due(prefix=prefix, dry_run=bool(options["dry_run"])) + self.stdout.write(f"Ran {count} due schedule(s).") + if options["once"]: + return + time.sleep(max(1, int(options["interval"]))) + + def _run_due(self, *, prefix: Path, dry_run: bool) -> int: + now = timezone.localtime(timezone.now()) + current_due_key = due_key(now) + ran = 0 + + schedules = ( + ScheduleConfig.objects.select_related("host") + .filter(enabled=True, host__enabled=True) + .order_by("host__host") + ) + for schedule in schedules: + if schedule.last_due_key == current_due_key: + continue + if not is_due(schedule.cron_expr, now): + continue + + with transaction.atomic(): + locked = ScheduleConfig.objects.select_for_update().get(pk=schedule.pk) + if locked.last_due_key == current_due_key: + continue + locked.last_due_key = current_due_key + locked.last_started_at = timezone.now() + locked.last_status = "running" + locked.save(update_fields=["last_due_key", "last_started_at", "last_status", "updated_at"]) + + status = "success" + try: + call_command( + "run_pobsync_backup", + schedule.host.host, + prefix=str(prefix), + dry_run=dry_run, + prune=schedule.prune, + prune_max_delete=schedule.prune_max_delete, + prune_protect_bases=schedule.prune_protect_bases, + ) + except Exception as exc: + status = "failed" + self.stderr.write(f"{schedule.host.host}: {type(exc).__name__}: {exc}") + finally: + ScheduleConfig.objects.filter(pk=schedule.pk).update( + last_finished_at=timezone.now(), + last_status=status, + ) + ran += 1 + + return ran diff --git a/src/pobsync_backend/migrations/0002_schedule_run_state.py b/src/pobsync_backend/migrations/0002_schedule_run_state.py new file mode 100644 index 0000000..c7756e0 --- /dev/null +++ b/src/pobsync_backend/migrations/0002_schedule_run_state.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pobsync_backend", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="scheduleconfig", + name="last_due_key", + field=models.CharField(blank=True, max_length=32), + ), + migrations.AddField( + model_name="scheduleconfig", + name="last_finished_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="scheduleconfig", + name="last_started_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="scheduleconfig", + name="last_status", + field=models.CharField(blank=True, max_length=16), + ), + ] diff --git a/src/pobsync_backend/models.py b/src/pobsync_backend/models.py index a2a8308..42d90db 100644 --- a/src/pobsync_backend/models.py +++ b/src/pobsync_backend/models.py @@ -102,6 +102,10 @@ class ScheduleConfig(TimestampedModel): prune = models.BooleanField(default=False) prune_max_delete = models.PositiveIntegerField(default=10) prune_protect_bases = models.BooleanField(default=False) + last_due_key = models.CharField(max_length=32, blank=True) + last_started_at = models.DateTimeField(null=True, blank=True) + last_finished_at = models.DateTimeField(null=True, blank=True) + last_status = models.CharField(max_length=16, blank=True) class Meta: ordering = ["host__host"] diff --git a/src/pobsync_backend/scheduler.py b/src/pobsync_backend/scheduler.py new file mode 100644 index 0000000..95a4f40 --- /dev/null +++ b/src/pobsync_backend/scheduler.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class CronSchedule: + minute: str + hour: str + day_of_month: str + month: str + day_of_week: str + + +def parse_cron_expr(expr: str) -> CronSchedule: + parts = expr.strip().split() + if len(parts) != 5: + raise ValueError("cron expression must have exactly 5 fields") + return CronSchedule(*parts) + + +def due_key(moment: datetime) -> str: + return moment.strftime("%Y%m%d%H%M") + + +def is_due(expr: str, moment: datetime) -> bool: + schedule = parse_cron_expr(expr) + cron_dow = (moment.weekday() + 1) % 7 + return ( + _field_matches(schedule.minute, moment.minute, 0, 59) + and _field_matches(schedule.hour, moment.hour, 0, 23) + and _field_matches(schedule.day_of_month, moment.day, 1, 31) + and _field_matches(schedule.month, moment.month, 1, 12) + and _field_matches(schedule.day_of_week, cron_dow, 0, 7, sunday_alias=True) + ) + + +def _field_matches(field: str, value: int, min_value: int, max_value: int, sunday_alias: bool = False) -> bool: + for part in field.split(","): + if _part_matches(part.strip(), value, min_value, max_value, sunday_alias=sunday_alias): + return True + return False + + +def _part_matches(part: str, value: int, min_value: int, max_value: int, sunday_alias: bool) -> bool: + if not part: + return False + + step = 1 + base = part + if "/" in part: + base, step_s = part.split("/", 1) + if not step_s.isdigit(): + return False + step = int(step_s) + if step <= 0: + return False + + if base == "*": + start = min_value + end = max_value + elif "-" in base: + start_s, end_s = base.split("-", 1) + if not start_s.isdigit() or not end_s.isdigit(): + return False + start = int(start_s) + end = int(end_s) + elif base.isdigit(): + start = end = int(base) + else: + return False + + values = _normalize_values(range(start, end + 1), sunday_alias=sunday_alias) + normalized_value = 0 if sunday_alias and value == 7 else value + if normalized_value not in values: + return False + return (normalized_value - min(values)) % step == 0 + + +def _normalize_values(values: range, sunday_alias: bool) -> set[int]: + out: set[int] = set() + for value in values: + if sunday_alias and value == 7: + out.add(0) + else: + out.add(value) + return out diff --git a/src/pobsync_backend/tests/__init__.py b/src/pobsync_backend/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/pobsync_backend/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/src/pobsync_backend/tests/test_config_repository.py b/src/pobsync_backend/tests/test_config_repository.py new file mode 100644 index 0000000..cccedf8 --- /dev/null +++ b/src/pobsync_backend/tests/test_config_repository.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import tempfile +from pathlib import Path + +from django.test import TestCase + +from pobsync.config.load import load_global_config, load_host_config +from pobsync_backend.config_repository import export_runtime_configs +from pobsync_backend.models import GlobalConfig, HostConfig + + +class ConfigRepositoryTests(TestCase): + def test_exports_database_configs_to_engine_yaml(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + prefix = Path(tmp) + GlobalConfig.objects.create( + name="default", + backup_root="/backups", + pobsync_home=str(prefix), + data={ + "backup_root": "/ignored", + "pobsync_home": "/ignored", + "retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + }, + ) + HostConfig.objects.create( + host="web-01", + address="web-01.example.test", + config={ + "host": "ignored", + "address": "ignored", + "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + "includes": [], + "excludes_add": ["/tmp/***"], + }, + ) + + written = export_runtime_configs(prefix=prefix, host="web-01") + + self.assertEqual(len(written), 2) + global_cfg = load_global_config(prefix / "config" / "global.yaml") + host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml") + self.assertEqual(global_cfg["backup_root"], "/backups") + self.assertEqual(global_cfg["pobsync_home"], str(prefix)) + self.assertEqual(host_cfg["host"], "web-01") + self.assertEqual(host_cfg["address"], "web-01.example.test") diff --git a/src/pobsync_backend/tests/test_retention.py b/src/pobsync_backend/tests/test_retention.py new file mode 100644 index 0000000..fe03815 --- /dev/null +++ b/src/pobsync_backend/tests/test_retention.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from django.test import SimpleTestCase + +from pobsync.retention import Snapshot, build_retention_plan + + +class RetentionTests(SimpleTestCase): + def test_always_keeps_newest_snapshot(self) -> None: + snapshots = [ + Snapshot("scheduled", "old", "/x/old", datetime(2026, 5, 18, tzinfo=timezone.utc), "success", None), + Snapshot("scheduled", "new", "/x/new", datetime(2026, 5, 19, tzinfo=timezone.utc), "failed", None), + ] + + plan = build_retention_plan( + snapshots, + retention={"daily": 0, "weekly": 0, "monthly": 0, "yearly": 0}, + now=datetime(2026, 5, 19, tzinfo=timezone.utc), + ) + + self.assertEqual(plan.keep, {"new"}) + self.assertEqual(plan.reasons["new"], ["newest"]) diff --git a/src/pobsync_backend/tests/test_scheduler.py b/src/pobsync_backend/tests/test_scheduler.py new file mode 100644 index 0000000..f5f2815 --- /dev/null +++ b/src/pobsync_backend/tests/test_scheduler.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from unittest.mock import patch +from zoneinfo import ZoneInfo + +from django.test import SimpleTestCase, TestCase + +from pobsync_backend.management.commands.run_pobsync_scheduler import Command +from pobsync_backend.models import HostConfig, ScheduleConfig +from pobsync_backend.scheduler import due_key, is_due + + +class SchedulerTests(SimpleTestCase): + def test_daily_time_is_due_only_on_matching_minute(self) -> None: + moment = datetime(2026, 5, 19, 2, 15, tzinfo=ZoneInfo("UTC")) + + self.assertTrue(is_due("15 2 * * *", moment)) + self.assertFalse(is_due("16 2 * * *", moment)) + + def test_step_values_are_supported(self) -> None: + moment = datetime(2026, 5, 19, 2, 30, tzinfo=ZoneInfo("UTC")) + + self.assertTrue(is_due("*/15 * * * *", moment)) + self.assertFalse(is_due("*/20 * * * *", moment)) + + def test_sunday_allows_zero_and_seven(self) -> None: + sunday = datetime(2026, 5, 24, 2, 0, tzinfo=ZoneInfo("UTC")) + + self.assertTrue(is_due("0 2 * * 0", sunday)) + self.assertTrue(is_due("0 2 * * 7", sunday)) + + def test_due_key_has_minute_granularity(self) -> None: + moment = datetime(2026, 5, 19, 2, 15, 45, tzinfo=ZoneInfo("UTC")) + + self.assertEqual(due_key(moment), "202605190215") + + +class SchedulerCommandTests(TestCase): + def test_run_due_executes_schedule_once_per_minute(self) -> None: + host = HostConfig.objects.create( + host="web-01", + address="web-01.example.test", + config={ + "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + }, + ) + ScheduleConfig.objects.create(host=host, cron_expr="* * * * *") + + command = Command() + with patch("pobsync_backend.management.commands.run_pobsync_scheduler.call_command") as call: + first_count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=True) + second_count = command._run_due(prefix=Path("/opt/pobsync"), dry_run=True) + + self.assertEqual(first_count, 1) + self.assertEqual(second_count, 0) + self.assertEqual(call.call_count, 1) + schedule = ScheduleConfig.objects.get(host=host) + self.assertEqual(schedule.last_status, "success")