diff --git a/README.md b/README.md index aefea24..913fc8e 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,8 @@ Run a backup through Django while still using the existing pobsync engine: python3 manage.py run_pobsync_backup --prefix /opt/pobsync --prune ``` +The Django backup command reads config from SQL directly. Runtime YAML export is kept as a compatibility tool for older CLI flows and retention planning during the transition. + Export database configs to the runtime YAML files consumed by the current engine: ``` @@ -193,8 +195,8 @@ The MariaDB profile is optional. SQLite remains the default because it is enough Recommended next steps: -- Move config reading/writing behind a repository interface that can use YAML or Django models. -- Record `run-scheduled` results into `BackupRun`. +- Continue moving config reading/writing behind repository interfaces so YAML export can eventually disappear. +- Record more engine-side run details into `BackupRun` and `SnapshotRecord`. - Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine. - Run schedules from Django/Docker instead of writing host cron files. - Add a snapshot discovery command that syncs existing snapshot metadata into `SnapshotRecord`. diff --git a/src/pobsync/commands/run_scheduled.py b/src/pobsync/commands/run_scheduled.py index e5a2b16..6bc8cb5 100644 --- a/src/pobsync/commands/run_scheduled.py +++ b/src/pobsync/commands/run_scheduled.py @@ -3,8 +3,7 @@ from __future__ import annotations from pathlib import Path from typing import Any -from ..config.load import load_global_config, load_host_config -from ..config.merge import build_effective_config +from ..config.source import ConfigSource, FileConfigSource from ..errors import ConfigError from ..lock import acquire_host_lock from ..paths import PobsyncPaths @@ -81,15 +80,21 @@ def _base_meta_from_path(base_dir: Path | None, link_dest: str | None) -> dict[s } -def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, prune_max_delete: int | None = None, prune_protect_bases: bool = False, ) -> dict[str, Any]: +def run_scheduled( + prefix: Path, + host: str, + dry_run: bool, + prune: bool = False, + prune_max_delete: int | None = None, + prune_protect_bases: bool = False, + config_source: ConfigSource | None = None, +) -> dict[str, Any]: host = sanitize_host(host) paths = PobsyncPaths(home=prefix) - # Load and merge config - global_cfg = load_global_config(paths.global_config_path) - host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml") - cfg = build_effective_config(global_cfg, host_cfg) + source = config_source or FileConfigSource(prefix=paths.home) + cfg = source.effective_config_for_host(host) backup_root = cfg.get("backup_root") if not isinstance(backup_root, str) or not backup_root.startswith("/"): @@ -259,6 +264,20 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p final_dir = dirs.scheduled / snap_name incomplete_dir.rename(final_dir) + prune_result = None + if prune: + from .retention_apply import run_retention_apply + + prune_result = run_retention_apply( + prefix=paths.home, + host=host, + kind="scheduled", + protect_bases=bool(prune_protect_bases), + yes=True, + max_delete=10 if prune_max_delete is None else int(prune_max_delete), + acquire_lock=False, + ) + return { "ok": True, "dry_run": False, @@ -266,5 +285,5 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p "snapshot": str(final_dir), "base": str(base_dir) if base_dir else None, "rsync": {"exit_code": result.exit_code}, + "prune": prune_result, } - diff --git a/src/pobsync/config/source.py b/src/pobsync/config/source.py new file mode 100644 index 0000000..bcd6ee6 --- /dev/null +++ b/src/pobsync/config/source.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Protocol + +from .load import load_global_config, load_host_config +from .merge import build_effective_config + + +class ConfigSource(Protocol): + def effective_config_for_host(self, host: str) -> dict[str, Any]: + """Return the fully merged effective config for a host.""" + + +class FileConfigSource: + def __init__(self, prefix: Path) -> None: + self.prefix = prefix + + def effective_config_for_host(self, host: str) -> dict[str, Any]: + global_cfg = load_global_config(self.prefix / "config" / "global.yaml") + host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml") + return build_effective_config(global_cfg, host_cfg) diff --git a/src/pobsync_backend/admin.py b/src/pobsync_backend/admin.py index ee4c7fc..4f562fb 100644 --- a/src/pobsync_backend/admin.py +++ b/src/pobsync_backend/admin.py @@ -35,6 +35,6 @@ class SnapshotRecordAdmin(admin.ModelAdmin): @admin.register(ScheduleConfig) class ScheduleConfigAdmin(admin.ModelAdmin): - list_display = ("host", "cron_expr", "enabled", "prune", "updated_at") - list_filter = ("enabled", "prune") + list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at") + list_filter = ("enabled", "prune", "last_status") search_fields = ("host__host", "cron_expr") diff --git a/src/pobsync_backend/config_repository.py b/src/pobsync_backend/config_repository.py index 626cce4..6d6a98e 100644 --- a/src/pobsync_backend/config_repository.py +++ b/src/pobsync_backend/config_repository.py @@ -31,6 +31,22 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]: return validate_dict(data, HOST_SCHEMA, path="host") +def global_config_data(name: str = "default") -> dict[str, Any]: + try: + global_config = GlobalConfig.objects.get(name=name) + except ObjectDoesNotExist as exc: + raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc + return _global_yaml_data(global_config) + + +def host_config_data(host: str) -> dict[str, Any]: + try: + host_config = HostConfig.objects.get(host=host, enabled=True) + except ObjectDoesNotExist as exc: + raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc + return _host_yaml_data(host_config) + + def export_global_config(prefix: Path, name: str = "default") -> Path: try: global_config = GlobalConfig.objects.get(name=name) diff --git a/src/pobsync_backend/config_source.py b/src/pobsync_backend/config_source.py new file mode 100644 index 0000000..a80f070 --- /dev/null +++ b/src/pobsync_backend/config_source.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from typing import Any + +from pobsync.config.merge import build_effective_config + +from .config_repository import global_config_data, host_config_data + + +class DjangoConfigSource: + def effective_config_for_host(self, host: str) -> dict[str, Any]: + return build_effective_config(global_config_data(), host_config_data(host)) diff --git a/src/pobsync_backend/management/commands/run_pobsync_backup.py b/src/pobsync_backend/management/commands/run_pobsync_backup.py index 4234458..d5e6266 100644 --- a/src/pobsync_backend/management/commands/run_pobsync_backup.py +++ b/src/pobsync_backend/management/commands/run_pobsync_backup.py @@ -10,6 +10,7 @@ from django.utils import timezone from pobsync.commands.run_scheduled import run_scheduled from pobsync.paths import PobsyncPaths from pobsync_backend.config_repository import export_runtime_configs +from pobsync_backend.config_source import DjangoConfigSource from pobsync_backend.models import BackupRun, HostConfig @@ -32,7 +33,10 @@ class Command(BaseCommand): except HostConfig.DoesNotExist as exc: raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc - export_runtime_configs(prefix=paths.home, host=host.host) + # Compatibility bridge: retention planning still reads runtime YAML. + # The backup run itself receives config directly from Django. + if options["prune"]: + export_runtime_configs(prefix=paths.home, host=host.host) run = BackupRun.objects.create( host=host, @@ -49,6 +53,7 @@ class Command(BaseCommand): prune=bool(options["prune"]), prune_max_delete=int(options["prune_max_delete"]), prune_protect_bases=bool(options["prune_protect_bases"]), + config_source=DjangoConfigSource(), ) except Exception as exc: run.status = BackupRun.Status.FAILED diff --git a/src/pobsync_backend/tests/test_django_config_source.py b/src/pobsync_backend/tests/test_django_config_source.py new file mode 100644 index 0000000..e9ca1ad --- /dev/null +++ b/src/pobsync_backend/tests/test_django_config_source.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from django.test import TestCase + +from pobsync_backend.config_source import DjangoConfigSource +from pobsync_backend.models import GlobalConfig, HostConfig + + +class DjangoConfigSourceTests(TestCase): + def test_returns_effective_config_from_database(self) -> None: + GlobalConfig.objects.create( + name="default", + backup_root="/backups", + pobsync_home="/opt/pobsync", + data={ + "backup_root": "/ignored", + "pobsync_home": "/ignored", + "ssh": {"user": "root", "port": 22, "options": []}, + "rsync": { + "binary": "rsync", + "args": ["--archive"], + "timeout_seconds": 0, + "bwlimit_kbps": 0, + "extra_args": ["--numeric-ids"], + }, + "defaults": {"source_root": "/", "destination_subdir": ""}, + "excludes_default": ["/proc/***"], + "retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + }, + ) + HostConfig.objects.create( + host="web-01", + address="web-01.example.test", + config={ + "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + "excludes_add": ["/tmp/***"], + "includes": [], + "rsync": {"extra_args": ["--delete"]}, + }, + ) + + cfg = DjangoConfigSource().effective_config_for_host("web-01") + + self.assertEqual(cfg["backup_root"], "/backups") + self.assertEqual(cfg["host"], "web-01") + self.assertEqual(cfg["address"], "web-01.example.test") + self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"]) + self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"]) diff --git a/src/pobsync_backend/tests/test_run_scheduled_config_source.py b/src/pobsync_backend/tests/test_run_scheduled_config_source.py new file mode 100644 index 0000000..790bf5a --- /dev/null +++ b/src/pobsync_backend/tests/test_run_scheduled_config_source.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from django.test import SimpleTestCase + +from pobsync.commands.run_scheduled import run_scheduled +from pobsync.rsync import RsyncResult + + +class FakeConfigSource: + def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None: + self.backup_root = backup_root + + def effective_config_for_host(self, host: str) -> dict: + return { + "backup_root": self.backup_root, + "host": host, + "address": "example.test", + "ssh": {"user": "root", "port": 22, "options": []}, + "rsync": { + "binary": "rsync", + "args_effective": ["--archive"], + "timeout_seconds": 0, + "bwlimit_kbps": 0, + }, + "source_root": "/", + "includes": [], + "excludes_effective": [], + "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + } + + +class RunScheduledConfigSourceTests(SimpleTestCase): + def test_dry_run_uses_injected_config_source(self) -> None: + with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync: + run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"]) + + result = run_scheduled( + prefix=Path("/missing-prefix"), + host="web-01", + dry_run=True, + config_source=FakeConfigSource(), + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["host"], "web-01") + run_rsync.assert_called_once() + + def test_successful_real_run_applies_prune_when_requested(self) -> None: + with TemporaryDirectory() as tmp: + prefix = Path(tmp) / "home" + with ( + patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync, + patch("pobsync.commands.retention_apply.run_retention_plan") as plan, + ): + run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"]) + plan.return_value = { + "ok": True, + "delete": [], + "keep": [], + "reasons": {}, + "protect_bases": False, + } + + result = run_scheduled( + prefix=prefix, + host="web-01", + dry_run=False, + prune=True, + prune_max_delete=10, + config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")), + ) + + self.assertTrue(result["ok"]) + self.assertIsNotNone(result["prune"]) + self.assertEqual(result["prune"]["deleted"], [])