refactor: inject config sources into scheduled backups
Introduce a ConfigSource interface so scheduled backups no longer need to load host configuration directly from runtime YAML. Add a Django-backed config source for SQL-driven backup runs, keep file-based config as the CLI default, and make scheduled prune execution actually apply retention after successful runs.
This commit is contained in:
@@ -3,8 +3,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..config.load import load_global_config, load_host_config
|
||||
from ..config.merge import build_effective_config
|
||||
from ..config.source import ConfigSource, FileConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..lock import acquire_host_lock
|
||||
from ..paths import PobsyncPaths
|
||||
@@ -81,15 +80,21 @@ def _base_meta_from_path(base_dir: Path | None, link_dest: str | None) -> dict[s
|
||||
}
|
||||
|
||||
|
||||
def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, prune_max_delete: int | None = None, prune_protect_bases: bool = False, ) -> dict[str, Any]:
|
||||
def run_scheduled(
|
||||
prefix: Path,
|
||||
host: str,
|
||||
dry_run: bool,
|
||||
prune: bool = False,
|
||||
prune_max_delete: int | None = None,
|
||||
prune_protect_bases: bool = False,
|
||||
config_source: ConfigSource | None = None,
|
||||
) -> dict[str, Any]:
|
||||
|
||||
host = sanitize_host(host)
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
# Load and merge config
|
||||
global_cfg = load_global_config(paths.global_config_path)
|
||||
host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml")
|
||||
cfg = build_effective_config(global_cfg, host_cfg)
|
||||
source = config_source or FileConfigSource(prefix=paths.home)
|
||||
cfg = source.effective_config_for_host(host)
|
||||
|
||||
backup_root = cfg.get("backup_root")
|
||||
if not isinstance(backup_root, str) or not backup_root.startswith("/"):
|
||||
@@ -259,6 +264,20 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
|
||||
final_dir = dirs.scheduled / snap_name
|
||||
incomplete_dir.rename(final_dir)
|
||||
|
||||
prune_result = None
|
||||
if prune:
|
||||
from .retention_apply import run_retention_apply
|
||||
|
||||
prune_result = run_retention_apply(
|
||||
prefix=paths.home,
|
||||
host=host,
|
||||
kind="scheduled",
|
||||
protect_bases=bool(prune_protect_bases),
|
||||
yes=True,
|
||||
max_delete=10 if prune_max_delete is None else int(prune_max_delete),
|
||||
acquire_lock=False,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
@@ -266,5 +285,5 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
|
||||
"snapshot": str(final_dir),
|
||||
"base": str(base_dir) if base_dir else None,
|
||||
"rsync": {"exit_code": result.exit_code},
|
||||
"prune": prune_result,
|
||||
}
|
||||
|
||||
|
||||
22
src/pobsync/config/source.py
Normal file
22
src/pobsync/config/source.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .load import load_global_config, load_host_config
|
||||
from .merge import build_effective_config
|
||||
|
||||
|
||||
class ConfigSource(Protocol):
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
"""Return the fully merged effective config for a host."""
|
||||
|
||||
|
||||
class FileConfigSource:
|
||||
def __init__(self, prefix: Path) -> None:
|
||||
self.prefix = prefix
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
global_cfg = load_global_config(self.prefix / "config" / "global.yaml")
|
||||
host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml")
|
||||
return build_effective_config(global_cfg, host_cfg)
|
||||
@@ -35,6 +35,6 @@ class SnapshotRecordAdmin(admin.ModelAdmin):
|
||||
|
||||
@admin.register(ScheduleConfig)
|
||||
class ScheduleConfigAdmin(admin.ModelAdmin):
|
||||
list_display = ("host", "cron_expr", "enabled", "prune", "updated_at")
|
||||
list_filter = ("enabled", "prune")
|
||||
list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at")
|
||||
list_filter = ("enabled", "prune", "last_status")
|
||||
search_fields = ("host__host", "cron_expr")
|
||||
|
||||
@@ -31,6 +31,22 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
return validate_dict(data, HOST_SCHEMA, path="host")
|
||||
|
||||
|
||||
def global_config_data(name: str = "default") -> dict[str, Any]:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
|
||||
return _global_yaml_data(global_config)
|
||||
|
||||
|
||||
def host_config_data(host: str) -> dict[str, Any]:
|
||||
try:
|
||||
host_config = HostConfig.objects.get(host=host, enabled=True)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
|
||||
return _host_yaml_data(host_config)
|
||||
|
||||
|
||||
def export_global_config(prefix: Path, name: str = "default") -> Path:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
|
||||
12
src/pobsync_backend/config_source.py
Normal file
12
src/pobsync_backend/config_source.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pobsync.config.merge import build_effective_config
|
||||
|
||||
from .config_repository import global_config_data, host_config_data
|
||||
|
||||
|
||||
class DjangoConfigSource:
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
return build_effective_config(global_config_data(), host_config_data(host))
|
||||
@@ -10,6 +10,7 @@ from django.utils import timezone
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.paths import PobsyncPaths
|
||||
from pobsync_backend.config_repository import export_runtime_configs
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import BackupRun, HostConfig
|
||||
|
||||
|
||||
@@ -32,7 +33,10 @@ class Command(BaseCommand):
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
|
||||
|
||||
export_runtime_configs(prefix=paths.home, host=host.host)
|
||||
# Compatibility bridge: retention planning still reads runtime YAML.
|
||||
# The backup run itself receives config directly from Django.
|
||||
if options["prune"]:
|
||||
export_runtime_configs(prefix=paths.home, host=host.host)
|
||||
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
@@ -49,6 +53,7 @@ class Command(BaseCommand):
|
||||
prune=bool(options["prune"]),
|
||||
prune_max_delete=int(options["prune_max_delete"]),
|
||||
prune_protect_bases=bool(options["prune_protect_bases"]),
|
||||
config_source=DjangoConfigSource(),
|
||||
)
|
||||
except Exception as exc:
|
||||
run.status = BackupRun.Status.FAILED
|
||||
|
||||
48
src/pobsync_backend/tests/test_django_config_source.py
Normal file
48
src/pobsync_backend/tests/test_django_config_source.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import GlobalConfig, HostConfig
|
||||
|
||||
|
||||
class DjangoConfigSourceTests(TestCase):
|
||||
def test_returns_effective_config_from_database(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
data={
|
||||
"backup_root": "/ignored",
|
||||
"pobsync_home": "/ignored",
|
||||
"ssh": {"user": "root", "port": 22, "options": []},
|
||||
"rsync": {
|
||||
"binary": "rsync",
|
||||
"args": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
"extra_args": ["--numeric-ids"],
|
||||
},
|
||||
"defaults": {"source_root": "/", "destination_subdir": ""},
|
||||
"excludes_default": ["/proc/***"],
|
||||
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
},
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
config={
|
||||
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
"excludes_add": ["/tmp/***"],
|
||||
"includes": [],
|
||||
"rsync": {"extra_args": ["--delete"]},
|
||||
},
|
||||
)
|
||||
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertEqual(cfg["backup_root"], "/backups")
|
||||
self.assertEqual(cfg["host"], "web-01")
|
||||
self.assertEqual(cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"])
|
||||
self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"])
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.rsync import RsyncResult
|
||||
|
||||
|
||||
class FakeConfigSource:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None:
|
||||
self.backup_root = backup_root
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict:
|
||||
return {
|
||||
"backup_root": self.backup_root,
|
||||
"host": host,
|
||||
"address": "example.test",
|
||||
"ssh": {"user": "root", "port": 22, "options": []},
|
||||
"rsync": {
|
||||
"binary": "rsync",
|
||||
"args_effective": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
},
|
||||
"source_root": "/",
|
||||
"includes": [],
|
||||
"excludes_effective": [],
|
||||
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
}
|
||||
|
||||
|
||||
class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
def test_dry_run_uses_injected_config_source(self) -> None:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
dry_run=True,
|
||||
config_source=FakeConfigSource(),
|
||||
)
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["host"], "web-01")
|
||||
run_rsync.assert_called_once()
|
||||
|
||||
def test_successful_real_run_applies_prune_when_requested(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
with (
|
||||
patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync,
|
||||
patch("pobsync.commands.retention_apply.run_retention_plan") as plan,
|
||||
):
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
|
||||
plan.return_value = {
|
||||
"ok": True,
|
||||
"delete": [],
|
||||
"keep": [],
|
||||
"reasons": {},
|
||||
"protect_bases": False,
|
||||
}
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=prefix,
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
prune=True,
|
||||
prune_max_delete=10,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
)
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertIsNotNone(result["prune"])
|
||||
self.assertEqual(result["prune"]["deleted"], [])
|
||||
Reference in New Issue
Block a user