3 Commits

Author SHA1 Message Date
a0eb5dcc8f refactor: promote backup configuration to structured SQL fields
Add explicit Django model fields for global and host backup settings,
including SSH, rsync, source, excludes, and retention configuration.
Populate them from legacy JSON during migration, make the config
repository prefer structured fields, and update import/admin/tests around
the SQL-first configuration model.
2026-05-19 05:04:49 +02:00
100215bf11 refactor: use injected config sources for retention
Allow retention planning and pruning to use the same ConfigSource
abstraction as scheduled backups. This removes the remaining SQL-to-YAML
export dependency from Django backup runs with pruning, keeping YAML only
as a legacy CLI compatibility path.
2026-05-19 05:00:15 +02:00
bb44f8a09c refactor: inject config sources into scheduled backups
Introduce a ConfigSource interface so scheduled backups no longer need
to load host configuration directly from runtime YAML. Add a Django-backed
config source for SQL-driven backup runs, keep file-based config as the
CLI default, and make scheduled prune execution actually apply retention
after successful runs.
2026-05-19 04:57:10 +02:00
16 changed files with 654 additions and 30 deletions

View File

@@ -115,7 +115,7 @@ For production use, always use the canonical entrypoint:
## Django backend (early refactor layer) ## Django backend (early refactor layer)
The Django backend is a management layer around the existing pobsync engine. The current CLI remains the source of truth for executing backups; Django stores configs, schedules, backup runs, and snapshot metadata so the project can grow toward a web/admin/API surface without rewriting rsync behavior in one risky step. The Django backend is becoming the management layer and source of truth for pobsync. Structured SQL fields store backup, SSH, rsync, retention, schedule, run, and snapshot state; legacy JSON/YAML remains only as an import/export compatibility path while the engine is being refactored.
### Local SQLite development ### Local SQLite development
@@ -145,6 +145,8 @@ Run a backup through Django while still using the existing pobsync engine:
python3 manage.py run_pobsync_backup <host> --prefix /opt/pobsync --prune python3 manage.py run_pobsync_backup <host> --prefix /opt/pobsync --prune
``` ```
The Django backup command reads backup and retention config from SQL directly. Runtime YAML export is kept as a compatibility tool for older CLI flows during the transition.
Export database configs to the runtime YAML files consumed by the current engine: Export database configs to the runtime YAML files consumed by the current engine:
``` ```
@@ -193,8 +195,8 @@ The MariaDB profile is optional. SQLite remains the default because it is enough
Recommended next steps: Recommended next steps:
- Move config reading/writing behind a repository interface that can use YAML or Django models. - Continue moving config reading/writing behind repository interfaces so YAML export can eventually disappear.
- Record `run-scheduled` results into `BackupRun`. - Record more engine-side run details into `BackupRun` and `SnapshotRecord`.
- Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine. - Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine.
- Run schedules from Django/Docker instead of writing host cron files. - Run schedules from Django/Docker instead of writing host cron files.
- Add a snapshot discovery command that syncs existing snapshot metadata into `SnapshotRecord`. - Add a snapshot discovery command that syncs existing snapshot metadata into `SnapshotRecord`.

View File

@@ -4,6 +4,7 @@ import shutil
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List from typing import Any, Dict, List
from ..config.source import ConfigSource
from ..errors import ConfigError from ..errors import ConfigError
from ..lock import acquire_host_lock from ..lock import acquire_host_lock
from ..paths import PobsyncPaths from ..paths import PobsyncPaths
@@ -19,6 +20,7 @@ def run_retention_apply(
yes: bool, yes: bool,
max_delete: int, max_delete: int,
acquire_lock: bool = True, acquire_lock: bool = True,
config_source: ConfigSource | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
host = sanitize_host(host) host = sanitize_host(host)
@@ -34,7 +36,13 @@ def run_retention_apply(
paths = PobsyncPaths(home=prefix) paths = PobsyncPaths(home=prefix)
def _do_apply() -> dict[str, Any]: def _do_apply() -> dict[str, Any]:
plan = run_retention_plan(prefix=prefix, host=host, kind=kind, protect_bases=bool(protect_bases)) plan = run_retention_plan(
prefix=prefix,
host=host,
kind=kind,
protect_bases=bool(protect_bases),
config_source=config_source,
)
delete_list = plan.get("delete") or [] delete_list = plan.get("delete") or []
if not isinstance(delete_list, list): if not isinstance(delete_list, list):
@@ -100,4 +108,3 @@ def run_retention_apply(
# Caller guarantees locking (used by run-scheduled) # Caller guarantees locking (used by run-scheduled)
return _do_apply() return _do_apply()

View File

@@ -4,8 +4,7 @@ from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from ..config.load import load_global_config, load_host_config from ..config.source import ConfigSource, FileConfigSource
from ..config.merge import build_effective_config
from ..errors import ConfigError from ..errors import ConfigError
from ..paths import PobsyncPaths from ..paths import PobsyncPaths
from ..retention import Snapshot, build_retention_plan from ..retention import Snapshot, build_retention_plan
@@ -82,7 +81,13 @@ def _apply_base_protection(
return keep, reasons return keep, reasons
def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool) -> dict[str, Any]: def run_retention_plan(
prefix: Path,
host: str,
kind: str,
protect_bases: bool,
config_source: ConfigSource | None = None,
) -> dict[str, Any]:
host = sanitize_host(host) host = sanitize_host(host)
if kind not in {"scheduled", "manual", "all"}: if kind not in {"scheduled", "manual", "all"}:
@@ -90,9 +95,8 @@ def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool)
paths = PobsyncPaths(home=prefix) paths = PobsyncPaths(home=prefix)
global_cfg = load_global_config(paths.global_config_path) source = config_source or FileConfigSource(prefix=paths.home)
host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml") cfg = source.effective_config_for_host(host)
cfg = build_effective_config(global_cfg, host_cfg)
retention = cfg.get("retention") retention = cfg.get("retention")
if not isinstance(retention, dict): if not isinstance(retention, dict):
@@ -161,4 +165,3 @@ def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool)
], ],
"reasons": reasons, "reasons": reasons,
} }

View File

@@ -3,8 +3,7 @@ from __future__ import annotations
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from ..config.load import load_global_config, load_host_config from ..config.source import ConfigSource, FileConfigSource
from ..config.merge import build_effective_config
from ..errors import ConfigError from ..errors import ConfigError
from ..lock import acquire_host_lock from ..lock import acquire_host_lock
from ..paths import PobsyncPaths from ..paths import PobsyncPaths
@@ -81,15 +80,21 @@ def _base_meta_from_path(base_dir: Path | None, link_dest: str | None) -> dict[s
} }
def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, prune_max_delete: int | None = None, prune_protect_bases: bool = False, ) -> dict[str, Any]: def run_scheduled(
prefix: Path,
host: str,
dry_run: bool,
prune: bool = False,
prune_max_delete: int | None = None,
prune_protect_bases: bool = False,
config_source: ConfigSource | None = None,
) -> dict[str, Any]:
host = sanitize_host(host) host = sanitize_host(host)
paths = PobsyncPaths(home=prefix) paths = PobsyncPaths(home=prefix)
# Load and merge config source = config_source or FileConfigSource(prefix=paths.home)
global_cfg = load_global_config(paths.global_config_path) cfg = source.effective_config_for_host(host)
host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml")
cfg = build_effective_config(global_cfg, host_cfg)
backup_root = cfg.get("backup_root") backup_root = cfg.get("backup_root")
if not isinstance(backup_root, str) or not backup_root.startswith("/"): if not isinstance(backup_root, str) or not backup_root.startswith("/"):
@@ -259,6 +264,21 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
final_dir = dirs.scheduled / snap_name final_dir = dirs.scheduled / snap_name
incomplete_dir.rename(final_dir) incomplete_dir.rename(final_dir)
prune_result = None
if prune:
from .retention_apply import run_retention_apply
prune_result = run_retention_apply(
prefix=paths.home,
host=host,
kind="scheduled",
protect_bases=bool(prune_protect_bases),
yes=True,
max_delete=10 if prune_max_delete is None else int(prune_max_delete),
acquire_lock=False,
config_source=source,
)
return { return {
"ok": True, "ok": True,
"dry_run": False, "dry_run": False,
@@ -266,5 +286,5 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
"snapshot": str(final_dir), "snapshot": str(final_dir),
"base": str(base_dir) if base_dir else None, "base": str(base_dir) if base_dir else None,
"rsync": {"exit_code": result.exit_code}, "rsync": {"exit_code": result.exit_code},
"prune": prune_result,
} }

View File

@@ -0,0 +1,22 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Protocol
from .load import load_global_config, load_host_config
from .merge import build_effective_config
class ConfigSource(Protocol):
def effective_config_for_host(self, host: str) -> dict[str, Any]:
"""Return the fully merged effective config for a host."""
class FileConfigSource:
def __init__(self, prefix: Path) -> None:
self.prefix = prefix
def effective_config_for_host(self, host: str) -> dict[str, Any]:
global_cfg = load_global_config(self.prefix / "config" / "global.yaml")
host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml")
return build_effective_config(global_cfg, host_cfg)

View File

@@ -7,8 +7,28 @@ from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, Snapsho
@admin.register(GlobalConfig) @admin.register(GlobalConfig)
class GlobalConfigAdmin(admin.ModelAdmin): class GlobalConfigAdmin(admin.ModelAdmin):
list_display = ("name", "backup_root", "updated_at") list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at")
readonly_fields = ("created_at", "updated_at") readonly_fields = ("created_at", "updated_at")
fieldsets = (
(None, {"fields": ("name", "backup_root", "pobsync_home")}),
("SSH", {"fields": ("ssh_user", "ssh_port", "ssh_options")}),
(
"Rsync",
{
"fields": (
"rsync_binary",
"rsync_args",
"rsync_extra_args",
"rsync_timeout_seconds",
"rsync_bwlimit_kbps",
)
},
),
("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}),
("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
@admin.register(HostConfig) @admin.register(HostConfig)
@@ -17,6 +37,15 @@ class HostConfigAdmin(admin.ModelAdmin):
list_filter = ("enabled",) list_filter = ("enabled",)
search_fields = ("host", "address") search_fields = ("host", "address")
readonly_fields = ("created_at", "updated_at") readonly_fields = ("created_at", "updated_at")
fieldsets = (
(None, {"fields": ("host", "address", "enabled")}),
("SSH override", {"fields": ("ssh_user", "ssh_port")}),
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
("Rsync override", {"fields": ("rsync_extra_args",)}),
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
@admin.register(BackupRun) @admin.register(BackupRun)
@@ -35,6 +64,6 @@ class SnapshotRecordAdmin(admin.ModelAdmin):
@admin.register(ScheduleConfig) @admin.register(ScheduleConfig)
class ScheduleConfigAdmin(admin.ModelAdmin): class ScheduleConfigAdmin(admin.ModelAdmin):
list_display = ("host", "cron_expr", "enabled", "prune", "updated_at") list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at")
list_filter = ("enabled", "prune") list_filter = ("enabled", "prune", "last_status")
search_fields = ("host__host", "cron_expr") search_fields = ("host__host", "cron_expr")

View File

@@ -21,6 +21,29 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
data = dict(global_config.data or {}) data = dict(global_config.data or {})
data["backup_root"] = global_config.backup_root data["backup_root"] = global_config.backup_root
data["pobsync_home"] = global_config.pobsync_home data["pobsync_home"] = global_config.pobsync_home
data["ssh"] = {
"user": global_config.ssh_user,
"port": global_config.ssh_port,
"options": list(global_config.ssh_options or []),
}
data["rsync"] = {
"binary": global_config.rsync_binary,
"args": list(global_config.rsync_args or []),
"timeout_seconds": global_config.rsync_timeout_seconds,
"bwlimit_kbps": global_config.rsync_bwlimit_kbps,
"extra_args": list(global_config.rsync_extra_args or []),
}
data["defaults"] = {
"source_root": global_config.default_source_root,
"destination_subdir": global_config.default_destination_subdir,
}
data["excludes_default"] = list(global_config.excludes_default or [])
data["retention_defaults"] = {
"daily": global_config.retention_daily,
"weekly": global_config.retention_weekly,
"monthly": global_config.retention_monthly,
"yearly": global_config.retention_yearly,
}
return validate_dict(data, GLOBAL_SCHEMA, path="global") return validate_dict(data, GLOBAL_SCHEMA, path="global")
@@ -28,9 +51,54 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
data = dict(host_config.config or {}) data = dict(host_config.config or {})
data["host"] = host_config.host data["host"] = host_config.host
data["address"] = host_config.address data["address"] = host_config.address
if host_config.ssh_user or host_config.ssh_port:
data["ssh"] = {}
if host_config.ssh_user:
data["ssh"]["user"] = host_config.ssh_user
if host_config.ssh_port is not None:
data["ssh"]["port"] = host_config.ssh_port
else:
data.pop("ssh", None)
if host_config.source_root:
data["source_root"] = host_config.source_root
else:
data.pop("source_root", None)
data["includes"] = list(host_config.includes or [])
if host_config.excludes_replace is not None:
data["excludes_replace"] = list(host_config.excludes_replace or [])
data.pop("excludes_add", None)
else:
data["excludes_add"] = list(host_config.excludes_add or [])
data.pop("excludes_replace", None)
if host_config.rsync_extra_args:
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
else:
data.pop("rsync", None)
data["retention"] = {
"daily": host_config.retention_daily,
"weekly": host_config.retention_weekly,
"monthly": host_config.retention_monthly,
"yearly": host_config.retention_yearly,
}
return validate_dict(data, HOST_SCHEMA, path="host") return validate_dict(data, HOST_SCHEMA, path="host")
def global_config_data(name: str = "default") -> dict[str, Any]:
try:
global_config = GlobalConfig.objects.get(name=name)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
return _global_yaml_data(global_config)
def host_config_data(host: str) -> dict[str, Any]:
try:
host_config = HostConfig.objects.get(host=host, enabled=True)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
return _host_yaml_data(host_config)
def export_global_config(prefix: Path, name: str = "default") -> Path: def export_global_config(prefix: Path, name: str = "default") -> Path:
try: try:
global_config = GlobalConfig.objects.get(name=name) global_config = GlobalConfig.objects.get(name=name)

View File

@@ -0,0 +1,12 @@
from __future__ import annotations
from typing import Any
from pobsync.config.merge import build_effective_config
from .config_repository import global_config_data, host_config_data
class DjangoConfigSource:
def effective_config_for_host(self, host: str) -> dict[str, Any]:
return build_effective_config(global_config_data(), host_config_data(host))

View File

@@ -23,11 +23,30 @@ class Command(BaseCommand):
raise CommandError(f"Missing global config: {paths.global_config_path}") raise CommandError(f"Missing global config: {paths.global_config_path}")
global_cfg = load_global_config(paths.global_config_path) global_cfg = load_global_config(paths.global_config_path)
global_ssh = global_cfg.get("ssh") or {}
global_rsync = global_cfg.get("rsync") or {}
global_defaults = global_cfg.get("defaults") or {}
retention_defaults = global_cfg.get("retention_defaults") or {}
GlobalConfig.objects.update_or_create( GlobalConfig.objects.update_or_create(
name="default", name="default",
defaults={ defaults={
"backup_root": global_cfg["backup_root"], "backup_root": global_cfg["backup_root"],
"pobsync_home": global_cfg.get("pobsync_home", str(paths.home)), "pobsync_home": global_cfg.get("pobsync_home", str(paths.home)),
"ssh_user": global_ssh.get("user") or "root",
"ssh_port": global_ssh.get("port") or 22,
"ssh_options": global_ssh.get("options") or [],
"rsync_binary": global_rsync.get("binary") or "rsync",
"rsync_args": global_rsync.get("args") or [],
"rsync_extra_args": global_rsync.get("extra_args") or [],
"rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0,
"rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0,
"default_source_root": global_defaults.get("source_root") or "/",
"default_destination_subdir": global_defaults.get("destination_subdir") or "",
"excludes_default": global_cfg.get("excludes_default") or [],
"retention_daily": retention_defaults.get("daily", 14),
"retention_weekly": retention_defaults.get("weekly", 8),
"retention_monthly": retention_defaults.get("monthly", 12),
"retention_yearly": retention_defaults.get("yearly", 0),
"data": global_cfg, "data": global_cfg,
}, },
) )
@@ -35,10 +54,24 @@ class Command(BaseCommand):
count = 0 count = 0
for host_path in sorted(paths.hosts_dir.glob("*.yaml")): for host_path in sorted(paths.hosts_dir.glob("*.yaml")):
host_cfg = load_host_config(host_path) host_cfg = load_host_config(host_path)
host_ssh = host_cfg.get("ssh") or {}
host_rsync = host_cfg.get("rsync") or {}
host_retention = host_cfg.get("retention") or {}
HostConfig.objects.update_or_create( HostConfig.objects.update_or_create(
host=host_cfg["host"], host=host_cfg["host"],
defaults={ defaults={
"address": host_cfg["address"], "address": host_cfg["address"],
"ssh_user": host_ssh.get("user") or "",
"ssh_port": host_ssh.get("port"),
"source_root": host_cfg.get("source_root") or "",
"includes": host_cfg.get("includes") or [],
"excludes_add": host_cfg.get("excludes_add") or [],
"excludes_replace": host_cfg.get("excludes_replace"),
"rsync_extra_args": host_rsync.get("extra_args") or [],
"retention_daily": host_retention.get("daily", 14),
"retention_weekly": host_retention.get("weekly", 8),
"retention_monthly": host_retention.get("monthly", 12),
"retention_yearly": host_retention.get("yearly", 0),
"config": host_cfg, "config": host_cfg,
"enabled": True, "enabled": True,
}, },

View File

@@ -9,7 +9,7 @@ from django.utils import timezone
from pobsync.commands.run_scheduled import run_scheduled from pobsync.commands.run_scheduled import run_scheduled
from pobsync.paths import PobsyncPaths from pobsync.paths import PobsyncPaths
from pobsync_backend.config_repository import export_runtime_configs from pobsync_backend.config_source import DjangoConfigSource
from pobsync_backend.models import BackupRun, HostConfig from pobsync_backend.models import BackupRun, HostConfig
@@ -32,8 +32,6 @@ class Command(BaseCommand):
except HostConfig.DoesNotExist as exc: except HostConfig.DoesNotExist as exc:
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
export_runtime_configs(prefix=paths.home, host=host.host)
run = BackupRun.objects.create( run = BackupRun.objects.create(
host=host, host=host,
run_type=BackupRun.RunType.SCHEDULED, run_type=BackupRun.RunType.SCHEDULED,
@@ -49,6 +47,7 @@ class Command(BaseCommand):
prune=bool(options["prune"]), prune=bool(options["prune"]),
prune_max_delete=int(options["prune_max_delete"]), prune_max_delete=int(options["prune_max_delete"]),
prune_protect_bases=bool(options["prune_protect_bases"]), prune_protect_bases=bool(options["prune_protect_bases"]),
config_source=DjangoConfigSource(),
) )
except Exception as exc: except Exception as exc:
run.status = BackupRun.Status.FAILED run.status = BackupRun.Status.FAILED

View File

@@ -0,0 +1,191 @@
from __future__ import annotations
from django.db import migrations, models
def copy_json_config_to_fields(apps, schema_editor) -> None:
GlobalConfig = apps.get_model("pobsync_backend", "GlobalConfig")
HostConfig = apps.get_model("pobsync_backend", "HostConfig")
for global_config in GlobalConfig.objects.all():
data = global_config.data or {}
ssh = data.get("ssh") or {}
rsync = data.get("rsync") or {}
defaults = data.get("defaults") or {}
retention = data.get("retention_defaults") or {}
global_config.ssh_user = ssh.get("user") or global_config.ssh_user
global_config.ssh_port = ssh.get("port") or global_config.ssh_port
global_config.ssh_options = ssh.get("options") or []
global_config.rsync_binary = rsync.get("binary") or global_config.rsync_binary
global_config.rsync_args = rsync.get("args") or []
global_config.rsync_extra_args = rsync.get("extra_args") or []
global_config.rsync_timeout_seconds = rsync.get("timeout_seconds") or 0
global_config.rsync_bwlimit_kbps = rsync.get("bwlimit_kbps") or 0
global_config.default_source_root = defaults.get("source_root") or "/"
global_config.default_destination_subdir = defaults.get("destination_subdir") or ""
global_config.excludes_default = data.get("excludes_default") or []
global_config.retention_daily = retention.get("daily", global_config.retention_daily)
global_config.retention_weekly = retention.get("weekly", global_config.retention_weekly)
global_config.retention_monthly = retention.get("monthly", global_config.retention_monthly)
global_config.retention_yearly = retention.get("yearly", global_config.retention_yearly)
global_config.save()
for host_config in HostConfig.objects.all():
config = host_config.config or {}
ssh = config.get("ssh") or {}
rsync = config.get("rsync") or {}
retention = config.get("retention") or {}
host_config.ssh_user = ssh.get("user") or ""
host_config.ssh_port = ssh.get("port")
host_config.source_root = config.get("source_root") or ""
host_config.includes = config.get("includes") or []
host_config.excludes_add = config.get("excludes_add") or []
host_config.excludes_replace = config.get("excludes_replace")
host_config.rsync_extra_args = rsync.get("extra_args") or []
host_config.retention_daily = retention.get("daily", host_config.retention_daily)
host_config.retention_weekly = retention.get("weekly", host_config.retention_weekly)
host_config.retention_monthly = retention.get("monthly", host_config.retention_monthly)
host_config.retention_yearly = retention.get("yearly", host_config.retention_yearly)
host_config.save()
class Migration(migrations.Migration):
dependencies = [
("pobsync_backend", "0002_schedule_run_state"),
]
operations = [
migrations.AddField(
model_name="globalconfig",
name="default_destination_subdir",
field=models.CharField(blank=True, default="", max_length=512),
),
migrations.AddField(
model_name="globalconfig",
name="default_source_root",
field=models.CharField(default="/", max_length=512),
),
migrations.AddField(
model_name="globalconfig",
name="excludes_default",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="retention_daily",
field=models.PositiveIntegerField(default=14),
),
migrations.AddField(
model_name="globalconfig",
name="retention_monthly",
field=models.PositiveIntegerField(default=12),
),
migrations.AddField(
model_name="globalconfig",
name="retention_weekly",
field=models.PositiveIntegerField(default=8),
),
migrations.AddField(
model_name="globalconfig",
name="retention_yearly",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_binary",
field=models.CharField(default="rsync", max_length=128),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_bwlimit_kbps",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_extra_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_timeout_seconds",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_options",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_port",
field=models.PositiveIntegerField(default=22),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_user",
field=models.CharField(default="root", max_length=64),
),
migrations.AddField(
model_name="hostconfig",
name="excludes_add",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="excludes_replace",
field=models.JSONField(blank=True, null=True),
),
migrations.AddField(
model_name="hostconfig",
name="includes",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="retention_daily",
field=models.PositiveIntegerField(default=14),
),
migrations.AddField(
model_name="hostconfig",
name="retention_monthly",
field=models.PositiveIntegerField(default=12),
),
migrations.AddField(
model_name="hostconfig",
name="retention_weekly",
field=models.PositiveIntegerField(default=8),
),
migrations.AddField(
model_name="hostconfig",
name="retention_yearly",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="hostconfig",
name="rsync_extra_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="source_root",
field=models.CharField(blank=True, max_length=512),
),
migrations.AddField(
model_name="hostconfig",
name="ssh_port",
field=models.PositiveIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="hostconfig",
name="ssh_user",
field=models.CharField(blank=True, max_length=64),
),
migrations.RunPython(copy_json_config_to_fields, migrations.RunPython.noop),
]

View File

@@ -15,6 +15,21 @@ class GlobalConfig(TimestampedModel):
name = models.CharField(max_length=64, default="default", unique=True) name = models.CharField(max_length=64, default="default", unique=True)
backup_root = models.CharField(max_length=512) backup_root = models.CharField(max_length=512)
pobsync_home = models.CharField(max_length=512, default="/opt/pobsync") pobsync_home = models.CharField(max_length=512, default="/opt/pobsync")
ssh_user = models.CharField(max_length=64, default="root")
ssh_port = models.PositiveIntegerField(default=22)
ssh_options = models.JSONField(default=list, blank=True)
rsync_binary = models.CharField(max_length=128, default="rsync")
rsync_args = models.JSONField(default=list, blank=True)
rsync_extra_args = models.JSONField(default=list, blank=True)
rsync_timeout_seconds = models.PositiveIntegerField(default=0)
rsync_bwlimit_kbps = models.PositiveIntegerField(default=0)
default_source_root = models.CharField(max_length=512, default="/")
default_destination_subdir = models.CharField(max_length=512, default="", blank=True)
excludes_default = models.JSONField(default=list, blank=True)
retention_daily = models.PositiveIntegerField(default=14)
retention_weekly = models.PositiveIntegerField(default=8)
retention_monthly = models.PositiveIntegerField(default=12)
retention_yearly = models.PositiveIntegerField(default=0)
data = models.JSONField(default=dict, blank=True) data = models.JSONField(default=dict, blank=True)
class Meta: class Meta:
@@ -29,6 +44,17 @@ class HostConfig(TimestampedModel):
host = models.CharField(max_length=255, unique=True) host = models.CharField(max_length=255, unique=True)
address = models.CharField(max_length=255) address = models.CharField(max_length=255)
enabled = models.BooleanField(default=True) enabled = models.BooleanField(default=True)
ssh_user = models.CharField(max_length=64, blank=True)
ssh_port = models.PositiveIntegerField(null=True, blank=True)
source_root = models.CharField(max_length=512, blank=True)
includes = models.JSONField(default=list, blank=True)
excludes_add = models.JSONField(default=list, blank=True)
excludes_replace = models.JSONField(null=True, blank=True)
rsync_extra_args = models.JSONField(default=list, blank=True)
retention_daily = models.PositiveIntegerField(default=14)
retention_weekly = models.PositiveIntegerField(default=8)
retention_monthly = models.PositiveIntegerField(default=12)
retention_yearly = models.PositiveIntegerField(default=0)
config = models.JSONField(default=dict, blank=True) config = models.JSONField(default=dict, blank=True)
class Meta: class Meta:

View File

@@ -18,21 +18,36 @@ class ConfigRepositoryTests(TestCase):
name="default", name="default",
backup_root="/backups", backup_root="/backups",
pobsync_home=str(prefix), pobsync_home=str(prefix),
ssh_user="backup",
ssh_port=2222,
rsync_args=["--archive"],
excludes_default=["/proc/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={ data={
"backup_root": "/ignored", "backup_root": "/ignored",
"pobsync_home": "/ignored", "pobsync_home": "/ignored",
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, "ssh": {"user": "ignored", "port": 22, "options": []},
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
}, },
) )
HostConfig.objects.create( HostConfig.objects.create(
host="web-01", host="web-01",
address="web-01.example.test", address="web-01.example.test",
ssh_user="root",
includes=[],
excludes_add=["/tmp/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
config={ config={
"host": "ignored", "host": "ignored",
"address": "ignored", "address": "ignored",
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
"includes": [], "excludes_add": ["/ignored/***"],
"excludes_add": ["/tmp/***"],
}, },
) )
@@ -43,5 +58,10 @@ class ConfigRepositoryTests(TestCase):
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml") host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
self.assertEqual(global_cfg["backup_root"], "/backups") self.assertEqual(global_cfg["backup_root"], "/backups")
self.assertEqual(global_cfg["pobsync_home"], str(prefix)) self.assertEqual(global_cfg["pobsync_home"], str(prefix))
self.assertEqual(global_cfg["ssh"]["user"], "backup")
self.assertEqual(global_cfg["ssh"]["port"], 2222)
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
self.assertEqual(host_cfg["host"], "web-01") self.assertEqual(host_cfg["host"], "web-01")
self.assertEqual(host_cfg["address"], "web-01.example.test") self.assertEqual(host_cfg["address"], "web-01.example.test")
self.assertEqual(host_cfg["retention"]["daily"], 7)
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])

View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from django.test import TestCase
from pobsync_backend.config_source import DjangoConfigSource
from pobsync_backend.models import GlobalConfig, HostConfig
class DjangoConfigSourceTests(TestCase):
def test_returns_effective_config_from_database(self) -> None:
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
rsync_args=["--archive"],
rsync_extra_args=["--numeric-ids"],
excludes_default=["/proc/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"ssh": {"user": "root", "port": 22, "options": []},
"rsync": {
"binary": "rsync",
"args": ["--archive"],
"timeout_seconds": 0,
"bwlimit_kbps": 0,
"extra_args": ["--numeric-ids"],
},
"defaults": {"source_root": "/", "destination_subdir": ""},
"excludes_default": ["/proc/***"],
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
},
)
HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
excludes_add=["/tmp/***"],
rsync_extra_args=["--delete"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
config={
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
"excludes_add": ["/ignored/***"],
"rsync": {"extra_args": ["--ignored"]},
},
)
cfg = DjangoConfigSource().effective_config_for_host("web-01")
self.assertEqual(cfg["backup_root"], "/backups")
self.assertEqual(cfg["host"], "web-01")
self.assertEqual(cfg["address"], "web-01.example.test")
self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"])
self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"])

View File

@@ -0,0 +1,53 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from tempfile import TemporaryDirectory
from django.test import SimpleTestCase
from pobsync.commands.retention_plan import run_retention_plan
from pobsync.util import write_yaml_atomic
class FakeConfigSource:
def __init__(self, backup_root: str) -> None:
self.backup_root = backup_root
def effective_config_for_host(self, host: str) -> dict:
return {
"backup_root": self.backup_root,
"host": host,
"address": "example.test",
"retention": {"daily": 1, "weekly": 0, "monthly": 0, "yearly": 0},
}
class RetentionConfigSourceTests(SimpleTestCase):
def test_retention_plan_uses_injected_config_source(self) -> None:
with TemporaryDirectory() as tmp:
root = Path(tmp) / "backups"
snap_dir = root / "web-01" / "scheduled" / "20260519-021500Z__ABCDEFGH"
meta_dir = snap_dir / "meta"
meta_dir.mkdir(parents=True)
write_yaml_atomic(
meta_dir / "meta.yaml",
{
"status": "success",
"started_at": datetime(2026, 5, 19, 2, 15, tzinfo=timezone.utc)
.isoformat()
.replace("+00:00", "Z"),
},
)
plan = run_retention_plan(
prefix=Path("/missing-prefix"),
host="web-01",
kind="scheduled",
protect_bases=False,
config_source=FakeConfigSource(str(root)),
)
self.assertTrue(plan["ok"])
self.assertEqual(plan["keep"], ["20260519-021500Z__ABCDEFGH"])
self.assertEqual(plan["delete"], [])

View File

@@ -0,0 +1,79 @@
from __future__ import annotations
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from django.test import SimpleTestCase
from pobsync.commands.run_scheduled import run_scheduled
from pobsync.rsync import RsyncResult
class FakeConfigSource:
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None:
self.backup_root = backup_root
def effective_config_for_host(self, host: str) -> dict:
return {
"backup_root": self.backup_root,
"host": host,
"address": "example.test",
"ssh": {"user": "root", "port": 22, "options": []},
"rsync": {
"binary": "rsync",
"args_effective": ["--archive"],
"timeout_seconds": 0,
"bwlimit_kbps": 0,
},
"source_root": "/",
"includes": [],
"excludes_effective": [],
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
}
class RunScheduledConfigSourceTests(SimpleTestCase):
def test_dry_run_uses_injected_config_source(self) -> None:
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
result = run_scheduled(
prefix=Path("/missing-prefix"),
host="web-01",
dry_run=True,
config_source=FakeConfigSource(),
)
self.assertTrue(result["ok"])
self.assertEqual(result["host"], "web-01")
run_rsync.assert_called_once()
def test_successful_real_run_applies_prune_when_requested(self) -> None:
with TemporaryDirectory() as tmp:
prefix = Path(tmp) / "home"
with (
patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync,
patch("pobsync.commands.retention_apply.run_retention_plan") as plan,
):
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
plan.return_value = {
"ok": True,
"delete": [],
"keep": [],
"reasons": {},
"protect_bases": False,
}
result = run_scheduled(
prefix=prefix,
host="web-01",
dry_run=False,
prune=True,
prune_max_delete=10,
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
)
self.assertTrue(result["ok"])
self.assertIsNotNone(result["prune"])
self.assertEqual(result["prune"]["deleted"], [])