Compare commits
3 Commits
18082496e4
...
a0eb5dcc8f
| Author | SHA1 | Date | |
|---|---|---|---|
| a0eb5dcc8f | |||
| 100215bf11 | |||
| bb44f8a09c |
@@ -115,7 +115,7 @@ For production use, always use the canonical entrypoint:
|
||||
|
||||
## Django backend (early refactor layer)
|
||||
|
||||
The Django backend is a management layer around the existing pobsync engine. The current CLI remains the source of truth for executing backups; Django stores configs, schedules, backup runs, and snapshot metadata so the project can grow toward a web/admin/API surface without rewriting rsync behavior in one risky step.
|
||||
The Django backend is becoming the management layer and source of truth for pobsync. Structured SQL fields store backup, SSH, rsync, retention, schedule, run, and snapshot state; legacy JSON/YAML remains only as an import/export compatibility path while the engine is being refactored.
|
||||
|
||||
### Local SQLite development
|
||||
|
||||
@@ -145,6 +145,8 @@ Run a backup through Django while still using the existing pobsync engine:
|
||||
python3 manage.py run_pobsync_backup <host> --prefix /opt/pobsync --prune
|
||||
```
|
||||
|
||||
The Django backup command reads backup and retention config from SQL directly. Runtime YAML export is kept as a compatibility tool for older CLI flows during the transition.
|
||||
|
||||
Export database configs to the runtime YAML files consumed by the current engine:
|
||||
|
||||
```
|
||||
@@ -193,8 +195,8 @@ The MariaDB profile is optional. SQLite remains the default because it is enough
|
||||
|
||||
Recommended next steps:
|
||||
|
||||
- Move config reading/writing behind a repository interface that can use YAML or Django models.
|
||||
- Record `run-scheduled` results into `BackupRun`.
|
||||
- Continue moving config reading/writing behind repository interfaces so YAML export can eventually disappear.
|
||||
- Record more engine-side run details into `BackupRun` and `SnapshotRecord`.
|
||||
- Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine.
|
||||
- Run schedules from Django/Docker instead of writing host cron files.
|
||||
- Add a snapshot discovery command that syncs existing snapshot metadata into `SnapshotRecord`.
|
||||
|
||||
@@ -4,6 +4,7 @@ import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ..config.source import ConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..lock import acquire_host_lock
|
||||
from ..paths import PobsyncPaths
|
||||
@@ -19,6 +20,7 @@ def run_retention_apply(
|
||||
yes: bool,
|
||||
max_delete: int,
|
||||
acquire_lock: bool = True,
|
||||
config_source: ConfigSource | None = None,
|
||||
) -> dict[str, Any]:
|
||||
host = sanitize_host(host)
|
||||
|
||||
@@ -34,7 +36,13 @@ def run_retention_apply(
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
def _do_apply() -> dict[str, Any]:
|
||||
plan = run_retention_plan(prefix=prefix, host=host, kind=kind, protect_bases=bool(protect_bases))
|
||||
plan = run_retention_plan(
|
||||
prefix=prefix,
|
||||
host=host,
|
||||
kind=kind,
|
||||
protect_bases=bool(protect_bases),
|
||||
config_source=config_source,
|
||||
)
|
||||
|
||||
delete_list = plan.get("delete") or []
|
||||
if not isinstance(delete_list, list):
|
||||
@@ -100,4 +108,3 @@ def run_retention_apply(
|
||||
|
||||
# Caller guarantees locking (used by run-scheduled)
|
||||
return _do_apply()
|
||||
|
||||
|
||||
@@ -4,8 +4,7 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..config.load import load_global_config, load_host_config
|
||||
from ..config.merge import build_effective_config
|
||||
from ..config.source import ConfigSource, FileConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..paths import PobsyncPaths
|
||||
from ..retention import Snapshot, build_retention_plan
|
||||
@@ -82,7 +81,13 @@ def _apply_base_protection(
|
||||
return keep, reasons
|
||||
|
||||
|
||||
def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool) -> dict[str, Any]:
|
||||
def run_retention_plan(
|
||||
prefix: Path,
|
||||
host: str,
|
||||
kind: str,
|
||||
protect_bases: bool,
|
||||
config_source: ConfigSource | None = None,
|
||||
) -> dict[str, Any]:
|
||||
host = sanitize_host(host)
|
||||
|
||||
if kind not in {"scheduled", "manual", "all"}:
|
||||
@@ -90,9 +95,8 @@ def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool)
|
||||
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
global_cfg = load_global_config(paths.global_config_path)
|
||||
host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml")
|
||||
cfg = build_effective_config(global_cfg, host_cfg)
|
||||
source = config_source or FileConfigSource(prefix=paths.home)
|
||||
cfg = source.effective_config_for_host(host)
|
||||
|
||||
retention = cfg.get("retention")
|
||||
if not isinstance(retention, dict):
|
||||
@@ -161,4 +165,3 @@ def run_retention_plan(prefix: Path, host: str, kind: str, protect_bases: bool)
|
||||
],
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..config.load import load_global_config, load_host_config
|
||||
from ..config.merge import build_effective_config
|
||||
from ..config.source import ConfigSource, FileConfigSource
|
||||
from ..errors import ConfigError
|
||||
from ..lock import acquire_host_lock
|
||||
from ..paths import PobsyncPaths
|
||||
@@ -81,15 +80,21 @@ def _base_meta_from_path(base_dir: Path | None, link_dest: str | None) -> dict[s
|
||||
}
|
||||
|
||||
|
||||
def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, prune_max_delete: int | None = None, prune_protect_bases: bool = False, ) -> dict[str, Any]:
|
||||
def run_scheduled(
|
||||
prefix: Path,
|
||||
host: str,
|
||||
dry_run: bool,
|
||||
prune: bool = False,
|
||||
prune_max_delete: int | None = None,
|
||||
prune_protect_bases: bool = False,
|
||||
config_source: ConfigSource | None = None,
|
||||
) -> dict[str, Any]:
|
||||
|
||||
host = sanitize_host(host)
|
||||
paths = PobsyncPaths(home=prefix)
|
||||
|
||||
# Load and merge config
|
||||
global_cfg = load_global_config(paths.global_config_path)
|
||||
host_cfg = load_host_config(paths.hosts_dir / f"{host}.yaml")
|
||||
cfg = build_effective_config(global_cfg, host_cfg)
|
||||
source = config_source or FileConfigSource(prefix=paths.home)
|
||||
cfg = source.effective_config_for_host(host)
|
||||
|
||||
backup_root = cfg.get("backup_root")
|
||||
if not isinstance(backup_root, str) or not backup_root.startswith("/"):
|
||||
@@ -259,6 +264,21 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
|
||||
final_dir = dirs.scheduled / snap_name
|
||||
incomplete_dir.rename(final_dir)
|
||||
|
||||
prune_result = None
|
||||
if prune:
|
||||
from .retention_apply import run_retention_apply
|
||||
|
||||
prune_result = run_retention_apply(
|
||||
prefix=paths.home,
|
||||
host=host,
|
||||
kind="scheduled",
|
||||
protect_bases=bool(prune_protect_bases),
|
||||
yes=True,
|
||||
max_delete=10 if prune_max_delete is None else int(prune_max_delete),
|
||||
acquire_lock=False,
|
||||
config_source=source,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"dry_run": False,
|
||||
@@ -266,5 +286,5 @@ def run_scheduled(prefix: Path, host: str, dry_run: bool, prune: bool = False, p
|
||||
"snapshot": str(final_dir),
|
||||
"base": str(base_dir) if base_dir else None,
|
||||
"rsync": {"exit_code": result.exit_code},
|
||||
"prune": prune_result,
|
||||
}
|
||||
|
||||
|
||||
22
src/pobsync/config/source.py
Normal file
22
src/pobsync/config/source.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .load import load_global_config, load_host_config
|
||||
from .merge import build_effective_config
|
||||
|
||||
|
||||
class ConfigSource(Protocol):
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
"""Return the fully merged effective config for a host."""
|
||||
|
||||
|
||||
class FileConfigSource:
|
||||
def __init__(self, prefix: Path) -> None:
|
||||
self.prefix = prefix
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
global_cfg = load_global_config(self.prefix / "config" / "global.yaml")
|
||||
host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml")
|
||||
return build_effective_config(global_cfg, host_cfg)
|
||||
@@ -7,8 +7,28 @@ from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, Snapsho
|
||||
|
||||
@admin.register(GlobalConfig)
|
||||
class GlobalConfigAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "backup_root", "updated_at")
|
||||
list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at")
|
||||
readonly_fields = ("created_at", "updated_at")
|
||||
fieldsets = (
|
||||
(None, {"fields": ("name", "backup_root", "pobsync_home")}),
|
||||
("SSH", {"fields": ("ssh_user", "ssh_port", "ssh_options")}),
|
||||
(
|
||||
"Rsync",
|
||||
{
|
||||
"fields": (
|
||||
"rsync_binary",
|
||||
"rsync_args",
|
||||
"rsync_extra_args",
|
||||
"rsync_timeout_seconds",
|
||||
"rsync_bwlimit_kbps",
|
||||
)
|
||||
},
|
||||
),
|
||||
("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}),
|
||||
("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
|
||||
("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
|
||||
@admin.register(HostConfig)
|
||||
@@ -17,6 +37,15 @@ class HostConfigAdmin(admin.ModelAdmin):
|
||||
list_filter = ("enabled",)
|
||||
search_fields = ("host", "address")
|
||||
readonly_fields = ("created_at", "updated_at")
|
||||
fieldsets = (
|
||||
(None, {"fields": ("host", "address", "enabled")}),
|
||||
("SSH override", {"fields": ("ssh_user", "ssh_port")}),
|
||||
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
|
||||
("Rsync override", {"fields": ("rsync_extra_args",)}),
|
||||
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
|
||||
("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}),
|
||||
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
|
||||
)
|
||||
|
||||
|
||||
@admin.register(BackupRun)
|
||||
@@ -35,6 +64,6 @@ class SnapshotRecordAdmin(admin.ModelAdmin):
|
||||
|
||||
@admin.register(ScheduleConfig)
|
||||
class ScheduleConfigAdmin(admin.ModelAdmin):
|
||||
list_display = ("host", "cron_expr", "enabled", "prune", "updated_at")
|
||||
list_filter = ("enabled", "prune")
|
||||
list_display = ("host", "cron_expr", "enabled", "prune", "last_status", "last_started_at", "updated_at")
|
||||
list_filter = ("enabled", "prune", "last_status")
|
||||
search_fields = ("host__host", "cron_expr")
|
||||
|
||||
@@ -21,6 +21,29 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
|
||||
data = dict(global_config.data or {})
|
||||
data["backup_root"] = global_config.backup_root
|
||||
data["pobsync_home"] = global_config.pobsync_home
|
||||
data["ssh"] = {
|
||||
"user": global_config.ssh_user,
|
||||
"port": global_config.ssh_port,
|
||||
"options": list(global_config.ssh_options or []),
|
||||
}
|
||||
data["rsync"] = {
|
||||
"binary": global_config.rsync_binary,
|
||||
"args": list(global_config.rsync_args or []),
|
||||
"timeout_seconds": global_config.rsync_timeout_seconds,
|
||||
"bwlimit_kbps": global_config.rsync_bwlimit_kbps,
|
||||
"extra_args": list(global_config.rsync_extra_args or []),
|
||||
}
|
||||
data["defaults"] = {
|
||||
"source_root": global_config.default_source_root,
|
||||
"destination_subdir": global_config.default_destination_subdir,
|
||||
}
|
||||
data["excludes_default"] = list(global_config.excludes_default or [])
|
||||
data["retention_defaults"] = {
|
||||
"daily": global_config.retention_daily,
|
||||
"weekly": global_config.retention_weekly,
|
||||
"monthly": global_config.retention_monthly,
|
||||
"yearly": global_config.retention_yearly,
|
||||
}
|
||||
return validate_dict(data, GLOBAL_SCHEMA, path="global")
|
||||
|
||||
|
||||
@@ -28,9 +51,54 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
|
||||
data = dict(host_config.config or {})
|
||||
data["host"] = host_config.host
|
||||
data["address"] = host_config.address
|
||||
if host_config.ssh_user or host_config.ssh_port:
|
||||
data["ssh"] = {}
|
||||
if host_config.ssh_user:
|
||||
data["ssh"]["user"] = host_config.ssh_user
|
||||
if host_config.ssh_port is not None:
|
||||
data["ssh"]["port"] = host_config.ssh_port
|
||||
else:
|
||||
data.pop("ssh", None)
|
||||
if host_config.source_root:
|
||||
data["source_root"] = host_config.source_root
|
||||
else:
|
||||
data.pop("source_root", None)
|
||||
data["includes"] = list(host_config.includes or [])
|
||||
if host_config.excludes_replace is not None:
|
||||
data["excludes_replace"] = list(host_config.excludes_replace or [])
|
||||
data.pop("excludes_add", None)
|
||||
else:
|
||||
data["excludes_add"] = list(host_config.excludes_add or [])
|
||||
data.pop("excludes_replace", None)
|
||||
if host_config.rsync_extra_args:
|
||||
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
|
||||
else:
|
||||
data.pop("rsync", None)
|
||||
data["retention"] = {
|
||||
"daily": host_config.retention_daily,
|
||||
"weekly": host_config.retention_weekly,
|
||||
"monthly": host_config.retention_monthly,
|
||||
"yearly": host_config.retention_yearly,
|
||||
}
|
||||
return validate_dict(data, HOST_SCHEMA, path="host")
|
||||
|
||||
|
||||
def global_config_data(name: str = "default") -> dict[str, Any]:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
|
||||
return _global_yaml_data(global_config)
|
||||
|
||||
|
||||
def host_config_data(host: str) -> dict[str, Any]:
|
||||
try:
|
||||
host_config = HostConfig.objects.get(host=host, enabled=True)
|
||||
except ObjectDoesNotExist as exc:
|
||||
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
|
||||
return _host_yaml_data(host_config)
|
||||
|
||||
|
||||
def export_global_config(prefix: Path, name: str = "default") -> Path:
|
||||
try:
|
||||
global_config = GlobalConfig.objects.get(name=name)
|
||||
|
||||
12
src/pobsync_backend/config_source.py
Normal file
12
src/pobsync_backend/config_source.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pobsync.config.merge import build_effective_config
|
||||
|
||||
from .config_repository import global_config_data, host_config_data
|
||||
|
||||
|
||||
class DjangoConfigSource:
|
||||
def effective_config_for_host(self, host: str) -> dict[str, Any]:
|
||||
return build_effective_config(global_config_data(), host_config_data(host))
|
||||
@@ -23,11 +23,30 @@ class Command(BaseCommand):
|
||||
raise CommandError(f"Missing global config: {paths.global_config_path}")
|
||||
|
||||
global_cfg = load_global_config(paths.global_config_path)
|
||||
global_ssh = global_cfg.get("ssh") or {}
|
||||
global_rsync = global_cfg.get("rsync") or {}
|
||||
global_defaults = global_cfg.get("defaults") or {}
|
||||
retention_defaults = global_cfg.get("retention_defaults") or {}
|
||||
GlobalConfig.objects.update_or_create(
|
||||
name="default",
|
||||
defaults={
|
||||
"backup_root": global_cfg["backup_root"],
|
||||
"pobsync_home": global_cfg.get("pobsync_home", str(paths.home)),
|
||||
"ssh_user": global_ssh.get("user") or "root",
|
||||
"ssh_port": global_ssh.get("port") or 22,
|
||||
"ssh_options": global_ssh.get("options") or [],
|
||||
"rsync_binary": global_rsync.get("binary") or "rsync",
|
||||
"rsync_args": global_rsync.get("args") or [],
|
||||
"rsync_extra_args": global_rsync.get("extra_args") or [],
|
||||
"rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0,
|
||||
"rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0,
|
||||
"default_source_root": global_defaults.get("source_root") or "/",
|
||||
"default_destination_subdir": global_defaults.get("destination_subdir") or "",
|
||||
"excludes_default": global_cfg.get("excludes_default") or [],
|
||||
"retention_daily": retention_defaults.get("daily", 14),
|
||||
"retention_weekly": retention_defaults.get("weekly", 8),
|
||||
"retention_monthly": retention_defaults.get("monthly", 12),
|
||||
"retention_yearly": retention_defaults.get("yearly", 0),
|
||||
"data": global_cfg,
|
||||
},
|
||||
)
|
||||
@@ -35,10 +54,24 @@ class Command(BaseCommand):
|
||||
count = 0
|
||||
for host_path in sorted(paths.hosts_dir.glob("*.yaml")):
|
||||
host_cfg = load_host_config(host_path)
|
||||
host_ssh = host_cfg.get("ssh") or {}
|
||||
host_rsync = host_cfg.get("rsync") or {}
|
||||
host_retention = host_cfg.get("retention") or {}
|
||||
HostConfig.objects.update_or_create(
|
||||
host=host_cfg["host"],
|
||||
defaults={
|
||||
"address": host_cfg["address"],
|
||||
"ssh_user": host_ssh.get("user") or "",
|
||||
"ssh_port": host_ssh.get("port"),
|
||||
"source_root": host_cfg.get("source_root") or "",
|
||||
"includes": host_cfg.get("includes") or [],
|
||||
"excludes_add": host_cfg.get("excludes_add") or [],
|
||||
"excludes_replace": host_cfg.get("excludes_replace"),
|
||||
"rsync_extra_args": host_rsync.get("extra_args") or [],
|
||||
"retention_daily": host_retention.get("daily", 14),
|
||||
"retention_weekly": host_retention.get("weekly", 8),
|
||||
"retention_monthly": host_retention.get("monthly", 12),
|
||||
"retention_yearly": host_retention.get("yearly", 0),
|
||||
"config": host_cfg,
|
||||
"enabled": True,
|
||||
},
|
||||
|
||||
@@ -9,7 +9,7 @@ from django.utils import timezone
|
||||
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.paths import PobsyncPaths
|
||||
from pobsync_backend.config_repository import export_runtime_configs
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import BackupRun, HostConfig
|
||||
|
||||
|
||||
@@ -32,8 +32,6 @@ class Command(BaseCommand):
|
||||
except HostConfig.DoesNotExist as exc:
|
||||
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
|
||||
|
||||
export_runtime_configs(prefix=paths.home, host=host.host)
|
||||
|
||||
run = BackupRun.objects.create(
|
||||
host=host,
|
||||
run_type=BackupRun.RunType.SCHEDULED,
|
||||
@@ -49,6 +47,7 @@ class Command(BaseCommand):
|
||||
prune=bool(options["prune"]),
|
||||
prune_max_delete=int(options["prune_max_delete"]),
|
||||
prune_protect_bases=bool(options["prune_protect_bases"]),
|
||||
config_source=DjangoConfigSource(),
|
||||
)
|
||||
except Exception as exc:
|
||||
run.status = BackupRun.Status.FAILED
|
||||
|
||||
191
src/pobsync_backend/migrations/0003_structured_config_fields.py
Normal file
191
src/pobsync_backend/migrations/0003_structured_config_fields.py
Normal file
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def copy_json_config_to_fields(apps, schema_editor) -> None:
|
||||
GlobalConfig = apps.get_model("pobsync_backend", "GlobalConfig")
|
||||
HostConfig = apps.get_model("pobsync_backend", "HostConfig")
|
||||
|
||||
for global_config in GlobalConfig.objects.all():
|
||||
data = global_config.data or {}
|
||||
ssh = data.get("ssh") or {}
|
||||
rsync = data.get("rsync") or {}
|
||||
defaults = data.get("defaults") or {}
|
||||
retention = data.get("retention_defaults") or {}
|
||||
|
||||
global_config.ssh_user = ssh.get("user") or global_config.ssh_user
|
||||
global_config.ssh_port = ssh.get("port") or global_config.ssh_port
|
||||
global_config.ssh_options = ssh.get("options") or []
|
||||
global_config.rsync_binary = rsync.get("binary") or global_config.rsync_binary
|
||||
global_config.rsync_args = rsync.get("args") or []
|
||||
global_config.rsync_extra_args = rsync.get("extra_args") or []
|
||||
global_config.rsync_timeout_seconds = rsync.get("timeout_seconds") or 0
|
||||
global_config.rsync_bwlimit_kbps = rsync.get("bwlimit_kbps") or 0
|
||||
global_config.default_source_root = defaults.get("source_root") or "/"
|
||||
global_config.default_destination_subdir = defaults.get("destination_subdir") or ""
|
||||
global_config.excludes_default = data.get("excludes_default") or []
|
||||
global_config.retention_daily = retention.get("daily", global_config.retention_daily)
|
||||
global_config.retention_weekly = retention.get("weekly", global_config.retention_weekly)
|
||||
global_config.retention_monthly = retention.get("monthly", global_config.retention_monthly)
|
||||
global_config.retention_yearly = retention.get("yearly", global_config.retention_yearly)
|
||||
global_config.save()
|
||||
|
||||
for host_config in HostConfig.objects.all():
|
||||
config = host_config.config or {}
|
||||
ssh = config.get("ssh") or {}
|
||||
rsync = config.get("rsync") or {}
|
||||
retention = config.get("retention") or {}
|
||||
|
||||
host_config.ssh_user = ssh.get("user") or ""
|
||||
host_config.ssh_port = ssh.get("port")
|
||||
host_config.source_root = config.get("source_root") or ""
|
||||
host_config.includes = config.get("includes") or []
|
||||
host_config.excludes_add = config.get("excludes_add") or []
|
||||
host_config.excludes_replace = config.get("excludes_replace")
|
||||
host_config.rsync_extra_args = rsync.get("extra_args") or []
|
||||
host_config.retention_daily = retention.get("daily", host_config.retention_daily)
|
||||
host_config.retention_weekly = retention.get("weekly", host_config.retention_weekly)
|
||||
host_config.retention_monthly = retention.get("monthly", host_config.retention_monthly)
|
||||
host_config.retention_yearly = retention.get("yearly", host_config.retention_yearly)
|
||||
host_config.save()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("pobsync_backend", "0002_schedule_run_state"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="default_destination_subdir",
|
||||
field=models.CharField(blank=True, default="", max_length=512),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="default_source_root",
|
||||
field=models.CharField(default="/", max_length=512),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="excludes_default",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="retention_daily",
|
||||
field=models.PositiveIntegerField(default=14),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="retention_monthly",
|
||||
field=models.PositiveIntegerField(default=12),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="retention_weekly",
|
||||
field=models.PositiveIntegerField(default=8),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="retention_yearly",
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="rsync_args",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="rsync_binary",
|
||||
field=models.CharField(default="rsync", max_length=128),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="rsync_bwlimit_kbps",
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="rsync_extra_args",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="rsync_timeout_seconds",
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="ssh_options",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="ssh_port",
|
||||
field=models.PositiveIntegerField(default=22),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="globalconfig",
|
||||
name="ssh_user",
|
||||
field=models.CharField(default="root", max_length=64),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="excludes_add",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="excludes_replace",
|
||||
field=models.JSONField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="includes",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="retention_daily",
|
||||
field=models.PositiveIntegerField(default=14),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="retention_monthly",
|
||||
field=models.PositiveIntegerField(default=12),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="retention_weekly",
|
||||
field=models.PositiveIntegerField(default=8),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="retention_yearly",
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="rsync_extra_args",
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="source_root",
|
||||
field=models.CharField(blank=True, max_length=512),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="ssh_port",
|
||||
field=models.PositiveIntegerField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="hostconfig",
|
||||
name="ssh_user",
|
||||
field=models.CharField(blank=True, max_length=64),
|
||||
),
|
||||
migrations.RunPython(copy_json_config_to_fields, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -15,6 +15,21 @@ class GlobalConfig(TimestampedModel):
|
||||
name = models.CharField(max_length=64, default="default", unique=True)
|
||||
backup_root = models.CharField(max_length=512)
|
||||
pobsync_home = models.CharField(max_length=512, default="/opt/pobsync")
|
||||
ssh_user = models.CharField(max_length=64, default="root")
|
||||
ssh_port = models.PositiveIntegerField(default=22)
|
||||
ssh_options = models.JSONField(default=list, blank=True)
|
||||
rsync_binary = models.CharField(max_length=128, default="rsync")
|
||||
rsync_args = models.JSONField(default=list, blank=True)
|
||||
rsync_extra_args = models.JSONField(default=list, blank=True)
|
||||
rsync_timeout_seconds = models.PositiveIntegerField(default=0)
|
||||
rsync_bwlimit_kbps = models.PositiveIntegerField(default=0)
|
||||
default_source_root = models.CharField(max_length=512, default="/")
|
||||
default_destination_subdir = models.CharField(max_length=512, default="", blank=True)
|
||||
excludes_default = models.JSONField(default=list, blank=True)
|
||||
retention_daily = models.PositiveIntegerField(default=14)
|
||||
retention_weekly = models.PositiveIntegerField(default=8)
|
||||
retention_monthly = models.PositiveIntegerField(default=12)
|
||||
retention_yearly = models.PositiveIntegerField(default=0)
|
||||
data = models.JSONField(default=dict, blank=True)
|
||||
|
||||
class Meta:
|
||||
@@ -29,6 +44,17 @@ class HostConfig(TimestampedModel):
|
||||
host = models.CharField(max_length=255, unique=True)
|
||||
address = models.CharField(max_length=255)
|
||||
enabled = models.BooleanField(default=True)
|
||||
ssh_user = models.CharField(max_length=64, blank=True)
|
||||
ssh_port = models.PositiveIntegerField(null=True, blank=True)
|
||||
source_root = models.CharField(max_length=512, blank=True)
|
||||
includes = models.JSONField(default=list, blank=True)
|
||||
excludes_add = models.JSONField(default=list, blank=True)
|
||||
excludes_replace = models.JSONField(null=True, blank=True)
|
||||
rsync_extra_args = models.JSONField(default=list, blank=True)
|
||||
retention_daily = models.PositiveIntegerField(default=14)
|
||||
retention_weekly = models.PositiveIntegerField(default=8)
|
||||
retention_monthly = models.PositiveIntegerField(default=12)
|
||||
retention_yearly = models.PositiveIntegerField(default=0)
|
||||
config = models.JSONField(default=dict, blank=True)
|
||||
|
||||
class Meta:
|
||||
|
||||
@@ -18,21 +18,36 @@ class ConfigRepositoryTests(TestCase):
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home=str(prefix),
|
||||
ssh_user="backup",
|
||||
ssh_port=2222,
|
||||
rsync_args=["--archive"],
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
data={
|
||||
"backup_root": "/ignored",
|
||||
"pobsync_home": "/ignored",
|
||||
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
"ssh": {"user": "ignored", "port": 22, "options": []},
|
||||
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
},
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
ssh_user="root",
|
||||
includes=[],
|
||||
excludes_add=["/tmp/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
config={
|
||||
"host": "ignored",
|
||||
"address": "ignored",
|
||||
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
"includes": [],
|
||||
"excludes_add": ["/tmp/***"],
|
||||
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
"excludes_add": ["/ignored/***"],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -43,5 +58,10 @@ class ConfigRepositoryTests(TestCase):
|
||||
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
|
||||
self.assertEqual(global_cfg["backup_root"], "/backups")
|
||||
self.assertEqual(global_cfg["pobsync_home"], str(prefix))
|
||||
self.assertEqual(global_cfg["ssh"]["user"], "backup")
|
||||
self.assertEqual(global_cfg["ssh"]["port"], 2222)
|
||||
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["host"], "web-01")
|
||||
self.assertEqual(host_cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(host_cfg["retention"]["daily"], 7)
|
||||
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])
|
||||
|
||||
60
src/pobsync_backend/tests/test_django_config_source.py
Normal file
60
src/pobsync_backend/tests/test_django_config_source.py
Normal file
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from pobsync_backend.config_source import DjangoConfigSource
|
||||
from pobsync_backend.models import GlobalConfig, HostConfig
|
||||
|
||||
|
||||
class DjangoConfigSourceTests(TestCase):
|
||||
def test_returns_effective_config_from_database(self) -> None:
|
||||
GlobalConfig.objects.create(
|
||||
name="default",
|
||||
backup_root="/backups",
|
||||
pobsync_home="/opt/pobsync",
|
||||
rsync_args=["--archive"],
|
||||
rsync_extra_args=["--numeric-ids"],
|
||||
excludes_default=["/proc/***"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
data={
|
||||
"backup_root": "/ignored",
|
||||
"pobsync_home": "/ignored",
|
||||
"ssh": {"user": "root", "port": 22, "options": []},
|
||||
"rsync": {
|
||||
"binary": "rsync",
|
||||
"args": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
"extra_args": ["--numeric-ids"],
|
||||
},
|
||||
"defaults": {"source_root": "/", "destination_subdir": ""},
|
||||
"excludes_default": ["/proc/***"],
|
||||
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
},
|
||||
)
|
||||
HostConfig.objects.create(
|
||||
host="web-01",
|
||||
address="web-01.example.test",
|
||||
excludes_add=["/tmp/***"],
|
||||
rsync_extra_args=["--delete"],
|
||||
retention_daily=7,
|
||||
retention_weekly=4,
|
||||
retention_monthly=3,
|
||||
retention_yearly=1,
|
||||
config={
|
||||
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
|
||||
"excludes_add": ["/ignored/***"],
|
||||
"rsync": {"extra_args": ["--ignored"]},
|
||||
},
|
||||
)
|
||||
|
||||
cfg = DjangoConfigSource().effective_config_for_host("web-01")
|
||||
|
||||
self.assertEqual(cfg["backup_root"], "/backups")
|
||||
self.assertEqual(cfg["host"], "web-01")
|
||||
self.assertEqual(cfg["address"], "web-01.example.test")
|
||||
self.assertEqual(cfg["excludes_effective"], ["/proc/***", "/tmp/***"])
|
||||
self.assertEqual(cfg["rsync"]["args_effective"], ["--archive", "--numeric-ids", "--delete"])
|
||||
53
src/pobsync_backend/tests/test_retention_config_source.py
Normal file
53
src/pobsync_backend/tests/test_retention_config_source.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync.commands.retention_plan import run_retention_plan
|
||||
from pobsync.util import write_yaml_atomic
|
||||
|
||||
|
||||
class FakeConfigSource:
|
||||
def __init__(self, backup_root: str) -> None:
|
||||
self.backup_root = backup_root
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict:
|
||||
return {
|
||||
"backup_root": self.backup_root,
|
||||
"host": host,
|
||||
"address": "example.test",
|
||||
"retention": {"daily": 1, "weekly": 0, "monthly": 0, "yearly": 0},
|
||||
}
|
||||
|
||||
|
||||
class RetentionConfigSourceTests(SimpleTestCase):
|
||||
def test_retention_plan_uses_injected_config_source(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
root = Path(tmp) / "backups"
|
||||
snap_dir = root / "web-01" / "scheduled" / "20260519-021500Z__ABCDEFGH"
|
||||
meta_dir = snap_dir / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
write_yaml_atomic(
|
||||
meta_dir / "meta.yaml",
|
||||
{
|
||||
"status": "success",
|
||||
"started_at": datetime(2026, 5, 19, 2, 15, tzinfo=timezone.utc)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
},
|
||||
)
|
||||
|
||||
plan = run_retention_plan(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
kind="scheduled",
|
||||
protect_bases=False,
|
||||
config_source=FakeConfigSource(str(root)),
|
||||
)
|
||||
|
||||
self.assertTrue(plan["ok"])
|
||||
self.assertEqual(plan["keep"], ["20260519-021500Z__ABCDEFGH"])
|
||||
self.assertEqual(plan["delete"], [])
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from pobsync.commands.run_scheduled import run_scheduled
|
||||
from pobsync.rsync import RsyncResult
|
||||
|
||||
|
||||
class FakeConfigSource:
|
||||
def __init__(self, backup_root: str = "/tmp/pobsync-test-backups") -> None:
|
||||
self.backup_root = backup_root
|
||||
|
||||
def effective_config_for_host(self, host: str) -> dict:
|
||||
return {
|
||||
"backup_root": self.backup_root,
|
||||
"host": host,
|
||||
"address": "example.test",
|
||||
"ssh": {"user": "root", "port": 22, "options": []},
|
||||
"rsync": {
|
||||
"binary": "rsync",
|
||||
"args_effective": ["--archive"],
|
||||
"timeout_seconds": 0,
|
||||
"bwlimit_kbps": 0,
|
||||
},
|
||||
"source_root": "/",
|
||||
"includes": [],
|
||||
"excludes_effective": [],
|
||||
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
|
||||
}
|
||||
|
||||
|
||||
class RunScheduledConfigSourceTests(SimpleTestCase):
|
||||
def test_dry_run_uses_injected_config_source(self) -> None:
|
||||
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=Path("/missing-prefix"),
|
||||
host="web-01",
|
||||
dry_run=True,
|
||||
config_source=FakeConfigSource(),
|
||||
)
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["host"], "web-01")
|
||||
run_rsync.assert_called_once()
|
||||
|
||||
def test_successful_real_run_applies_prune_when_requested(self) -> None:
|
||||
with TemporaryDirectory() as tmp:
|
||||
prefix = Path(tmp) / "home"
|
||||
with (
|
||||
patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync,
|
||||
patch("pobsync.commands.retention_apply.run_retention_plan") as plan,
|
||||
):
|
||||
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])
|
||||
plan.return_value = {
|
||||
"ok": True,
|
||||
"delete": [],
|
||||
"keep": [],
|
||||
"reasons": {},
|
||||
"protect_bases": False,
|
||||
}
|
||||
|
||||
result = run_scheduled(
|
||||
prefix=prefix,
|
||||
host="web-01",
|
||||
dry_run=False,
|
||||
prune=True,
|
||||
prune_max_delete=10,
|
||||
config_source=FakeConfigSource(backup_root=str(Path(tmp) / "backups")),
|
||||
)
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertIsNotNone(result["prune"])
|
||||
self.assertEqual(result["prune"]["deleted"], [])
|
||||
Reference in New Issue
Block a user