hippogrief/pobsync
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor: stop using legacy JSON for runtime config

Browse Source 
Build runtime pobsync configuration exclusively from structured SQL
fields, leaving legacy JSON only for import and audit context. Add
SQL-first management commands for global and host configuration and
cover them with tests.
This commit is contained in:
Peter van Arkel
2026-05-19 05:08:37 +02:00
parent a0eb5dcc8f
commit 6d9ddc4457
6 changed files with 231 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/pobsync_backend/config_repository.py
View File

@@ -18,68 +18,62 @@ class ConfigRepositoryError(RuntimeError):
def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
data = dict(global_config.data or {})
data["backup_root"] = global_config.backup_root
data["pobsync_home"] = global_config.pobsync_home
data["ssh"] = {
"user": global_config.ssh_user,
"port": global_config.ssh_port,
"options": list(global_config.ssh_options or []),
}
data["rsync"] = {
"binary": global_config.rsync_binary,
"args": list(global_config.rsync_args or []),
"timeout_seconds": global_config.rsync_timeout_seconds,
"bwlimit_kbps": global_config.rsync_bwlimit_kbps,
"extra_args": list(global_config.rsync_extra_args or []),
}
data["defaults"] = {
"source_root": global_config.default_source_root,
"destination_subdir": global_config.default_destination_subdir,
}
data["excludes_default"] = list(global_config.excludes_default or [])
data["retention_defaults"] = {
"daily": global_config.retention_daily,
"weekly": global_config.retention_weekly,
"monthly": global_config.retention_monthly,
"yearly": global_config.retention_yearly,
data = {
"backup_root": global_config.backup_root,
"pobsync_home": global_config.pobsync_home,
"ssh": {
"user": global_config.ssh_user,
"port": global_config.ssh_port,
"options": list(global_config.ssh_options or []),
},
"rsync": {
"binary": global_config.rsync_binary,
"args": list(global_config.rsync_args or []),
"timeout_seconds": global_config.rsync_timeout_seconds,
"bwlimit_kbps": global_config.rsync_bwlimit_kbps,
"extra_args": list(global_config.rsync_extra_args or []),
},
"defaults": {
"source_root": global_config.default_source_root,
"destination_subdir": global_config.default_destination_subdir,
},
"excludes_default": list(global_config.excludes_default or []),
"retention_defaults": {
"daily": global_config.retention_daily,
"weekly": global_config.retention_weekly,
"monthly": global_config.retention_monthly,
"yearly": global_config.retention_yearly,
},
}
return validate_dict(data, GLOBAL_SCHEMA, path="global")
def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
data = dict(host_config.config or {})
data["host"] = host_config.host
data["address"] = host_config.address
data: dict[str, Any] = {
"host": host_config.host,
"address": host_config.address,
"includes": list(host_config.includes or []),
"retention": {
"daily": host_config.retention_daily,
"weekly": host_config.retention_weekly,
"monthly": host_config.retention_monthly,
"yearly": host_config.retention_yearly,
},
}
if host_config.ssh_user or host_config.ssh_port:
data["ssh"] = {}
if host_config.ssh_user:
data["ssh"]["user"] = host_config.ssh_user
if host_config.ssh_port is not None:
data["ssh"]["port"] = host_config.ssh_port
else:
data.pop("ssh", None)
if host_config.source_root:
data["source_root"] = host_config.source_root
else:
data.pop("source_root", None)
data["includes"] = list(host_config.includes or [])
if host_config.excludes_replace is not None:
data["excludes_replace"] = list(host_config.excludes_replace or [])
data.pop("excludes_add", None)
else:
data["excludes_add"] = list(host_config.excludes_add or [])
data.pop("excludes_replace", None)
if host_config.rsync_extra_args:
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
else:
data.pop("rsync", None)
data["retention"] = {
"daily": host_config.retention_daily,
"weekly": host_config.retention_weekly,
"monthly": host_config.retention_monthly,
"yearly": host_config.retention_yearly,
}
return validate_dict(data, HOST_SCHEMA, path="host")

60
src/pobsync_backend/management/commands/configure_pobsync_global.py Normal file
View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from pobsync.cli import parse_retention
from pobsync.commands.install import DEFAULT_EXCLUDES, DEFAULT_RSYNC_ARGS
from pobsync.util import is_absolute_non_root
from pobsync_backend.models import GlobalConfig
class Command(BaseCommand):
help = "Create or update the SQL-backed global pobsync configuration."
def add_arguments(self, parser) -> None:
parser.add_argument("--name", default="default")
parser.add_argument("--backup-root", required=True)
parser.add_argument("--pobsync-home", default=settings.POBSYNC_HOME)
parser.add_argument("--ssh-user", default="root")
parser.add_argument("--ssh-port", type=int, default=22)
parser.add_argument("--source-root", default="/")
parser.add_argument("--retention", default="daily=14,weekly=8,monthly=12,yearly=0")
parser.add_argument("--force", action="store_true", help="Update existing config")
def handle(self, *args: Any, **options: Any) -> None:
backup_root = options["backup_root"]
if not is_absolute_non_root(backup_root):
raise CommandError("--backup-root must be an absolute path and must not be '/'")
pobsync_home = str(Path(options["pobsync_home"]))
retention = parse_retention(options["retention"])
defaults = {
"backup_root": backup_root,
"pobsync_home": pobsync_home,
"ssh_user": options["ssh_user"],
"ssh_port": options["ssh_port"],
"ssh_options": ["-oBatchMode=yes", "-oStrictHostKeyChecking=accept-new"],
"rsync_binary": "rsync",
"rsync_args": DEFAULT_RSYNC_ARGS,
"rsync_extra_args": [],
"rsync_timeout_seconds": 0,
"rsync_bwlimit_kbps": 0,
"default_source_root": options["source_root"],
"default_destination_subdir": "",
"excludes_default": DEFAULT_EXCLUDES,
"retention_daily": retention["daily"],
"retention_weekly": retention["weekly"],
"retention_monthly": retention["monthly"],
"retention_yearly": retention["yearly"],
}
if GlobalConfig.objects.filter(name=options["name"]).exists() and not options["force"]:
raise CommandError(f"GlobalConfig {options['name']!r} already exists; use --force to update")
_obj, created = GlobalConfig.objects.update_or_create(name=options["name"], defaults=defaults)
action = "Created" if created else "Updated"
self.stdout.write(self.style.SUCCESS(f"{action} GlobalConfig {options['name']!r}."))

65
src/pobsync_backend/management/commands/configure_pobsync_host.py Normal file
View File

@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import Any
from django.core.management.base import BaseCommand, CommandError
from pobsync.cli import parse_retention
from pobsync.util import sanitize_host
from pobsync_backend.models import GlobalConfig, HostConfig
class Command(BaseCommand):
help = "Create or update a SQL-backed host pobsync configuration."
def add_arguments(self, parser) -> None:
parser.add_argument("host")
parser.add_argument("--address", required=True)
parser.add_argument("--ssh-user", default="")
parser.add_argument("--ssh-port", type=int, default=None)
parser.add_argument("--source-root", default="")
parser.add_argument("--include", action="append", default=[])
parser.add_argument("--exclude-add", action="append", default=[])
parser.add_argument("--exclude-replace", action="append", default=None)
parser.add_argument("--rsync-extra-arg", action="append", default=[])
parser.add_argument("--retention", default=None)
parser.add_argument("--disabled", action="store_true")
parser.add_argument("--force", action="store_true", help="Update existing host")
def handle(self, *args: Any, **options: Any) -> None:
host = sanitize_host(options["host"])
if HostConfig.objects.filter(host=host).exists() and not options["force"]:
raise CommandError(f"HostConfig {host!r} already exists; use --force to update")
retention = self._retention(options["retention"])
defaults = {
"address": options["address"],
"enabled": not options["disabled"],
"ssh_user": options["ssh_user"],
"ssh_port": options["ssh_port"],
"source_root": options["source_root"],
"includes": list(options["include"]),
"excludes_add": [] if options["exclude_replace"] is not None else list(options["exclude_add"]),
"excludes_replace": options["exclude_replace"],
"rsync_extra_args": list(options["rsync_extra_arg"]),
"retention_daily": retention["daily"],
"retention_weekly": retention["weekly"],
"retention_monthly": retention["monthly"],
"retention_yearly": retention["yearly"],
}
_obj, created = HostConfig.objects.update_or_create(host=host, defaults=defaults)
action = "Created" if created else "Updated"
self.stdout.write(self.style.SUCCESS(f"{action} HostConfig {host!r}."))
def _retention(self, value: str | None) -> dict[str, int]:
if value:
return parse_retention(value)
global_config = GlobalConfig.objects.filter(name="default").first()
if global_config is None:
return {"daily": 14, "weekly": 8, "monthly": 12, "yearly": 0}
return {
"daily": global_config.retention_daily,
"weekly": global_config.retention_weekly,
"monthly": global_config.retention_monthly,
"yearly": global_config.retention_yearly,
}

4
src/pobsync_backend/tests/test_config_repository.py
View File

@@ -30,6 +30,7 @@ class ConfigRepositoryTests(TestCase):
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"ssh": {"user": "ignored", "port": 22, "options": []},
"unknown": "must-not-leak",
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
},
)
@@ -48,6 +49,7 @@ class ConfigRepositoryTests(TestCase):
"address": "ignored",
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
"excludes_add": ["/ignored/***"],
"unknown": "must-not-leak",
},
)
@@ -65,3 +67,5 @@ class ConfigRepositoryTests(TestCase):
self.assertEqual(host_cfg["address"], "web-01.example.test")
self.assertEqual(host_cfg["retention"]["daily"], 7)
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])
self.assertNotIn("unknown", global_cfg)
self.assertNotIn("unknown", host_cfg)

56
src/pobsync_backend/tests/test_configure_commands.py Normal file
View File

@@ -0,0 +1,56 @@
from __future__ import annotations
from io import StringIO
from django.core.management import call_command
from django.test import TestCase
from pobsync_backend.config_source import DjangoConfigSource
from pobsync_backend.models import GlobalConfig, HostConfig
class ConfigureCommandsTests(TestCase):
def test_configure_global_creates_structured_config(self) -> None:
out = StringIO()
call_command(
"configure_pobsync_global",
backup_root="/backups",
pobsync_home="/opt/pobsync",
retention="daily=3,weekly=2,monthly=1,yearly=0",
stdout=out,
)
config = GlobalConfig.objects.get(name="default")
self.assertEqual(config.backup_root, "/backups")
self.assertEqual(config.retention_daily, 3)
self.assertIn("Created GlobalConfig", out.getvalue())
def test_configure_host_uses_global_retention_defaults(self) -> None:
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
retention_daily=5,
retention_weekly=4,
retention_monthly=3,
retention_yearly=2,
)
out = StringIO()
call_command(
"configure_pobsync_host",
"web-01",
address="web-01.example.test",
exclude_add=["/tmp/***"],
rsync_extra_arg=["--delete"],
stdout=out,
)
host = HostConfig.objects.get(host="web-01")
self.assertEqual(host.retention_daily, 5)
self.assertEqual(host.excludes_add, ["/tmp/***"])
self.assertEqual(host.rsync_extra_args, ["--delete"])
effective = DjangoConfigSource().effective_config_for_host("web-01")
self.assertEqual(effective["retention"]["yearly"], 2)
self.assertEqual(effective["excludes_effective"], ["/tmp/***"])