From 6d9ddc4457044d4d0fc41c4cf2088b2a9f600e11 Mon Sep 17 00:00:00 2001 From: Peter van Arkel Date: Tue, 19 May 2026 05:08:37 +0200 Subject: [PATCH] refactor: stop using legacy JSON for runtime config Build runtime pobsync configuration exclusively from structured SQL fields, leaving legacy JSON only for import and audit context. Add SQL-first management commands for global and host configuration and cover them with tests. --- README.md | 11 ++- src/pobsync_backend/config_repository.py | 80 +++++++++---------- .../commands/configure_pobsync_global.py | 60 ++++++++++++++ .../commands/configure_pobsync_host.py | 65 +++++++++++++++ .../tests/test_config_repository.py | 4 + .../tests/test_configure_commands.py | 56 +++++++++++++ 6 files changed, 231 insertions(+), 45 deletions(-) create mode 100644 src/pobsync_backend/management/commands/configure_pobsync_global.py create mode 100644 src/pobsync_backend/management/commands/configure_pobsync_host.py create mode 100644 src/pobsync_backend/tests/test_configure_commands.py diff --git a/README.md b/README.md index 6a87651..7fdcc0e 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,13 @@ Import existing YAML configs into the database: python3 manage.py import_pobsync_configs --prefix /opt/pobsync ``` +Create SQL-backed configuration directly: + +``` +python3 manage.py configure_pobsync_global --backup-root /mnt/backups/pobsync +python3 manage.py configure_pobsync_host --address +``` + Run a backup through Django while still using the existing pobsync engine: ``` @@ -147,7 +154,7 @@ python3 manage.py run_pobsync_backup --prefix /opt/pobsync --prune The Django backup command reads backup and retention config from SQL directly. Runtime YAML export is kept as a compatibility tool for older CLI flows during the transition. -Export database configs to the runtime YAML files consumed by the current engine: +Export database configs to runtime YAML for legacy CLI compatibility: ``` python3 manage.py export_pobsync_configs --prefix /opt/pobsync @@ -195,7 +202,7 @@ The MariaDB profile is optional. SQLite remains the default because it is enough Recommended next steps: -- Continue moving config reading/writing behind repository interfaces so YAML export can eventually disappear. +- Remove remaining legacy YAML-first commands after SQL-first setup covers all workflows. - Record more engine-side run details into `BackupRun` and `SnapshotRecord`. - Treat SQL as the source of truth and export YAML only as a compatibility layer for the current engine. - Run schedules from Django/Docker instead of writing host cron files. diff --git a/src/pobsync_backend/config_repository.py b/src/pobsync_backend/config_repository.py index ab7b947..5e2b822 100644 --- a/src/pobsync_backend/config_repository.py +++ b/src/pobsync_backend/config_repository.py @@ -18,68 +18,62 @@ class ConfigRepositoryError(RuntimeError): def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]: - data = dict(global_config.data or {}) - data["backup_root"] = global_config.backup_root - data["pobsync_home"] = global_config.pobsync_home - data["ssh"] = { - "user": global_config.ssh_user, - "port": global_config.ssh_port, - "options": list(global_config.ssh_options or []), - } - data["rsync"] = { - "binary": global_config.rsync_binary, - "args": list(global_config.rsync_args or []), - "timeout_seconds": global_config.rsync_timeout_seconds, - "bwlimit_kbps": global_config.rsync_bwlimit_kbps, - "extra_args": list(global_config.rsync_extra_args or []), - } - data["defaults"] = { - "source_root": global_config.default_source_root, - "destination_subdir": global_config.default_destination_subdir, - } - data["excludes_default"] = list(global_config.excludes_default or []) - data["retention_defaults"] = { - "daily": global_config.retention_daily, - "weekly": global_config.retention_weekly, - "monthly": global_config.retention_monthly, - "yearly": global_config.retention_yearly, + data = { + "backup_root": global_config.backup_root, + "pobsync_home": global_config.pobsync_home, + "ssh": { + "user": global_config.ssh_user, + "port": global_config.ssh_port, + "options": list(global_config.ssh_options or []), + }, + "rsync": { + "binary": global_config.rsync_binary, + "args": list(global_config.rsync_args or []), + "timeout_seconds": global_config.rsync_timeout_seconds, + "bwlimit_kbps": global_config.rsync_bwlimit_kbps, + "extra_args": list(global_config.rsync_extra_args or []), + }, + "defaults": { + "source_root": global_config.default_source_root, + "destination_subdir": global_config.default_destination_subdir, + }, + "excludes_default": list(global_config.excludes_default or []), + "retention_defaults": { + "daily": global_config.retention_daily, + "weekly": global_config.retention_weekly, + "monthly": global_config.retention_monthly, + "yearly": global_config.retention_yearly, + }, } return validate_dict(data, GLOBAL_SCHEMA, path="global") def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]: - data = dict(host_config.config or {}) - data["host"] = host_config.host - data["address"] = host_config.address + data: dict[str, Any] = { + "host": host_config.host, + "address": host_config.address, + "includes": list(host_config.includes or []), + "retention": { + "daily": host_config.retention_daily, + "weekly": host_config.retention_weekly, + "monthly": host_config.retention_monthly, + "yearly": host_config.retention_yearly, + }, + } if host_config.ssh_user or host_config.ssh_port: data["ssh"] = {} if host_config.ssh_user: data["ssh"]["user"] = host_config.ssh_user if host_config.ssh_port is not None: data["ssh"]["port"] = host_config.ssh_port - else: - data.pop("ssh", None) if host_config.source_root: data["source_root"] = host_config.source_root - else: - data.pop("source_root", None) - data["includes"] = list(host_config.includes or []) if host_config.excludes_replace is not None: data["excludes_replace"] = list(host_config.excludes_replace or []) - data.pop("excludes_add", None) else: data["excludes_add"] = list(host_config.excludes_add or []) - data.pop("excludes_replace", None) if host_config.rsync_extra_args: data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])} - else: - data.pop("rsync", None) - data["retention"] = { - "daily": host_config.retention_daily, - "weekly": host_config.retention_weekly, - "monthly": host_config.retention_monthly, - "yearly": host_config.retention_yearly, - } return validate_dict(data, HOST_SCHEMA, path="host") diff --git a/src/pobsync_backend/management/commands/configure_pobsync_global.py b/src/pobsync_backend/management/commands/configure_pobsync_global.py new file mode 100644 index 0000000..8b732c5 --- /dev/null +++ b/src/pobsync_backend/management/commands/configure_pobsync_global.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from pobsync.cli import parse_retention +from pobsync.commands.install import DEFAULT_EXCLUDES, DEFAULT_RSYNC_ARGS +from pobsync.util import is_absolute_non_root +from pobsync_backend.models import GlobalConfig + + +class Command(BaseCommand): + help = "Create or update the SQL-backed global pobsync configuration." + + def add_arguments(self, parser) -> None: + parser.add_argument("--name", default="default") + parser.add_argument("--backup-root", required=True) + parser.add_argument("--pobsync-home", default=settings.POBSYNC_HOME) + parser.add_argument("--ssh-user", default="root") + parser.add_argument("--ssh-port", type=int, default=22) + parser.add_argument("--source-root", default="/") + parser.add_argument("--retention", default="daily=14,weekly=8,monthly=12,yearly=0") + parser.add_argument("--force", action="store_true", help="Update existing config") + + def handle(self, *args: Any, **options: Any) -> None: + backup_root = options["backup_root"] + if not is_absolute_non_root(backup_root): + raise CommandError("--backup-root must be an absolute path and must not be '/'") + + pobsync_home = str(Path(options["pobsync_home"])) + retention = parse_retention(options["retention"]) + defaults = { + "backup_root": backup_root, + "pobsync_home": pobsync_home, + "ssh_user": options["ssh_user"], + "ssh_port": options["ssh_port"], + "ssh_options": ["-oBatchMode=yes", "-oStrictHostKeyChecking=accept-new"], + "rsync_binary": "rsync", + "rsync_args": DEFAULT_RSYNC_ARGS, + "rsync_extra_args": [], + "rsync_timeout_seconds": 0, + "rsync_bwlimit_kbps": 0, + "default_source_root": options["source_root"], + "default_destination_subdir": "", + "excludes_default": DEFAULT_EXCLUDES, + "retention_daily": retention["daily"], + "retention_weekly": retention["weekly"], + "retention_monthly": retention["monthly"], + "retention_yearly": retention["yearly"], + } + + if GlobalConfig.objects.filter(name=options["name"]).exists() and not options["force"]: + raise CommandError(f"GlobalConfig {options['name']!r} already exists; use --force to update") + + _obj, created = GlobalConfig.objects.update_or_create(name=options["name"], defaults=defaults) + action = "Created" if created else "Updated" + self.stdout.write(self.style.SUCCESS(f"{action} GlobalConfig {options['name']!r}.")) diff --git a/src/pobsync_backend/management/commands/configure_pobsync_host.py b/src/pobsync_backend/management/commands/configure_pobsync_host.py new file mode 100644 index 0000000..692061d --- /dev/null +++ b/src/pobsync_backend/management/commands/configure_pobsync_host.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import Any + +from django.core.management.base import BaseCommand, CommandError + +from pobsync.cli import parse_retention +from pobsync.util import sanitize_host +from pobsync_backend.models import GlobalConfig, HostConfig + + +class Command(BaseCommand): + help = "Create or update a SQL-backed host pobsync configuration." + + def add_arguments(self, parser) -> None: + parser.add_argument("host") + parser.add_argument("--address", required=True) + parser.add_argument("--ssh-user", default="") + parser.add_argument("--ssh-port", type=int, default=None) + parser.add_argument("--source-root", default="") + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude-add", action="append", default=[]) + parser.add_argument("--exclude-replace", action="append", default=None) + parser.add_argument("--rsync-extra-arg", action="append", default=[]) + parser.add_argument("--retention", default=None) + parser.add_argument("--disabled", action="store_true") + parser.add_argument("--force", action="store_true", help="Update existing host") + + def handle(self, *args: Any, **options: Any) -> None: + host = sanitize_host(options["host"]) + if HostConfig.objects.filter(host=host).exists() and not options["force"]: + raise CommandError(f"HostConfig {host!r} already exists; use --force to update") + + retention = self._retention(options["retention"]) + defaults = { + "address": options["address"], + "enabled": not options["disabled"], + "ssh_user": options["ssh_user"], + "ssh_port": options["ssh_port"], + "source_root": options["source_root"], + "includes": list(options["include"]), + "excludes_add": [] if options["exclude_replace"] is not None else list(options["exclude_add"]), + "excludes_replace": options["exclude_replace"], + "rsync_extra_args": list(options["rsync_extra_arg"]), + "retention_daily": retention["daily"], + "retention_weekly": retention["weekly"], + "retention_monthly": retention["monthly"], + "retention_yearly": retention["yearly"], + } + _obj, created = HostConfig.objects.update_or_create(host=host, defaults=defaults) + action = "Created" if created else "Updated" + self.stdout.write(self.style.SUCCESS(f"{action} HostConfig {host!r}.")) + + def _retention(self, value: str | None) -> dict[str, int]: + if value: + return parse_retention(value) + global_config = GlobalConfig.objects.filter(name="default").first() + if global_config is None: + return {"daily": 14, "weekly": 8, "monthly": 12, "yearly": 0} + return { + "daily": global_config.retention_daily, + "weekly": global_config.retention_weekly, + "monthly": global_config.retention_monthly, + "yearly": global_config.retention_yearly, + } diff --git a/src/pobsync_backend/tests/test_config_repository.py b/src/pobsync_backend/tests/test_config_repository.py index 1f51c0b..312aab2 100644 --- a/src/pobsync_backend/tests/test_config_repository.py +++ b/src/pobsync_backend/tests/test_config_repository.py @@ -30,6 +30,7 @@ class ConfigRepositoryTests(TestCase): "backup_root": "/ignored", "pobsync_home": "/ignored", "ssh": {"user": "ignored", "port": 22, "options": []}, + "unknown": "must-not-leak", "retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, }, ) @@ -48,6 +49,7 @@ class ConfigRepositoryTests(TestCase): "address": "ignored", "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, "excludes_add": ["/ignored/***"], + "unknown": "must-not-leak", }, ) @@ -65,3 +67,5 @@ class ConfigRepositoryTests(TestCase): self.assertEqual(host_cfg["address"], "web-01.example.test") self.assertEqual(host_cfg["retention"]["daily"], 7) self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"]) + self.assertNotIn("unknown", global_cfg) + self.assertNotIn("unknown", host_cfg) diff --git a/src/pobsync_backend/tests/test_configure_commands.py b/src/pobsync_backend/tests/test_configure_commands.py new file mode 100644 index 0000000..0c42001 --- /dev/null +++ b/src/pobsync_backend/tests/test_configure_commands.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from io import StringIO + +from django.core.management import call_command +from django.test import TestCase + +from pobsync_backend.config_source import DjangoConfigSource +from pobsync_backend.models import GlobalConfig, HostConfig + + +class ConfigureCommandsTests(TestCase): + def test_configure_global_creates_structured_config(self) -> None: + out = StringIO() + + call_command( + "configure_pobsync_global", + backup_root="/backups", + pobsync_home="/opt/pobsync", + retention="daily=3,weekly=2,monthly=1,yearly=0", + stdout=out, + ) + + config = GlobalConfig.objects.get(name="default") + self.assertEqual(config.backup_root, "/backups") + self.assertEqual(config.retention_daily, 3) + self.assertIn("Created GlobalConfig", out.getvalue()) + + def test_configure_host_uses_global_retention_defaults(self) -> None: + GlobalConfig.objects.create( + name="default", + backup_root="/backups", + retention_daily=5, + retention_weekly=4, + retention_monthly=3, + retention_yearly=2, + ) + out = StringIO() + + call_command( + "configure_pobsync_host", + "web-01", + address="web-01.example.test", + exclude_add=["/tmp/***"], + rsync_extra_arg=["--delete"], + stdout=out, + ) + + host = HostConfig.objects.get(host="web-01") + self.assertEqual(host.retention_daily, 5) + self.assertEqual(host.excludes_add, ["/tmp/***"]) + self.assertEqual(host.rsync_extra_args, ["--delete"]) + + effective = DjangoConfigSource().effective_config_for_host("web-01") + self.assertEqual(effective["retention"]["yearly"], 2) + self.assertEqual(effective["excludes_effective"], ["/tmp/***"])