From a0eb5dcc8fa08c8a4539079dfb35748d2e7e94ef Mon Sep 17 00:00:00 2001 From: Peter van Arkel Date: Tue, 19 May 2026 05:04:49 +0200 Subject: [PATCH] refactor: promote backup configuration to structured SQL fields Add explicit Django model fields for global and host backup settings, including SSH, rsync, source, excludes, and retention configuration. Populate them from legacy JSON during migration, make the config repository prefer structured fields, and update import/admin/tests around the SQL-first configuration model. --- README.md | 2 +- src/pobsync_backend/admin.py | 31 ++- src/pobsync_backend/config_repository.py | 52 +++++ .../commands/import_pobsync_configs.py | 33 +++ .../0003_structured_config_fields.py | 191 ++++++++++++++++++ src/pobsync_backend/models.py | 26 +++ .../tests/test_config_repository.py | 28 ++- .../tests/test_django_config_source.py | 20 +- 8 files changed, 373 insertions(+), 10 deletions(-) create mode 100644 src/pobsync_backend/migrations/0003_structured_config_fields.py diff --git a/README.md b/README.md index 269bb9a..6a87651 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ For production use, always use the canonical entrypoint: ## Django backend (early refactor layer) -The Django backend is a management layer around the existing pobsync engine. The current CLI remains the source of truth for executing backups; Django stores configs, schedules, backup runs, and snapshot metadata so the project can grow toward a web/admin/API surface without rewriting rsync behavior in one risky step. +The Django backend is becoming the management layer and source of truth for pobsync. Structured SQL fields store backup, SSH, rsync, retention, schedule, run, and snapshot state; legacy JSON/YAML remains only as an import/export compatibility path while the engine is being refactored. ### Local SQLite development diff --git a/src/pobsync_backend/admin.py b/src/pobsync_backend/admin.py index 4f562fb..9fcca30 100644 --- a/src/pobsync_backend/admin.py +++ b/src/pobsync_backend/admin.py @@ -7,8 +7,28 @@ from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, Snapsho @admin.register(GlobalConfig) class GlobalConfigAdmin(admin.ModelAdmin): - list_display = ("name", "backup_root", "updated_at") + list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at") readonly_fields = ("created_at", "updated_at") + fieldsets = ( + (None, {"fields": ("name", "backup_root", "pobsync_home")}), + ("SSH", {"fields": ("ssh_user", "ssh_port", "ssh_options")}), + ( + "Rsync", + { + "fields": ( + "rsync_binary", + "rsync_args", + "rsync_extra_args", + "rsync_timeout_seconds", + "rsync_bwlimit_kbps", + ) + }, + ), + ("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}), + ("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}), + ("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}), + ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), + ) @admin.register(HostConfig) @@ -17,6 +37,15 @@ class HostConfigAdmin(admin.ModelAdmin): list_filter = ("enabled",) search_fields = ("host", "address") readonly_fields = ("created_at", "updated_at") + fieldsets = ( + (None, {"fields": ("host", "address", "enabled")}), + ("SSH override", {"fields": ("ssh_user", "ssh_port")}), + ("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}), + ("Rsync override", {"fields": ("rsync_extra_args",)}), + ("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}), + ("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}), + ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), + ) @admin.register(BackupRun) diff --git a/src/pobsync_backend/config_repository.py b/src/pobsync_backend/config_repository.py index 6d6a98e..ab7b947 100644 --- a/src/pobsync_backend/config_repository.py +++ b/src/pobsync_backend/config_repository.py @@ -21,6 +21,29 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]: data = dict(global_config.data or {}) data["backup_root"] = global_config.backup_root data["pobsync_home"] = global_config.pobsync_home + data["ssh"] = { + "user": global_config.ssh_user, + "port": global_config.ssh_port, + "options": list(global_config.ssh_options or []), + } + data["rsync"] = { + "binary": global_config.rsync_binary, + "args": list(global_config.rsync_args or []), + "timeout_seconds": global_config.rsync_timeout_seconds, + "bwlimit_kbps": global_config.rsync_bwlimit_kbps, + "extra_args": list(global_config.rsync_extra_args or []), + } + data["defaults"] = { + "source_root": global_config.default_source_root, + "destination_subdir": global_config.default_destination_subdir, + } + data["excludes_default"] = list(global_config.excludes_default or []) + data["retention_defaults"] = { + "daily": global_config.retention_daily, + "weekly": global_config.retention_weekly, + "monthly": global_config.retention_monthly, + "yearly": global_config.retention_yearly, + } return validate_dict(data, GLOBAL_SCHEMA, path="global") @@ -28,6 +51,35 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]: data = dict(host_config.config or {}) data["host"] = host_config.host data["address"] = host_config.address + if host_config.ssh_user or host_config.ssh_port: + data["ssh"] = {} + if host_config.ssh_user: + data["ssh"]["user"] = host_config.ssh_user + if host_config.ssh_port is not None: + data["ssh"]["port"] = host_config.ssh_port + else: + data.pop("ssh", None) + if host_config.source_root: + data["source_root"] = host_config.source_root + else: + data.pop("source_root", None) + data["includes"] = list(host_config.includes or []) + if host_config.excludes_replace is not None: + data["excludes_replace"] = list(host_config.excludes_replace or []) + data.pop("excludes_add", None) + else: + data["excludes_add"] = list(host_config.excludes_add or []) + data.pop("excludes_replace", None) + if host_config.rsync_extra_args: + data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])} + else: + data.pop("rsync", None) + data["retention"] = { + "daily": host_config.retention_daily, + "weekly": host_config.retention_weekly, + "monthly": host_config.retention_monthly, + "yearly": host_config.retention_yearly, + } return validate_dict(data, HOST_SCHEMA, path="host") diff --git a/src/pobsync_backend/management/commands/import_pobsync_configs.py b/src/pobsync_backend/management/commands/import_pobsync_configs.py index a1ad500..c64e8ce 100644 --- a/src/pobsync_backend/management/commands/import_pobsync_configs.py +++ b/src/pobsync_backend/management/commands/import_pobsync_configs.py @@ -23,11 +23,30 @@ class Command(BaseCommand): raise CommandError(f"Missing global config: {paths.global_config_path}") global_cfg = load_global_config(paths.global_config_path) + global_ssh = global_cfg.get("ssh") or {} + global_rsync = global_cfg.get("rsync") or {} + global_defaults = global_cfg.get("defaults") or {} + retention_defaults = global_cfg.get("retention_defaults") or {} GlobalConfig.objects.update_or_create( name="default", defaults={ "backup_root": global_cfg["backup_root"], "pobsync_home": global_cfg.get("pobsync_home", str(paths.home)), + "ssh_user": global_ssh.get("user") or "root", + "ssh_port": global_ssh.get("port") or 22, + "ssh_options": global_ssh.get("options") or [], + "rsync_binary": global_rsync.get("binary") or "rsync", + "rsync_args": global_rsync.get("args") or [], + "rsync_extra_args": global_rsync.get("extra_args") or [], + "rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0, + "rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0, + "default_source_root": global_defaults.get("source_root") or "/", + "default_destination_subdir": global_defaults.get("destination_subdir") or "", + "excludes_default": global_cfg.get("excludes_default") or [], + "retention_daily": retention_defaults.get("daily", 14), + "retention_weekly": retention_defaults.get("weekly", 8), + "retention_monthly": retention_defaults.get("monthly", 12), + "retention_yearly": retention_defaults.get("yearly", 0), "data": global_cfg, }, ) @@ -35,10 +54,24 @@ class Command(BaseCommand): count = 0 for host_path in sorted(paths.hosts_dir.glob("*.yaml")): host_cfg = load_host_config(host_path) + host_ssh = host_cfg.get("ssh") or {} + host_rsync = host_cfg.get("rsync") or {} + host_retention = host_cfg.get("retention") or {} HostConfig.objects.update_or_create( host=host_cfg["host"], defaults={ "address": host_cfg["address"], + "ssh_user": host_ssh.get("user") or "", + "ssh_port": host_ssh.get("port"), + "source_root": host_cfg.get("source_root") or "", + "includes": host_cfg.get("includes") or [], + "excludes_add": host_cfg.get("excludes_add") or [], + "excludes_replace": host_cfg.get("excludes_replace"), + "rsync_extra_args": host_rsync.get("extra_args") or [], + "retention_daily": host_retention.get("daily", 14), + "retention_weekly": host_retention.get("weekly", 8), + "retention_monthly": host_retention.get("monthly", 12), + "retention_yearly": host_retention.get("yearly", 0), "config": host_cfg, "enabled": True, }, diff --git a/src/pobsync_backend/migrations/0003_structured_config_fields.py b/src/pobsync_backend/migrations/0003_structured_config_fields.py new file mode 100644 index 0000000..6462ff1 --- /dev/null +++ b/src/pobsync_backend/migrations/0003_structured_config_fields.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from django.db import migrations, models + + +def copy_json_config_to_fields(apps, schema_editor) -> None: + GlobalConfig = apps.get_model("pobsync_backend", "GlobalConfig") + HostConfig = apps.get_model("pobsync_backend", "HostConfig") + + for global_config in GlobalConfig.objects.all(): + data = global_config.data or {} + ssh = data.get("ssh") or {} + rsync = data.get("rsync") or {} + defaults = data.get("defaults") or {} + retention = data.get("retention_defaults") or {} + + global_config.ssh_user = ssh.get("user") or global_config.ssh_user + global_config.ssh_port = ssh.get("port") or global_config.ssh_port + global_config.ssh_options = ssh.get("options") or [] + global_config.rsync_binary = rsync.get("binary") or global_config.rsync_binary + global_config.rsync_args = rsync.get("args") or [] + global_config.rsync_extra_args = rsync.get("extra_args") or [] + global_config.rsync_timeout_seconds = rsync.get("timeout_seconds") or 0 + global_config.rsync_bwlimit_kbps = rsync.get("bwlimit_kbps") or 0 + global_config.default_source_root = defaults.get("source_root") or "/" + global_config.default_destination_subdir = defaults.get("destination_subdir") or "" + global_config.excludes_default = data.get("excludes_default") or [] + global_config.retention_daily = retention.get("daily", global_config.retention_daily) + global_config.retention_weekly = retention.get("weekly", global_config.retention_weekly) + global_config.retention_monthly = retention.get("monthly", global_config.retention_monthly) + global_config.retention_yearly = retention.get("yearly", global_config.retention_yearly) + global_config.save() + + for host_config in HostConfig.objects.all(): + config = host_config.config or {} + ssh = config.get("ssh") or {} + rsync = config.get("rsync") or {} + retention = config.get("retention") or {} + + host_config.ssh_user = ssh.get("user") or "" + host_config.ssh_port = ssh.get("port") + host_config.source_root = config.get("source_root") or "" + host_config.includes = config.get("includes") or [] + host_config.excludes_add = config.get("excludes_add") or [] + host_config.excludes_replace = config.get("excludes_replace") + host_config.rsync_extra_args = rsync.get("extra_args") or [] + host_config.retention_daily = retention.get("daily", host_config.retention_daily) + host_config.retention_weekly = retention.get("weekly", host_config.retention_weekly) + host_config.retention_monthly = retention.get("monthly", host_config.retention_monthly) + host_config.retention_yearly = retention.get("yearly", host_config.retention_yearly) + host_config.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("pobsync_backend", "0002_schedule_run_state"), + ] + + operations = [ + migrations.AddField( + model_name="globalconfig", + name="default_destination_subdir", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AddField( + model_name="globalconfig", + name="default_source_root", + field=models.CharField(default="/", max_length=512), + ), + migrations.AddField( + model_name="globalconfig", + name="excludes_default", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_daily", + field=models.PositiveIntegerField(default=14), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_monthly", + field=models.PositiveIntegerField(default=12), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_weekly", + field=models.PositiveIntegerField(default=8), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_yearly", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_binary", + field=models.CharField(default="rsync", max_length=128), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_bwlimit_kbps", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_extra_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_timeout_seconds", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_options", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_port", + field=models.PositiveIntegerField(default=22), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_user", + field=models.CharField(default="root", max_length=64), + ), + migrations.AddField( + model_name="hostconfig", + name="excludes_add", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="excludes_replace", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="hostconfig", + name="includes", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_daily", + field=models.PositiveIntegerField(default=14), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_monthly", + field=models.PositiveIntegerField(default=12), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_weekly", + field=models.PositiveIntegerField(default=8), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_yearly", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="hostconfig", + name="rsync_extra_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="source_root", + field=models.CharField(blank=True, max_length=512), + ), + migrations.AddField( + model_name="hostconfig", + name="ssh_port", + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="hostconfig", + name="ssh_user", + field=models.CharField(blank=True, max_length=64), + ), + migrations.RunPython(copy_json_config_to_fields, migrations.RunPython.noop), + ] diff --git a/src/pobsync_backend/models.py b/src/pobsync_backend/models.py index 42d90db..ee65a11 100644 --- a/src/pobsync_backend/models.py +++ b/src/pobsync_backend/models.py @@ -15,6 +15,21 @@ class GlobalConfig(TimestampedModel): name = models.CharField(max_length=64, default="default", unique=True) backup_root = models.CharField(max_length=512) pobsync_home = models.CharField(max_length=512, default="/opt/pobsync") + ssh_user = models.CharField(max_length=64, default="root") + ssh_port = models.PositiveIntegerField(default=22) + ssh_options = models.JSONField(default=list, blank=True) + rsync_binary = models.CharField(max_length=128, default="rsync") + rsync_args = models.JSONField(default=list, blank=True) + rsync_extra_args = models.JSONField(default=list, blank=True) + rsync_timeout_seconds = models.PositiveIntegerField(default=0) + rsync_bwlimit_kbps = models.PositiveIntegerField(default=0) + default_source_root = models.CharField(max_length=512, default="/") + default_destination_subdir = models.CharField(max_length=512, default="", blank=True) + excludes_default = models.JSONField(default=list, blank=True) + retention_daily = models.PositiveIntegerField(default=14) + retention_weekly = models.PositiveIntegerField(default=8) + retention_monthly = models.PositiveIntegerField(default=12) + retention_yearly = models.PositiveIntegerField(default=0) data = models.JSONField(default=dict, blank=True) class Meta: @@ -29,6 +44,17 @@ class HostConfig(TimestampedModel): host = models.CharField(max_length=255, unique=True) address = models.CharField(max_length=255) enabled = models.BooleanField(default=True) + ssh_user = models.CharField(max_length=64, blank=True) + ssh_port = models.PositiveIntegerField(null=True, blank=True) + source_root = models.CharField(max_length=512, blank=True) + includes = models.JSONField(default=list, blank=True) + excludes_add = models.JSONField(default=list, blank=True) + excludes_replace = models.JSONField(null=True, blank=True) + rsync_extra_args = models.JSONField(default=list, blank=True) + retention_daily = models.PositiveIntegerField(default=14) + retention_weekly = models.PositiveIntegerField(default=8) + retention_monthly = models.PositiveIntegerField(default=12) + retention_yearly = models.PositiveIntegerField(default=0) config = models.JSONField(default=dict, blank=True) class Meta: diff --git a/src/pobsync_backend/tests/test_config_repository.py b/src/pobsync_backend/tests/test_config_repository.py index cccedf8..1f51c0b 100644 --- a/src/pobsync_backend/tests/test_config_repository.py +++ b/src/pobsync_backend/tests/test_config_repository.py @@ -18,21 +18,36 @@ class ConfigRepositoryTests(TestCase): name="default", backup_root="/backups", pobsync_home=str(prefix), + ssh_user="backup", + ssh_port=2222, + rsync_args=["--archive"], + excludes_default=["/proc/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, data={ "backup_root": "/ignored", "pobsync_home": "/ignored", - "retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + "ssh": {"user": "ignored", "port": 22, "options": []}, + "retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, }, ) HostConfig.objects.create( host="web-01", address="web-01.example.test", + ssh_user="root", + includes=[], + excludes_add=["/tmp/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, config={ "host": "ignored", "address": "ignored", - "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, - "includes": [], - "excludes_add": ["/tmp/***"], + "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, + "excludes_add": ["/ignored/***"], }, ) @@ -43,5 +58,10 @@ class ConfigRepositoryTests(TestCase): host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml") self.assertEqual(global_cfg["backup_root"], "/backups") self.assertEqual(global_cfg["pobsync_home"], str(prefix)) + self.assertEqual(global_cfg["ssh"]["user"], "backup") + self.assertEqual(global_cfg["ssh"]["port"], 2222) + self.assertEqual(global_cfg["retention_defaults"]["daily"], 7) self.assertEqual(host_cfg["host"], "web-01") self.assertEqual(host_cfg["address"], "web-01.example.test") + self.assertEqual(host_cfg["retention"]["daily"], 7) + self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"]) diff --git a/src/pobsync_backend/tests/test_django_config_source.py b/src/pobsync_backend/tests/test_django_config_source.py index e9ca1ad..c2aebd9 100644 --- a/src/pobsync_backend/tests/test_django_config_source.py +++ b/src/pobsync_backend/tests/test_django_config_source.py @@ -12,6 +12,13 @@ class DjangoConfigSourceTests(TestCase): name="default", backup_root="/backups", pobsync_home="/opt/pobsync", + rsync_args=["--archive"], + rsync_extra_args=["--numeric-ids"], + excludes_default=["/proc/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, data={ "backup_root": "/ignored", "pobsync_home": "/ignored", @@ -31,11 +38,16 @@ class DjangoConfigSourceTests(TestCase): HostConfig.objects.create( host="web-01", address="web-01.example.test", + excludes_add=["/tmp/***"], + rsync_extra_args=["--delete"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, config={ - "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, - "excludes_add": ["/tmp/***"], - "includes": [], - "rsync": {"extra_args": ["--delete"]}, + "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, + "excludes_add": ["/ignored/***"], + "rsync": {"extra_args": ["--ignored"]}, }, )