diff --git a/README.md b/README.md index 269bb9a..6a87651 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ For production use, always use the canonical entrypoint: ## Django backend (early refactor layer) -The Django backend is a management layer around the existing pobsync engine. The current CLI remains the source of truth for executing backups; Django stores configs, schedules, backup runs, and snapshot metadata so the project can grow toward a web/admin/API surface without rewriting rsync behavior in one risky step. +The Django backend is becoming the management layer and source of truth for pobsync. Structured SQL fields store backup, SSH, rsync, retention, schedule, run, and snapshot state; legacy JSON/YAML remains only as an import/export compatibility path while the engine is being refactored. ### Local SQLite development diff --git a/src/pobsync_backend/admin.py b/src/pobsync_backend/admin.py index 4f562fb..9fcca30 100644 --- a/src/pobsync_backend/admin.py +++ b/src/pobsync_backend/admin.py @@ -7,8 +7,28 @@ from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, Snapsho @admin.register(GlobalConfig) class GlobalConfigAdmin(admin.ModelAdmin): - list_display = ("name", "backup_root", "updated_at") + list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at") readonly_fields = ("created_at", "updated_at") + fieldsets = ( + (None, {"fields": ("name", "backup_root", "pobsync_home")}), + ("SSH", {"fields": ("ssh_user", "ssh_port", "ssh_options")}), + ( + "Rsync", + { + "fields": ( + "rsync_binary", + "rsync_args", + "rsync_extra_args", + "rsync_timeout_seconds", + "rsync_bwlimit_kbps", + ) + }, + ), + ("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}), + ("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}), + ("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}), + ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), + ) @admin.register(HostConfig) @@ -17,6 +37,15 @@ class HostConfigAdmin(admin.ModelAdmin): list_filter = ("enabled",) search_fields = ("host", "address") readonly_fields = ("created_at", "updated_at") + fieldsets = ( + (None, {"fields": ("host", "address", "enabled")}), + ("SSH override", {"fields": ("ssh_user", "ssh_port")}), + ("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}), + ("Rsync override", {"fields": ("rsync_extra_args",)}), + ("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}), + ("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}), + ("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), + ) @admin.register(BackupRun) diff --git a/src/pobsync_backend/config_repository.py b/src/pobsync_backend/config_repository.py index 6d6a98e..ab7b947 100644 --- a/src/pobsync_backend/config_repository.py +++ b/src/pobsync_backend/config_repository.py @@ -21,6 +21,29 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]: data = dict(global_config.data or {}) data["backup_root"] = global_config.backup_root data["pobsync_home"] = global_config.pobsync_home + data["ssh"] = { + "user": global_config.ssh_user, + "port": global_config.ssh_port, + "options": list(global_config.ssh_options or []), + } + data["rsync"] = { + "binary": global_config.rsync_binary, + "args": list(global_config.rsync_args or []), + "timeout_seconds": global_config.rsync_timeout_seconds, + "bwlimit_kbps": global_config.rsync_bwlimit_kbps, + "extra_args": list(global_config.rsync_extra_args or []), + } + data["defaults"] = { + "source_root": global_config.default_source_root, + "destination_subdir": global_config.default_destination_subdir, + } + data["excludes_default"] = list(global_config.excludes_default or []) + data["retention_defaults"] = { + "daily": global_config.retention_daily, + "weekly": global_config.retention_weekly, + "monthly": global_config.retention_monthly, + "yearly": global_config.retention_yearly, + } return validate_dict(data, GLOBAL_SCHEMA, path="global") @@ -28,6 +51,35 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]: data = dict(host_config.config or {}) data["host"] = host_config.host data["address"] = host_config.address + if host_config.ssh_user or host_config.ssh_port: + data["ssh"] = {} + if host_config.ssh_user: + data["ssh"]["user"] = host_config.ssh_user + if host_config.ssh_port is not None: + data["ssh"]["port"] = host_config.ssh_port + else: + data.pop("ssh", None) + if host_config.source_root: + data["source_root"] = host_config.source_root + else: + data.pop("source_root", None) + data["includes"] = list(host_config.includes or []) + if host_config.excludes_replace is not None: + data["excludes_replace"] = list(host_config.excludes_replace or []) + data.pop("excludes_add", None) + else: + data["excludes_add"] = list(host_config.excludes_add or []) + data.pop("excludes_replace", None) + if host_config.rsync_extra_args: + data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])} + else: + data.pop("rsync", None) + data["retention"] = { + "daily": host_config.retention_daily, + "weekly": host_config.retention_weekly, + "monthly": host_config.retention_monthly, + "yearly": host_config.retention_yearly, + } return validate_dict(data, HOST_SCHEMA, path="host") diff --git a/src/pobsync_backend/management/commands/import_pobsync_configs.py b/src/pobsync_backend/management/commands/import_pobsync_configs.py index a1ad500..c64e8ce 100644 --- a/src/pobsync_backend/management/commands/import_pobsync_configs.py +++ b/src/pobsync_backend/management/commands/import_pobsync_configs.py @@ -23,11 +23,30 @@ class Command(BaseCommand): raise CommandError(f"Missing global config: {paths.global_config_path}") global_cfg = load_global_config(paths.global_config_path) + global_ssh = global_cfg.get("ssh") or {} + global_rsync = global_cfg.get("rsync") or {} + global_defaults = global_cfg.get("defaults") or {} + retention_defaults = global_cfg.get("retention_defaults") or {} GlobalConfig.objects.update_or_create( name="default", defaults={ "backup_root": global_cfg["backup_root"], "pobsync_home": global_cfg.get("pobsync_home", str(paths.home)), + "ssh_user": global_ssh.get("user") or "root", + "ssh_port": global_ssh.get("port") or 22, + "ssh_options": global_ssh.get("options") or [], + "rsync_binary": global_rsync.get("binary") or "rsync", + "rsync_args": global_rsync.get("args") or [], + "rsync_extra_args": global_rsync.get("extra_args") or [], + "rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0, + "rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0, + "default_source_root": global_defaults.get("source_root") or "/", + "default_destination_subdir": global_defaults.get("destination_subdir") or "", + "excludes_default": global_cfg.get("excludes_default") or [], + "retention_daily": retention_defaults.get("daily", 14), + "retention_weekly": retention_defaults.get("weekly", 8), + "retention_monthly": retention_defaults.get("monthly", 12), + "retention_yearly": retention_defaults.get("yearly", 0), "data": global_cfg, }, ) @@ -35,10 +54,24 @@ class Command(BaseCommand): count = 0 for host_path in sorted(paths.hosts_dir.glob("*.yaml")): host_cfg = load_host_config(host_path) + host_ssh = host_cfg.get("ssh") or {} + host_rsync = host_cfg.get("rsync") or {} + host_retention = host_cfg.get("retention") or {} HostConfig.objects.update_or_create( host=host_cfg["host"], defaults={ "address": host_cfg["address"], + "ssh_user": host_ssh.get("user") or "", + "ssh_port": host_ssh.get("port"), + "source_root": host_cfg.get("source_root") or "", + "includes": host_cfg.get("includes") or [], + "excludes_add": host_cfg.get("excludes_add") or [], + "excludes_replace": host_cfg.get("excludes_replace"), + "rsync_extra_args": host_rsync.get("extra_args") or [], + "retention_daily": host_retention.get("daily", 14), + "retention_weekly": host_retention.get("weekly", 8), + "retention_monthly": host_retention.get("monthly", 12), + "retention_yearly": host_retention.get("yearly", 0), "config": host_cfg, "enabled": True, }, diff --git a/src/pobsync_backend/migrations/0003_structured_config_fields.py b/src/pobsync_backend/migrations/0003_structured_config_fields.py new file mode 100644 index 0000000..6462ff1 --- /dev/null +++ b/src/pobsync_backend/migrations/0003_structured_config_fields.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from django.db import migrations, models + + +def copy_json_config_to_fields(apps, schema_editor) -> None: + GlobalConfig = apps.get_model("pobsync_backend", "GlobalConfig") + HostConfig = apps.get_model("pobsync_backend", "HostConfig") + + for global_config in GlobalConfig.objects.all(): + data = global_config.data or {} + ssh = data.get("ssh") or {} + rsync = data.get("rsync") or {} + defaults = data.get("defaults") or {} + retention = data.get("retention_defaults") or {} + + global_config.ssh_user = ssh.get("user") or global_config.ssh_user + global_config.ssh_port = ssh.get("port") or global_config.ssh_port + global_config.ssh_options = ssh.get("options") or [] + global_config.rsync_binary = rsync.get("binary") or global_config.rsync_binary + global_config.rsync_args = rsync.get("args") or [] + global_config.rsync_extra_args = rsync.get("extra_args") or [] + global_config.rsync_timeout_seconds = rsync.get("timeout_seconds") or 0 + global_config.rsync_bwlimit_kbps = rsync.get("bwlimit_kbps") or 0 + global_config.default_source_root = defaults.get("source_root") or "/" + global_config.default_destination_subdir = defaults.get("destination_subdir") or "" + global_config.excludes_default = data.get("excludes_default") or [] + global_config.retention_daily = retention.get("daily", global_config.retention_daily) + global_config.retention_weekly = retention.get("weekly", global_config.retention_weekly) + global_config.retention_monthly = retention.get("monthly", global_config.retention_monthly) + global_config.retention_yearly = retention.get("yearly", global_config.retention_yearly) + global_config.save() + + for host_config in HostConfig.objects.all(): + config = host_config.config or {} + ssh = config.get("ssh") or {} + rsync = config.get("rsync") or {} + retention = config.get("retention") or {} + + host_config.ssh_user = ssh.get("user") or "" + host_config.ssh_port = ssh.get("port") + host_config.source_root = config.get("source_root") or "" + host_config.includes = config.get("includes") or [] + host_config.excludes_add = config.get("excludes_add") or [] + host_config.excludes_replace = config.get("excludes_replace") + host_config.rsync_extra_args = rsync.get("extra_args") or [] + host_config.retention_daily = retention.get("daily", host_config.retention_daily) + host_config.retention_weekly = retention.get("weekly", host_config.retention_weekly) + host_config.retention_monthly = retention.get("monthly", host_config.retention_monthly) + host_config.retention_yearly = retention.get("yearly", host_config.retention_yearly) + host_config.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("pobsync_backend", "0002_schedule_run_state"), + ] + + operations = [ + migrations.AddField( + model_name="globalconfig", + name="default_destination_subdir", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AddField( + model_name="globalconfig", + name="default_source_root", + field=models.CharField(default="/", max_length=512), + ), + migrations.AddField( + model_name="globalconfig", + name="excludes_default", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_daily", + field=models.PositiveIntegerField(default=14), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_monthly", + field=models.PositiveIntegerField(default=12), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_weekly", + field=models.PositiveIntegerField(default=8), + ), + migrations.AddField( + model_name="globalconfig", + name="retention_yearly", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_binary", + field=models.CharField(default="rsync", max_length=128), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_bwlimit_kbps", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_extra_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="rsync_timeout_seconds", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_options", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_port", + field=models.PositiveIntegerField(default=22), + ), + migrations.AddField( + model_name="globalconfig", + name="ssh_user", + field=models.CharField(default="root", max_length=64), + ), + migrations.AddField( + model_name="hostconfig", + name="excludes_add", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="excludes_replace", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="hostconfig", + name="includes", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_daily", + field=models.PositiveIntegerField(default=14), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_monthly", + field=models.PositiveIntegerField(default=12), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_weekly", + field=models.PositiveIntegerField(default=8), + ), + migrations.AddField( + model_name="hostconfig", + name="retention_yearly", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="hostconfig", + name="rsync_extra_args", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="hostconfig", + name="source_root", + field=models.CharField(blank=True, max_length=512), + ), + migrations.AddField( + model_name="hostconfig", + name="ssh_port", + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="hostconfig", + name="ssh_user", + field=models.CharField(blank=True, max_length=64), + ), + migrations.RunPython(copy_json_config_to_fields, migrations.RunPython.noop), + ] diff --git a/src/pobsync_backend/models.py b/src/pobsync_backend/models.py index 42d90db..ee65a11 100644 --- a/src/pobsync_backend/models.py +++ b/src/pobsync_backend/models.py @@ -15,6 +15,21 @@ class GlobalConfig(TimestampedModel): name = models.CharField(max_length=64, default="default", unique=True) backup_root = models.CharField(max_length=512) pobsync_home = models.CharField(max_length=512, default="/opt/pobsync") + ssh_user = models.CharField(max_length=64, default="root") + ssh_port = models.PositiveIntegerField(default=22) + ssh_options = models.JSONField(default=list, blank=True) + rsync_binary = models.CharField(max_length=128, default="rsync") + rsync_args = models.JSONField(default=list, blank=True) + rsync_extra_args = models.JSONField(default=list, blank=True) + rsync_timeout_seconds = models.PositiveIntegerField(default=0) + rsync_bwlimit_kbps = models.PositiveIntegerField(default=0) + default_source_root = models.CharField(max_length=512, default="/") + default_destination_subdir = models.CharField(max_length=512, default="", blank=True) + excludes_default = models.JSONField(default=list, blank=True) + retention_daily = models.PositiveIntegerField(default=14) + retention_weekly = models.PositiveIntegerField(default=8) + retention_monthly = models.PositiveIntegerField(default=12) + retention_yearly = models.PositiveIntegerField(default=0) data = models.JSONField(default=dict, blank=True) class Meta: @@ -29,6 +44,17 @@ class HostConfig(TimestampedModel): host = models.CharField(max_length=255, unique=True) address = models.CharField(max_length=255) enabled = models.BooleanField(default=True) + ssh_user = models.CharField(max_length=64, blank=True) + ssh_port = models.PositiveIntegerField(null=True, blank=True) + source_root = models.CharField(max_length=512, blank=True) + includes = models.JSONField(default=list, blank=True) + excludes_add = models.JSONField(default=list, blank=True) + excludes_replace = models.JSONField(null=True, blank=True) + rsync_extra_args = models.JSONField(default=list, blank=True) + retention_daily = models.PositiveIntegerField(default=14) + retention_weekly = models.PositiveIntegerField(default=8) + retention_monthly = models.PositiveIntegerField(default=12) + retention_yearly = models.PositiveIntegerField(default=0) config = models.JSONField(default=dict, blank=True) class Meta: diff --git a/src/pobsync_backend/tests/test_config_repository.py b/src/pobsync_backend/tests/test_config_repository.py index cccedf8..1f51c0b 100644 --- a/src/pobsync_backend/tests/test_config_repository.py +++ b/src/pobsync_backend/tests/test_config_repository.py @@ -18,21 +18,36 @@ class ConfigRepositoryTests(TestCase): name="default", backup_root="/backups", pobsync_home=str(prefix), + ssh_user="backup", + ssh_port=2222, + rsync_args=["--archive"], + excludes_default=["/proc/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, data={ "backup_root": "/ignored", "pobsync_home": "/ignored", - "retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, + "ssh": {"user": "ignored", "port": 22, "options": []}, + "retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, }, ) HostConfig.objects.create( host="web-01", address="web-01.example.test", + ssh_user="root", + includes=[], + excludes_add=["/tmp/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, config={ "host": "ignored", "address": "ignored", - "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, - "includes": [], - "excludes_add": ["/tmp/***"], + "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, + "excludes_add": ["/ignored/***"], }, ) @@ -43,5 +58,10 @@ class ConfigRepositoryTests(TestCase): host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml") self.assertEqual(global_cfg["backup_root"], "/backups") self.assertEqual(global_cfg["pobsync_home"], str(prefix)) + self.assertEqual(global_cfg["ssh"]["user"], "backup") + self.assertEqual(global_cfg["ssh"]["port"], 2222) + self.assertEqual(global_cfg["retention_defaults"]["daily"], 7) self.assertEqual(host_cfg["host"], "web-01") self.assertEqual(host_cfg["address"], "web-01.example.test") + self.assertEqual(host_cfg["retention"]["daily"], 7) + self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"]) diff --git a/src/pobsync_backend/tests/test_django_config_source.py b/src/pobsync_backend/tests/test_django_config_source.py index e9ca1ad..c2aebd9 100644 --- a/src/pobsync_backend/tests/test_django_config_source.py +++ b/src/pobsync_backend/tests/test_django_config_source.py @@ -12,6 +12,13 @@ class DjangoConfigSourceTests(TestCase): name="default", backup_root="/backups", pobsync_home="/opt/pobsync", + rsync_args=["--archive"], + rsync_extra_args=["--numeric-ids"], + excludes_default=["/proc/***"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, data={ "backup_root": "/ignored", "pobsync_home": "/ignored", @@ -31,11 +38,16 @@ class DjangoConfigSourceTests(TestCase): HostConfig.objects.create( host="web-01", address="web-01.example.test", + excludes_add=["/tmp/***"], + rsync_extra_args=["--delete"], + retention_daily=7, + retention_weekly=4, + retention_monthly=3, + retention_yearly=1, config={ - "retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1}, - "excludes_add": ["/tmp/***"], - "includes": [], - "rsync": {"extra_args": ["--delete"]}, + "retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99}, + "excludes_add": ["/ignored/***"], + "rsync": {"extra_args": ["--ignored"]}, }, )