hippogrief/pobsync
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor: promote backup configuration to structured SQL fields

Browse Source 
Add explicit Django model fields for global and host backup settings,
including SSH, rsync, source, excludes, and retention configuration.
Populate them from legacy JSON during migration, make the config
repository prefer structured fields, and update import/admin/tests around
the SQL-first configuration model.
This commit is contained in:
Peter van Arkel
2026-05-19 05:04:49 +02:00
parent 100215bf11
commit a0eb5dcc8f
8 changed files with 373 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/pobsync_backend/admin.py
View File

@@ -7,8 +7,28 @@ from .models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, Snapsho
@admin.register(GlobalConfig)
class GlobalConfigAdmin(admin.ModelAdmin):
list_display = ("name", "backup_root", "updated_at")
list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at")
readonly_fields = ("created_at", "updated_at")
fieldsets = (
(None, {"fields": ("name", "backup_root", "pobsync_home")}),
("SSH", {"fields": ("ssh_user", "ssh_port", "ssh_options")}),
(
"Rsync",
{
"fields": (
"rsync_binary",
"rsync_args",
"rsync_extra_args",
"rsync_timeout_seconds",
"rsync_bwlimit_kbps",
)
},
),
("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}),
("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
@admin.register(HostConfig)
@@ -17,6 +37,15 @@ class HostConfigAdmin(admin.ModelAdmin):
list_filter = ("enabled",)
search_fields = ("host", "address")
readonly_fields = ("created_at", "updated_at")
fieldsets = (
(None, {"fields": ("host", "address", "enabled")}),
("SSH override", {"fields": ("ssh_user", "ssh_port")}),
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
("Rsync override", {"fields": ("rsync_extra_args",)}),
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
@admin.register(BackupRun)

52
src/pobsync_backend/config_repository.py
View File

@@ -21,6 +21,29 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
data = dict(global_config.data or {})
data["backup_root"] = global_config.backup_root
data["pobsync_home"] = global_config.pobsync_home
data["ssh"] = {
"user": global_config.ssh_user,
"port": global_config.ssh_port,
"options": list(global_config.ssh_options or []),
}
data["rsync"] = {
"binary": global_config.rsync_binary,
"args": list(global_config.rsync_args or []),
"timeout_seconds": global_config.rsync_timeout_seconds,
"bwlimit_kbps": global_config.rsync_bwlimit_kbps,
"extra_args": list(global_config.rsync_extra_args or []),
}
data["defaults"] = {
"source_root": global_config.default_source_root,
"destination_subdir": global_config.default_destination_subdir,
}
data["excludes_default"] = list(global_config.excludes_default or [])
data["retention_defaults"] = {
"daily": global_config.retention_daily,
"weekly": global_config.retention_weekly,
"monthly": global_config.retention_monthly,
"yearly": global_config.retention_yearly,
}
return validate_dict(data, GLOBAL_SCHEMA, path="global")
@@ -28,6 +51,35 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
data = dict(host_config.config or {})
data["host"] = host_config.host
data["address"] = host_config.address
if host_config.ssh_user or host_config.ssh_port:
data["ssh"] = {}
if host_config.ssh_user:
data["ssh"]["user"] = host_config.ssh_user
if host_config.ssh_port is not None:
data["ssh"]["port"] = host_config.ssh_port
else:
data.pop("ssh", None)
if host_config.source_root:
data["source_root"] = host_config.source_root
else:
data.pop("source_root", None)
data["includes"] = list(host_config.includes or [])
if host_config.excludes_replace is not None:
data["excludes_replace"] = list(host_config.excludes_replace or [])
data.pop("excludes_add", None)
else:
data["excludes_add"] = list(host_config.excludes_add or [])
data.pop("excludes_replace", None)
if host_config.rsync_extra_args:
data["rsync"] = {"extra_args": list(host_config.rsync_extra_args or [])}
else:
data.pop("rsync", None)
data["retention"] = {
"daily": host_config.retention_daily,
"weekly": host_config.retention_weekly,
"monthly": host_config.retention_monthly,
"yearly": host_config.retention_yearly,
}
return validate_dict(data, HOST_SCHEMA, path="host")

33
src/pobsync_backend/management/commands/import_pobsync_configs.py
View File

@@ -23,11 +23,30 @@ class Command(BaseCommand):
raise CommandError(f"Missing global config: {paths.global_config_path}")
global_cfg = load_global_config(paths.global_config_path)
global_ssh = global_cfg.get("ssh") or {}
global_rsync = global_cfg.get("rsync") or {}
global_defaults = global_cfg.get("defaults") or {}
retention_defaults = global_cfg.get("retention_defaults") or {}
GlobalConfig.objects.update_or_create(
name="default",
defaults={
"backup_root": global_cfg["backup_root"],
"pobsync_home": global_cfg.get("pobsync_home", str(paths.home)),
"ssh_user": global_ssh.get("user") or "root",
"ssh_port": global_ssh.get("port") or 22,
"ssh_options": global_ssh.get("options") or [],
"rsync_binary": global_rsync.get("binary") or "rsync",
"rsync_args": global_rsync.get("args") or [],
"rsync_extra_args": global_rsync.get("extra_args") or [],
"rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0,
"rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0,
"default_source_root": global_defaults.get("source_root") or "/",
"default_destination_subdir": global_defaults.get("destination_subdir") or "",
"excludes_default": global_cfg.get("excludes_default") or [],
"retention_daily": retention_defaults.get("daily", 14),
"retention_weekly": retention_defaults.get("weekly", 8),
"retention_monthly": retention_defaults.get("monthly", 12),
"retention_yearly": retention_defaults.get("yearly", 0),
"data": global_cfg,
},
)
@@ -35,10 +54,24 @@ class Command(BaseCommand):
count = 0
for host_path in sorted(paths.hosts_dir.glob("*.yaml")):
host_cfg = load_host_config(host_path)
host_ssh = host_cfg.get("ssh") or {}
host_rsync = host_cfg.get("rsync") or {}
host_retention = host_cfg.get("retention") or {}
HostConfig.objects.update_or_create(
host=host_cfg["host"],
defaults={
"address": host_cfg["address"],
"ssh_user": host_ssh.get("user") or "",
"ssh_port": host_ssh.get("port"),
"source_root": host_cfg.get("source_root") or "",
"includes": host_cfg.get("includes") or [],
"excludes_add": host_cfg.get("excludes_add") or [],
"excludes_replace": host_cfg.get("excludes_replace"),
"rsync_extra_args": host_rsync.get("extra_args") or [],
"retention_daily": host_retention.get("daily", 14),
"retention_weekly": host_retention.get("weekly", 8),
"retention_monthly": host_retention.get("monthly", 12),
"retention_yearly": host_retention.get("yearly", 0),
"config": host_cfg,
"enabled": True,
},

191
src/pobsync_backend/migrations/0003_structured_config_fields.py Normal file
View File

@@ -0,0 +1,191 @@
from __future__ import annotations
from django.db import migrations, models
def copy_json_config_to_fields(apps, schema_editor) -> None:
GlobalConfig = apps.get_model("pobsync_backend", "GlobalConfig")
HostConfig = apps.get_model("pobsync_backend", "HostConfig")
for global_config in GlobalConfig.objects.all():
data = global_config.data or {}
ssh = data.get("ssh") or {}
rsync = data.get("rsync") or {}
defaults = data.get("defaults") or {}
retention = data.get("retention_defaults") or {}
global_config.ssh_user = ssh.get("user") or global_config.ssh_user
global_config.ssh_port = ssh.get("port") or global_config.ssh_port
global_config.ssh_options = ssh.get("options") or []
global_config.rsync_binary = rsync.get("binary") or global_config.rsync_binary
global_config.rsync_args = rsync.get("args") or []
global_config.rsync_extra_args = rsync.get("extra_args") or []
global_config.rsync_timeout_seconds = rsync.get("timeout_seconds") or 0
global_config.rsync_bwlimit_kbps = rsync.get("bwlimit_kbps") or 0
global_config.default_source_root = defaults.get("source_root") or "/"
global_config.default_destination_subdir = defaults.get("destination_subdir") or ""
global_config.excludes_default = data.get("excludes_default") or []
global_config.retention_daily = retention.get("daily", global_config.retention_daily)
global_config.retention_weekly = retention.get("weekly", global_config.retention_weekly)
global_config.retention_monthly = retention.get("monthly", global_config.retention_monthly)
global_config.retention_yearly = retention.get("yearly", global_config.retention_yearly)
global_config.save()
for host_config in HostConfig.objects.all():
config = host_config.config or {}
ssh = config.get("ssh") or {}
rsync = config.get("rsync") or {}
retention = config.get("retention") or {}
host_config.ssh_user = ssh.get("user") or ""
host_config.ssh_port = ssh.get("port")
host_config.source_root = config.get("source_root") or ""
host_config.includes = config.get("includes") or []
host_config.excludes_add = config.get("excludes_add") or []
host_config.excludes_replace = config.get("excludes_replace")
host_config.rsync_extra_args = rsync.get("extra_args") or []
host_config.retention_daily = retention.get("daily", host_config.retention_daily)
host_config.retention_weekly = retention.get("weekly", host_config.retention_weekly)
host_config.retention_monthly = retention.get("monthly", host_config.retention_monthly)
host_config.retention_yearly = retention.get("yearly", host_config.retention_yearly)
host_config.save()
class Migration(migrations.Migration):
dependencies = [
("pobsync_backend", "0002_schedule_run_state"),
]
operations = [
migrations.AddField(
model_name="globalconfig",
name="default_destination_subdir",
field=models.CharField(blank=True, default="", max_length=512),
),
migrations.AddField(
model_name="globalconfig",
name="default_source_root",
field=models.CharField(default="/", max_length=512),
),
migrations.AddField(
model_name="globalconfig",
name="excludes_default",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="retention_daily",
field=models.PositiveIntegerField(default=14),
),
migrations.AddField(
model_name="globalconfig",
name="retention_monthly",
field=models.PositiveIntegerField(default=12),
),
migrations.AddField(
model_name="globalconfig",
name="retention_weekly",
field=models.PositiveIntegerField(default=8),
),
migrations.AddField(
model_name="globalconfig",
name="retention_yearly",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_binary",
field=models.CharField(default="rsync", max_length=128),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_bwlimit_kbps",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_extra_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="rsync_timeout_seconds",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_options",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_port",
field=models.PositiveIntegerField(default=22),
),
migrations.AddField(
model_name="globalconfig",
name="ssh_user",
field=models.CharField(default="root", max_length=64),
),
migrations.AddField(
model_name="hostconfig",
name="excludes_add",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="excludes_replace",
field=models.JSONField(blank=True, null=True),
),
migrations.AddField(
model_name="hostconfig",
name="includes",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="retention_daily",
field=models.PositiveIntegerField(default=14),
),
migrations.AddField(
model_name="hostconfig",
name="retention_monthly",
field=models.PositiveIntegerField(default=12),
),
migrations.AddField(
model_name="hostconfig",
name="retention_weekly",
field=models.PositiveIntegerField(default=8),
),
migrations.AddField(
model_name="hostconfig",
name="retention_yearly",
field=models.PositiveIntegerField(default=0),
),
migrations.AddField(
model_name="hostconfig",
name="rsync_extra_args",
field=models.JSONField(blank=True, default=list),
),
migrations.AddField(
model_name="hostconfig",
name="source_root",
field=models.CharField(blank=True, max_length=512),
),
migrations.AddField(
model_name="hostconfig",
name="ssh_port",
field=models.PositiveIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="hostconfig",
name="ssh_user",
field=models.CharField(blank=True, max_length=64),
),
migrations.RunPython(copy_json_config_to_fields, migrations.RunPython.noop),
]

26
src/pobsync_backend/models.py
View File

@@ -15,6 +15,21 @@ class GlobalConfig(TimestampedModel):
name = models.CharField(max_length=64, default="default", unique=True)
backup_root = models.CharField(max_length=512)
pobsync_home = models.CharField(max_length=512, default="/opt/pobsync")
ssh_user = models.CharField(max_length=64, default="root")
ssh_port = models.PositiveIntegerField(default=22)
ssh_options = models.JSONField(default=list, blank=True)
rsync_binary = models.CharField(max_length=128, default="rsync")
rsync_args = models.JSONField(default=list, blank=True)
rsync_extra_args = models.JSONField(default=list, blank=True)
rsync_timeout_seconds = models.PositiveIntegerField(default=0)
rsync_bwlimit_kbps = models.PositiveIntegerField(default=0)
default_source_root = models.CharField(max_length=512, default="/")
default_destination_subdir = models.CharField(max_length=512, default="", blank=True)
excludes_default = models.JSONField(default=list, blank=True)
retention_daily = models.PositiveIntegerField(default=14)
retention_weekly = models.PositiveIntegerField(default=8)
retention_monthly = models.PositiveIntegerField(default=12)
retention_yearly = models.PositiveIntegerField(default=0)
data = models.JSONField(default=dict, blank=True)
class Meta:
@@ -29,6 +44,17 @@ class HostConfig(TimestampedModel):
host = models.CharField(max_length=255, unique=True)
address = models.CharField(max_length=255)
enabled = models.BooleanField(default=True)
ssh_user = models.CharField(max_length=64, blank=True)
ssh_port = models.PositiveIntegerField(null=True, blank=True)
source_root = models.CharField(max_length=512, blank=True)
includes = models.JSONField(default=list, blank=True)
excludes_add = models.JSONField(default=list, blank=True)
excludes_replace = models.JSONField(null=True, blank=True)
rsync_extra_args = models.JSONField(default=list, blank=True)
retention_daily = models.PositiveIntegerField(default=14)
retention_weekly = models.PositiveIntegerField(default=8)
retention_monthly = models.PositiveIntegerField(default=12)
retention_yearly = models.PositiveIntegerField(default=0)
config = models.JSONField(default=dict, blank=True)
class Meta:

28
src/pobsync_backend/tests/test_config_repository.py
View File

@@ -18,21 +18,36 @@ class ConfigRepositoryTests(TestCase):
name="default",
backup_root="/backups",
pobsync_home=str(prefix),
ssh_user="backup",
ssh_port=2222,
rsync_args=["--archive"],
excludes_default=["/proc/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
"ssh": {"user": "ignored", "port": 22, "options": []},
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
},
)
HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
ssh_user="root",
includes=[],
excludes_add=["/tmp/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
config={
"host": "ignored",
"address": "ignored",
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
"includes": [],
"excludes_add": ["/tmp/***"],
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
"excludes_add": ["/ignored/***"],
},
)
@@ -43,5 +58,10 @@ class ConfigRepositoryTests(TestCase):
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
self.assertEqual(global_cfg["backup_root"], "/backups")
self.assertEqual(global_cfg["pobsync_home"], str(prefix))
self.assertEqual(global_cfg["ssh"]["user"], "backup")
self.assertEqual(global_cfg["ssh"]["port"], 2222)
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
self.assertEqual(host_cfg["host"], "web-01")
self.assertEqual(host_cfg["address"], "web-01.example.test")
self.assertEqual(host_cfg["retention"]["daily"], 7)
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])

20
src/pobsync_backend/tests/test_django_config_source.py
View File

@@ -12,6 +12,13 @@ class DjangoConfigSourceTests(TestCase):
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
rsync_args=["--archive"],
rsync_extra_args=["--numeric-ids"],
excludes_default=["/proc/***"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
@@ -31,11 +38,16 @@ class DjangoConfigSourceTests(TestCase):
HostConfig.objects.create(
host="web-01",
address="web-01.example.test",
excludes_add=["/tmp/***"],
rsync_extra_args=["--delete"],
retention_daily=7,
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
config={
"retention": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
"excludes_add": ["/tmp/***"],
"includes": [],
"rsync": {"extra_args": ["--delete"]},
"retention": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
"excludes_add": ["/ignored/***"],
"rsync": {"extra_args": ["--ignored"]},
},
)