hippogrief/pobsync
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge pull request 'issue-7-config-cleanup-legacy-removal' (#18) from issue-7-config-cleanup-legacy-removal into master

Browse Source 
Reviewed-on: #18
This commit was merged in pull request #18.
This commit is contained in:
Peter van Arkel
2026-05-21 02:57:38 +02:00
parent 2d9f453767 a73d34ac9f
commit 362a9dde62
36 changed files with 202 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -196,8 +196,8 @@ rsync.
## SSH Keys
SSH keys can be managed from `/ssh-credentials/`. The recommended flow is to generate keys from Django or during the
installer. pobsync stores the private key on disk under `POBSYNC_HOME`, keeps the public key visible in the UI, and lets
you select a credential either as the global default or as a per-host override.
installer. pobsync stores the private key on disk under the runtime state root (`POBSYNC_HOME`), keeps the public key
visible in the UI, and lets you select a credential either as the global default or as a per-host override.
Generated private keys are stored at:

29
docs/development.md
View File

@@ -47,6 +47,9 @@ pobsync django check
python3 manage.py showmigrations pobsync_backend
```
The short `pobsync` aliases are limited to operational actions that are useful while debugging a running install.
Configuration aliases are intentionally not public commands; use the Django UI or explicit management commands instead.
Worker and scheduler commands are normally run by systemd services:
```
@@ -62,6 +65,14 @@ pobsync discover-snapshots --host <host>
pobsync retention <host>
```
For scripted configuration changes, call the Django management command explicitly so it is clear that this is an
automation/debugging path rather than the normal UI workflow:
```
pobsync django configure_pobsync_host <host> --address <host.example>
pobsync django configure_pobsync_schedule <host> --schedule-expression "15 2 * * *"
```
## Installer Development
The native installer is interactive by default when stdin is a terminal. It should keep every prompt backed by a command
@@ -85,23 +96,6 @@ The updater is intentionally a small wrapper around the installer for routine pr
non-interactive, preserve the existing environment file, skip OS package installation, skip superuser creation, and still
run the Django/runtime refresh steps needed after a code update.
## Migration Helpers
Import existing legacy YAML configs:
```
python3 manage.py import_pobsync_configs --prefix /opt/pobsync
```
Export SQL config to legacy runtime YAML for inspection or one-off compatibility:
```
python3 manage.py export_pobsync_configs --prefix /opt/pobsync
```
These commands are migration helpers, not the normal operating model. After import, review and continue operating from
the Django control panel.
## Docker With SQLite
Docker Compose is useful for local development and disposable test installs. Native systemd is preferred for production
@@ -181,4 +175,3 @@ Next refactor targets:
- Move more snapshot lifecycle details into typed domain objects.
- Replace remaining dictionary-shaped config at engine boundaries.
- Remove legacy YAML import/export once production migration no longer needs it.

6
src/pobsync/cli.py
View File

@@ -8,9 +8,6 @@ from django.core.management import execute_from_command_line
COMMAND_ALIASES = {
"configure-global": "configure_pobsync_global",
"configure-host": "configure_pobsync_host",
"schedule": "configure_pobsync_schedule",
"backup": "run_pobsync_backup",
"retention": "run_pobsync_retention",
"discover-snapshots": "discover_pobsync_snapshots",
@@ -29,6 +26,9 @@ Usage:
Commands:
{commands}
Configuration is managed from the Django control panel. Use
`pobsync django <management-command>` for automation or debugging.
"""

10
src/pobsync/commands/retention_plan.py
View File

@@ -4,9 +4,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, List
from ..config.source import ConfigSource, FileConfigSource
from ..config.source import ConfigSource
from ..errors import ConfigError
from ..paths import PobsyncPaths
from ..retention import Snapshot, apply_base_protection, build_retention_plan
from ..snapshot_meta import iter_snapshot_dirs, read_snapshot_meta, resolve_host_root
from ..util import sanitize_host
@@ -40,10 +39,9 @@ def run_retention_plan(
if kind not in {"scheduled", "manual", "all"}:
raise ConfigError("kind must be scheduled, manual, or all")
paths = PobsyncPaths(home=prefix)
source = config_source or FileConfigSource(prefix=paths.home)
cfg = source.effective_config_for_host(host)
if config_source is None:
raise ConfigError("A Django config source is required.")
cfg = config_source.effective_config_for_host(host)
retention = cfg.get("retention")
if not isinstance(retention, dict):

8
src/pobsync/commands/run_scheduled.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from pathlib import Path
from typing import Any, Callable
from ..config.source import ConfigSource, FileConfigSource
from ..config.source import ConfigSource
from ..errors import ConfigError
from ..lock import acquire_host_lock
from ..paths import PobsyncPaths
@@ -163,8 +163,9 @@ def run_scheduled(
host = sanitize_host(host)
paths = PobsyncPaths(home=prefix)
source = config_source or FileConfigSource(prefix=paths.home)
cfg = source.effective_config_for_host(host)
if config_source is None:
raise ConfigError("A Django config source is required.")
cfg = config_source.effective_config_for_host(host)
backup_root = cfg.get("backup_root")
if not isinstance(backup_root, str) or not backup_root.startswith("/"):
@@ -316,7 +317,6 @@ def run_scheduled(
"duration_seconds": None,
"base": _base_meta_from_path(base_dir, link_dest),
"rsync": {"exit_code": None, "command": cmd, "stats": {}},
# Keep existing fields for future expansion / compatibility with current structure.
"overrides": {"includes": [], "excludes": [], "base": None},
}

53
src/pobsync/config/load.py
View File

@@ -1,53 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
import yaml
from ..errors import ConfigError, ValidationError
from ..validate import validate_dict
from .schemas import GLOBAL_SCHEMA, HOST_SCHEMA
def load_yaml_file(path: Path) -> dict[str, Any]:
if not path.exists():
raise ConfigError(f"Missing config file: {path}")
try:
raw = path.read_text(encoding="utf-8")
except OSError as e:
raise ConfigError(f"Cannot read config file: {path}: {e}") from e
try:
data = yaml.safe_load(raw)
except yaml.YAMLError as e:
raise ConfigError(f"Invalid YAML in {path}: {e}") from e
if data is None:
data = {}
if not isinstance(data, dict):
raise ConfigError(f"Config root must be a mapping in {path}")
return data
def load_global_config(path: Path) -> dict[str, Any]:
data = load_yaml_file(path)
try:
return validate_dict(data, GLOBAL_SCHEMA, path="global")
except ValidationError as e:
raise ConfigError(f"Invalid global config at {path}: {format_validation_error(e)}") from e
def load_host_config(path: Path) -> dict[str, Any]:
data = load_yaml_file(path)
try:
return validate_dict(data, HOST_SCHEMA, path="host")
except ValidationError as e:
raise ConfigError(f"Invalid host config at {path}: {format_validation_error(e)}") from e
def format_validation_error(err: ValidationError) -> str:
if err.path:
return f"{err.path}: {err}"
return str(err)

3
src/pobsync/config/schemas.py
View File

@@ -83,7 +83,6 @@ OUTPUT_SCHEMA = Schema(
GLOBAL_SCHEMA = Schema(
fields={
"backup_root": FieldSpec(str, required=True),
"pobsync_home": FieldSpec(str, required=False, default="/opt/pobsync"),
"ssh": FieldSpec(dict, required=False, schema=SSH_SCHEMA),
"rsync": FieldSpec(dict, required=False, schema=RSYNC_SCHEMA),
"defaults": FieldSpec(dict, required=False, schema=DEFAULTS_SCHEMA),
@@ -95,7 +94,6 @@ GLOBAL_SCHEMA = Schema(
),
"logging": FieldSpec(dict, required=False, schema=LOGGING_SCHEMA),
"output": FieldSpec(dict, required=False, schema=OUTPUT_SCHEMA),
# Used by `init-host` as a convenience default
"retention_defaults": FieldSpec(
dict,
required=False,
@@ -131,4 +129,3 @@ HOST_SCHEMA = Schema(
},
allow_unknown=False,
)

14
src/pobsync/config/source.py
View File

@@ -1,22 +1,8 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Protocol
from .load import load_global_config, load_host_config
from .merge import build_effective_config
class ConfigSource(Protocol):
def effective_config_for_host(self, host: str) -> dict[str, Any]:
"""Return the fully merged effective config for a host."""
class FileConfigSource:
def __init__(self, prefix: Path) -> None:
self.prefix = prefix
def effective_config_for_host(self, host: str) -> dict[str, Any]:
global_cfg = load_global_config(self.prefix / "config" / "global.yaml")
host_cfg = load_host_config(self.prefix / "config" / "hosts" / f"{host}.yaml")
return build_effective_config(global_cfg, host_cfg)

13
src/pobsync/paths.py
View File

@@ -8,14 +8,6 @@ from pathlib import Path
class PobsyncPaths:
home: Path # usually /opt/pobsync
@property
def config_dir(self) -> Path:
return self.home / "config"
@property
def hosts_dir(self) -> Path:
return self.config_dir / "hosts"
@property
def state_dir(self) -> Path:
return self.home / "state"
@@ -28,11 +20,6 @@ class PobsyncPaths:
def logs_dir(self) -> Path:
return self.home / "logs"
@property
def global_config_path(self) -> Path:
return self.config_dir / "global.yaml"
@property
def central_log_path(self) -> Path:
return self.logs_dir / "pobsync.log"

5
src/pobsync_backend/admin.py
View File

@@ -34,7 +34,7 @@ class GlobalConfigAdmin(admin.ModelAdmin):
list_display = ("name", "backup_root", "ssh_user", "ssh_port", "updated_at")
readonly_fields = ("created_at", "updated_at")
fieldsets = (
(None, {"fields": ("name", "backup_root", "pobsync_home")}),
(None, {"fields": ("name", "backup_root")}),
("SSH", {"fields": ("default_ssh_credential", "ssh_user", "ssh_port", "ssh_options")}),
(
"Rsync",
@@ -50,7 +50,6 @@ class GlobalConfigAdmin(admin.ModelAdmin):
),
("Defaults", {"fields": ("default_source_root", "default_destination_subdir", "excludes_default")}),
("Retention defaults", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("data",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
@@ -76,7 +75,7 @@ class HostConfigAdmin(admin.ModelAdmin):
("Source", {"fields": ("source_root", "includes", "excludes_add", "excludes_replace")}),
("Rsync override", {"fields": ("rsync_extra_args",)}),
("Retention", {"fields": ("retention_daily", "retention_weekly", "retention_monthly", "retention_yearly")}),
("Legacy JSON", {"fields": ("config",), "classes": ("collapse",)}),
("Runtime state", {"fields": ("config",), "classes": ("collapse",)}),
("Timestamps", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)

4
src/pobsync_backend/config_checks.py
View File

@@ -17,7 +17,7 @@ CRITICAL_ROOT_EXCLUDES = ("/proc/***", "/sys/***", "/dev/***", "/run/***", "/tmp
def collect_global_config_checks(global_config: GlobalConfig) -> list[SelfCheck]:
checks = [
_absolute_path_check("Global backup root", global_config.backup_root),
_absolute_path_check("Global pobsync home", global_config.pobsync_home),
_absolute_path_check("Runtime state root", settings.POBSYNC_HOME),
_runtime_backup_root_check(global_config),
_rsync_binary_check(global_config.rsync_binary),
_rsync_recursion_check(
@@ -97,7 +97,7 @@ def _runtime_backup_root_check(global_config: GlobalConfig) -> SelfCheck:
return SelfCheck(
"Runtime backup root",
"warning",
"Database backup root differs from runtime POBSYNC_BACKUP_ROOT.",
"Database backup root differs from the runtime backup root.",
f"database={global_config.backup_root} runtime={settings.POBSYNC_BACKUP_ROOT}",
)

53
src/pobsync_backend/config_repository.py
View File

@@ -1,13 +1,10 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.core.exceptions import ObjectDoesNotExist
from pobsync.config.schemas import GLOBAL_SCHEMA, HOST_SCHEMA
from pobsync.paths import PobsyncPaths
from pobsync.util import write_yaml_atomic
from pobsync.validate import validate_dict
from .models import GlobalConfig, HostConfig
@@ -17,10 +14,9 @@ class ConfigRepositoryError(RuntimeError):
pass
def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
def _global_runtime_data(global_config: GlobalConfig) -> dict[str, Any]:
data = {
"backup_root": global_config.backup_root,
"pobsync_home": global_config.pobsync_home,
"ssh": {
"user": global_config.ssh_user,
"port": global_config.ssh_port,
@@ -48,7 +44,7 @@ def _global_yaml_data(global_config: GlobalConfig) -> dict[str, Any]:
return validate_dict(data, GLOBAL_SCHEMA, path="global")
def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
def _host_runtime_data(host_config: HostConfig) -> dict[str, Any]:
data: dict[str, Any] = {
"host": host_config.host,
"address": host_config.address,
@@ -78,57 +74,24 @@ def _host_yaml_data(host_config: HostConfig) -> dict[str, Any]:
def global_config_object_data(global_config: GlobalConfig) -> dict[str, Any]:
return _global_yaml_data(global_config)
return _global_runtime_data(global_config)
def host_config_object_data(host_config: HostConfig) -> dict[str, Any]:
return _host_yaml_data(host_config)
return _host_runtime_data(host_config)
def global_config_data(name: str = "default") -> dict[str, Any]:
try:
global_config = GlobalConfig.objects.get(name=name)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
return _global_yaml_data(global_config)
raise ConfigRepositoryError(f"Missing global config {name!r}") from exc
return _global_runtime_data(global_config)
def host_config_data(host: str) -> dict[str, Any]:
try:
host_config = HostConfig.objects.get(host=host, enabled=True)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
return _host_yaml_data(host_config)
def export_global_config(prefix: Path, name: str = "default") -> Path:
try:
global_config = GlobalConfig.objects.get(name=name)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing GlobalConfig {name!r}") from exc
paths = PobsyncPaths(home=prefix)
write_yaml_atomic(paths.global_config_path, _global_yaml_data(global_config))
return paths.global_config_path
def export_host_config(prefix: Path, host: str) -> Path:
try:
host_config = HostConfig.objects.get(host=host, enabled=True)
except ObjectDoesNotExist as exc:
raise ConfigRepositoryError(f"Missing enabled HostConfig {host!r}") from exc
paths = PobsyncPaths(home=prefix)
target = paths.hosts_dir / f"{host_config.host}.yaml"
write_yaml_atomic(target, _host_yaml_data(host_config))
return target
def export_runtime_configs(prefix: Path, host: str | None = None) -> list[Path]:
written = [export_global_config(prefix)]
hosts = HostConfig.objects.filter(enabled=True).order_by("host")
if host is not None:
hosts = hosts.filter(host=host)
for host_config in hosts:
written.append(export_host_config(prefix, host_config.host))
return written
raise ConfigRepositoryError(f"Missing enabled host {host!r}") from exc
return _host_runtime_data(host_config)

1
src/pobsync_backend/forms.py
View File

@@ -119,7 +119,6 @@ class GlobalConfigForm(forms.ModelForm):
def save(self, commit: bool = True):
instance = super().save(commit=False)
instance.backup_root = settings.POBSYNC_BACKUP_ROOT
instance.pobsync_home = settings.POBSYNC_HOME
if commit:
instance.save()
self.save_m2m()

11
src/pobsync_backend/management/commands/configure_pobsync_global.py
View File

@@ -1,9 +1,7 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from pobsync.config.retention import parse_retention
@@ -13,12 +11,11 @@ from pobsync_backend.models import GlobalConfig
class Command(BaseCommand):
help = "Create or update the SQL-backed global pobsync configuration."
help = "Create or update the default global backup configuration."
def add_arguments(self, parser) -> None:
parser.add_argument("--name", default="default")
parser.add_argument("--backup-root", required=True)
parser.add_argument("--pobsync-home", default=settings.POBSYNC_HOME)
parser.add_argument("--ssh-user", default="root")
parser.add_argument("--ssh-port", type=int, default=22)
parser.add_argument("--source-root", default="/")
@@ -30,11 +27,9 @@ class Command(BaseCommand):
if not is_absolute_non_root(backup_root):
raise CommandError("--backup-root must be an absolute path and must not be '/'")
pobsync_home = str(Path(options["pobsync_home"]))
retention = parse_retention(options["retention"])
defaults = {
"backup_root": backup_root,
"pobsync_home": pobsync_home,
"ssh_user": options["ssh_user"],
"ssh_port": options["ssh_port"],
"ssh_options": ["-oBatchMode=yes", "-oStrictHostKeyChecking=accept-new"],
@@ -53,8 +48,8 @@ class Command(BaseCommand):
}
if GlobalConfig.objects.filter(name=options["name"]).exists() and not options["force"]:
raise CommandError(f"GlobalConfig {options['name']!r} already exists; use --force to update")
raise CommandError(f"Global config {options['name']!r} already exists; use --force to update")
_obj, created = GlobalConfig.objects.update_or_create(name=options["name"], defaults=defaults)
action = "Created" if created else "Updated"
self.stdout.write(self.style.SUCCESS(f"{action} GlobalConfig {options['name']!r}."))
self.stdout.write(self.style.SUCCESS(f"{action} global config {options['name']!r}."))

6
src/pobsync_backend/management/commands/configure_pobsync_host.py
View File

@@ -10,7 +10,7 @@ from pobsync_backend.models import GlobalConfig, HostConfig
class Command(BaseCommand):
help = "Create or update a SQL-backed host pobsync configuration."
help = "Create or update a host backup configuration."
def add_arguments(self, parser) -> None:
parser.add_argument("host")
@@ -29,7 +29,7 @@ class Command(BaseCommand):
def handle(self, *args: Any, **options: Any) -> None:
host = sanitize_host(options["host"])
if HostConfig.objects.filter(host=host).exists() and not options["force"]:
raise CommandError(f"HostConfig {host!r} already exists; use --force to update")
raise CommandError(f"Host {host!r} already exists; use --force to update")
retention = self._retention(options["retention"])
defaults = {
@@ -49,7 +49,7 @@ class Command(BaseCommand):
}
_obj, created = HostConfig.objects.update_or_create(host=host, defaults=defaults)
action = "Created" if created else "Updated"
self.stdout.write(self.style.SUCCESS(f"{action} HostConfig {host!r}."))
self.stdout.write(self.style.SUCCESS(f"{action} host {host!r}."))
def _retention(self, value: str | None) -> dict[str, int]:
if value:

20
src/pobsync_backend/management/commands/configure_pobsync_schedule.py
View File

@@ -9,11 +9,16 @@ from pobsync_backend.scheduler import parse_cron_expr
class Command(BaseCommand):
help = "Create, update, disable, or remove a SQL-backed pobsync schedule."
help = "Create, update, disable, or remove a scheduler-managed host schedule."
def add_arguments(self, parser) -> None:
parser.add_argument("host")
parser.add_argument("--cron", help='Cron expression, e.g. "15 2 * * *"')
parser.add_argument(
"--schedule-expression",
"--cron",
dest="schedule_expression",
help='Five-field schedule expression, e.g. "15 2 * * *"',
)
parser.add_argument("--prune", action="store_true")
parser.add_argument("--prune-max-delete", type=int, default=10)
parser.add_argument("--prune-protect-bases", action="store_true")
@@ -24,24 +29,25 @@ class Command(BaseCommand):
try:
host = HostConfig.objects.get(host=options["host"])
except HostConfig.DoesNotExist as exc:
raise CommandError(f"Missing HostConfig {options['host']!r}") from exc
raise CommandError(f"Missing host {options['host']!r}") from exc
if options["delete"]:
deleted, _details = ScheduleConfig.objects.filter(host=host).delete()
self.stdout.write(self.style.SUCCESS(f"Deleted {deleted} schedule row(s) for {host.host!r}."))
return
if not options["cron"]:
raise CommandError("--cron is required unless --delete is used")
schedule_expression = options["schedule_expression"]
if not schedule_expression:
raise CommandError("--schedule-expression is required unless --delete is used")
try:
parse_cron_expr(options["cron"])
parse_cron_expr(schedule_expression)
except ValueError as exc:
raise CommandError(str(exc)) from exc
schedule, created = ScheduleConfig.objects.update_or_create(
host=host,
defaults={
"cron_expr": options["cron"],
"cron_expr": schedule_expression,
"enabled": not options["disabled"],
"prune": bool(options["prune"]),
"prune_max_delete": int(options["prune_max_delete"]),

4
src/pobsync_backend/management/commands/discover_pobsync_snapshots.py
View File

@@ -20,14 +20,14 @@ class Command(BaseCommand):
try:
global_config = GlobalConfig.objects.get(name="default")
except GlobalConfig.DoesNotExist as exc:
raise CommandError("Missing GlobalConfig 'default'") from exc
raise CommandError("Missing default global config") from exc
host = None
if options["host"]:
try:
host = HostConfig.objects.get(host=options["host"], enabled=True)
except HostConfig.DoesNotExist as exc:
raise CommandError(f"Missing enabled HostConfig {options['host']!r}") from exc
raise CommandError(f"Missing enabled host {options['host']!r}") from exc
kind = normalize_kind(options["kind"])
kinds = ["scheduled", "manual", "incomplete"] if kind == "all" else [kind]

23
src/pobsync_backend/management/commands/export_pobsync_configs.py
View File

@@ -1,23 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from pobsync_backend.config_repository import export_runtime_configs
class Command(BaseCommand):
help = "Export Django database configs to pobsync runtime YAML files."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--host", default=None, help="Export only one enabled host")
def handle(self, *args: Any, **options: Any) -> None:
written = export_runtime_configs(prefix=Path(options["prefix"]), host=options["host"])
for path in written:
self.stdout.write(str(path))
self.stdout.write(self.style.SUCCESS(f"Exported {len(written)} config file(s)."))

81
src/pobsync_backend/management/commands/import_pobsync_configs.py
View File

@@ -1,81 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from pobsync.config.load import load_global_config, load_host_config
from pobsync.paths import PobsyncPaths
from pobsync_backend.models import GlobalConfig, HostConfig
class Command(BaseCommand):
help = "Import pobsync YAML configs into the Django database."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
def handle(self, *args: Any, **options: Any) -> None:
paths = PobsyncPaths(home=Path(options["prefix"]))
if not paths.global_config_path.exists():
raise CommandError(f"Missing global config: {paths.global_config_path}")
global_cfg = load_global_config(paths.global_config_path)
global_ssh = global_cfg.get("ssh") or {}
global_rsync = global_cfg.get("rsync") or {}
global_defaults = global_cfg.get("defaults") or {}
retention_defaults = global_cfg.get("retention_defaults") or {}
GlobalConfig.objects.update_or_create(
name="default",
defaults={
"backup_root": global_cfg["backup_root"],
"pobsync_home": global_cfg.get("pobsync_home", str(paths.home)),
"ssh_user": global_ssh.get("user") or "root",
"ssh_port": global_ssh.get("port") or 22,
"ssh_options": global_ssh.get("options") or [],
"rsync_binary": global_rsync.get("binary") or "rsync",
"rsync_args": global_rsync.get("args") or [],
"rsync_extra_args": global_rsync.get("extra_args") or [],
"rsync_timeout_seconds": global_rsync.get("timeout_seconds") or 0,
"rsync_bwlimit_kbps": global_rsync.get("bwlimit_kbps") or 0,
"default_source_root": global_defaults.get("source_root") or "/",
"default_destination_subdir": global_defaults.get("destination_subdir") or "",
"excludes_default": global_cfg.get("excludes_default") or [],
"retention_daily": retention_defaults.get("daily", 14),
"retention_weekly": retention_defaults.get("weekly", 8),
"retention_monthly": retention_defaults.get("monthly", 12),
"retention_yearly": retention_defaults.get("yearly", 0),
"data": global_cfg,
},
)
count = 0
for host_path in sorted(paths.hosts_dir.glob("*.yaml")):
host_cfg = load_host_config(host_path)
host_ssh = host_cfg.get("ssh") or {}
host_rsync = host_cfg.get("rsync") or {}
host_retention = host_cfg.get("retention") or {}
HostConfig.objects.update_or_create(
host=host_cfg["host"],
defaults={
"address": host_cfg["address"],
"ssh_user": host_ssh.get("user") or "",
"ssh_port": host_ssh.get("port"),
"source_root": host_cfg.get("source_root") or "",
"includes": host_cfg.get("includes") or [],
"excludes_add": host_cfg.get("excludes_add") or [],
"excludes_replace": host_cfg.get("excludes_replace"),
"rsync_extra_args": host_rsync.get("extra_args") or [],
"retention_daily": host_retention.get("daily", 14),
"retention_weekly": host_retention.get("weekly", 8),
"retention_monthly": host_retention.get("monthly", 12),
"retention_yearly": host_retention.get("yearly", 0),
"config": host_cfg,
"enabled": True,
},
)
count += 1
self.stdout.write(self.style.SUCCESS(f"Imported global config and {count} host config(s)."))

4
src/pobsync_backend/management/commands/run_pobsync_backup.py
View File

@@ -16,7 +16,7 @@ class Command(BaseCommand):
def add_arguments(self, parser) -> None:
parser.add_argument("host", help="Host to back up")
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
parser.add_argument("--dry-run", action="store_true", help="Run rsync --dry-run")
parser.add_argument("--verbose-rsync", action="store_true", help="Write itemized rsync output to the run log")
parser.add_argument("--prune", action="store_true", help="Apply retention after a successful run")
@@ -30,7 +30,7 @@ class Command(BaseCommand):
try:
host = HostConfig.objects.get(host=host_name, enabled=True)
except HostConfig.DoesNotExist as exc:
raise CommandError(f"Missing enabled HostConfig {host_name!r}") from exc
raise CommandError(f"Missing enabled host {host_name!r}") from exc
run = BackupRun.objects.create(
host=host,

4
src/pobsync_backend/management/commands/run_pobsync_retention.py
View File

@@ -12,11 +12,11 @@ from pobsync_backend.retention import run_sql_retention_apply, run_sql_retention
class Command(BaseCommand):
help = "Plan or apply retention using SQL-backed pobsync configuration."
help = "Plan or apply retention using the Django backup configuration."
def add_arguments(self, parser) -> None:
parser.add_argument("host")
parser.add_argument("--prefix", default=settings.POBSYNC_HOME)
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
parser.add_argument("--kind", default="scheduled", choices=["scheduled", "manual", "all"])
parser.add_argument("--protect-bases", action="store_true")
parser.add_argument("--apply", action="store_true")

2
src/pobsync_backend/management/commands/run_pobsync_scheduler.py
View File

@@ -18,7 +18,7 @@ class Command(BaseCommand):
help = "Run due pobsync schedules from the Django database."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
parser.add_argument("--once", action="store_true", help="Check once and exit")
parser.add_argument("--loop", action="store_true", help="Keep checking schedules")
parser.add_argument("--interval", type=int, default=60, help="Loop interval in seconds")

2
src/pobsync_backend/management/commands/run_pobsync_worker.py
View File

@@ -15,7 +15,7 @@ class Command(BaseCommand):
help = "Run queued pobsync backup jobs from the Django database."
def add_arguments(self, parser) -> None:
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Pobsync home directory")
parser.add_argument("--prefix", default=settings.POBSYNC_HOME, help="Runtime state root")
parser.add_argument("--once", action="store_true", help="Process one queued run and exit")
parser.add_argument("--loop", action="store_true", help="Keep checking for queued runs")
parser.add_argument("--interval", type=int, default=15, help="Loop interval in seconds")

14
src/pobsync_backend/migrations/0010_remove_globalconfig_pobsync_home.py Normal file
View File

@@ -0,0 +1,14 @@
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("pobsync_backend", "0009_remove_scheduleconfig_user"),
]
operations = [
migrations.RemoveField(
model_name="globalconfig",
name="pobsync_home",
),
]

14
src/pobsync_backend/migrations/0011_remove_globalconfig_data.py Normal file
View File

@@ -0,0 +1,14 @@
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("pobsync_backend", "0010_remove_globalconfig_pobsync_home"),
]
operations = [
migrations.RemoveField(
model_name="globalconfig",
name="data",
),
]

2
src/pobsync_backend/models.py
View File

@@ -14,7 +14,6 @@ class TimestampedModel(models.Model):
class GlobalConfig(TimestampedModel):
name = models.CharField(max_length=64, default="default", unique=True)
backup_root = models.CharField(max_length=512)
pobsync_home = models.CharField(max_length=512, default="/opt/pobsync")
default_ssh_credential = models.ForeignKey(
"SshCredential",
on_delete=models.SET_NULL,
@@ -37,7 +36,6 @@ class GlobalConfig(TimestampedModel):
retention_weekly = models.PositiveIntegerField(default=8)
retention_monthly = models.PositiveIntegerField(default=12)
retention_yearly = models.PositiveIntegerField(default=0)
data = models.JSONField(default=dict, blank=True)
class Meta:
verbose_name = "global config"

2
src/pobsync_backend/retention.py
View File

@@ -135,7 +135,7 @@ def _enabled_host_config(host: str) -> HostConfig:
try:
return HostConfig.objects.get(host=host, enabled=True)
except HostConfig.DoesNotExist as exc:
raise ConfigError(f"Missing enabled HostConfig {host!r}") from exc
raise ConfigError(f"Missing enabled host {host!r}") from exc
def _retention_for_host(host_config: HostConfig) -> dict[str, int]:

13
src/pobsync_backend/self_check.py
View File

@@ -76,10 +76,17 @@ def _django_checks() -> list[SelfCheck]:
def _path_checks() -> list[SelfCheck]:
checks = []
checks.append(_path_check("POBSYNC_HOME", Path(settings.POBSYNC_HOME), must_be_absolute=True, must_be_writable=True))
checks.append(
_path_check(
"POBSYNC_BACKUP_ROOT",
"State root",
Path(settings.POBSYNC_HOME),
must_be_absolute=True,
must_be_writable=True,
)
)
checks.append(
_path_check(
"Backup root",
Path(settings.POBSYNC_BACKUP_ROOT),
must_be_absolute=True,
must_exist=True,
@@ -259,7 +266,7 @@ def _config_checks() -> list[SelfCheck]:
message = "Default global config exists."
if global_config.backup_root != settings.POBSYNC_BACKUP_ROOT:
status = "warning"
message = "Global config backup root differs from runtime POBSYNC_BACKUP_ROOT."
message = "Global config backup root differs from the runtime backup root."
return [
SelfCheck(
"Global config",

20
src/pobsync_backend/tests/test_admin.py
View File

@@ -5,11 +5,27 @@ from datetime import datetime, timezone
from django.contrib.admin.sites import AdminSite
from django.test import TestCase
from pobsync_backend.admin import BackupRunAdmin, HostConfigAdmin, SnapshotRecordAdmin
from pobsync_backend.models import BackupRun, HostConfig, ScheduleConfig, SnapshotRecord
from pobsync_backend.admin import BackupRunAdmin, GlobalConfigAdmin, HostConfigAdmin, SnapshotRecordAdmin
from pobsync_backend.models import BackupRun, GlobalConfig, HostConfig, ScheduleConfig, SnapshotRecord
class AdminDisplayTests(TestCase):
def test_admin_hides_old_global_state_fields_and_labels_host_runtime_state(self) -> None:
site = AdminSite()
global_admin = GlobalConfigAdmin(GlobalConfig, site)
host_admin = HostConfigAdmin(HostConfig, site)
global_fieldsets = list(global_admin.fieldsets)
host_fieldsets = list(host_admin.fieldsets)
global_fields = [field for _name, options in global_fieldsets for field in options["fields"]]
fieldset_names = [name for name, _options in [*global_fieldsets, *host_fieldsets]]
self.assertNotIn("pobsync_home", global_fields)
self.assertNotIn("data", global_fields)
self.assertIn("Runtime state", fieldset_names)
self.assertNotIn("Compatibility data", fieldset_names)
self.assertNotIn("Legacy JSON", fieldset_names)
def test_host_admin_links_to_related_snapshots_and_runs(self) -> None:
site = AdminSite()
admin = HostConfigAdmin(HostConfig, site)

34
src/pobsync_backend/tests/test_config_repository.py
View File

@@ -1,23 +1,16 @@
from __future__ import annotations
import tempfile
from pathlib import Path
from django.test import TestCase
from pobsync.config.load import load_global_config, load_host_config
from pobsync_backend.config_repository import export_runtime_configs
from pobsync_backend.config_repository import ConfigRepositoryError, global_config_data, host_config_data
from pobsync_backend.models import GlobalConfig, HostConfig
class ConfigRepositoryTests(TestCase):
def test_exports_database_configs_to_engine_yaml(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
prefix = Path(tmp)
def test_builds_runtime_config_from_database_fields(self) -> None:
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home=str(prefix),
ssh_user="backup",
ssh_port=2222,
rsync_args=["--archive"],
@@ -26,13 +19,6 @@ class ConfigRepositoryTests(TestCase):
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"ssh": {"user": "ignored", "port": 22, "options": []},
"unknown": "must-not-leak",
"retention_defaults": {"daily": 99, "weekly": 99, "monthly": 99, "yearly": 99},
},
)
HostConfig.objects.create(
host="web-01",
@@ -53,13 +39,10 @@ class ConfigRepositoryTests(TestCase):
},
)
written = export_runtime_configs(prefix=prefix, host="web-01")
global_cfg = global_config_data()
host_cfg = host_config_data("web-01")
self.assertEqual(len(written), 2)
global_cfg = load_global_config(prefix / "config" / "global.yaml")
host_cfg = load_host_config(prefix / "config" / "hosts" / "web-01.yaml")
self.assertEqual(global_cfg["backup_root"], "/backups")
self.assertEqual(global_cfg["pobsync_home"], str(prefix))
self.assertEqual(global_cfg["ssh"]["user"], "backup")
self.assertEqual(global_cfg["ssh"]["port"], 2222)
self.assertEqual(global_cfg["retention_defaults"]["daily"], 7)
@@ -69,3 +52,12 @@ class ConfigRepositoryTests(TestCase):
self.assertEqual(host_cfg["excludes_add"], ["/tmp/***"])
self.assertNotIn("unknown", global_cfg)
self.assertNotIn("unknown", host_cfg)
def test_missing_config_errors_use_operator_labels(self) -> None:
with self.assertRaisesMessage(ConfigRepositoryError, "Missing global config 'default'"):
global_config_data()
GlobalConfig.objects.create(name="default", backup_root="/backups")
with self.assertRaisesMessage(ConfigRepositoryError, "Missing enabled host 'web-01'"):
host_config_data("web-01")

5
src/pobsync_backend/tests/test_configure_commands.py
View File

@@ -16,7 +16,6 @@ class ConfigureCommandsTests(TestCase):
call_command(
"configure_pobsync_global",
backup_root="/backups",
pobsync_home="/opt/pobsync",
retention="daily=3,weekly=2,monthly=1,yearly=0",
stdout=out,
)
@@ -24,7 +23,7 @@ class ConfigureCommandsTests(TestCase):
config = GlobalConfig.objects.get(name="default")
self.assertEqual(config.backup_root, "/backups")
self.assertEqual(config.retention_daily, 3)
self.assertIn("Created GlobalConfig", out.getvalue())
self.assertIn("Created global config", out.getvalue())
def test_configure_host_uses_global_retention_defaults(self) -> None:
GlobalConfig.objects.create(
@@ -62,7 +61,7 @@ class ConfigureCommandsTests(TestCase):
call_command(
"configure_pobsync_schedule",
host.host,
cron="15 2 * * *",
schedule_expression="15 2 * * *",
prune=True,
stdout=out,
)

18
src/pobsync_backend/tests/test_console_entrypoint.py
View File

@@ -31,15 +31,6 @@ class ConsoleEntrypointTests(SimpleTestCase):
self.assertEqual(exit_code, 0)
execute.assert_called_once_with(["pobsync", "check"])
def test_maps_schedule_alias_to_django_command(self) -> None:
with patch("pobsync.cli.execute_from_command_line") as execute:
exit_code = main(["schedule", "web-01", "--cron", "15 2 * * *"])
self.assertEqual(exit_code, 0)
execute.assert_called_once_with(
["pobsync", "configure_pobsync_schedule", "web-01", "--cron", "15 2 * * *"]
)
def test_maps_discover_snapshots_alias_to_django_command(self) -> None:
with patch("pobsync.cli.execute_from_command_line") as execute:
exit_code = main(["discover-snapshots", "--host", "web-01"])
@@ -53,3 +44,12 @@ class ConsoleEntrypointTests(SimpleTestCase):
self.assertEqual(exit_code, 0)
execute.assert_called_once_with(["pobsync", "run_pobsync_worker", "--once"])
def test_configuration_aliases_are_not_public_commands(self) -> None:
stderr = StringIO()
with patch("sys.stderr", stderr):
exit_code = main(["schedule", "web-01", "--cron", "15 2 * * *"])
self.assertEqual(exit_code, 2)
self.assertIn("Unknown pobsync command", stderr.getvalue())
self.assertIn("pobsync django <management-command>", stderr.getvalue())

20
src/pobsync_backend/tests/test_django_config_source.py
View File

@@ -15,7 +15,6 @@ class DjangoConfigSourceTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
rsync_args=["--archive"],
rsync_extra_args=["--numeric-ids"],
excludes_default=["/proc/***"],
@@ -23,21 +22,6 @@ class DjangoConfigSourceTests(TestCase):
retention_weekly=4,
retention_monthly=3,
retention_yearly=1,
data={
"backup_root": "/ignored",
"pobsync_home": "/ignored",
"ssh": {"user": "root", "port": 22, "options": []},
"rsync": {
"binary": "rsync",
"args": ["--archive"],
"timeout_seconds": 0,
"bwlimit_kbps": 0,
"extra_args": ["--numeric-ids"],
},
"defaults": {"source_root": "/", "destination_subdir": ""},
"excludes_default": ["/proc/***"],
"retention_defaults": {"daily": 7, "weekly": 4, "monthly": 3, "yearly": 1},
},
)
HostConfig.objects.create(
host="web-01",
@@ -72,7 +56,6 @@ class DjangoConfigSourceTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
default_ssh_credential=credential,
ssh_options=["-oBatchMode=yes"],
)
@@ -99,7 +82,6 @@ class DjangoConfigSourceTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
default_ssh_credential=global_credential,
)
HostConfig.objects.create(
@@ -127,7 +109,6 @@ class DjangoConfigSourceTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
default_ssh_credential=credential,
)
HostConfig.objects.create(host="web-01", address="web-01.example.test")
@@ -146,7 +127,6 @@ class DjangoConfigSourceTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/backups",
pobsync_home="/opt/pobsync",
default_ssh_credential=credential,
)
HostConfig.objects.create(host="web-01", address="web-01.example.test")

10
src/pobsync_backend/tests/test_retention_config_source.py
View File

@@ -7,6 +7,7 @@ from tempfile import TemporaryDirectory
from django.test import SimpleTestCase
from pobsync.commands.retention_plan import run_retention_plan
from pobsync.errors import ConfigError
from pobsync.util import write_yaml_atomic
@@ -24,6 +25,15 @@ class FakeConfigSource:
class RetentionConfigSourceTests(SimpleTestCase):
def test_retention_plan_requires_explicit_config_source(self) -> None:
with self.assertRaisesMessage(ConfigError, "A Django config source is required."):
run_retention_plan(
prefix=Path("/missing-prefix"),
host="web-01",
kind="scheduled",
protect_bases=False,
)
def test_retention_plan_uses_injected_config_source(self) -> None:
with TemporaryDirectory() as tmp:
root = Path(tmp) / "backups"

5
src/pobsync_backend/tests/test_run_scheduled_config_source.py
View File

@@ -7,6 +7,7 @@ from unittest.mock import patch
from django.test import SimpleTestCase
from pobsync.commands.run_scheduled import run_scheduled
from pobsync.errors import ConfigError
from pobsync.rsync import RsyncResult
@@ -34,6 +35,10 @@ class FakeConfigSource:
class RunScheduledConfigSourceTests(SimpleTestCase):
def test_requires_explicit_config_source(self) -> None:
with self.assertRaisesMessage(ConfigError, "A Django config source is required."):
run_scheduled(prefix=Path("/missing-prefix"), host="web-01", dry_run=True)
def test_dry_run_uses_injected_config_source(self) -> None:
with patch("pobsync.commands.run_scheduled.run_rsync") as run_rsync:
run_rsync.return_value = RsyncResult(exit_code=0, command=["rsync", "--archive"])

8
src/pobsync_backend/tests/test_views.py
View File

@@ -237,7 +237,7 @@ class ViewTests(TestCase):
self.assertContains(response, "Self Check")
self.assertContains(response, "Django debug")
self.assertContains(response, "Database connection")
self.assertContains(response, "POBSYNC_HOME")
self.assertContains(response, "State root")
def test_logs_view_renders_filtered_journal_messages(self) -> None:
self.client.force_login(self.staff_user)
@@ -475,7 +475,6 @@ class ViewTests(TestCase):
self.assertContains(response, "Global config saved for default.")
config = GlobalConfig.objects.get(name="default")
self.assertEqual(config.backup_root, "/backups")
self.assertEqual(config.pobsync_home, "/opt/pobsync")
self.assertEqual(config.default_ssh_credential, credential)
self.assertEqual(config.ssh_user, "backup")
self.assertEqual(config.ssh_port, 2222)
@@ -502,7 +501,6 @@ class ViewTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/mnt/pobsync/backups",
pobsync_home="/custom/legacy/home",
)
response = self.client.get(reverse("edit_global_config"))
@@ -512,8 +510,10 @@ class ViewTests(TestCase):
self.assertContains(response, "/backups")
self.assertContains(response, "Config Check")
self.assertContains(response, "Runtime backup root")
self.assertContains(response, "Runtime state root")
self.assertNotContains(response, "/opt/pobsync/backups")
self.assertNotContains(response, "Pobsync home")
self.assertNotContains(response, "Global pobsync home")
def test_global_config_form_renders_config_check_for_non_recursive_rsync(self) -> None:
self.client.force_login(self.staff_user)
@@ -530,7 +530,6 @@ class ViewTests(TestCase):
GlobalConfig.objects.create(
name="default",
backup_root="/mnt/pobsync/backups",
pobsync_home="/custom/legacy/home",
)
response = self.client.post(
@@ -559,7 +558,6 @@ class ViewTests(TestCase):
self.assertRedirects(response, reverse("dashboard"))
config = GlobalConfig.objects.get(name="default")
self.assertEqual(config.backup_root, "/backups")
self.assertEqual(config.pobsync_home, "/opt/pobsync")
def test_create_host_config_form_creates_host(self) -> None:
self.client.force_login(self.staff_user)