From 851f967f12e30c87d85c70d48413ce17b9851dd6 Mon Sep 17 00:00:00 2001 From: Peter van Arkel Date: Wed, 20 May 2026 01:44:51 +0200 Subject: [PATCH] (ops) Expand native install self checks and recovery docs Extend the runtime self check with native install diagnostics for the environment file, service user, backup root ownership, and SQLite database path. Export install metadata from the systemd units and pobsync-manage wrapper so custom env files and service users are visible to Django checks. Document restart, journal log inspection, and rollback steps in the README so production updates have a clear recovery path. --- README.md | 24 +++++ deploy/bin/pobsync-manage | 5 + deploy/pobsync.env.example | 3 + deploy/systemd/pobsync-scheduler.service | 3 + deploy/systemd/pobsync-web.service | 3 + deploy/systemd/pobsync-worker.service | 3 + scripts/install-systemd | 5 + src/pobsync_backend/self_check.py | 97 +++++++++++++++++++- src/pobsync_backend/tests/test_self_check.py | 54 ++++++++++- src/pobsync_server/settings.py | 3 + 10 files changed, 196 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5c46739..527c817 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,30 @@ sudo -u pobsync pobsync-manage check sudo -u pobsync pobsync-manage check_pobsync_install ``` +Restart services manually after environment or reverse proxy changes: + +``` +sudo systemctl restart pobsync-web pobsync-worker pobsync-scheduler +``` + +Inspect service logs with: + +``` +journalctl -u pobsync-web -n 100 --no-pager +journalctl -u pobsync-worker -f +journalctl -u pobsync-scheduler -n 100 --no-pager +``` + +Rollback to a previous revision by checking out the known-good commit or tag, then running the updater again: + +``` +git switch master +git pull +git checkout +sudo scripts/update-systemd +sudo -u pobsync pobsync-manage check_pobsync_install +``` + ## Development Development, Docker, maintainer tooling, and architecture notes live in: diff --git a/deploy/bin/pobsync-manage b/deploy/bin/pobsync-manage index f55c96a..4499b2f 100644 --- a/deploy/bin/pobsync-manage +++ b/deploy/bin/pobsync-manage @@ -4,6 +4,8 @@ set -eu APP_DIR="@POBSYNC_APP_DIR@" VENV_DIR="@POBSYNC_VENV_DIR@" ENV_FILE="@POBSYNC_ENV_FILE@" +SERVICE_USER="@POBSYNC_USER@" +SERVICE_GROUP="@POBSYNC_GROUP@" if [ ! -f "$ENV_FILE" ]; then echo "pobsync environment file not found: $ENV_FILE" >&2 @@ -14,6 +16,9 @@ set -a # shellcheck disable=SC1090 . "$ENV_FILE" set +a +export POBSYNC_ENV_FILE="$ENV_FILE" +export POBSYNC_SERVICE_USER="$SERVICE_USER" +export POBSYNC_SERVICE_GROUP="$SERVICE_GROUP" cd "$APP_DIR" exec "$VENV_DIR/bin/python" "$APP_DIR/manage.py" "$@" diff --git a/deploy/pobsync.env.example b/deploy/pobsync.env.example index 076b7c0..317d9a5 100644 --- a/deploy/pobsync.env.example +++ b/deploy/pobsync.env.example @@ -7,6 +7,9 @@ POBSYNC_HOME=/var/lib/pobsync POBSYNC_BACKUP_ROOT=/backups POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3 POBSYNC_STATIC_ROOT=/var/lib/pobsync/static +POBSYNC_ENV_FILE=/etc/pobsync/pobsync.env +POBSYNC_SERVICE_USER=pobsync +POBSYNC_SERVICE_GROUP=pobsync POBSYNC_WEB_BIND=127.0.0.1:8010 POBSYNC_GUNICORN_WORKERS=2 diff --git a/deploy/systemd/pobsync-scheduler.service b/deploy/systemd/pobsync-scheduler.service index a3a5dcf..ff07093 100644 --- a/deploy/systemd/pobsync-scheduler.service +++ b/deploy/systemd/pobsync-scheduler.service @@ -9,6 +9,9 @@ User=@POBSYNC_USER@ Group=@POBSYNC_GROUP@ WorkingDirectory=@POBSYNC_APP_DIR@ EnvironmentFile=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@ +Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@ ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_scheduler --loop --interval "${POBSYNC_SCHEDULER_INTERVAL:-60}"' Restart=on-failure RestartSec=5 diff --git a/deploy/systemd/pobsync-web.service b/deploy/systemd/pobsync-web.service index 3f99b0b..f07e353 100644 --- a/deploy/systemd/pobsync-web.service +++ b/deploy/systemd/pobsync-web.service @@ -9,6 +9,9 @@ User=@POBSYNC_USER@ Group=@POBSYNC_GROUP@ WorkingDirectory=@POBSYNC_APP_DIR@ EnvironmentFile=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@ +Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@ ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py migrate --noinput ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py collectstatic --noinput --clear ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/gunicorn pobsync_server.wsgi:application --bind "${POBSYNC_WEB_BIND:-127.0.0.1:8010}" --workers "${POBSYNC_GUNICORN_WORKERS:-2}" --timeout "${POBSYNC_GUNICORN_TIMEOUT:-120}"' diff --git a/deploy/systemd/pobsync-worker.service b/deploy/systemd/pobsync-worker.service index a1c4837..9a7efa1 100644 --- a/deploy/systemd/pobsync-worker.service +++ b/deploy/systemd/pobsync-worker.service @@ -9,6 +9,9 @@ User=@POBSYNC_USER@ Group=@POBSYNC_GROUP@ WorkingDirectory=@POBSYNC_APP_DIR@ EnvironmentFile=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@ +Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@ +Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@ ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_worker --loop --interval "${POBSYNC_WORKER_INTERVAL:-15}"' Restart=on-failure RestartSec=5 diff --git a/scripts/install-systemd b/scripts/install-systemd index f309834..d77a521 100755 --- a/scripts/install-systemd +++ b/scripts/install-systemd @@ -463,6 +463,9 @@ POBSYNC_BACKUP_ROOT=$BACKUP_ROOT POBSYNC_TIME_ZONE=$TIME_ZONE POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3 POBSYNC_STATIC_ROOT=/var/lib/pobsync/static +POBSYNC_ENV_FILE=$ENV_FILE +POBSYNC_SERVICE_USER=$SERVICE_USER +POBSYNC_SERVICE_GROUP=$SERVICE_GROUP POBSYNC_WEB_BIND=$WEB_BIND POBSYNC_GUNICORN_WORKERS=2 @@ -509,6 +512,8 @@ install_manage_wrapper() { -e "s|@POBSYNC_APP_DIR@|$APP_DIR|g" \ -e "s|@POBSYNC_VENV_DIR@|$VENV_DIR|g" \ -e "s|@POBSYNC_ENV_FILE@|$ENV_FILE|g" \ + -e "s|@POBSYNC_USER@|$SERVICE_USER|g" \ + -e "s|@POBSYNC_GROUP@|$SERVICE_GROUP|g" \ "$APP_DIR/deploy/bin/pobsync-manage" > /usr/local/bin/pobsync-manage chmod 0755 /usr/local/bin/pobsync-manage } diff --git a/src/pobsync_backend/self_check.py b/src/pobsync_backend/self_check.py index d354c57..ed7a9ff 100644 --- a/src/pobsync_backend/self_check.py +++ b/src/pobsync_backend/self_check.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import pwd import shutil import subprocess import sys @@ -28,6 +29,7 @@ class SelfCheck: def collect_self_checks() -> list[SelfCheck]: checks: list[SelfCheck] = [] checks.extend(_django_checks()) + checks.extend(_install_checks()) checks.extend(_path_checks()) checks.extend(_binary_checks()) checks.extend(_database_checks()) @@ -36,6 +38,10 @@ def collect_self_checks() -> list[SelfCheck]: return checks +def _native_runtime_available() -> bool: + return Path("/run/systemd/system").exists() and shutil.which("systemctl") is not None + + def summarize_self_checks(checks: list[SelfCheck]) -> dict[str, int]: return { "ok": sum(1 for check in checks if check.status == "ok"), @@ -91,18 +97,105 @@ def _path_checks() -> list[SelfCheck]: ) db_settings = settings.DATABASES["default"] if db_settings["ENGINE"] == "django.db.backends.sqlite3": + sqlite_path = Path(str(db_settings["NAME"])) checks.append( _path_check( "SQLite directory", - Path(str(db_settings["NAME"])).parent, + sqlite_path.parent, must_be_absolute=True, must_exist=True, must_be_writable=True, ) ) + checks.append(_sqlite_database_check(sqlite_path)) return checks +def _install_checks() -> list[SelfCheck]: + if not _native_runtime_available() and not Path(settings.POBSYNC_ENV_FILE).exists(): + return [ + SelfCheck( + "Environment file", + "skipped", + "Native environment file is not configured in this runtime.", + "This is expected inside Docker or local development.", + ), + SelfCheck( + "Service user", + "skipped", + "Native service user check is not available in this runtime.", + "This is expected inside Docker or local development.", + ), + SelfCheck( + "Backup root owner", + "skipped", + "Native backup root ownership check is not available in this runtime.", + "This is expected inside Docker or local development.", + ), + ] + + checks = [_env_file_check(Path(settings.POBSYNC_ENV_FILE)), _service_user_check()] + checks.append(_backup_root_owner_check(Path(settings.POBSYNC_BACKUP_ROOT))) + return checks + + +def _env_file_check(path: Path) -> SelfCheck: + if not path.is_absolute(): + return SelfCheck("Environment file", "failed", f"{path} is not absolute.") + if not path.exists(): + return SelfCheck("Environment file", "failed", f"{path} does not exist.") + if not path.is_file(): + return SelfCheck("Environment file", "failed", f"{path} is not a regular file.") + if not os.access(path, os.R_OK): + return SelfCheck("Environment file", "failed", f"{path} is not readable by this process.") + return SelfCheck("Environment file", "ok", str(path)) + + +def _service_user_check() -> SelfCheck: + expected_user = settings.POBSYNC_SERVICE_USER + try: + current_user = pwd.getpwuid(os.geteuid()).pw_name + except KeyError: + return SelfCheck("Service user", "failed", f"Current uid {os.geteuid()} has no passwd entry.") + if current_user != expected_user: + return SelfCheck( + "Service user", + "warning", + f"Current process runs as {current_user}, expected {expected_user}.", + "Run terminal checks with sudo -u pobsync-manage check_pobsync_install.", + ) + return SelfCheck("Service user", "ok", current_user) + + +def _backup_root_owner_check(path: Path) -> SelfCheck: + if not path.exists(): + return SelfCheck("Backup root owner", "failed", f"{path} does not exist.") + expected_user = settings.POBSYNC_SERVICE_USER + try: + owner = pwd.getpwuid(path.stat().st_uid).pw_name + except KeyError: + return SelfCheck("Backup root owner", "warning", f"{path} owner uid {path.stat().st_uid} has no passwd entry.") + if owner != expected_user: + return SelfCheck( + "Backup root owner", + "warning", + f"{path} is owned by {owner}, expected {expected_user}.", + ) + return SelfCheck("Backup root owner", "ok", f"{path} owner={owner}") + + +def _sqlite_database_check(path: Path) -> SelfCheck: + if not path.is_absolute(): + return SelfCheck("SQLite database", "failed", f"{path} is not absolute.") + if not path.exists(): + return SelfCheck("SQLite database", "warning", f"{path} does not exist yet.") + if not path.is_file(): + return SelfCheck("SQLite database", "failed", f"{path} is not a regular file.") + if not os.access(path, os.R_OK | os.W_OK): + return SelfCheck("SQLite database", "failed", f"{path} is not readable and writable by this process.") + return SelfCheck("SQLite database", "ok", str(path)) + + def _path_check( name: str, path: Path, @@ -178,7 +271,7 @@ def _config_checks() -> list[SelfCheck]: def _systemd_checks() -> list[SelfCheck]: - if not Path("/run/systemd/system").exists() or shutil.which("systemctl") is None: + if not _native_runtime_available(): return [ SelfCheck( "Systemd services", diff --git a/src/pobsync_backend/tests/test_self_check.py b/src/pobsync_backend/tests/test_self_check.py index 76dff6c..28e0361 100644 --- a/src/pobsync_backend/tests/test_self_check.py +++ b/src/pobsync_backend/tests/test_self_check.py @@ -2,13 +2,15 @@ from __future__ import annotations import subprocess from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory from unittest.mock import patch from django.core.management import call_command from django.core.management.base import CommandError -from django.test import SimpleTestCase +from django.test import SimpleTestCase, override_settings -from pobsync_backend.self_check import SelfCheck, _systemd_checks +from pobsync_backend.self_check import SelfCheck, _install_checks, _sqlite_database_check, _systemd_checks class SystemdSelfCheckTests(SimpleTestCase): @@ -45,6 +47,54 @@ class SystemdSelfCheckTests(SimpleTestCase): self.assertEqual(journal_check.message, "pobsync cannot read service logs.") +class InstallSelfCheckTests(SimpleTestCase): + def test_install_checks_skip_native_paths_in_development_runtime(self) -> None: + with override_settings(POBSYNC_ENV_FILE="/missing/pobsync.env"), patch( + "pobsync_backend.self_check._native_runtime_available", + return_value=False, + ): + checks = _install_checks() + + self.assertEqual([check.status for check in checks], ["skipped", "skipped", "skipped"]) + self.assertEqual(checks[0].name, "Environment file") + self.assertEqual(checks[1].name, "Service user") + self.assertEqual(checks[2].name, "Backup root owner") + + def test_service_user_warns_when_current_user_differs(self) -> None: + with override_settings( + POBSYNC_ENV_FILE="/etc/pobsync/pobsync.env", + POBSYNC_SERVICE_USER="pobsync", + POBSYNC_BACKUP_ROOT="/backups", + ), patch("pobsync_backend.self_check._native_runtime_available", return_value=True), patch( + "pobsync_backend.self_check._env_file_check", + return_value=SelfCheck("Environment file", "ok", "/etc/pobsync/pobsync.env"), + ), patch( + "pobsync_backend.self_check._backup_root_owner_check", + return_value=SelfCheck("Backup root owner", "ok", "/backups owner=pobsync"), + ), patch( + "pobsync_backend.self_check.os.geteuid", + return_value=0, + ), patch( + "pobsync_backend.self_check.pwd.getpwuid", + ) as getpwuid: + getpwuid.return_value.pw_name = "root" + checks = _install_checks() + + service_user_check = next(check for check in checks if check.name == "Service user") + self.assertEqual(service_user_check.status, "warning") + self.assertIn("expected pobsync", service_user_check.message) + + def test_sqlite_database_check_reports_existing_database(self) -> None: + with TemporaryDirectory() as tmp: + db_path = Path(tmp) / "pobsync.sqlite3" + db_path.write_text("", encoding="utf-8") + + check = _sqlite_database_check(db_path) + + self.assertEqual(check.status, "ok") + self.assertEqual(check.name, "SQLite database") + + class CheckPobsyncInstallCommandTests(SimpleTestCase): def test_command_prints_summary_for_successful_checks(self) -> None: stdout = StringIO() diff --git a/src/pobsync_server/settings.py b/src/pobsync_server/settings.py index 24d49d4..b000ab9 100644 --- a/src/pobsync_server/settings.py +++ b/src/pobsync_server/settings.py @@ -99,3 +99,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" POBSYNC_HOME = os.getenv("POBSYNC_HOME", "/opt/pobsync") POBSYNC_BACKUP_ROOT = os.getenv("POBSYNC_BACKUP_ROOT", "/backups") +POBSYNC_ENV_FILE = os.getenv("POBSYNC_ENV_FILE", "/etc/pobsync/pobsync.env") +POBSYNC_SERVICE_USER = os.getenv("POBSYNC_SERVICE_USER", "pobsync") +POBSYNC_SERVICE_GROUP = os.getenv("POBSYNC_SERVICE_GROUP", "pobsync")