(ops) Expand native install self checks and recovery docs

Extend the runtime self check with native install diagnostics for the
environment file, service user, backup root ownership, and SQLite database
path. Export install metadata from the systemd units and pobsync-manage wrapper
so custom env files and service users are visible to Django checks.

Document restart, journal log inspection, and rollback steps in the README so
production updates have a clear recovery path.

README.md

@@ -202,6 +202,30 @@ sudo -u pobsync pobsync-manage check
 sudo -u pobsync pobsync-manage check_pobsync_install
 ```
 
+Restart services manually after environment or reverse proxy changes:
+
+```
+sudo systemctl restart pobsync-web pobsync-worker pobsync-scheduler
+```
+
+Inspect service logs with:
+
+```
+journalctl -u pobsync-web -n 100 --no-pager
+journalctl -u pobsync-worker -f
+journalctl -u pobsync-scheduler -n 100 --no-pager
+```
+
+Rollback to a previous revision by checking out the known-good commit or tag, then running the updater again:
+
+```
+git switch master
+git pull
+git checkout <known-good-commit-or-tag>
+sudo scripts/update-systemd
+sudo -u pobsync pobsync-manage check_pobsync_install
+```
+
 ## Development
 
 Development, Docker, maintainer tooling, and architecture notes live in:

deploy/bin/pobsync-manage

@@ -4,6 +4,8 @@ set -eu
 APP_DIR="@POBSYNC_APP_DIR@"
 VENV_DIR="@POBSYNC_VENV_DIR@"
 ENV_FILE="@POBSYNC_ENV_FILE@"
+SERVICE_USER="@POBSYNC_USER@"
+SERVICE_GROUP="@POBSYNC_GROUP@"
 
 if [ ! -f "$ENV_FILE" ]; then
     echo "pobsync environment file not found: $ENV_FILE" >&2
@@ -14,6 +16,9 @@ set -a
 # shellcheck disable=SC1090
 . "$ENV_FILE"
 set +a
+export POBSYNC_ENV_FILE="$ENV_FILE"
+export POBSYNC_SERVICE_USER="$SERVICE_USER"
+export POBSYNC_SERVICE_GROUP="$SERVICE_GROUP"
 
 cd "$APP_DIR"
 exec "$VENV_DIR/bin/python" "$APP_DIR/manage.py" "$@"

deploy/pobsync.env.example

@@ -7,6 +7,9 @@ POBSYNC_HOME=/var/lib/pobsync
 POBSYNC_BACKUP_ROOT=/backups
 POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
 POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
+POBSYNC_ENV_FILE=/etc/pobsync/pobsync.env
+POBSYNC_SERVICE_USER=pobsync
+POBSYNC_SERVICE_GROUP=pobsync
 
 POBSYNC_WEB_BIND=127.0.0.1:8010
 POBSYNC_GUNICORN_WORKERS=2

deploy/systemd/pobsync-scheduler.service

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
 Group=@POBSYNC_GROUP@
 WorkingDirectory=@POBSYNC_APP_DIR@
 EnvironmentFile=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
+Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
 ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_scheduler --loop --interval "${POBSYNC_SCHEDULER_INTERVAL:-60}"'
 Restart=on-failure
 RestartSec=5

deploy/systemd/pobsync-web.service

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
 Group=@POBSYNC_GROUP@
 WorkingDirectory=@POBSYNC_APP_DIR@
 EnvironmentFile=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
+Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
 ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py migrate --noinput
 ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py collectstatic --noinput --clear
 ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/gunicorn pobsync_server.wsgi:application --bind "${POBSYNC_WEB_BIND:-127.0.0.1:8010}" --workers "${POBSYNC_GUNICORN_WORKERS:-2}" --timeout "${POBSYNC_GUNICORN_TIMEOUT:-120}"'

deploy/systemd/pobsync-worker.service

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
 Group=@POBSYNC_GROUP@
 WorkingDirectory=@POBSYNC_APP_DIR@
 EnvironmentFile=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
+Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
+Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
 ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_worker --loop --interval "${POBSYNC_WORKER_INTERVAL:-15}"'
 Restart=on-failure
 RestartSec=5

scripts/install-systemd

@@ -463,6 +463,9 @@ POBSYNC_BACKUP_ROOT=$BACKUP_ROOT
 POBSYNC_TIME_ZONE=$TIME_ZONE
 POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
 POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
+POBSYNC_ENV_FILE=$ENV_FILE
+POBSYNC_SERVICE_USER=$SERVICE_USER
+POBSYNC_SERVICE_GROUP=$SERVICE_GROUP
 
 POBSYNC_WEB_BIND=$WEB_BIND
 POBSYNC_GUNICORN_WORKERS=2
@@ -509,6 +512,8 @@ install_manage_wrapper() {
         -e "s|@POBSYNC_APP_DIR@|$APP_DIR|g" \
         -e "s|@POBSYNC_VENV_DIR@|$VENV_DIR|g" \
         -e "s|@POBSYNC_ENV_FILE@|$ENV_FILE|g" \
+        -e "s|@POBSYNC_USER@|$SERVICE_USER|g" \
+        -e "s|@POBSYNC_GROUP@|$SERVICE_GROUP|g" \
         "$APP_DIR/deploy/bin/pobsync-manage" > /usr/local/bin/pobsync-manage
     chmod 0755 /usr/local/bin/pobsync-manage
 }

src/pobsync_backend/self_check.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import pwd
 import shutil
 import subprocess
 import sys
@@ -28,6 +29,7 @@ class SelfCheck:
 def collect_self_checks() -> list[SelfCheck]:
     checks: list[SelfCheck] = []
     checks.extend(_django_checks())
+    checks.extend(_install_checks())
     checks.extend(_path_checks())
     checks.extend(_binary_checks())
     checks.extend(_database_checks())
@@ -36,6 +38,10 @@ def collect_self_checks() -> list[SelfCheck]:
     return checks
 
 
+def _native_runtime_available() -> bool:
+    return Path("/run/systemd/system").exists() and shutil.which("systemctl") is not None
+
+
 def summarize_self_checks(checks: list[SelfCheck]) -> dict[str, int]:
     return {
         "ok": sum(1 for check in checks if check.status == "ok"),
@@ -91,18 +97,105 @@ def _path_checks() -> list[SelfCheck]:
     )
     db_settings = settings.DATABASES["default"]
     if db_settings["ENGINE"] == "django.db.backends.sqlite3":
+        sqlite_path = Path(str(db_settings["NAME"]))
         checks.append(
             _path_check(
                 "SQLite directory",
-                Path(str(db_settings["NAME"])).parent,
+                sqlite_path.parent,
                 must_be_absolute=True,
                 must_exist=True,
                 must_be_writable=True,
             )
         )
+        checks.append(_sqlite_database_check(sqlite_path))
     return checks
 
 
+def _install_checks() -> list[SelfCheck]:
+    if not _native_runtime_available() and not Path(settings.POBSYNC_ENV_FILE).exists():
+        return [
+            SelfCheck(
+                "Environment file",
+                "skipped",
+                "Native environment file is not configured in this runtime.",
+                "This is expected inside Docker or local development.",
+            ),
+            SelfCheck(
+                "Service user",
+                "skipped",
+                "Native service user check is not available in this runtime.",
+                "This is expected inside Docker or local development.",
+            ),
+            SelfCheck(
+                "Backup root owner",
+                "skipped",
+                "Native backup root ownership check is not available in this runtime.",
+                "This is expected inside Docker or local development.",
+            ),
+        ]
+
+    checks = [_env_file_check(Path(settings.POBSYNC_ENV_FILE)), _service_user_check()]
+    checks.append(_backup_root_owner_check(Path(settings.POBSYNC_BACKUP_ROOT)))
+    return checks
+
+
+def _env_file_check(path: Path) -> SelfCheck:
+    if not path.is_absolute():
+        return SelfCheck("Environment file", "failed", f"{path} is not absolute.")
+    if not path.exists():
+        return SelfCheck("Environment file", "failed", f"{path} does not exist.")
+    if not path.is_file():
+        return SelfCheck("Environment file", "failed", f"{path} is not a regular file.")
+    if not os.access(path, os.R_OK):
+        return SelfCheck("Environment file", "failed", f"{path} is not readable by this process.")
+    return SelfCheck("Environment file", "ok", str(path))
+
+
+def _service_user_check() -> SelfCheck:
+    expected_user = settings.POBSYNC_SERVICE_USER
+    try:
+        current_user = pwd.getpwuid(os.geteuid()).pw_name
+    except KeyError:
+        return SelfCheck("Service user", "failed", f"Current uid {os.geteuid()} has no passwd entry.")
+    if current_user != expected_user:
+        return SelfCheck(
+            "Service user",
+            "warning",
+            f"Current process runs as {current_user}, expected {expected_user}.",
+            "Run terminal checks with sudo -u <service-user> pobsync-manage check_pobsync_install.",
+        )
+    return SelfCheck("Service user", "ok", current_user)
+
+
+def _backup_root_owner_check(path: Path) -> SelfCheck:
+    if not path.exists():
+        return SelfCheck("Backup root owner", "failed", f"{path} does not exist.")
+    expected_user = settings.POBSYNC_SERVICE_USER
+    try:
+        owner = pwd.getpwuid(path.stat().st_uid).pw_name
+    except KeyError:
+        return SelfCheck("Backup root owner", "warning", f"{path} owner uid {path.stat().st_uid} has no passwd entry.")
+    if owner != expected_user:
+        return SelfCheck(
+            "Backup root owner",
+            "warning",
+            f"{path} is owned by {owner}, expected {expected_user}.",
+        )
+    return SelfCheck("Backup root owner", "ok", f"{path} owner={owner}")
+
+
+def _sqlite_database_check(path: Path) -> SelfCheck:
+    if not path.is_absolute():
+        return SelfCheck("SQLite database", "failed", f"{path} is not absolute.")
+    if not path.exists():
+        return SelfCheck("SQLite database", "warning", f"{path} does not exist yet.")
+    if not path.is_file():
+        return SelfCheck("SQLite database", "failed", f"{path} is not a regular file.")
+    if not os.access(path, os.R_OK | os.W_OK):
+        return SelfCheck("SQLite database", "failed", f"{path} is not readable and writable by this process.")
+    return SelfCheck("SQLite database", "ok", str(path))
+
+
 def _path_check(
     name: str,
     path: Path,
@@ -178,7 +271,7 @@ def _config_checks() -> list[SelfCheck]:
 
 
 def _systemd_checks() -> list[SelfCheck]:
-    if not Path("/run/systemd/system").exists() or shutil.which("systemctl") is None:
+    if not _native_runtime_available():
         return [
             SelfCheck(
                 "Systemd services",

src/pobsync_backend/tests/test_self_check.py

@@ -2,13 +2,15 @@ from __future__ import annotations
 
 import subprocess
 from io import StringIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
 from unittest.mock import patch
 
 from django.core.management import call_command
 from django.core.management.base import CommandError
-from django.test import SimpleTestCase
+from django.test import SimpleTestCase, override_settings
 
-from pobsync_backend.self_check import SelfCheck, _systemd_checks
+from pobsync_backend.self_check import SelfCheck, _install_checks, _sqlite_database_check, _systemd_checks
 
 
 class SystemdSelfCheckTests(SimpleTestCase):
@@ -45,6 +47,54 @@ class SystemdSelfCheckTests(SimpleTestCase):
         self.assertEqual(journal_check.message, "pobsync cannot read service logs.")
 
 
+class InstallSelfCheckTests(SimpleTestCase):
+    def test_install_checks_skip_native_paths_in_development_runtime(self) -> None:
+        with override_settings(POBSYNC_ENV_FILE="/missing/pobsync.env"), patch(
+            "pobsync_backend.self_check._native_runtime_available",
+            return_value=False,
+        ):
+            checks = _install_checks()
+
+        self.assertEqual([check.status for check in checks], ["skipped", "skipped", "skipped"])
+        self.assertEqual(checks[0].name, "Environment file")
+        self.assertEqual(checks[1].name, "Service user")
+        self.assertEqual(checks[2].name, "Backup root owner")
+
+    def test_service_user_warns_when_current_user_differs(self) -> None:
+        with override_settings(
+            POBSYNC_ENV_FILE="/etc/pobsync/pobsync.env",
+            POBSYNC_SERVICE_USER="pobsync",
+            POBSYNC_BACKUP_ROOT="/backups",
+        ), patch("pobsync_backend.self_check._native_runtime_available", return_value=True), patch(
+            "pobsync_backend.self_check._env_file_check",
+            return_value=SelfCheck("Environment file", "ok", "/etc/pobsync/pobsync.env"),
+        ), patch(
+            "pobsync_backend.self_check._backup_root_owner_check",
+            return_value=SelfCheck("Backup root owner", "ok", "/backups owner=pobsync"),
+        ), patch(
+            "pobsync_backend.self_check.os.geteuid",
+            return_value=0,
+        ), patch(
+            "pobsync_backend.self_check.pwd.getpwuid",
+        ) as getpwuid:
+            getpwuid.return_value.pw_name = "root"
+            checks = _install_checks()
+
+        service_user_check = next(check for check in checks if check.name == "Service user")
+        self.assertEqual(service_user_check.status, "warning")
+        self.assertIn("expected pobsync", service_user_check.message)
+
+    def test_sqlite_database_check_reports_existing_database(self) -> None:
+        with TemporaryDirectory() as tmp:
+            db_path = Path(tmp) / "pobsync.sqlite3"
+            db_path.write_text("", encoding="utf-8")
+
+            check = _sqlite_database_check(db_path)
+
+        self.assertEqual(check.status, "ok")
+        self.assertEqual(check.name, "SQLite database")
+
+
 class CheckPobsyncInstallCommandTests(SimpleTestCase):
     def test_command_prints_summary_for_successful_checks(self) -> None:
         stdout = StringIO()

src/pobsync_server/settings.py

@@ -99,3 +99,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
 POBSYNC_HOME = os.getenv("POBSYNC_HOME", "/opt/pobsync")
 POBSYNC_BACKUP_ROOT = os.getenv("POBSYNC_BACKUP_ROOT", "/backups")
+POBSYNC_ENV_FILE = os.getenv("POBSYNC_ENV_FILE", "/etc/pobsync/pobsync.env")
+POBSYNC_SERVICE_USER = os.getenv("POBSYNC_SERVICE_USER", "pobsync")
+POBSYNC_SERVICE_GROUP = os.getenv("POBSYNC_SERVICE_GROUP", "pobsync")