(ops) Complete native production install and update flow #9

Merged
parkel merged 5 commits from issue-1-production-install-update into master 2026-05-20 01:50:23 +02:00
10 changed files with 196 additions and 4 deletions
Showing only changes of commit 851f967f12 - Show all commits

View File

@@ -202,6 +202,30 @@ sudo -u pobsync pobsync-manage check
sudo -u pobsync pobsync-manage check_pobsync_install sudo -u pobsync pobsync-manage check_pobsync_install
``` ```
Restart services manually after environment or reverse proxy changes:
```
sudo systemctl restart pobsync-web pobsync-worker pobsync-scheduler
```
Inspect service logs with:
```
journalctl -u pobsync-web -n 100 --no-pager
journalctl -u pobsync-worker -f
journalctl -u pobsync-scheduler -n 100 --no-pager
```
Rollback to a previous revision by checking out the known-good commit or tag, then running the updater again:
```
git switch master
git pull
git checkout <known-good-commit-or-tag>
sudo scripts/update-systemd
sudo -u pobsync pobsync-manage check_pobsync_install
```
## Development ## Development
Development, Docker, maintainer tooling, and architecture notes live in: Development, Docker, maintainer tooling, and architecture notes live in:

View File

@@ -4,6 +4,8 @@ set -eu
APP_DIR="@POBSYNC_APP_DIR@" APP_DIR="@POBSYNC_APP_DIR@"
VENV_DIR="@POBSYNC_VENV_DIR@" VENV_DIR="@POBSYNC_VENV_DIR@"
ENV_FILE="@POBSYNC_ENV_FILE@" ENV_FILE="@POBSYNC_ENV_FILE@"
SERVICE_USER="@POBSYNC_USER@"
SERVICE_GROUP="@POBSYNC_GROUP@"
if [ ! -f "$ENV_FILE" ]; then if [ ! -f "$ENV_FILE" ]; then
echo "pobsync environment file not found: $ENV_FILE" >&2 echo "pobsync environment file not found: $ENV_FILE" >&2
@@ -14,6 +16,9 @@ set -a
# shellcheck disable=SC1090 # shellcheck disable=SC1090
. "$ENV_FILE" . "$ENV_FILE"
set +a set +a
export POBSYNC_ENV_FILE="$ENV_FILE"
export POBSYNC_SERVICE_USER="$SERVICE_USER"
export POBSYNC_SERVICE_GROUP="$SERVICE_GROUP"
cd "$APP_DIR" cd "$APP_DIR"
exec "$VENV_DIR/bin/python" "$APP_DIR/manage.py" "$@" exec "$VENV_DIR/bin/python" "$APP_DIR/manage.py" "$@"

View File

@@ -7,6 +7,9 @@ POBSYNC_HOME=/var/lib/pobsync
POBSYNC_BACKUP_ROOT=/backups POBSYNC_BACKUP_ROOT=/backups
POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3 POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
POBSYNC_STATIC_ROOT=/var/lib/pobsync/static POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
POBSYNC_ENV_FILE=/etc/pobsync/pobsync.env
POBSYNC_SERVICE_USER=pobsync
POBSYNC_SERVICE_GROUP=pobsync
POBSYNC_WEB_BIND=127.0.0.1:8010 POBSYNC_WEB_BIND=127.0.0.1:8010
POBSYNC_GUNICORN_WORKERS=2 POBSYNC_GUNICORN_WORKERS=2

View File

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
Group=@POBSYNC_GROUP@ Group=@POBSYNC_GROUP@
WorkingDirectory=@POBSYNC_APP_DIR@ WorkingDirectory=@POBSYNC_APP_DIR@
EnvironmentFile=@POBSYNC_ENV_FILE@ EnvironmentFile=@POBSYNC_ENV_FILE@
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_scheduler --loop --interval "${POBSYNC_SCHEDULER_INTERVAL:-60}"' ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_scheduler --loop --interval "${POBSYNC_SCHEDULER_INTERVAL:-60}"'
Restart=on-failure Restart=on-failure
RestartSec=5 RestartSec=5

View File

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
Group=@POBSYNC_GROUP@ Group=@POBSYNC_GROUP@
WorkingDirectory=@POBSYNC_APP_DIR@ WorkingDirectory=@POBSYNC_APP_DIR@
EnvironmentFile=@POBSYNC_ENV_FILE@ EnvironmentFile=@POBSYNC_ENV_FILE@
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py migrate --noinput ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py migrate --noinput
ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py collectstatic --noinput --clear ExecStartPre=@POBSYNC_VENV_DIR@/bin/python manage.py collectstatic --noinput --clear
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/gunicorn pobsync_server.wsgi:application --bind "${POBSYNC_WEB_BIND:-127.0.0.1:8010}" --workers "${POBSYNC_GUNICORN_WORKERS:-2}" --timeout "${POBSYNC_GUNICORN_TIMEOUT:-120}"' ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/gunicorn pobsync_server.wsgi:application --bind "${POBSYNC_WEB_BIND:-127.0.0.1:8010}" --workers "${POBSYNC_GUNICORN_WORKERS:-2}" --timeout "${POBSYNC_GUNICORN_TIMEOUT:-120}"'

View File

@@ -9,6 +9,9 @@ User=@POBSYNC_USER@
Group=@POBSYNC_GROUP@ Group=@POBSYNC_GROUP@
WorkingDirectory=@POBSYNC_APP_DIR@ WorkingDirectory=@POBSYNC_APP_DIR@
EnvironmentFile=@POBSYNC_ENV_FILE@ EnvironmentFile=@POBSYNC_ENV_FILE@
Environment=POBSYNC_ENV_FILE=@POBSYNC_ENV_FILE@
Environment=POBSYNC_SERVICE_USER=@POBSYNC_USER@
Environment=POBSYNC_SERVICE_GROUP=@POBSYNC_GROUP@
ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_worker --loop --interval "${POBSYNC_WORKER_INTERVAL:-15}"' ExecStart=/bin/sh -c 'exec @POBSYNC_VENV_DIR@/bin/python manage.py run_pobsync_worker --loop --interval "${POBSYNC_WORKER_INTERVAL:-15}"'
Restart=on-failure Restart=on-failure
RestartSec=5 RestartSec=5

View File

@@ -463,6 +463,9 @@ POBSYNC_BACKUP_ROOT=$BACKUP_ROOT
POBSYNC_TIME_ZONE=$TIME_ZONE POBSYNC_TIME_ZONE=$TIME_ZONE
POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3 POBSYNC_SQLITE_PATH=/var/lib/pobsync/pobsync.sqlite3
POBSYNC_STATIC_ROOT=/var/lib/pobsync/static POBSYNC_STATIC_ROOT=/var/lib/pobsync/static
POBSYNC_ENV_FILE=$ENV_FILE
POBSYNC_SERVICE_USER=$SERVICE_USER
POBSYNC_SERVICE_GROUP=$SERVICE_GROUP
POBSYNC_WEB_BIND=$WEB_BIND POBSYNC_WEB_BIND=$WEB_BIND
POBSYNC_GUNICORN_WORKERS=2 POBSYNC_GUNICORN_WORKERS=2
@@ -509,6 +512,8 @@ install_manage_wrapper() {
-e "s|@POBSYNC_APP_DIR@|$APP_DIR|g" \ -e "s|@POBSYNC_APP_DIR@|$APP_DIR|g" \
-e "s|@POBSYNC_VENV_DIR@|$VENV_DIR|g" \ -e "s|@POBSYNC_VENV_DIR@|$VENV_DIR|g" \
-e "s|@POBSYNC_ENV_FILE@|$ENV_FILE|g" \ -e "s|@POBSYNC_ENV_FILE@|$ENV_FILE|g" \
-e "s|@POBSYNC_USER@|$SERVICE_USER|g" \
-e "s|@POBSYNC_GROUP@|$SERVICE_GROUP|g" \
"$APP_DIR/deploy/bin/pobsync-manage" > /usr/local/bin/pobsync-manage "$APP_DIR/deploy/bin/pobsync-manage" > /usr/local/bin/pobsync-manage
chmod 0755 /usr/local/bin/pobsync-manage chmod 0755 /usr/local/bin/pobsync-manage
} }

View File

@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import os import os
import pwd
import shutil import shutil
import subprocess import subprocess
import sys import sys
@@ -28,6 +29,7 @@ class SelfCheck:
def collect_self_checks() -> list[SelfCheck]: def collect_self_checks() -> list[SelfCheck]:
checks: list[SelfCheck] = [] checks: list[SelfCheck] = []
checks.extend(_django_checks()) checks.extend(_django_checks())
checks.extend(_install_checks())
checks.extend(_path_checks()) checks.extend(_path_checks())
checks.extend(_binary_checks()) checks.extend(_binary_checks())
checks.extend(_database_checks()) checks.extend(_database_checks())
@@ -36,6 +38,10 @@ def collect_self_checks() -> list[SelfCheck]:
return checks return checks
def _native_runtime_available() -> bool:
return Path("/run/systemd/system").exists() and shutil.which("systemctl") is not None
def summarize_self_checks(checks: list[SelfCheck]) -> dict[str, int]: def summarize_self_checks(checks: list[SelfCheck]) -> dict[str, int]:
return { return {
"ok": sum(1 for check in checks if check.status == "ok"), "ok": sum(1 for check in checks if check.status == "ok"),
@@ -91,18 +97,105 @@ def _path_checks() -> list[SelfCheck]:
) )
db_settings = settings.DATABASES["default"] db_settings = settings.DATABASES["default"]
if db_settings["ENGINE"] == "django.db.backends.sqlite3": if db_settings["ENGINE"] == "django.db.backends.sqlite3":
sqlite_path = Path(str(db_settings["NAME"]))
checks.append( checks.append(
_path_check( _path_check(
"SQLite directory", "SQLite directory",
Path(str(db_settings["NAME"])).parent, sqlite_path.parent,
must_be_absolute=True, must_be_absolute=True,
must_exist=True, must_exist=True,
must_be_writable=True, must_be_writable=True,
) )
) )
checks.append(_sqlite_database_check(sqlite_path))
return checks return checks
def _install_checks() -> list[SelfCheck]:
if not _native_runtime_available() and not Path(settings.POBSYNC_ENV_FILE).exists():
return [
SelfCheck(
"Environment file",
"skipped",
"Native environment file is not configured in this runtime.",
"This is expected inside Docker or local development.",
),
SelfCheck(
"Service user",
"skipped",
"Native service user check is not available in this runtime.",
"This is expected inside Docker or local development.",
),
SelfCheck(
"Backup root owner",
"skipped",
"Native backup root ownership check is not available in this runtime.",
"This is expected inside Docker or local development.",
),
]
checks = [_env_file_check(Path(settings.POBSYNC_ENV_FILE)), _service_user_check()]
checks.append(_backup_root_owner_check(Path(settings.POBSYNC_BACKUP_ROOT)))
return checks
def _env_file_check(path: Path) -> SelfCheck:
if not path.is_absolute():
return SelfCheck("Environment file", "failed", f"{path} is not absolute.")
if not path.exists():
return SelfCheck("Environment file", "failed", f"{path} does not exist.")
if not path.is_file():
return SelfCheck("Environment file", "failed", f"{path} is not a regular file.")
if not os.access(path, os.R_OK):
return SelfCheck("Environment file", "failed", f"{path} is not readable by this process.")
return SelfCheck("Environment file", "ok", str(path))
def _service_user_check() -> SelfCheck:
expected_user = settings.POBSYNC_SERVICE_USER
try:
current_user = pwd.getpwuid(os.geteuid()).pw_name
except KeyError:
return SelfCheck("Service user", "failed", f"Current uid {os.geteuid()} has no passwd entry.")
if current_user != expected_user:
return SelfCheck(
"Service user",
"warning",
f"Current process runs as {current_user}, expected {expected_user}.",
"Run terminal checks with sudo -u <service-user> pobsync-manage check_pobsync_install.",
)
return SelfCheck("Service user", "ok", current_user)
def _backup_root_owner_check(path: Path) -> SelfCheck:
if not path.exists():
return SelfCheck("Backup root owner", "failed", f"{path} does not exist.")
expected_user = settings.POBSYNC_SERVICE_USER
try:
owner = pwd.getpwuid(path.stat().st_uid).pw_name
except KeyError:
return SelfCheck("Backup root owner", "warning", f"{path} owner uid {path.stat().st_uid} has no passwd entry.")
if owner != expected_user:
return SelfCheck(
"Backup root owner",
"warning",
f"{path} is owned by {owner}, expected {expected_user}.",
)
return SelfCheck("Backup root owner", "ok", f"{path} owner={owner}")
def _sqlite_database_check(path: Path) -> SelfCheck:
if not path.is_absolute():
return SelfCheck("SQLite database", "failed", f"{path} is not absolute.")
if not path.exists():
return SelfCheck("SQLite database", "warning", f"{path} does not exist yet.")
if not path.is_file():
return SelfCheck("SQLite database", "failed", f"{path} is not a regular file.")
if not os.access(path, os.R_OK | os.W_OK):
return SelfCheck("SQLite database", "failed", f"{path} is not readable and writable by this process.")
return SelfCheck("SQLite database", "ok", str(path))
def _path_check( def _path_check(
name: str, name: str,
path: Path, path: Path,
@@ -178,7 +271,7 @@ def _config_checks() -> list[SelfCheck]:
def _systemd_checks() -> list[SelfCheck]: def _systemd_checks() -> list[SelfCheck]:
if not Path("/run/systemd/system").exists() or shutil.which("systemctl") is None: if not _native_runtime_available():
return [ return [
SelfCheck( SelfCheck(
"Systemd services", "Systemd services",

View File

@@ -2,13 +2,15 @@ from __future__ import annotations
import subprocess import subprocess
from io import StringIO from io import StringIO
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch from unittest.mock import patch
from django.core.management import call_command from django.core.management import call_command
from django.core.management.base import CommandError from django.core.management.base import CommandError
from django.test import SimpleTestCase from django.test import SimpleTestCase, override_settings
from pobsync_backend.self_check import SelfCheck, _systemd_checks from pobsync_backend.self_check import SelfCheck, _install_checks, _sqlite_database_check, _systemd_checks
class SystemdSelfCheckTests(SimpleTestCase): class SystemdSelfCheckTests(SimpleTestCase):
@@ -45,6 +47,54 @@ class SystemdSelfCheckTests(SimpleTestCase):
self.assertEqual(journal_check.message, "pobsync cannot read service logs.") self.assertEqual(journal_check.message, "pobsync cannot read service logs.")
class InstallSelfCheckTests(SimpleTestCase):
def test_install_checks_skip_native_paths_in_development_runtime(self) -> None:
with override_settings(POBSYNC_ENV_FILE="/missing/pobsync.env"), patch(
"pobsync_backend.self_check._native_runtime_available",
return_value=False,
):
checks = _install_checks()
self.assertEqual([check.status for check in checks], ["skipped", "skipped", "skipped"])
self.assertEqual(checks[0].name, "Environment file")
self.assertEqual(checks[1].name, "Service user")
self.assertEqual(checks[2].name, "Backup root owner")
def test_service_user_warns_when_current_user_differs(self) -> None:
with override_settings(
POBSYNC_ENV_FILE="/etc/pobsync/pobsync.env",
POBSYNC_SERVICE_USER="pobsync",
POBSYNC_BACKUP_ROOT="/backups",
), patch("pobsync_backend.self_check._native_runtime_available", return_value=True), patch(
"pobsync_backend.self_check._env_file_check",
return_value=SelfCheck("Environment file", "ok", "/etc/pobsync/pobsync.env"),
), patch(
"pobsync_backend.self_check._backup_root_owner_check",
return_value=SelfCheck("Backup root owner", "ok", "/backups owner=pobsync"),
), patch(
"pobsync_backend.self_check.os.geteuid",
return_value=0,
), patch(
"pobsync_backend.self_check.pwd.getpwuid",
) as getpwuid:
getpwuid.return_value.pw_name = "root"
checks = _install_checks()
service_user_check = next(check for check in checks if check.name == "Service user")
self.assertEqual(service_user_check.status, "warning")
self.assertIn("expected pobsync", service_user_check.message)
def test_sqlite_database_check_reports_existing_database(self) -> None:
with TemporaryDirectory() as tmp:
db_path = Path(tmp) / "pobsync.sqlite3"
db_path.write_text("", encoding="utf-8")
check = _sqlite_database_check(db_path)
self.assertEqual(check.status, "ok")
self.assertEqual(check.name, "SQLite database")
class CheckPobsyncInstallCommandTests(SimpleTestCase): class CheckPobsyncInstallCommandTests(SimpleTestCase):
def test_command_prints_summary_for_successful_checks(self) -> None: def test_command_prints_summary_for_successful_checks(self) -> None:
stdout = StringIO() stdout = StringIO()

View File

@@ -99,3 +99,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
POBSYNC_HOME = os.getenv("POBSYNC_HOME", "/opt/pobsync") POBSYNC_HOME = os.getenv("POBSYNC_HOME", "/opt/pobsync")
POBSYNC_BACKUP_ROOT = os.getenv("POBSYNC_BACKUP_ROOT", "/backups") POBSYNC_BACKUP_ROOT = os.getenv("POBSYNC_BACKUP_ROOT", "/backups")
POBSYNC_ENV_FILE = os.getenv("POBSYNC_ENV_FILE", "/etc/pobsync/pobsync.env")
POBSYNC_SERVICE_USER = os.getenv("POBSYNC_SERVICE_USER", "pobsync")
POBSYNC_SERVICE_GROUP = os.getenv("POBSYNC_SERVICE_GROUP", "pobsync")