diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..fa7cfb1 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +# Copy to .env and replace all values with independently generated secrets. +DJANGO_SECRET_KEY=replace-with-random-secret +POSTGRES_PASSWORD=replace-with-bootstrap-secret +MIGRATION_PASSWORD=replace-with-migration-secret +RUNTIME_PASSWORD=replace-with-runtime-secret +TEST_PASSWORD=replace-with-test-secret diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ed10f4e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.13-slim +COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + && . /etc/os-release \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client-17 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PATH="/app/.venv/bin:$PATH" +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-install-project +COPY config ./config +COPY core ./core +COPY finance ./finance +COPY ingestion ./ingestion +COPY tests ./tests +COPY manage.py ./ +RUN useradd --uid 10001 --create-home app +USER app +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000", "--noreload"] diff --git a/compose.yaml b/compose.yaml index ae8b805..78ff3e3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -42,13 +42,14 @@ services: volumes: - ./imports:/app/imports - ./exports:/app/exports + - ./backups:/app/backups entrypoint: ["python", "manage.py"] command: ["check"] depends_on: db: {condition: service_healthy} test-db: image: postgres:17 - profiles: ["test"] + profiles: ["test", "backup"] environment: POSTGRES_DB: gizmoduck_tests POSTGRES_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} @@ -79,5 +80,40 @@ services: command: ["python", "tests/check_permissions.py"] depends_on: db: {condition: service_healthy} + restore-db: + image: postgres:17 + profiles: ["backup"] + environment: + POSTGRES_DB: gizmoduck_restore + POSTGRES_USER: gizmoduck_restore + POSTGRES_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} + tmpfs: [/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U gizmoduck_restore -d gizmoduck_restore"] + interval: 2s + timeout: 3s + retries: 30 + verify-backup: + <<: *app + profiles: ["backup"] + environment: + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?Set DJANGO_SECRET_KEY in .env} + DB_HOST: restore-db + DB_NAME: gizmoduck_restore + DB_USER: gizmoduck_restore + DB_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} + BACKUP_ROOT: /app/backups + BACKUP_VERIFY_ALLOWED: "1" + LIVE_DB_HOST: db + LIVE_DB_NAME: gizmoduck + BACKUP_SOURCE_HOST: test-db + BACKUP_SOURCE_NAME: gizmoduck_tests + BACKUP_SOURCE_USER: gizmoduck_test + BACKUP_SOURCE_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} + volumes: + - ./backups:/app/backups + depends_on: + restore-db: {condition: service_healthy} + test-db: {condition: service_healthy} volumes: postgres_data: diff --git a/config/settings.py b/config/settings.py index c27cf43..4f3ace8 100644 --- a/config/settings.py +++ b/config/settings.py @@ -52,3 +52,9 @@ TIME_ZONE = "UTC" IMPORT_ROOT = "/app/imports" EXPORT_ROOT = os.environ.get("EXPORT_ROOT", "/app/exports") +BACKUP_ROOT = os.environ.get("BACKUP_ROOT", "/app/backups") +BACKUP_VERIFY_ALLOWED = os.environ.get("BACKUP_VERIFY_ALLOWED", "").casefold() in { + "1", "true", "yes" +} +LIVE_DATABASE_HOST = os.environ.get("LIVE_DB_HOST") +LIVE_DATABASE_NAME = os.environ.get("LIVE_DB_NAME") diff --git a/core/backup.py b/core/backup.py new file mode 100644 index 0000000..d98a6fa --- /dev/null +++ b/core/backup.py @@ -0,0 +1,603 @@ +import json +import os +import re +import subprocess +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone as datetime_timezone +from hashlib import sha256 +from pathlib import Path + +import psycopg +from psycopg import sql +from django.conf import settings + + +BACKUP_FORMAT = "gizmoduck-postgresql-backup" +BACKUP_VERSION = 1 +APPLICATION_VERSION = "0.1.0" + +_PLAIN_DUMP_NAME = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.dump\Z") +_TABLE_NAME = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") +_MANIFEST_KEYS = { + "application_version", + "byte_size", + "created_at", + "database_name", + "format", + "migrations", + "postgresql_client_version", + "row_counts", + "sha256", + "version", +} + + +class BackupError(Exception): + def __init__(self, code): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class DatabaseTarget: + host: str + port: str + name: str + user: str + password: str + + @classmethod + def from_default(cls, database): + return cls( + host=str(database.get("HOST", "")), + port=str(database.get("PORT") or "5432"), + name=str(database.get("NAME", "")), + user=str(database.get("USER", "")), + password=str(database.get("PASSWORD", "")), + ) + + +@dataclass(frozen=True) +class DatabaseSnapshot: + migrations: tuple[tuple[str, str], ...] + row_counts: dict[str, int] + + +@dataclass(frozen=True) +class BackupResult: + name: str + path: Path + manifest_path: Path + version: int + created_at: datetime + postgresql_client_version: str + row_counts: dict[str, int] + + +@dataclass(frozen=True) +class VerificationResult: + name: str + version: int + row_counts: dict[str, int] + + +def create_backup(name=None, *, runner=None, _target=None): + runner = runner or subprocess.run + root = _backup_root() + if name is None: + now = datetime.now(datetime_timezone.utc) + name = f"gizmoduck-backup-{now:%Y%m%dT%H%M%SZ}.dump" + else: + now = datetime.now(datetime_timezone.utc) + _validate_name(name) + + destination = root / name + manifest_destination = root / f"{name}.json" + _ensure_available(destination, manifest_destination) + + target = _target or DatabaseTarget.from_default(settings.DATABASES["default"]) + snapshot = _read_snapshot(target) + dump_path = None + manifest_path = None + published = [] + try: + dump_fd, dump_name = tempfile.mkstemp( + prefix=".gizmoduck-backup-", suffix=".dump.tmp", dir=root + ) + dump_path = Path(dump_name) + with os.fdopen(dump_fd, "wb") as stream: + runner( + _pg_dump_argv(target), + env=_database_env(target), + check=True, + stdout=stream, + capture_output=False, + ) + stream.flush() + os.fsync(stream.fileno()) + + content_digest, byte_size = _file_digest(dump_path) + version_process = runner( + ["pg_dump", "--version"], + env=_database_env(target), + check=True, + stdout=None, + capture_output=True, + ) + client_version = _safe_client_version(getattr(version_process, "stdout", b"")) + manifest = { + "application_version": APPLICATION_VERSION, + "byte_size": byte_size, + "created_at": now.isoformat(), + "database_name": target.name, + "format": BACKUP_FORMAT, + "migrations": [list(item) for item in snapshot.migrations], + "postgresql_client_version": client_version, + "row_counts": snapshot.row_counts, + "sha256": content_digest, + "version": BACKUP_VERSION, + } + manifest_fd, manifest_name = tempfile.mkstemp( + prefix=".gizmoduck-backup-", suffix=".json.tmp", dir=root + ) + manifest_path = Path(manifest_name) + with os.fdopen(manifest_fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(manifest, stream, ensure_ascii=False, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + _publish_no_clobber(manifest_path, manifest_destination) + published.append((manifest_destination, manifest_path)) + _publish_no_clobber(dump_path, destination) + published.append((destination, dump_path)) + _fsync_directory(root) + manifest_path.unlink() + manifest_path = None + dump_path.unlink() + dump_path = None + return BackupResult( + name=name, + path=destination, + manifest_path=manifest_destination, + version=BACKUP_VERSION, + created_at=now, + postgresql_client_version=client_version, + row_counts=dict(snapshot.row_counts), + ) + except BackupError: + _remove_our_publications(published) + raise + except Exception: + _remove_our_publications(published) + raise BackupError("backup_failed") from None + finally: + _unlink_private(manifest_path) + _unlink_private(dump_path) + + +def verify_backup(name, target, *, runner=None): + runner = runner or subprocess.run + _validate_name(name) + live = DatabaseTarget( + host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or settings.DATABASES["default"].get("HOST", "")), + port=str(settings.DATABASES["default"].get("PORT") or "5432"), + name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or settings.DATABASES["default"].get("NAME", "")), + user="", + password="", + ) + if _same_database(live, target): + raise BackupError("live_target") + if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): + raise BackupError("verification_not_allowed") + + root = _backup_root() + dump_path = root / name + manifest_path = root / f"{name}.json" + manifest = _load_manifest(manifest_path) + if not dump_path.is_file() or dump_path.is_symlink(): + raise BackupError("missing_backup") + digest, byte_size = _file_digest(dump_path) + if manifest["byte_size"] != byte_size: + raise BackupError("size_mismatch") + if manifest["sha256"] != digest: + raise BackupError("digest_mismatch") + + try: + before = _read_snapshot(target) + if before.migrations or before.row_counts: + raise BackupError("target_not_empty") + runner( + _pg_restore_argv(target, dump_path), + env=_database_env(target), + check=True, + stdout=None, + capture_output=True, + ) + after = _read_snapshot(target) + except BackupError: + raise + except Exception: + raise BackupError("verification_failed") from None + + expected_migrations = tuple(tuple(item) for item in manifest["migrations"]) + if after.migrations != expected_migrations: + raise BackupError("migration_mismatch") + if after.row_counts != manifest["row_counts"]: + raise BackupError("row_count_mismatch") + return VerificationResult(name=name, version=BACKUP_VERSION, row_counts=after.row_counts) + + +def _pg_dump_argv(target): + return [ + "pg_dump", + "--format=custom", + "--no-owner", + "--no-acl", + "--host", + target.host, + "--port", + target.port, + "--username", + target.user, + "--dbname", + target.name, + ] + + +def _pg_restore_argv(target, dump_path): + return [ + "pg_restore", + "--exit-on-error", + "--single-transaction", + "--no-owner", + "--no-acl", + "--host", + target.host, + "--port", + target.port, + "--username", + target.user, + "--dbname", + target.name, + str(dump_path), + ] + + +def _database_env(target): + env = os.environ.copy() + env["PGPASSWORD"] = target.password + return env + + +def _backup_root(): + try: + root = Path(settings.BACKUP_ROOT).resolve(strict=True) + except (AttributeError, OSError, RuntimeError, TypeError): + raise BackupError("invalid_root") from None + if not root.is_dir(): + raise BackupError("invalid_root") + return root + + +def _validate_name(name): + if not isinstance(name, str) or not _PLAIN_DUMP_NAME.fullmatch(name): + raise BackupError("invalid_name") + + +def _ensure_available(*paths): + if any(path.exists() or path.is_symlink() for path in paths): + raise BackupError("existing_destination") + + +def _publish_no_clobber(source, destination): + try: + os.link(source, destination) + except FileExistsError: + raise BackupError("existing_destination") from None + except OSError: + raise BackupError("publication_failed") from None + + +def _remove_our_publications(publications): + for destination, source in reversed(publications): + try: + if destination.exists() and os.path.samefile(destination, source): + destination.unlink() + except OSError: + pass + + +def _unlink_private(path): + if path is not None: + try: + path.unlink(missing_ok=True) + except OSError: + pass + + +def _fsync_directory(root): + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(root, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _file_digest(path): + digest = sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _safe_client_version(output): + if isinstance(output, bytes): + output = output.decode("ascii", errors="replace") + version = str(output).strip() + if not version.startswith("pg_dump (PostgreSQL) 17.") or len(version) > 80: + raise BackupError("client_version_failed") + return version + + +def _load_manifest(path): + try: + if path.is_symlink(): + raise BackupError("invalid_manifest") + manifest = json.loads(path.read_text(encoding="utf-8")) + except BackupError: + raise + except (OSError, UnicodeError, json.JSONDecodeError): + raise BackupError("invalid_manifest") from None + if not isinstance(manifest, dict) or set(manifest) != _MANIFEST_KEYS: + raise BackupError("invalid_manifest") + if manifest["format"] != BACKUP_FORMAT or manifest["version"] != BACKUP_VERSION: + raise BackupError("invalid_manifest") + if ( + not isinstance(manifest["application_version"], str) + or not manifest["application_version"] + or not isinstance(manifest["created_at"], str) + or not isinstance(manifest["database_name"], str) + or not manifest["database_name"] + or not isinstance(manifest["postgresql_client_version"], str) + or not manifest["postgresql_client_version"].startswith("pg_dump (PostgreSQL) 17.") + or not isinstance(manifest["byte_size"], int) + or manifest["byte_size"] < 0 + or not isinstance(manifest["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", manifest["sha256"]) + or not isinstance(manifest["migrations"], list) + or not isinstance(manifest["row_counts"], dict) + ): + raise BackupError("invalid_manifest") + try: + created_at = datetime.fromisoformat(manifest["created_at"]) + except ValueError: + raise BackupError("invalid_manifest") from None + if created_at.tzinfo is None: + raise BackupError("invalid_manifest") + migrations = manifest["migrations"] + if any( + not isinstance(item, list) + or len(item) != 2 + or not all(isinstance(value, str) and value for value in item) + for item in migrations + ): + raise BackupError("invalid_manifest") + if any( + not isinstance(table, str) + or not _TABLE_NAME.fullmatch(table) + or not isinstance(count, int) + or count < 0 + for table, count in manifest["row_counts"].items() + ): + raise BackupError("invalid_manifest") + return manifest + + +def _read_snapshot(target): + with psycopg.connect( + host=target.host, + port=target.port, + dbname=target.name, + user=target.user, + password=target.password, + connect_timeout=3, + ) as database: + with database.cursor() as cursor: + cursor.execute( + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + ORDER BY table_name + """ + ) + tables = [row[0] for row in cursor.fetchall()] + migrations = () + if "django_migrations" in tables: + cursor.execute( + "SELECT app, name FROM django_migrations ORDER BY app, name" + ) + migrations = tuple((row[0], row[1]) for row in cursor.fetchall()) + row_counts = {} + for table in tables: + cursor.execute( + sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)) + ) + row_counts[table] = cursor.fetchone()[0] + return DatabaseSnapshot(migrations=migrations, row_counts=row_counts) + + +def _same_database(first, second): + return ( + first.host.strip().casefold(), + first.name.strip().casefold(), + ) == ( + second.host.strip().casefold(), + second.name.strip().casefold(), + ) + + +def run_synthetic_restore_verification(): + """Exercise backup and restore only against disposable synthetic databases.""" + from datetime import date + from decimal import Decimal + from uuid import uuid4 + + from django.core.management import call_command + from django.db import connections + + from finance.models import Account, AuditEvent, Category, Payee, SourceRecord, Transaction + from ingestion.models import ImportBatch, ImportItem + + if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): + raise BackupError("verification_not_allowed") + required = ( + "BACKUP_SOURCE_HOST", + "BACKUP_SOURCE_NAME", + "BACKUP_SOURCE_USER", + "BACKUP_SOURCE_PASSWORD", + ) + if any(not os.environ.get(name) for name in required): + raise BackupError("source_not_configured") + + restore_target = DatabaseTarget.from_default(settings.DATABASES["default"]) + source_target = DatabaseTarget( + host=os.environ["BACKUP_SOURCE_HOST"], + port=os.environ.get("BACKUP_SOURCE_PORT", "5432"), + name=os.environ["BACKUP_SOURCE_NAME"], + user=os.environ["BACKUP_SOURCE_USER"], + password=os.environ["BACKUP_SOURCE_PASSWORD"], + ) + live_target = DatabaseTarget( + host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or ""), + port="5432", + name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or ""), + user="", + password="", + ) + if _same_database(source_target, restore_target): + raise BackupError("source_is_target") + if live_target.host and ( + _same_database(source_target, live_target) + or _same_database(restore_target, live_target) + ): + raise BackupError("live_target") + if _read_snapshot(source_target).row_counts: + raise BackupError("source_not_empty") + restore_before = _read_snapshot(restore_target) + if restore_before.migrations or restore_before.row_counts: + raise BackupError("target_not_empty") + + default_connection = connections["default"] + original_database = dict(default_connection.settings_dict) + artifact = None + try: + default_connection.close() + _configure_connection(default_connection, source_target) + call_command("migrate", interactive=False, verbosity=0) + + account = Account.objects.create( + name="Synthetic Restore Bank", kind=Account.Kind.BANK, currency="BRL" + ) + payee = Payee.objects.create(name="Synthetic Restore Merchant") + category = Category.objects.create( + name="Synthetic Restore Expense", kind=Category.Kind.EXPENSE + ) + transaction = Transaction.objects.create( + account=account, + amount=Decimal("-12.34"), + posting_date=date(2026, 9, 8), + description="Synthetic restore purchase", + kind=Transaction.Kind.EXPENSE, + payee=payee, + category=category, + ) + SourceRecord.objects.create( + transaction=transaction, + document_hash="a" * 64, + source_format=SourceRecord.Format.BANK_CSV, + parser_version="synthetic-v1", + item_locator="row:1", + ) + AuditEvent.objects.create( + actor_kind=AuditEvent.ActorKind.AGENT, + actor_identifier="synthetic-backup-verification", + operation="synthetic_backup_verification", + target_id=transaction.id, + outcome=AuditEvent.Outcome.SUCCESS, + ) + batch = ImportBatch.objects.create( + document_hash="b" * 64, + source_format="BANK_CSV", + profile="SYNTHETIC", + encoding="utf-8", + actor_kind="AGENT", + actor_identifier="synthetic-backup-verification", + ) + ImportItem.objects.create( + batch=batch, + locator="row:1", + candidate={"description": "Synthetic restore candidate"}, + ) + before = _read_snapshot(source_target) + artifact = create_backup( + f"synthetic-restore-check-{uuid4().hex}.dump", _target=source_target + ) + + default_connection.close() + default_connection.settings_dict.clear() + default_connection.settings_dict.update(original_database) + result = verify_backup(artifact.name, restore_target) + call_command("check", databases=["default"], verbosity=0) + if _read_snapshot(source_target) != before: + raise BackupError("source_changed") + return result + except BackupError: + raise + except Exception: + raise BackupError("integration_verification_failed") from None + finally: + default_connection.close() + default_connection.settings_dict.clear() + default_connection.settings_dict.update(original_database) + if artifact is not None: + artifact.path.unlink(missing_ok=True) + artifact.manifest_path.unlink(missing_ok=True) + _clear_disposable_database(source_target) + _clear_disposable_database(restore_target) + + +def _configure_connection(connection, target): + connection.settings_dict.update( + { + "HOST": target.host, + "PORT": target.port, + "NAME": target.name, + "USER": target.user, + "PASSWORD": target.password, + } + ) + + +def _clear_disposable_database(target): + try: + with psycopg.connect( + host=target.host, + port=target.port, + dbname=target.name, + user=target.user, + password=target.password, + connect_timeout=3, + autocommit=True, + ) as database: + with database.cursor() as cursor: + cursor.execute("DROP SCHEMA public CASCADE") + cursor.execute("CREATE SCHEMA public") + except psycopg.Error: + pass diff --git a/core/management/__init__.py b/core/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/management/commands/__init__.py b/core/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/management/commands/backup_database.py b/core/management/commands/backup_database.py new file mode 100644 index 0000000..aa18244 --- /dev/null +++ b/core/management/commands/backup_database.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand, CommandError + +from core.backup import BackupError, create_backup + + +class Command(BaseCommand): + help = "Create a local PostgreSQL backup and checksum manifest." + + def add_arguments(self, parser): + parser.add_argument("--output", dest="output") + + def handle(self, *args, **options): + try: + result = create_backup(options["output"]) + except BackupError as error: + raise CommandError(error.code) from None + self.stdout.write(f"backup={result.name} format={result.version}") + for table, count in sorted(result.row_counts.items()): + self.stdout.write(f"table={table} rows={count}") diff --git a/core/management/commands/test_backup_restore.py b/core/management/commands/test_backup_restore.py new file mode 100644 index 0000000..e24957a --- /dev/null +++ b/core/management/commands/test_backup_restore.py @@ -0,0 +1,16 @@ +from django.core.management.base import BaseCommand, CommandError + +from core.backup import BackupError, run_synthetic_restore_verification + + +class Command(BaseCommand): + help = "Run the synthetic backup and disposable restore acceptance check." + + def handle(self, *args, **options): + try: + result = run_synthetic_restore_verification() + except BackupError as error: + raise CommandError(error.code) from None + self.stdout.write(f"backup={result.name} format={result.version} verified") + for table, count in sorted(result.row_counts.items()): + self.stdout.write(f"table={table} rows={count}") diff --git a/core/management/commands/verify_backup.py b/core/management/commands/verify_backup.py new file mode 100644 index 0000000..702917b --- /dev/null +++ b/core/management/commands/verify_backup.py @@ -0,0 +1,31 @@ +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from core.backup import BackupError, DatabaseTarget, verify_backup + + +class Command(BaseCommand): + help = "Restore a backup into an explicitly allowed disposable database and verify it." + + def add_arguments(self, parser): + parser.add_argument("--input", dest="input", required=True) + parser.add_argument("--target-name", required=True) + parser.add_argument("--target-host", required=True) + parser.add_argument("--target-user", required=True) + + def handle(self, *args, **options): + default = settings.DATABASES["default"] + target = DatabaseTarget( + host=options["target_host"], + port=str(default.get("PORT") or "5432"), + name=options["target_name"], + user=options["target_user"], + password=str(default.get("PASSWORD", "")), + ) + try: + result = verify_backup(options["input"], target) + except BackupError as error: + raise CommandError(error.code) from None + self.stdout.write(f"backup={result.name} format={result.version} verified") + for table, count in sorted(result.row_counts.items()): + self.stdout.write(f"table={table} rows={count}") diff --git a/tests/test_backup.py b/tests/test_backup.py new file mode 100644 index 0000000..d161d50 --- /dev/null +++ b/tests/test_backup.py @@ -0,0 +1,260 @@ +import json +from hashlib import sha256 +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from core.backup import ( + BACKUP_FORMAT, + BACKUP_VERSION, + BackupError, + DatabaseSnapshot, + DatabaseTarget, + create_backup, + verify_backup, +) + + +class RecordingRunner: + def __init__(self, output=b"PGDMP synthetic"): + self.calls = [] + self.output = output + + def __call__(self, argv, *, env, check, stdout=None, capture_output=False): + self.calls.append((list(argv), dict(env), check, stdout, capture_output)) + if stdout is not None: + stdout.write(self.output) + return SimpleNamespace(stdout=b"pg_dump (PostgreSQL) 17.6\n") + + +@pytest.fixture +def database_settings(settings): + settings.DATABASES["default"] = { + "ENGINE": "django.db.backends.postgresql", + "HOST": "live-db", + "PORT": "5432", + "NAME": "gizmoduck", + "USER": "gizmoduck_migrate", + "PASSWORD": "synthetic-secret", + } + return settings.DATABASES["default"] + + +@pytest.fixture +def backup_root(tmp_path, settings): + settings.BACKUP_ROOT = tmp_path + return tmp_path + + +@pytest.fixture +def synthetic_snapshot(monkeypatch): + snapshot = DatabaseSnapshot( + migrations=(("finance", "0001_initial"), ("ingestion", "0001_initial")), + row_counts={"finance_account": 2, "ingestion_importbatch": 1}, + ) + monkeypatch.setattr("core.backup._read_snapshot", lambda _target: snapshot) + return snapshot + + +def test_pg_dump_receives_password_only_in_environment( + backup_root, database_settings, synthetic_snapshot +): + runner = RecordingRunner() + + result = create_backup("synthetic.dump", runner=runner) + + argv, env, check, stdout, capture_output = runner.calls[0] + assert argv[:4] == ["pg_dump", "--format=custom", "--no-owner", "--no-acl"] + assert database_settings["PASSWORD"] not in argv + assert env["PGPASSWORD"] == database_settings["PASSWORD"] + assert check is True + assert stdout is not None + assert capture_output is False + assert result.path.read_bytes() == runner.output + + +def test_backup_writes_versioned_checksum_manifest( + backup_root, database_settings, synthetic_snapshot +): + runner = RecordingRunner() + + result = create_backup("synthetic.dump", runner=runner) + + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert manifest == { + "application_version": "0.1.0", + "byte_size": len(runner.output), + "created_at": result.created_at.isoformat(), + "database_name": "gizmoduck", + "format": BACKUP_FORMAT, + "migrations": [ + ["finance", "0001_initial"], + ["ingestion", "0001_initial"], + ], + "postgresql_client_version": result.postgresql_client_version, + "row_counts": { + "finance_account": 2, + "ingestion_importbatch": 1, + }, + "sha256": sha256(runner.output).hexdigest(), + "version": BACKUP_VERSION, + } + + +@pytest.mark.parametrize( + "name", + ["../outside.dump", "nested/file.dump", "nested\\file.dump", ".dump", "file.zip"], +) +def test_backup_rejects_non_plain_dump_names( + backup_root, database_settings, synthetic_snapshot, name +): + with pytest.raises(BackupError, match="invalid_name"): + create_backup(name, runner=RecordingRunner()) + + assert list(backup_root.iterdir()) == [] + + +def test_backup_does_not_replace_existing_artifacts( + backup_root, database_settings, synthetic_snapshot +): + dump = backup_root / "synthetic.dump" + manifest = backup_root / "synthetic.dump.json" + dump.write_bytes(b"existing dump") + manifest.write_text("existing manifest", encoding="utf-8") + + with pytest.raises(BackupError, match="existing_destination"): + create_backup("synthetic.dump", runner=RecordingRunner()) + + assert dump.read_bytes() == b"existing dump" + assert manifest.read_text(encoding="utf-8") == "existing manifest" + + +def test_backup_cleans_private_temporary_files_on_dump_failure( + backup_root, database_settings, synthetic_snapshot +): + def failing_runner(argv, **kwargs): + raise OSError("synthetic private detail") + + with pytest.raises(BackupError, match="backup_failed"): + create_backup("synthetic.dump", runner=failing_runner) + + assert list(backup_root.iterdir()) == [] + + +def test_verification_rejects_live_database( + backup_root, database_settings, synthetic_snapshot +): + target = DatabaseTarget.from_default(database_settings) + + with pytest.raises(BackupError, match="live_target"): + verify_backup("synthetic.dump", target, runner=RecordingRunner()) + + +def test_verification_requires_explicit_allow_flag( + backup_root, database_settings, synthetic_snapshot, settings +): + settings.BACKUP_VERIFY_ALLOWED = False + target = DatabaseTarget( + host="restore-db", + port="5432", + name="gizmoduck_restore", + user="restore_user", + password="restore-secret", + ) + + with pytest.raises(BackupError, match="verification_not_allowed"): + verify_backup("synthetic.dump", target, runner=RecordingRunner()) + + +def test_verification_rejects_changed_dump_before_restore( + backup_root, database_settings, synthetic_snapshot, settings +): + settings.BACKUP_VERIFY_ALLOWED = True + result = create_backup("synthetic.dump", runner=RecordingRunner()) + result.path.write_bytes(b"PGDMP tampered!") + restore_runner = RecordingRunner() + target = DatabaseTarget( + host="restore-db", + port="5432", + name="gizmoduck_restore", + user="restore_user", + password="restore-secret", + ) + + with pytest.raises(BackupError, match="digest_mismatch"): + verify_backup(result.name, target, runner=restore_runner) + + assert restore_runner.calls == [] + + +def test_verification_rejects_manifest_with_wrong_field_type( + backup_root, database_settings, synthetic_snapshot, settings +): + settings.BACKUP_VERIFY_ALLOWED = True + result = create_backup("synthetic.dump", runner=RecordingRunner()) + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + manifest["application_version"] = 1 + result.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + target = DatabaseTarget( + host="restore-db", + port="5432", + name="gizmoduck_restore", + user="restore_user", + password="restore-secret", + ) + + with pytest.raises(BackupError, match="invalid_manifest"): + verify_backup(result.name, target, runner=RecordingRunner()) + + +def test_backup_rejects_nonmatching_postgresql_client_major( + backup_root, database_settings, synthetic_snapshot +): + class WrongVersionRunner(RecordingRunner): + def __call__(self, argv, **kwargs): + result = super().__call__(argv, **kwargs) + if argv == ["pg_dump", "--version"]: + result.stdout = b"pg_dump (PostgreSQL) 16.9\n" + return result + + with pytest.raises(BackupError, match="client_version_failed"): + create_backup("synthetic.dump", runner=WrongVersionRunner()) + + assert list(backup_root.iterdir()) == [] + + +def test_pg_restore_receives_password_only_in_environment( + backup_root, database_settings, synthetic_snapshot, settings, monkeypatch +): + settings.BACKUP_VERIFY_ALLOWED = True + result = create_backup("synthetic.dump", runner=RecordingRunner()) + snapshots = iter( + [DatabaseSnapshot(migrations=(), row_counts={}), synthetic_snapshot] + ) + monkeypatch.setattr("core.backup._read_snapshot", lambda _target: next(snapshots)) + runner = RecordingRunner() + target = DatabaseTarget( + host="restore-db", + port="5432", + name="gizmoduck_restore", + user="restore_user", + password="synthetic-restore-secret", + ) + + verification = verify_backup(result.name, target, runner=runner) + + argv, env, check, stdout, capture_output = runner.calls[0] + assert argv[:5] == [ + "pg_restore", + "--exit-on-error", + "--single-transaction", + "--no-owner", + "--no-acl", + ] + assert target.password not in argv + assert env["PGPASSWORD"] == target.password + assert check is True + assert stdout is None + assert capture_output is True + assert verification.row_counts == synthetic_snapshot.row_counts diff --git a/tests/test_backup_commands.py b/tests/test_backup_commands.py new file mode 100644 index 0000000..ae5b3c1 --- /dev/null +++ b/tests/test_backup_commands.py @@ -0,0 +1,135 @@ +from datetime import datetime, timezone +from io import StringIO +from pathlib import Path + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError + +from core.backup import BackupError, BackupResult, VerificationResult + + +def backup_result(): + return BackupResult( + name="synthetic.dump", + path=Path("/private/backup/synthetic.dump"), + manifest_path=Path("/private/backup/synthetic.dump.json"), + version=1, + created_at=datetime(2026, 9, 8, tzinfo=timezone.utc), + postgresql_client_version="pg_dump (PostgreSQL) 17.6", + row_counts={"finance_account": 2, "finance_transaction": 4}, + ) + + +def test_backup_database_prints_only_safe_summary(monkeypatch): + monkeypatch.setattr( + "core.management.commands.backup_database.create_backup", + lambda name: backup_result(), + ) + stdout = StringIO() + + call_command("backup_database", output="synthetic.dump", stdout=stdout) + + assert stdout.getvalue().splitlines() == [ + "backup=synthetic.dump format=1", + "table=finance_account rows=2", + "table=finance_transaction rows=4", + ] + assert "/private/backup" not in stdout.getvalue() + + +def test_backup_database_converts_service_error_to_safe_code(monkeypatch): + def fail(_name): + raise BackupError("backup_failed") + + monkeypatch.setattr("core.management.commands.backup_database.create_backup", fail) + + with pytest.raises(CommandError, match=r"^backup_failed$"): + call_command("backup_database", output="synthetic.dump") + + +def test_verify_backup_uses_configured_password_and_prints_safe_summary( + monkeypatch, settings +): + settings.DATABASES["default"] = { + "HOST": "restore-db", + "PORT": "5432", + "NAME": "gizmoduck_restore", + "USER": "restore_user", + "PASSWORD": "synthetic-restore-secret", + } + captured = {} + + def verify(name, target): + captured["name"] = name + captured["target"] = target + return VerificationResult( + name=name, + version=1, + row_counts={"django_migrations": 12, "finance_account": 2}, + ) + + monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", verify) + stdout = StringIO() + + call_command( + "verify_backup", + input="synthetic.dump", + target_name="gizmoduck_restore", + target_host="restore-db", + target_user="restore_user", + stdout=stdout, + ) + + assert captured["name"] == "synthetic.dump" + assert captured["target"].password == "synthetic-restore-secret" + assert stdout.getvalue().splitlines() == [ + "backup=synthetic.dump format=1 verified", + "table=django_migrations rows=12", + "table=finance_account rows=2", + ] + assert "synthetic-restore-secret" not in stdout.getvalue() + + +def test_verify_backup_converts_service_error_to_safe_code(monkeypatch, settings): + settings.DATABASES["default"] = { + "HOST": "restore-db", + "PORT": "5432", + "NAME": "gizmoduck_restore", + "USER": "restore_user", + "PASSWORD": "synthetic-restore-secret", + } + + def fail(_name, _target): + raise BackupError("target_not_empty") + + monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", fail) + + with pytest.raises(CommandError, match=r"^target_not_empty$"): + call_command( + "verify_backup", + input="synthetic.dump", + target_name="gizmoduck_restore", + target_host="restore-db", + target_user="restore_user", + ) + + +def test_backup_restore_acceptance_command_prints_only_safe_summary(monkeypatch): + monkeypatch.setattr( + "core.management.commands.test_backup_restore.run_synthetic_restore_verification", + lambda: VerificationResult( + name="synthetic-restore-check.dump", + version=1, + row_counts={"finance_account": 1, "finance_transaction": 1}, + ), + ) + stdout = StringIO() + + call_command("test_backup_restore", stdout=stdout) + + assert stdout.getvalue().splitlines() == [ + "backup=synthetic-restore-check.dump format=1 verified", + "table=finance_account rows=1", + "table=finance_transaction rows=1", + ] diff --git a/tests/test_backup_restore.py b/tests/test_backup_restore.py new file mode 100644 index 0000000..d925023 --- /dev/null +++ b/tests/test_backup_restore.py @@ -0,0 +1,14 @@ +import subprocess + + +def test_backup_configuration_is_allow_gated(settings): + assert settings.BACKUP_ROOT == "/app/backups" + assert settings.BACKUP_VERIFY_ALLOWED is False + + +def test_application_image_installs_matching_postgresql_client(): + result = subprocess.run( + ["pg_dump", "--version"], check=True, capture_output=True, text=True + ) + + assert result.stdout.startswith("pg_dump (PostgreSQL) 17.")