Codex
2 months ago
13 changed files with 1153 additions and 1 deletions
@ -0,0 +1,6 @@
|
||||
# Copy to .env and replace all values with independently generated secrets. |
||||
DJANGO_SECRET_KEY=replace-with-random-secret |
||||
POSTGRES_PASSWORD=replace-with-bootstrap-secret |
||||
MIGRATION_PASSWORD=replace-with-migration-secret |
||||
RUNTIME_PASSWORD=replace-with-runtime-secret |
||||
TEST_PASSWORD=replace-with-test-secret |
||||
@ -0,0 +1,26 @@
|
||||
FROM python:3.13-slim |
||||
COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv |
||||
RUN apt-get update \ |
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \ |
||||
&& install -d /usr/share/postgresql-common/pgdg \ |
||||
&& curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ |
||||
-o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ |
||||
&& . /etc/os-release \ |
||||
&& echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" \ |
||||
> /etc/apt/sources.list.d/pgdg.list \ |
||||
&& apt-get update \ |
||||
&& apt-get install -y --no-install-recommends postgresql-client-17 \ |
||||
&& rm -rf /var/lib/apt/lists/* |
||||
WORKDIR /app |
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PATH="/app/.venv/bin:$PATH" |
||||
COPY pyproject.toml uv.lock ./ |
||||
RUN uv sync --frozen --no-install-project |
||||
COPY config ./config |
||||
COPY core ./core |
||||
COPY finance ./finance |
||||
COPY ingestion ./ingestion |
||||
COPY tests ./tests |
||||
COPY manage.py ./ |
||||
RUN useradd --uid 10001 --create-home app |
||||
USER app |
||||
CMD ["python", "manage.py", "runserver", "0.0.0.0:8000", "--noreload"] |
||||
@ -0,0 +1,603 @@
|
||||
import json |
||||
import os |
||||
import re |
||||
import subprocess |
||||
import tempfile |
||||
from dataclasses import dataclass |
||||
from datetime import datetime, timezone as datetime_timezone |
||||
from hashlib import sha256 |
||||
from pathlib import Path |
||||
|
||||
import psycopg |
||||
from psycopg import sql |
||||
from django.conf import settings |
||||
|
||||
|
||||
BACKUP_FORMAT = "gizmoduck-postgresql-backup" |
||||
BACKUP_VERSION = 1 |
||||
APPLICATION_VERSION = "0.1.0" |
||||
|
||||
_PLAIN_DUMP_NAME = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.dump\Z") |
||||
_TABLE_NAME = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z") |
||||
_MANIFEST_KEYS = { |
||||
"application_version", |
||||
"byte_size", |
||||
"created_at", |
||||
"database_name", |
||||
"format", |
||||
"migrations", |
||||
"postgresql_client_version", |
||||
"row_counts", |
||||
"sha256", |
||||
"version", |
||||
} |
||||
|
||||
|
||||
class BackupError(Exception): |
||||
def __init__(self, code): |
||||
self.code = code |
||||
super().__init__(code) |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class DatabaseTarget: |
||||
host: str |
||||
port: str |
||||
name: str |
||||
user: str |
||||
password: str |
||||
|
||||
@classmethod |
||||
def from_default(cls, database): |
||||
return cls( |
||||
host=str(database.get("HOST", "")), |
||||
port=str(database.get("PORT") or "5432"), |
||||
name=str(database.get("NAME", "")), |
||||
user=str(database.get("USER", "")), |
||||
password=str(database.get("PASSWORD", "")), |
||||
) |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class DatabaseSnapshot: |
||||
migrations: tuple[tuple[str, str], ...] |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class BackupResult: |
||||
name: str |
||||
path: Path |
||||
manifest_path: Path |
||||
version: int |
||||
created_at: datetime |
||||
postgresql_client_version: str |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class VerificationResult: |
||||
name: str |
||||
version: int |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
def create_backup(name=None, *, runner=None, _target=None): |
||||
runner = runner or subprocess.run |
||||
root = _backup_root() |
||||
if name is None: |
||||
now = datetime.now(datetime_timezone.utc) |
||||
name = f"gizmoduck-backup-{now:%Y%m%dT%H%M%SZ}.dump" |
||||
else: |
||||
now = datetime.now(datetime_timezone.utc) |
||||
_validate_name(name) |
||||
|
||||
destination = root / name |
||||
manifest_destination = root / f"{name}.json" |
||||
_ensure_available(destination, manifest_destination) |
||||
|
||||
target = _target or DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
snapshot = _read_snapshot(target) |
||||
dump_path = None |
||||
manifest_path = None |
||||
published = [] |
||||
try: |
||||
dump_fd, dump_name = tempfile.mkstemp( |
||||
prefix=".gizmoduck-backup-", suffix=".dump.tmp", dir=root |
||||
) |
||||
dump_path = Path(dump_name) |
||||
with os.fdopen(dump_fd, "wb") as stream: |
||||
runner( |
||||
_pg_dump_argv(target), |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=stream, |
||||
capture_output=False, |
||||
) |
||||
stream.flush() |
||||
os.fsync(stream.fileno()) |
||||
|
||||
content_digest, byte_size = _file_digest(dump_path) |
||||
version_process = runner( |
||||
["pg_dump", "--version"], |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=None, |
||||
capture_output=True, |
||||
) |
||||
client_version = _safe_client_version(getattr(version_process, "stdout", b"")) |
||||
manifest = { |
||||
"application_version": APPLICATION_VERSION, |
||||
"byte_size": byte_size, |
||||
"created_at": now.isoformat(), |
||||
"database_name": target.name, |
||||
"format": BACKUP_FORMAT, |
||||
"migrations": [list(item) for item in snapshot.migrations], |
||||
"postgresql_client_version": client_version, |
||||
"row_counts": snapshot.row_counts, |
||||
"sha256": content_digest, |
||||
"version": BACKUP_VERSION, |
||||
} |
||||
manifest_fd, manifest_name = tempfile.mkstemp( |
||||
prefix=".gizmoduck-backup-", suffix=".json.tmp", dir=root |
||||
) |
||||
manifest_path = Path(manifest_name) |
||||
with os.fdopen(manifest_fd, "w", encoding="utf-8", newline="\n") as stream: |
||||
json.dump(manifest, stream, ensure_ascii=False, separators=(",", ":")) |
||||
stream.write("\n") |
||||
stream.flush() |
||||
os.fsync(stream.fileno()) |
||||
|
||||
_publish_no_clobber(manifest_path, manifest_destination) |
||||
published.append((manifest_destination, manifest_path)) |
||||
_publish_no_clobber(dump_path, destination) |
||||
published.append((destination, dump_path)) |
||||
_fsync_directory(root) |
||||
manifest_path.unlink() |
||||
manifest_path = None |
||||
dump_path.unlink() |
||||
dump_path = None |
||||
return BackupResult( |
||||
name=name, |
||||
path=destination, |
||||
manifest_path=manifest_destination, |
||||
version=BACKUP_VERSION, |
||||
created_at=now, |
||||
postgresql_client_version=client_version, |
||||
row_counts=dict(snapshot.row_counts), |
||||
) |
||||
except BackupError: |
||||
_remove_our_publications(published) |
||||
raise |
||||
except Exception: |
||||
_remove_our_publications(published) |
||||
raise BackupError("backup_failed") from None |
||||
finally: |
||||
_unlink_private(manifest_path) |
||||
_unlink_private(dump_path) |
||||
|
||||
|
||||
def verify_backup(name, target, *, runner=None): |
||||
runner = runner or subprocess.run |
||||
_validate_name(name) |
||||
live = DatabaseTarget( |
||||
host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or settings.DATABASES["default"].get("HOST", "")), |
||||
port=str(settings.DATABASES["default"].get("PORT") or "5432"), |
||||
name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or settings.DATABASES["default"].get("NAME", "")), |
||||
user="", |
||||
password="", |
||||
) |
||||
if _same_database(live, target): |
||||
raise BackupError("live_target") |
||||
if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): |
||||
raise BackupError("verification_not_allowed") |
||||
|
||||
root = _backup_root() |
||||
dump_path = root / name |
||||
manifest_path = root / f"{name}.json" |
||||
manifest = _load_manifest(manifest_path) |
||||
if not dump_path.is_file() or dump_path.is_symlink(): |
||||
raise BackupError("missing_backup") |
||||
digest, byte_size = _file_digest(dump_path) |
||||
if manifest["byte_size"] != byte_size: |
||||
raise BackupError("size_mismatch") |
||||
if manifest["sha256"] != digest: |
||||
raise BackupError("digest_mismatch") |
||||
|
||||
try: |
||||
before = _read_snapshot(target) |
||||
if before.migrations or before.row_counts: |
||||
raise BackupError("target_not_empty") |
||||
runner( |
||||
_pg_restore_argv(target, dump_path), |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=None, |
||||
capture_output=True, |
||||
) |
||||
after = _read_snapshot(target) |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("verification_failed") from None |
||||
|
||||
expected_migrations = tuple(tuple(item) for item in manifest["migrations"]) |
||||
if after.migrations != expected_migrations: |
||||
raise BackupError("migration_mismatch") |
||||
if after.row_counts != manifest["row_counts"]: |
||||
raise BackupError("row_count_mismatch") |
||||
return VerificationResult(name=name, version=BACKUP_VERSION, row_counts=after.row_counts) |
||||
|
||||
|
||||
def _pg_dump_argv(target): |
||||
return [ |
||||
"pg_dump", |
||||
"--format=custom", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
"--host", |
||||
target.host, |
||||
"--port", |
||||
target.port, |
||||
"--username", |
||||
target.user, |
||||
"--dbname", |
||||
target.name, |
||||
] |
||||
|
||||
|
||||
def _pg_restore_argv(target, dump_path): |
||||
return [ |
||||
"pg_restore", |
||||
"--exit-on-error", |
||||
"--single-transaction", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
"--host", |
||||
target.host, |
||||
"--port", |
||||
target.port, |
||||
"--username", |
||||
target.user, |
||||
"--dbname", |
||||
target.name, |
||||
str(dump_path), |
||||
] |
||||
|
||||
|
||||
def _database_env(target): |
||||
env = os.environ.copy() |
||||
env["PGPASSWORD"] = target.password |
||||
return env |
||||
|
||||
|
||||
def _backup_root(): |
||||
try: |
||||
root = Path(settings.BACKUP_ROOT).resolve(strict=True) |
||||
except (AttributeError, OSError, RuntimeError, TypeError): |
||||
raise BackupError("invalid_root") from None |
||||
if not root.is_dir(): |
||||
raise BackupError("invalid_root") |
||||
return root |
||||
|
||||
|
||||
def _validate_name(name): |
||||
if not isinstance(name, str) or not _PLAIN_DUMP_NAME.fullmatch(name): |
||||
raise BackupError("invalid_name") |
||||
|
||||
|
||||
def _ensure_available(*paths): |
||||
if any(path.exists() or path.is_symlink() for path in paths): |
||||
raise BackupError("existing_destination") |
||||
|
||||
|
||||
def _publish_no_clobber(source, destination): |
||||
try: |
||||
os.link(source, destination) |
||||
except FileExistsError: |
||||
raise BackupError("existing_destination") from None |
||||
except OSError: |
||||
raise BackupError("publication_failed") from None |
||||
|
||||
|
||||
def _remove_our_publications(publications): |
||||
for destination, source in reversed(publications): |
||||
try: |
||||
if destination.exists() and os.path.samefile(destination, source): |
||||
destination.unlink() |
||||
except OSError: |
||||
pass |
||||
|
||||
|
||||
def _unlink_private(path): |
||||
if path is not None: |
||||
try: |
||||
path.unlink(missing_ok=True) |
||||
except OSError: |
||||
pass |
||||
|
||||
|
||||
def _fsync_directory(root): |
||||
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) |
||||
descriptor = os.open(root, flags) |
||||
try: |
||||
os.fsync(descriptor) |
||||
finally: |
||||
os.close(descriptor) |
||||
|
||||
|
||||
def _file_digest(path): |
||||
digest = sha256() |
||||
size = 0 |
||||
with path.open("rb") as stream: |
||||
while chunk := stream.read(1024 * 1024): |
||||
digest.update(chunk) |
||||
size += len(chunk) |
||||
return digest.hexdigest(), size |
||||
|
||||
|
||||
def _safe_client_version(output): |
||||
if isinstance(output, bytes): |
||||
output = output.decode("ascii", errors="replace") |
||||
version = str(output).strip() |
||||
if not version.startswith("pg_dump (PostgreSQL) 17.") or len(version) > 80: |
||||
raise BackupError("client_version_failed") |
||||
return version |
||||
|
||||
|
||||
def _load_manifest(path): |
||||
try: |
||||
if path.is_symlink(): |
||||
raise BackupError("invalid_manifest") |
||||
manifest = json.loads(path.read_text(encoding="utf-8")) |
||||
except BackupError: |
||||
raise |
||||
except (OSError, UnicodeError, json.JSONDecodeError): |
||||
raise BackupError("invalid_manifest") from None |
||||
if not isinstance(manifest, dict) or set(manifest) != _MANIFEST_KEYS: |
||||
raise BackupError("invalid_manifest") |
||||
if manifest["format"] != BACKUP_FORMAT or manifest["version"] != BACKUP_VERSION: |
||||
raise BackupError("invalid_manifest") |
||||
if ( |
||||
not isinstance(manifest["application_version"], str) |
||||
or not manifest["application_version"] |
||||
or not isinstance(manifest["created_at"], str) |
||||
or not isinstance(manifest["database_name"], str) |
||||
or not manifest["database_name"] |
||||
or not isinstance(manifest["postgresql_client_version"], str) |
||||
or not manifest["postgresql_client_version"].startswith("pg_dump (PostgreSQL) 17.") |
||||
or not isinstance(manifest["byte_size"], int) |
||||
or manifest["byte_size"] < 0 |
||||
or not isinstance(manifest["sha256"], str) |
||||
or not re.fullmatch(r"[0-9a-f]{64}", manifest["sha256"]) |
||||
or not isinstance(manifest["migrations"], list) |
||||
or not isinstance(manifest["row_counts"], dict) |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
try: |
||||
created_at = datetime.fromisoformat(manifest["created_at"]) |
||||
except ValueError: |
||||
raise BackupError("invalid_manifest") from None |
||||
if created_at.tzinfo is None: |
||||
raise BackupError("invalid_manifest") |
||||
migrations = manifest["migrations"] |
||||
if any( |
||||
not isinstance(item, list) |
||||
or len(item) != 2 |
||||
or not all(isinstance(value, str) and value for value in item) |
||||
for item in migrations |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
if any( |
||||
not isinstance(table, str) |
||||
or not _TABLE_NAME.fullmatch(table) |
||||
or not isinstance(count, int) |
||||
or count < 0 |
||||
for table, count in manifest["row_counts"].items() |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
return manifest |
||||
|
||||
|
||||
def _read_snapshot(target): |
||||
with psycopg.connect( |
||||
host=target.host, |
||||
port=target.port, |
||||
dbname=target.name, |
||||
user=target.user, |
||||
password=target.password, |
||||
connect_timeout=3, |
||||
) as database: |
||||
with database.cursor() as cursor: |
||||
cursor.execute( |
||||
""" |
||||
SELECT table_name |
||||
FROM information_schema.tables |
||||
WHERE table_schema = 'public' AND table_type = 'BASE TABLE' |
||||
ORDER BY table_name |
||||
""" |
||||
) |
||||
tables = [row[0] for row in cursor.fetchall()] |
||||
migrations = () |
||||
if "django_migrations" in tables: |
||||
cursor.execute( |
||||
"SELECT app, name FROM django_migrations ORDER BY app, name" |
||||
) |
||||
migrations = tuple((row[0], row[1]) for row in cursor.fetchall()) |
||||
row_counts = {} |
||||
for table in tables: |
||||
cursor.execute( |
||||
sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)) |
||||
) |
||||
row_counts[table] = cursor.fetchone()[0] |
||||
return DatabaseSnapshot(migrations=migrations, row_counts=row_counts) |
||||
|
||||
|
||||
def _same_database(first, second): |
||||
return ( |
||||
first.host.strip().casefold(), |
||||
first.name.strip().casefold(), |
||||
) == ( |
||||
second.host.strip().casefold(), |
||||
second.name.strip().casefold(), |
||||
) |
||||
|
||||
|
||||
def run_synthetic_restore_verification(): |
||||
"""Exercise backup and restore only against disposable synthetic databases.""" |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
from uuid import uuid4 |
||||
|
||||
from django.core.management import call_command |
||||
from django.db import connections |
||||
|
||||
from finance.models import Account, AuditEvent, Category, Payee, SourceRecord, Transaction |
||||
from ingestion.models import ImportBatch, ImportItem |
||||
|
||||
if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): |
||||
raise BackupError("verification_not_allowed") |
||||
required = ( |
||||
"BACKUP_SOURCE_HOST", |
||||
"BACKUP_SOURCE_NAME", |
||||
"BACKUP_SOURCE_USER", |
||||
"BACKUP_SOURCE_PASSWORD", |
||||
) |
||||
if any(not os.environ.get(name) for name in required): |
||||
raise BackupError("source_not_configured") |
||||
|
||||
restore_target = DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
source_target = DatabaseTarget( |
||||
host=os.environ["BACKUP_SOURCE_HOST"], |
||||
port=os.environ.get("BACKUP_SOURCE_PORT", "5432"), |
||||
name=os.environ["BACKUP_SOURCE_NAME"], |
||||
user=os.environ["BACKUP_SOURCE_USER"], |
||||
password=os.environ["BACKUP_SOURCE_PASSWORD"], |
||||
) |
||||
live_target = DatabaseTarget( |
||||
host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or ""), |
||||
port="5432", |
||||
name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or ""), |
||||
user="", |
||||
password="", |
||||
) |
||||
if _same_database(source_target, restore_target): |
||||
raise BackupError("source_is_target") |
||||
if live_target.host and ( |
||||
_same_database(source_target, live_target) |
||||
or _same_database(restore_target, live_target) |
||||
): |
||||
raise BackupError("live_target") |
||||
if _read_snapshot(source_target).row_counts: |
||||
raise BackupError("source_not_empty") |
||||
restore_before = _read_snapshot(restore_target) |
||||
if restore_before.migrations or restore_before.row_counts: |
||||
raise BackupError("target_not_empty") |
||||
|
||||
default_connection = connections["default"] |
||||
original_database = dict(default_connection.settings_dict) |
||||
artifact = None |
||||
try: |
||||
default_connection.close() |
||||
_configure_connection(default_connection, source_target) |
||||
call_command("migrate", interactive=False, verbosity=0) |
||||
|
||||
account = Account.objects.create( |
||||
name="Synthetic Restore Bank", kind=Account.Kind.BANK, currency="BRL" |
||||
) |
||||
payee = Payee.objects.create(name="Synthetic Restore Merchant") |
||||
category = Category.objects.create( |
||||
name="Synthetic Restore Expense", kind=Category.Kind.EXPENSE |
||||
) |
||||
transaction = Transaction.objects.create( |
||||
account=account, |
||||
amount=Decimal("-12.34"), |
||||
posting_date=date(2026, 9, 8), |
||||
description="Synthetic restore purchase", |
||||
kind=Transaction.Kind.EXPENSE, |
||||
payee=payee, |
||||
category=category, |
||||
) |
||||
SourceRecord.objects.create( |
||||
transaction=transaction, |
||||
document_hash="a" * 64, |
||||
source_format=SourceRecord.Format.BANK_CSV, |
||||
parser_version="synthetic-v1", |
||||
item_locator="row:1", |
||||
) |
||||
AuditEvent.objects.create( |
||||
actor_kind=AuditEvent.ActorKind.AGENT, |
||||
actor_identifier="synthetic-backup-verification", |
||||
operation="synthetic_backup_verification", |
||||
target_id=transaction.id, |
||||
outcome=AuditEvent.Outcome.SUCCESS, |
||||
) |
||||
batch = ImportBatch.objects.create( |
||||
document_hash="b" * 64, |
||||
source_format="BANK_CSV", |
||||
profile="SYNTHETIC", |
||||
encoding="utf-8", |
||||
actor_kind="AGENT", |
||||
actor_identifier="synthetic-backup-verification", |
||||
) |
||||
ImportItem.objects.create( |
||||
batch=batch, |
||||
locator="row:1", |
||||
candidate={"description": "Synthetic restore candidate"}, |
||||
) |
||||
before = _read_snapshot(source_target) |
||||
artifact = create_backup( |
||||
f"synthetic-restore-check-{uuid4().hex}.dump", _target=source_target |
||||
) |
||||
|
||||
default_connection.close() |
||||
default_connection.settings_dict.clear() |
||||
default_connection.settings_dict.update(original_database) |
||||
result = verify_backup(artifact.name, restore_target) |
||||
call_command("check", databases=["default"], verbosity=0) |
||||
if _read_snapshot(source_target) != before: |
||||
raise BackupError("source_changed") |
||||
return result |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("integration_verification_failed") from None |
||||
finally: |
||||
default_connection.close() |
||||
default_connection.settings_dict.clear() |
||||
default_connection.settings_dict.update(original_database) |
||||
if artifact is not None: |
||||
artifact.path.unlink(missing_ok=True) |
||||
artifact.manifest_path.unlink(missing_ok=True) |
||||
_clear_disposable_database(source_target) |
||||
_clear_disposable_database(restore_target) |
||||
|
||||
|
||||
def _configure_connection(connection, target): |
||||
connection.settings_dict.update( |
||||
{ |
||||
"HOST": target.host, |
||||
"PORT": target.port, |
||||
"NAME": target.name, |
||||
"USER": target.user, |
||||
"PASSWORD": target.password, |
||||
} |
||||
) |
||||
|
||||
|
||||
def _clear_disposable_database(target): |
||||
try: |
||||
with psycopg.connect( |
||||
host=target.host, |
||||
port=target.port, |
||||
dbname=target.name, |
||||
user=target.user, |
||||
password=target.password, |
||||
connect_timeout=3, |
||||
autocommit=True, |
||||
) as database: |
||||
with database.cursor() as cursor: |
||||
cursor.execute("DROP SCHEMA public CASCADE") |
||||
cursor.execute("CREATE SCHEMA public") |
||||
except psycopg.Error: |
||||
pass |
||||
@ -0,0 +1,19 @@
|
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, create_backup |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Create a local PostgreSQL backup and checksum manifest." |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument("--output", dest="output") |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
result = create_backup(options["output"]) |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version}") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,16 @@
|
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, run_synthetic_restore_verification |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Run the synthetic backup and disposable restore acceptance check." |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
result = run_synthetic_restore_verification() |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version} verified") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,31 @@
|
||||
from django.conf import settings |
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, DatabaseTarget, verify_backup |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Restore a backup into an explicitly allowed disposable database and verify it." |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument("--input", dest="input", required=True) |
||||
parser.add_argument("--target-name", required=True) |
||||
parser.add_argument("--target-host", required=True) |
||||
parser.add_argument("--target-user", required=True) |
||||
|
||||
def handle(self, *args, **options): |
||||
default = settings.DATABASES["default"] |
||||
target = DatabaseTarget( |
||||
host=options["target_host"], |
||||
port=str(default.get("PORT") or "5432"), |
||||
name=options["target_name"], |
||||
user=options["target_user"], |
||||
password=str(default.get("PASSWORD", "")), |
||||
) |
||||
try: |
||||
result = verify_backup(options["input"], target) |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version} verified") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,260 @@
|
||||
import json |
||||
from hashlib import sha256 |
||||
from pathlib import Path |
||||
from types import SimpleNamespace |
||||
|
||||
import pytest |
||||
|
||||
from core.backup import ( |
||||
BACKUP_FORMAT, |
||||
BACKUP_VERSION, |
||||
BackupError, |
||||
DatabaseSnapshot, |
||||
DatabaseTarget, |
||||
create_backup, |
||||
verify_backup, |
||||
) |
||||
|
||||
|
||||
class RecordingRunner: |
||||
def __init__(self, output=b"PGDMP synthetic"): |
||||
self.calls = [] |
||||
self.output = output |
||||
|
||||
def __call__(self, argv, *, env, check, stdout=None, capture_output=False): |
||||
self.calls.append((list(argv), dict(env), check, stdout, capture_output)) |
||||
if stdout is not None: |
||||
stdout.write(self.output) |
||||
return SimpleNamespace(stdout=b"pg_dump (PostgreSQL) 17.6\n") |
||||
|
||||
|
||||
@pytest.fixture |
||||
def database_settings(settings): |
||||
settings.DATABASES["default"] = { |
||||
"ENGINE": "django.db.backends.postgresql", |
||||
"HOST": "live-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck", |
||||
"USER": "gizmoduck_migrate", |
||||
"PASSWORD": "synthetic-secret", |
||||
} |
||||
return settings.DATABASES["default"] |
||||
|
||||
|
||||
@pytest.fixture |
||||
def backup_root(tmp_path, settings): |
||||
settings.BACKUP_ROOT = tmp_path |
||||
return tmp_path |
||||
|
||||
|
||||
@pytest.fixture |
||||
def synthetic_snapshot(monkeypatch): |
||||
snapshot = DatabaseSnapshot( |
||||
migrations=(("finance", "0001_initial"), ("ingestion", "0001_initial")), |
||||
row_counts={"finance_account": 2, "ingestion_importbatch": 1}, |
||||
) |
||||
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: snapshot) |
||||
return snapshot |
||||
|
||||
|
||||
def test_pg_dump_receives_password_only_in_environment( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
runner = RecordingRunner() |
||||
|
||||
result = create_backup("synthetic.dump", runner=runner) |
||||
|
||||
argv, env, check, stdout, capture_output = runner.calls[0] |
||||
assert argv[:4] == ["pg_dump", "--format=custom", "--no-owner", "--no-acl"] |
||||
assert database_settings["PASSWORD"] not in argv |
||||
assert env["PGPASSWORD"] == database_settings["PASSWORD"] |
||||
assert check is True |
||||
assert stdout is not None |
||||
assert capture_output is False |
||||
assert result.path.read_bytes() == runner.output |
||||
|
||||
|
||||
def test_backup_writes_versioned_checksum_manifest( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
runner = RecordingRunner() |
||||
|
||||
result = create_backup("synthetic.dump", runner=runner) |
||||
|
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
assert manifest == { |
||||
"application_version": "0.1.0", |
||||
"byte_size": len(runner.output), |
||||
"created_at": result.created_at.isoformat(), |
||||
"database_name": "gizmoduck", |
||||
"format": BACKUP_FORMAT, |
||||
"migrations": [ |
||||
["finance", "0001_initial"], |
||||
["ingestion", "0001_initial"], |
||||
], |
||||
"postgresql_client_version": result.postgresql_client_version, |
||||
"row_counts": { |
||||
"finance_account": 2, |
||||
"ingestion_importbatch": 1, |
||||
}, |
||||
"sha256": sha256(runner.output).hexdigest(), |
||||
"version": BACKUP_VERSION, |
||||
} |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name", |
||||
["../outside.dump", "nested/file.dump", "nested\\file.dump", ".dump", "file.zip"], |
||||
) |
||||
def test_backup_rejects_non_plain_dump_names( |
||||
backup_root, database_settings, synthetic_snapshot, name |
||||
): |
||||
with pytest.raises(BackupError, match="invalid_name"): |
||||
create_backup(name, runner=RecordingRunner()) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_backup_does_not_replace_existing_artifacts( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
dump = backup_root / "synthetic.dump" |
||||
manifest = backup_root / "synthetic.dump.json" |
||||
dump.write_bytes(b"existing dump") |
||||
manifest.write_text("existing manifest", encoding="utf-8") |
||||
|
||||
with pytest.raises(BackupError, match="existing_destination"): |
||||
create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
|
||||
assert dump.read_bytes() == b"existing dump" |
||||
assert manifest.read_text(encoding="utf-8") == "existing manifest" |
||||
|
||||
|
||||
def test_backup_cleans_private_temporary_files_on_dump_failure( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
def failing_runner(argv, **kwargs): |
||||
raise OSError("synthetic private detail") |
||||
|
||||
with pytest.raises(BackupError, match="backup_failed"): |
||||
create_backup("synthetic.dump", runner=failing_runner) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_verification_rejects_live_database( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
target = DatabaseTarget.from_default(database_settings) |
||||
|
||||
with pytest.raises(BackupError, match="live_target"): |
||||
verify_backup("synthetic.dump", target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_verification_requires_explicit_allow_flag( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = False |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="verification_not_allowed"): |
||||
verify_backup("synthetic.dump", target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_verification_rejects_changed_dump_before_restore( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
result.path.write_bytes(b"PGDMP tampered!") |
||||
restore_runner = RecordingRunner() |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="digest_mismatch"): |
||||
verify_backup(result.name, target, runner=restore_runner) |
||||
|
||||
assert restore_runner.calls == [] |
||||
|
||||
|
||||
def test_verification_rejects_manifest_with_wrong_field_type( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
manifest["application_version"] = 1 |
||||
result.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="invalid_manifest"): |
||||
verify_backup(result.name, target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_backup_rejects_nonmatching_postgresql_client_major( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
class WrongVersionRunner(RecordingRunner): |
||||
def __call__(self, argv, **kwargs): |
||||
result = super().__call__(argv, **kwargs) |
||||
if argv == ["pg_dump", "--version"]: |
||||
result.stdout = b"pg_dump (PostgreSQL) 16.9\n" |
||||
return result |
||||
|
||||
with pytest.raises(BackupError, match="client_version_failed"): |
||||
create_backup("synthetic.dump", runner=WrongVersionRunner()) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_pg_restore_receives_password_only_in_environment( |
||||
backup_root, database_settings, synthetic_snapshot, settings, monkeypatch |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
snapshots = iter( |
||||
[DatabaseSnapshot(migrations=(), row_counts={}), synthetic_snapshot] |
||||
) |
||||
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: next(snapshots)) |
||||
runner = RecordingRunner() |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="synthetic-restore-secret", |
||||
) |
||||
|
||||
verification = verify_backup(result.name, target, runner=runner) |
||||
|
||||
argv, env, check, stdout, capture_output = runner.calls[0] |
||||
assert argv[:5] == [ |
||||
"pg_restore", |
||||
"--exit-on-error", |
||||
"--single-transaction", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
] |
||||
assert target.password not in argv |
||||
assert env["PGPASSWORD"] == target.password |
||||
assert check is True |
||||
assert stdout is None |
||||
assert capture_output is True |
||||
assert verification.row_counts == synthetic_snapshot.row_counts |
||||
@ -0,0 +1,135 @@
|
||||
from datetime import datetime, timezone |
||||
from io import StringIO |
||||
from pathlib import Path |
||||
|
||||
import pytest |
||||
from django.core.management import call_command |
||||
from django.core.management.base import CommandError |
||||
|
||||
from core.backup import BackupError, BackupResult, VerificationResult |
||||
|
||||
|
||||
def backup_result(): |
||||
return BackupResult( |
||||
name="synthetic.dump", |
||||
path=Path("/private/backup/synthetic.dump"), |
||||
manifest_path=Path("/private/backup/synthetic.dump.json"), |
||||
version=1, |
||||
created_at=datetime(2026, 9, 8, tzinfo=timezone.utc), |
||||
postgresql_client_version="pg_dump (PostgreSQL) 17.6", |
||||
row_counts={"finance_account": 2, "finance_transaction": 4}, |
||||
) |
||||
|
||||
|
||||
def test_backup_database_prints_only_safe_summary(monkeypatch): |
||||
monkeypatch.setattr( |
||||
"core.management.commands.backup_database.create_backup", |
||||
lambda name: backup_result(), |
||||
) |
||||
stdout = StringIO() |
||||
|
||||
call_command("backup_database", output="synthetic.dump", stdout=stdout) |
||||
|
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic.dump format=1", |
||||
"table=finance_account rows=2", |
||||
"table=finance_transaction rows=4", |
||||
] |
||||
assert "/private/backup" not in stdout.getvalue() |
||||
|
||||
|
||||
def test_backup_database_converts_service_error_to_safe_code(monkeypatch): |
||||
def fail(_name): |
||||
raise BackupError("backup_failed") |
||||
|
||||
monkeypatch.setattr("core.management.commands.backup_database.create_backup", fail) |
||||
|
||||
with pytest.raises(CommandError, match=r"^backup_failed$"): |
||||
call_command("backup_database", output="synthetic.dump") |
||||
|
||||
|
||||
def test_verify_backup_uses_configured_password_and_prints_safe_summary( |
||||
monkeypatch, settings |
||||
): |
||||
settings.DATABASES["default"] = { |
||||
"HOST": "restore-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck_restore", |
||||
"USER": "restore_user", |
||||
"PASSWORD": "synthetic-restore-secret", |
||||
} |
||||
captured = {} |
||||
|
||||
def verify(name, target): |
||||
captured["name"] = name |
||||
captured["target"] = target |
||||
return VerificationResult( |
||||
name=name, |
||||
version=1, |
||||
row_counts={"django_migrations": 12, "finance_account": 2}, |
||||
) |
||||
|
||||
monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", verify) |
||||
stdout = StringIO() |
||||
|
||||
call_command( |
||||
"verify_backup", |
||||
input="synthetic.dump", |
||||
target_name="gizmoduck_restore", |
||||
target_host="restore-db", |
||||
target_user="restore_user", |
||||
stdout=stdout, |
||||
) |
||||
|
||||
assert captured["name"] == "synthetic.dump" |
||||
assert captured["target"].password == "synthetic-restore-secret" |
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic.dump format=1 verified", |
||||
"table=django_migrations rows=12", |
||||
"table=finance_account rows=2", |
||||
] |
||||
assert "synthetic-restore-secret" not in stdout.getvalue() |
||||
|
||||
|
||||
def test_verify_backup_converts_service_error_to_safe_code(monkeypatch, settings): |
||||
settings.DATABASES["default"] = { |
||||
"HOST": "restore-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck_restore", |
||||
"USER": "restore_user", |
||||
"PASSWORD": "synthetic-restore-secret", |
||||
} |
||||
|
||||
def fail(_name, _target): |
||||
raise BackupError("target_not_empty") |
||||
|
||||
monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", fail) |
||||
|
||||
with pytest.raises(CommandError, match=r"^target_not_empty$"): |
||||
call_command( |
||||
"verify_backup", |
||||
input="synthetic.dump", |
||||
target_name="gizmoduck_restore", |
||||
target_host="restore-db", |
||||
target_user="restore_user", |
||||
) |
||||
|
||||
|
||||
def test_backup_restore_acceptance_command_prints_only_safe_summary(monkeypatch): |
||||
monkeypatch.setattr( |
||||
"core.management.commands.test_backup_restore.run_synthetic_restore_verification", |
||||
lambda: VerificationResult( |
||||
name="synthetic-restore-check.dump", |
||||
version=1, |
||||
row_counts={"finance_account": 1, "finance_transaction": 1}, |
||||
), |
||||
) |
||||
stdout = StringIO() |
||||
|
||||
call_command("test_backup_restore", stdout=stdout) |
||||
|
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic-restore-check.dump format=1 verified", |
||||
"table=finance_account rows=1", |
||||
"table=finance_transaction rows=1", |
||||
] |
||||
@ -0,0 +1,14 @@
|
||||
import subprocess |
||||
|
||||
|
||||
def test_backup_configuration_is_allow_gated(settings): |
||||
assert settings.BACKUP_ROOT == "/app/backups" |
||||
assert settings.BACKUP_VERIFY_ALLOWED is False |
||||
|
||||
|
||||
def test_application_image_installs_matching_postgresql_client(): |
||||
result = subprocess.run( |
||||
["pg_dump", "--version"], check=True, capture_output=True, text=True |
||||
) |
||||
|
||||
assert result.stdout.startswith("pg_dump (PostgreSQL) 17.") |
||||
Loading…
Reference in new issue