You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
298 lines
14 KiB
298 lines
14 KiB
"""The local owner imports synthetic files without sending source rows to the UI.""" |
|
|
|
from pathlib import Path |
|
import random |
|
import string |
|
|
|
import pytest |
|
from django.core import signing |
|
from django.test import Client |
|
from django.urls import reverse |
|
|
|
from finance.models import AuditEvent, SourceRecord, Transaction |
|
from finance.services import Actor, archive, create_account |
|
from ingestion import services |
|
from ingestion.models import ImportBatch |
|
|
|
|
|
pytestmark = pytest.mark.django_db(transaction=True) |
|
ACTOR = Actor(kind="AGENT", identifier="synthetic-guided-web") |
|
DATA = Path("tests/fixtures/synthetic/ingestion/bank.csv").read_bytes() |
|
|
|
|
|
@pytest.fixture |
|
def source(tmp_path, settings): |
|
settings.IMPORT_ROOT = tmp_path |
|
(tmp_path / "bank.csv").write_bytes(DATA) |
|
return tmp_path |
|
|
|
|
|
@pytest.fixture |
|
def bank(): |
|
return create_account(name="Synthetic web bank", kind="BANK", actor=ACTOR) |
|
|
|
|
|
def scan(client): |
|
response = client.post(reverse("imports-index"), {"action": "scan"}, follow=True) |
|
assert response.status_code == 200 |
|
assert not response.redirect_chain |
|
return response |
|
|
|
|
|
def prepare(client, scanned, bank, **overrides): |
|
values = { |
|
"action": "prepare", "scan_token": scanned.context["scan_token"], |
|
"file-0-section_0": str(bank.pk), |
|
} |
|
values.update(overrides) |
|
return client.post(reverse("imports-index"), values, follow=True) |
|
|
|
|
|
def commit(client, prepared): |
|
return client.post(reverse("imports-index"), { |
|
"action": "commit", "prepared_token": prepared.context["prepared_token"], |
|
}, follow=True) |
|
|
|
|
|
def assert_private(response, source): |
|
content = response.content.decode() |
|
for forbidden in ("Synthetic purchase", "Synthetic income", "-12.34", "100.00", str(source)): |
|
assert forbidden not in content |
|
|
|
|
|
def test_navigation_opens_imports_without_automatically_scanning(client, source): |
|
home = client.get(reverse("home")) |
|
assert 'href="/imports/"' in home.content.decode() |
|
response = client.get(reverse("imports-index")) |
|
assert response.status_code == 200 |
|
assert "Scan files" in response.content.decode() |
|
assert "bank.csv" not in response.content.decode() |
|
assert not ImportBatch.objects.exists() |
|
|
|
|
|
@pytest.mark.parametrize("action", ["scan", "prepare", "commit"]) |
|
def test_import_actions_require_csrf(source, action): |
|
browser = Client(enforce_csrf_checks=True) |
|
response = browser.post(reverse("imports-index"), {"action": action}) |
|
assert response.status_code == 403 |
|
assert not ImportBatch.objects.exists() |
|
|
|
|
|
def test_scan_shows_safe_metadata_and_only_active_brl_account_options(client, source, bank): |
|
retired = create_account(name="Synthetic archived", kind="BANK", actor=ACTOR) |
|
archive(retired, actor=ACTOR) |
|
create_account(name="Synthetic foreign", kind="BANK", currency="USD", actor=ACTOR) |
|
(source / "unsupported.ofx").write_text("Synthetic private source text") |
|
response = scan(client) |
|
content = response.content.decode() |
|
assert "bank.csv" in content and "BANK_CSV_V1" in content and "Section 1" in content |
|
assert "unsupported.ofx" in content and "UNSUPPORTED_FILE" in content |
|
assert bank.name in content |
|
assert "Synthetic archived" not in content and "Synthetic foreign" not in content |
|
assert "Synthetic private source text" not in content |
|
assert_private(response, source) |
|
assert not ImportBatch.objects.exists() |
|
payload = signing.loads(response.context["scan_token"], salt="guided-imports-scan") |
|
assert len(payload) == 1 |
|
assert set(payload[0]) == {"name", "document_hash", "profile", "sections"} |
|
|
|
|
|
def test_mapping_prepares_then_explicit_commit_creates_with_local_human_actor(client, source, bank): |
|
prepared = prepare(client, scan(client), bank) |
|
assert prepared.status_code == 200 |
|
assert not prepared.redirect_chain |
|
assert prepared.context["files"][0]["state"] == "READY" |
|
assert prepared.context["files"][0]["counts"] == {"created": 2, "matched": 0, "skipped": 0} |
|
assert not Transaction.objects.exists() |
|
assert_private(prepared, source) |
|
payload = signing.loads(prepared.context["prepared_token"], salt="guided-imports-prepared") |
|
assert set(payload[0]) == {"name", "document_hash", "profile", "sections", "batch_id", "revision"} |
|
response = commit(client, prepared) |
|
assert response.redirect_chain == [(reverse("imports-index"), 302)] |
|
assert response.context["files"][0]["state"] == "COMMITTED" |
|
assert response.context["files"][0]["counts"]["created"] == 2 |
|
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
|
assert set(Transaction.objects.values_list("account_id", flat=True)) == {bank.pk} |
|
assert set(AuditEvent.objects.filter(operation__startswith="import_").values_list( |
|
"actor_kind", "actor_identifier")) == {("HUMAN", "local-owner")} |
|
assert_private(response, source) |
|
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
|
|
|
|
|
def test_scan_and_prepared_form_metadata_do_not_depend_on_session_payload(client, source, bank): |
|
scanned = scan(client) |
|
scan_token = scanned.context["scan_token"] |
|
assert f'name="scan_token" value="{scan_token}"' in scanned.content.decode() |
|
assert "scan_token" not in str(dict(client.session)) |
|
client.cookies.pop("sessionid", None) |
|
prepared = prepare(client, scanned, bank) |
|
prepared_token = prepared.context["prepared_token"] |
|
assert f'name="prepared_token" value="{prepared_token}"' in prepared.content.decode() |
|
assert "prepared_token" not in str(dict(client.session)) |
|
client.cookies.pop("sessionid", None) |
|
response = commit(client, prepared) |
|
assert response.context["files"][0]["counts"]["created"] == 2 |
|
assert "document_hash" not in str(dict(client.session)) |
|
assert "batch_id" not in str(dict(client.session)) |
|
|
|
|
|
def test_oversized_scan_has_safe_result_and_leaves_every_file_untouched(client, source, bank): |
|
for index in range(20): |
|
(source / f"synthetic-{index}.csv").write_bytes(DATA) |
|
response = scan(client) |
|
assert "SCAN_TOO_LARGE" in response.content.decode() |
|
assert not response.context.get("scan_token") |
|
assert not ImportBatch.objects.exists() |
|
assert len(list(source.iterdir())) == 21 |
|
if "sessionid" in client.cookies: |
|
assert len(client.cookies["sessionid"].value) < 4096 |
|
|
|
|
|
def test_long_filename_summaries_fit_browser_cookie_and_do_not_accumulate_across_tabs(client, source, bank): |
|
(source / "bank.csv").unlink() |
|
names = random.Random(41) |
|
filenames = ["".join(names.choices(string.ascii_letters + string.digits, k=250)) + ".csv" |
|
for _ in range(10)] |
|
for filename in filenames: |
|
(source / filename).write_bytes(DATA) |
|
scanned = scan(client) |
|
fields = {f"file-{index}-section_0": str(bank.pk) for index in range(10)} |
|
prepared = prepare(client, scanned, bank, **fields) |
|
# A second tab scans other files before the first tab finishes its old form. |
|
for filename in filenames: |
|
(source / filename).unlink() |
|
(source / filename.replace(".csv", ".ofx")).write_text("Synthetic unsupported") |
|
scan(client) |
|
response = commit(client, prepared) |
|
assert len(response.context["files"]) == 10 |
|
assert all(row["code"] == "UNREADABLE_FILE" for row in response.context["files"]) |
|
assert len(client.cookies["sessionid"].OutputString().encode()) < 4096 |
|
assert not Transaction.objects.exists() |
|
|
|
|
|
def test_exact_reimport_reports_skips_without_new_transactions(client, source, bank): |
|
commit(client, prepare(client, scan(client), bank)) |
|
prepared = prepare(client, scan(client), bank) |
|
assert prepared.context["files"][0]["state"] == "ALREADY_IMPORTED" |
|
result = commit(client, prepared) |
|
assert result.context["files"][0]["counts"] == {"created": 0, "matched": 0, "skipped": 2} |
|
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
|
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
|
|
|
|
|
def test_multi_section_card_has_one_typed_mapping_per_section(client, source): |
|
(source / "bank.csv").unlink() |
|
(source / "cards.txt").write_text( |
|
"1 - SYNTHETIC HOLDER\n" + f"15.01.2026{'Synthetic first':37}BR 12,50 0,00\n" |
|
+ "2 - SYNTHETIC OTHER\n" + f"16.01.2026{'Synthetic second':37}BR 7,25 0,00\n" |
|
) |
|
first = create_account(name="Synthetic first card", kind="CREDIT_CARD", actor=ACTOR) |
|
second = create_account(name="Synthetic second card", kind="CREDIT_CARD", actor=ACTOR) |
|
scanned = scan(client) |
|
assert "Section 1" in scanned.content.decode() and "Section 2" in scanned.content.decode() |
|
assert "SYNTHETIC HOLDER" not in scanned.content.decode() |
|
response = commit(client, prepare(client, scanned, first, **{"file-0-section_1": str(second.pk)})) |
|
assert response.context["files"][0]["counts"]["created"] == 2 |
|
assert set(Transaction.objects.values_list("account_id", flat=True)) == {first.pk, second.pk} |
|
|
|
|
|
@pytest.mark.parametrize("mapping", ["", "bad-id", "archived", "foreign"]) |
|
def test_invalid_mapping_is_safe_and_rejected_before_preview(client, source, bank, mapping): |
|
scanned = scan(client) |
|
if mapping == "archived": |
|
archive(bank, actor=ACTOR) |
|
mapping = str(bank.pk) |
|
elif mapping == "foreign": |
|
mapping = str(create_account(name="Synthetic USD", kind="BANK", currency="USD", actor=ACTOR).pk) |
|
result = prepare(client, scanned, bank, **{"file-0-section_0": mapping}) |
|
assert result.context["files"][0]["code"] == "INVALID_SECTION_MAPPING" |
|
assert not ImportBatch.objects.exists() |
|
assert_private(result, source) |
|
|
|
|
|
def test_review_exception_does_not_prevent_another_file_commit(client, source, bank): |
|
(source / "ambiguous.csv").write_bytes(DATA + DATA.splitlines(keepends=True)[1]) |
|
prepared = prepare(client, scan(client), bank, **{"file-1-section_0": str(bank.pk)}) |
|
assert [row["state"] for row in prepared.context["files"]] == ["NEEDS_ATTENTION", "READY"] |
|
result = commit(client, prepared) |
|
assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] |
|
assert result.context["files"][0]["code"] == "REVIEW_REQUIRED" |
|
assert Transaction.objects.count() == 2 |
|
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
|
|
|
|
|
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
|
def test_unexpected_per_file_error_never_exposes_source_or_blocks_next_file(client, source, bank, monkeypatch, stage): |
|
from core import views |
|
|
|
(source / "first.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) |
|
scanned = scan(client) |
|
if stage == "commit": |
|
prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) |
|
service_name = "prepare_import" if stage == "prepare" else "commit_prepared_import" |
|
original = getattr(views.guided_imports, service_name) |
|
|
|
def fail_one_file(**kwargs): |
|
row = kwargs["discovered" if stage == "prepare" else "prepared"] |
|
if row.name == "bank.csv": |
|
raise RuntimeError(f"{source}: Synthetic private source text 1234.56") |
|
return original(**kwargs) |
|
|
|
monkeypatch.setattr(views.guided_imports, service_name, fail_one_file) |
|
if stage == "prepare": |
|
prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) |
|
result = commit(client, prepared) |
|
assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] |
|
assert result.context["files"][0]["code"] == "IMPORT_FAILED" |
|
assert "Synthetic private source text" not in result.content.decode() |
|
assert "1234.56" not in result.content.decode() |
|
assert Transaction.objects.count() == 2 |
|
assert_private(result, source) |
|
|
|
|
|
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
|
@pytest.mark.parametrize("alteration", ["tamper", "expired", "wrong-stage"]) |
|
def test_signed_workflow_rejects_altered_expired_and_wrong_stage_tokens( |
|
client, source, bank, monkeypatch, stage, alteration |
|
): |
|
scanned = scan(client) |
|
prepared = prepare(client, scanned, bank) if stage == "commit" else None |
|
token_name = "scan_token" if stage == "prepare" else "prepared_token" |
|
token = (prepared if prepared else scanned).context[token_name] |
|
if alteration == "tamper": |
|
token += "altered" |
|
elif alteration == "expired": |
|
import time |
|
original_time = time.time() |
|
monkeypatch.setattr(signing.time, "time", lambda: original_time + 901) |
|
else: |
|
token = scanned.context["scan_token"] if stage == "commit" else signing.dumps( |
|
[], salt="guided-imports-prepared") |
|
response = client.post(reverse("imports-index"), { |
|
"action": stage, token_name: token, "file-0-section_0": str(bank.pk), |
|
}) |
|
assert response.status_code == 400 |
|
assert "Scan again" in response.content.decode() |
|
assert not Transaction.objects.exists() |
|
assert_private(response, source) |
|
|
|
|
|
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
|
def test_changed_source_rejects_previously_signed_metadata(client, source, bank, stage): |
|
scanned = scan(client) |
|
prepared = prepare(client, scanned, bank) if stage == "commit" else None |
|
(source / "bank.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) |
|
response = commit(client, prepared) if prepared else prepare(client, scanned, bank) |
|
assert response.context["files"][0]["code"] == "FILE_CHANGED" |
|
assert not Transaction.objects.exists() |
|
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
|
|
|
|
|
def test_stale_prepared_revision_requires_attention_without_writing(client, source, bank): |
|
prepared = prepare(client, scan(client), bank) |
|
batch = ImportBatch.objects.get() |
|
services.refresh_preview(batch_id=batch.pk, revision=batch.revision, actor=ACTOR) |
|
response = commit(client, prepared) |
|
assert response.context["files"][0]["code"] == "STALE_REVISION" |
|
assert not Transaction.objects.exists() |
|
assert not ImportBatch.objects.filter(state="PREVIEW").exists()
|
|
|