Files
accounts/tests/test_receipts.py
T
jimmyandClaude Opus 5.5 708ce8352e Add receipts, suppliers and a top navigation bar
Receipts and invoices: attach photos and PDFs to transactions, read the
supplier, date, total, GST and line items with Tesseract OCR, and match
them to bank transactions by amount and date. Receipts can have several
pages, and hand corrections survive re-reading.

Suppliers: saved suppliers that uploads are linked to by GST number or
name.

The sections (cashbook, receipts, invoices, customers, suppliers,
mileage, statements) move to a top bar, with the book controls in a row
beneath it, and the layout works on phones.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 19:05:44 +13:00

322 lines
16 KiB
Python

import os
import shutil
import sqlite3
import pytest
import ledger
from cashbook_app.services import receipts
from cashbook_app.services.receipts import parse_receipt
from conftest import SAMPLE_CSV
TILL_RECEIPT = """
BUNNINGS WAREHOUSE
Mt Wellington
TAX INVOICE GST No: 123-456-789
Date: 14/05/2026 Time 10:42
2 x Timber 90x45 24.50
Screws Galv 8g 12,99
Paint Brush 50mm 9.90
SUBTOTAL 47.39
TOTAL $47.39
GST INCLUDED 6.18
EFTPOS $47.39
CHANGE 0.00
"""
INVOICE = """Acme Consulting Ltd
Invoice INV-0042
Invoice Date: 3 Jun 2026
Due Date: 20 Jun 2026
GST Reg 98-765-432
Web design services 1,000.00
Subtotal (excl GST) 1,000.00
GST 15% 150.00
Total NZD 1,150.00
"""
def test_parse_till_receipt():
r = parse_receipt(TILL_RECEIPT)
assert r["vendor"] == "BUNNINGS WAREHOUSE"
assert r["date"] == "2026-05-14"
assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (4739, 618, "123-456-789")
assert r["items"] == [{"description": "Timber 90x45", "amount": 24.5},
{"description": "Screws Galv 8g", "amount": 12.99}, # OCR read the point as a comma
{"description": "Paint Brush 50mm", "amount": 9.9}]
def test_parse_invoice():
r = parse_receipt(INVOICE)
assert r["vendor"] == "Acme Consulting Ltd"
assert r["date"] == "2026-06-03" # the invoice date, not the due date
assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (115000, 15000, "098-765-432")
assert r["items"] == [{"description": "Web design services", "amount": 1000.0}]
def test_parse_fallbacks():
# no "total" line: the card payment is the total; GST far from 3/23 of it is ignored
r = parse_receipt("Corner Dairy\n12 Mar 2026\nMilk 3.50\nBread 4.20\nEFTPOS 7.70\nGST 5.00\n")
assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-03-12", 770, None)
assert parse_receipt("")["total_cents"] is None
# OCR noise inside the GST number
assert parse_receipt("TAX INVOICE GST No: 45-6/78-912\nTOTAL 10.00")["gst_number"] == "045-678-912"
assert parse_receipt("GST 150.00 150.00\nTOTAL 1150.00")["gst_cents"] == 15000
def png(seed: bytes) -> bytes:
return b"\x89PNG\r\n\x1a\n" + seed * 20 # only the header is checked; OCR is faked below
@pytest.fixture
def books(admin, monkeypatch):
"""A book with SAMPLE_CSV imported, and OCR replaced by whatever `fake_ocr` holds."""
admin.post("/api/books", json={"name": "b", "year": 2026})
path = admin.post("/api/upload?filename=s.csv", content=SAMPLE_CSV.encode()).json()["path"]
admin.post("/api/import", json={"book": "b", "statements": [path]})
admin.fake_ocr = {"status": "done", "text": "", "items": []}
monkeypatch.setattr(receipts, "read_document", lambda p, mime: dict(admin.fake_ocr))
return admin
def rows(client, sheet):
return client.get(f"/api/month?book=b&sheet={sheet}").json()["rows"]
def test_upload_suggest_and_link(books, tmp_path):
books.fake_ocr.update(vendor="Jaycar Electronics", date="2026-04-12", total_cents=11500, gst_cents=1500,
items=[{"description": "Soldering iron", "amount": 115.0}])
r = books.post("/api/attachments?filename=jaycar.png", content=png(b"j"))
assert r.status_code == 200, r.text
a = r.json()
assert (a["vendor"], a["total"], a["gst"], a["transaction"]) == ("Jaycar Electronics", 115.0, 15.0, None)
assert a["items"][0]["description"] == "Soldering iron"
[s] = a["suggestions"]
assert (s["reference"], s["amount"], s["days_apart"], s["sheet"]) == ("Jaycar Chris", -115.0, 1, "APR")
# the same file again is refused, and its copy isn't kept
assert books.post("/api/attachments?filename=again.png", content=png(b"j")).status_code == 400
assert len(os.listdir(tmp_path / "attachments")) == 1
unmatched = books.get("/api/attachments?unmatched=1").json()
assert [x["id"] for x in unmatched] == [a["id"]]
linked = books.post(f"/api/attachments/{a['id']}/link", json={"row": s["row"]}).json()
assert linked["transaction"]["reference"] == "Jaycar Chris"
assert linked["proposed"] == {} # already Incl GST, and nothing to go on for a category
assert books.get("/api/attachments?unmatched=1").json() == []
assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 1
assert [x["id"] for x in books.get(f"/api/row/{s['row']}/attachments").json()] == [a["id"]]
f = books.get(f"/api/attachments/{a['id']}/file")
assert f.status_code == 200 and f.content == png(b"j") and f.headers["content-type"] == "image/png"
books.post(f"/api/attachments/{a['id']}/unlink")
assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 0
assert books.delete(f"/api/attachments/{a['id']}").json() == {"ok": True}
assert os.listdir(tmp_path / "attachments") == []
assert books.get(f"/api/attachments/{a['id']}").status_code == 404
def test_link_offers_gst_and_category(books):
hetzner = next(r for r in rows(books, "MAY") if "Hetzner" in r["reference"])
assert hetzner["gst"] == "Excl GST"
# uploaded from the transaction's row: linked straight away, with changes to offer
books.fake_ocr.update(vendor="Hetzner Online", total_cents=553, gst_cents=72, gst_number="123-456-789")
a = books.post(f"/api/attachments?filename=h.pdf&kind=invoice&row={hetzner['row']}",
content=b"%PDF-1.4 hetzner").json()
assert (a["kind"], a["files"][0]["mime"], a["transaction"]["row"]) == ("invoice", "application/pdf", hetzner["row"])
assert a["proposed"] == {"gst": "Incl GST"}
a = books.post(f"/api/attachments/{a['id']}/link",
json={"row": hetzner["row"], "apply": {"gst": "Incl GST", "category": "Purchases"}}).json()
assert a["proposed"] == {}
hetzner = next(r for r in rows(books, "MAY") if r["row"] == hetzner["row"])
assert (hetzner["gst"], hetzner["category"], hetzner["attachments"]) == ("Incl GST", "Purchases", 1)
r = books.post(f"/api/attachments/{a['id']}/link", json={"row": hetzner["row"], "apply": {"amount": "1"}})
assert r.status_code == 400
def test_category_from_vendor_and_history(books, tmp_path):
con = sqlite3.connect(tmp_path / "cashbook.db")
with con:
con.execute("UPDATE transactions SET category = NULL")
fee = next(r for r in rows(books, "APR") if r["reference"] == "Monthly A/C Fee")
books.fake_ocr.update(vendor="Spark NZ", total_cents=850)
a = books.post(f"/api/attachments?filename=spark.png&row={fee['row']}", content=png(b"s")).json()
assert a["proposed"] == {"category": "Telephone & Power"}
def test_edit_details_updates_suggestions(books):
books.fake_ocr.update(status="failed", error="tesseract isn't installed")
r = books.post("/api/attachments?filename=blurry.jpg", content=b"\xff\xd8\xff" + b"x" * 50).json()
assert (r["ocr_status"], r["ocr_error"], r["suggestions"]) == ("failed", "tesseract isn't installed", [])
a = books.patch(f"/api/attachments/{r['id']}", json={"total": 1150, "doc_date": "2026-04-21",
"vendor": "Client Ltd"}).json()
assert (a["total_cents"], a["vendor"]) == (115000, "Client Ltd")
assert [s["reference"] for s in a["suggestions"]] == ["Client Ltd (Inv 101)"]
# too far from the transaction's date
a = books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "2026-05-21"}).json()
assert a["suggestions"] == []
assert books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "21/05/2026"}).status_code == 400
assert books.patch(f"/api/attachments/{r['id']}", json={"kind": "bill"}).status_code == 400
def test_several_photos_of_one_receipt(books, tmp_path, monkeypatch):
# a long receipt in two photos: the top has the shop and date, the bottom the total
pages = {b"top": "JAYCAR ELECTRONICS\nDate: 12/04/2026\nSoldering Station 89.00\n",
b"bottom": "Solder 26.00\nTOTAL $115.00\nGST INCLUDED 15.00\nGST No: 45-678-912\n"}
def read(p, mime):
text = pages[p.read_bytes()[8:]]
return {"status": "done", "text": text, **parse_receipt(text)}
monkeypatch.setattr(receipts, "read_document", read)
def photo(name):
return b"\x89PNG\r\n\x1a\n" + name
top = books.post("/api/attachments?filename=top.png", content=photo(b"top")).json()
# with no TOTAL line, the top alone gives a wrong guess at the total
assert (top["total"], top["suggestions"], len(top["files"])) == (89.0, [], 1)
books.patch(f"/api/attachments/{top['id']}", json={"vendor": "Jaycar"}) # a correction by hand
a = books.post(f"/api/attachments/{top['id']}/pages?filename=bottom.png", content=photo(b"bottom")).json()
assert [f["filename"] for f in a["files"]] == ["top.png", "bottom.png"]
# the details come from both pages read together, except the correction, which is kept
assert (a["vendor"], a["doc_date"], a["total"], a["gst"], a["gst_number"], a["edited"]) == \
("Jaycar", "2026-04-12", 115.0, 15.0, "045-678-912", ["vendor"])
assert [i["description"] for i in a["items"]] == ["Soldering Station", "Solder"]
assert a["raw_text"].startswith("JAYCAR") and "TOTAL" in a["raw_text"]
assert [s["reference"] for s in a["suggestions"]] == ["Jaycar Chris"]
assert len(books.get("/api/attachments").json()) == 1
bottom_id = a["files"][1]["id"]
f = books.get(f"/api/attachments/{a['id']}/files/{bottom_id}")
assert f.status_code == 200 and f.content == photo(b"bottom")
assert books.get(f"/api/attachments/{a['id']}/file").content == photo(b"top")
assert books.get(f"/api/attachments/999/files/{bottom_id}").status_code == 404
# the same photo can't go on twice, here or on another receipt
r = books.post(f"/api/attachments/{a['id']}/pages?filename=again.png", content=photo(b"bottom"))
assert r.status_code == 400
assert books.post(f"/api/attachments/999/pages?filename=x.png", content=photo(b"x")).status_code == 404
assert len(os.listdir(tmp_path / "attachments")) == 2
a = books.post(f"/api/attachments/{a['id']}/reread").json()
assert (a["vendor"], a["edited"]) == ("JAYCAR ELECTRONICS", []) # re-reading replaces corrections
# without the bottom, it's back to what the top says
books.patch(f"/api/attachments/{a['id']}", json={"gst": 15})
a = books.delete(f"/api/attachments/{a['id']}/pages/{bottom_id}").json()
assert ([f["filename"] for f in a["files"]], a["total"], a["gst"], a["gst_number"]) == \
(["top.png"], 89.0, 15.0, None)
assert len(os.listdir(tmp_path / "attachments")) == 1
r = books.delete(f"/api/attachments/{a['id']}/pages/{a['files'][0]['id']}")
assert r.status_code == 400 and "only page" in r.json()["detail"]
books.post(f"/api/attachments/{a['id']}/pages?filename=bottom.png", content=photo(b"bottom"))
books.delete(f"/api/attachments/{a['id']}")
assert os.listdir(tmp_path / "attachments") == []
def test_rejects_other_files(books, tmp_path):
r = books.post("/api/attachments?filename=a.txt", content=b"hello")
assert r.status_code == 400 and "JPEG" in r.json()["detail"]
r = books.post("/api/attachments?filename=a.heic", content=b"\0\0\0\x18ftypheic" + b"\0" * 20)
assert r.status_code == 400 and "HEIC" in r.json()["detail"]
assert books.post("/api/attachments?filename=a.png", content=b"").status_code == 400
assert books.post("/api/attachments?filename=a.png&row=9999", content=png(b"z")).status_code == 404
assert os.listdir(tmp_path / "attachments") == []
def test_needs_sign_in(client):
assert client.get("/api/attachments").status_code == 401
assert client.post("/api/attachments?filename=a.png", content=png(b"a")).status_code == 401
def test_upgrade_keeps_attachment_reference(tmp_path):
"""The v1 -> v2 table rebuild mustn't leave attachments pointing at the old table."""
path = tmp_path / "old.db"
con = sqlite3.connect(path)
con.executescript(ledger.SCHEMA.replace("'Zero-rated', ", ""))
con.close()
conn = ledger.connect(path)
sql = conn.execute("SELECT sql FROM sqlite_master WHERE name = 'attachments'").fetchone()[0]
assert "transactions_v1" not in sql
assert "Zero-rated" in conn.execute("SELECT sql FROM sqlite_master WHERE name = 'transactions'").fetchone()[0]
V2_ATTACHMENTS = """CREATE TABLE attachments (
id INTEGER PRIMARY KEY,
transaction_id INTEGER REFERENCES transactions(id) ON DELETE SET NULL,
kind TEXT NOT NULL DEFAULT 'receipt' CHECK (kind IN ('receipt', 'invoice')),
filename TEXT NOT NULL, stored_name TEXT NOT NULL UNIQUE, mime TEXT NOT NULL,
sha256 TEXT NOT NULL UNIQUE, size INTEGER NOT NULL,
vendor TEXT, doc_date TEXT, total_cents INTEGER, gst_cents INTEGER, gst_number TEXT,
items_json TEXT, raw_text TEXT,
ocr_status TEXT NOT NULL DEFAULT 'skipped' CHECK (ocr_status IN ('done', 'failed', 'skipped')),
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE INDEX attachments_transaction ON attachments (transaction_id);"""
def test_upgrade_moves_files_to_pages(tmp_path):
"""Version 2 databases had one file per attachment, in the attachments table."""
path = tmp_path / "v2.db"
con = sqlite3.connect(path)
con.executescript("""CREATE TABLE books (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE,
start_year INTEGER NOT NULL, opening_cents INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE transactions (id INTEGER PRIMARY KEY,
book_id INTEGER NOT NULL REFERENCES books(id) ON DELETE CASCADE, date TEXT NOT NULL,
reference TEXT NOT NULL DEFAULT '', origin TEXT NOT NULL DEFAULT '', amount_cents INTEGER NOT NULL,
gst TEXT NOT NULL CHECK (gst IN ('Incl GST', 'Zero-rated', 'Excl GST')), category TEXT, source TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);""" + V2_ATTACHMENTS + """
INSERT INTO books (id, name, start_year) VALUES (1, 'b', 2026);
INSERT INTO transactions (id, book_id, date, reference, amount_cents, gst) VALUES (5, 1, '2026-04-13', 'Jaycar', -11500, 'Incl GST');
INSERT INTO attachments (id, transaction_id, filename, stored_name, mime, sha256, size, vendor, total_cents,
items_json, raw_text, ocr_status) VALUES (7, 5, 'r.jpg', 'abc-1.jpg', 'image/jpeg', 'abc', 10, 'Jaycar',
11500, '[]', 'JAYCAR TOTAL 115.00', 'done');
PRAGMA user_version = 2;""")
con.close()
conn = ledger.connect(path)
assert conn.execute("PRAGMA user_version").fetchone()[0] == ledger.SCHEMA_VERSION
a = ledger.get_attachment(conn, 7)
assert a["edited"] == []
assert (a["transaction"]["row"], a["vendor"], a["total"], a["raw_text"], a["filename"]) == \
(5, "Jaycar", 115.0, "JAYCAR TOTAL 115.00", "r.jpg")
assert [(f["filename"], f["mime"], f["size"]) for f in a["files"]] == [("r.jpg", "image/jpeg", 10)]
assert ledger.get_file(conn, 7)["stored_name"] == "abc-1.jpg"
assert conn.execute("SELECT name FROM sqlite_master WHERE name = 'attachments_transaction'").fetchone()
assert "attachments_v2" not in conn.execute(
"SELECT sql FROM sqlite_master WHERE name = 'attachment_files'").fetchone()[0]
assert (tmp_path / "v2.db.v2.bak").exists()
# pages go with their attachment, and the link still works both ways
conn.execute("DELETE FROM transactions WHERE id = 5")
assert ledger.get_attachment(conn, 7)["transaction_id"] is None
ledger.delete_attachment(conn, 7)
assert conn.execute("SELECT COUNT(*) FROM attachment_files").fetchone()[0] == 0
conn.close()
ledger.connect(path).close() # and connecting again changes nothing
@pytest.mark.skipif(not shutil.which("tesseract"), reason="tesseract isn't installed")
def test_ocr_round_trip():
from PIL import Image, ImageDraw, ImageFont
img = Image.new("RGB", (900, 700), "white")
draw = ImageDraw.Draw(img)
try:
font = ImageFont.truetype("DejaVuSansMono.ttf", 34)
except OSError:
font = ImageFont.load_default(size=34)
for i, line in enumerate(TILL_RECEIPT.strip().splitlines()):
draw.text((40, 30 + i * 50), line.strip(), fill="black", font=font)
import io
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=90)
r = parse_receipt(receipts.ocr_image(buf.getvalue()))
assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-05-14", 4739, 618)
assert "BUNNINGS" in r["vendor"]