From 708ce8352eca106b87d40a425ce9fde25d117d50 Mon Sep 17 00:00:00 2001 From: jimmy Date: Sun, 25 Oct 2026 19:05:44 +1300 Subject: [PATCH] Add receipts, suppliers and a top navigation bar Receipts and invoices: attach photos and PDFs to transactions, read the supplier, date, total, GST and line items with Tesseract OCR, and match them to bank transactions by amount and date. Receipts can have several pages, and hand corrections survive re-reading. Suppliers: saved suppliers that uploads are linked to by GST number or name. The sections (cashbook, receipts, invoices, customers, suppliers, mileage, statements) move to a top bar, with the book controls in a row beneath it, and the layout works on phones. Co-Authored-By: Claude Opus 5.5 --- .claude/launch.json | 9 + Dockerfile | 3 +- README.md | 51 ++- src/cashbook_app/main.py | 4 + src/cashbook_app/routers/attachments.py | 171 ++++++++ src/cashbook_app/routers/suppliers.py | 52 +++ src/cashbook_app/services/receipts.py | 297 ++++++++++++++ src/ledger.py | 373 ++++++++++++++++- src/static/index.html | 510 ++++++++++++++++++++---- src/static/suppliers.js | 96 +++++ src/suppliers.py | 60 +++ tests/test_receipts.py | 321 +++++++++++++++ tests/test_suppliers.py | 84 ++++ 13 files changed, 1944 insertions(+), 87 deletions(-) create mode 100644 src/cashbook_app/routers/attachments.py create mode 100644 src/cashbook_app/routers/suppliers.py create mode 100644 src/cashbook_app/services/receipts.py create mode 100644 src/static/suppliers.js create mode 100644 src/suppliers.py create mode 100644 tests/test_receipts.py create mode 100644 tests/test_suppliers.py diff --git a/.claude/launch.json b/.claude/launch.json index a7e66c2..2976145 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -6,6 +6,15 @@ "runtimeExecutable": "pipenv", "runtimeArgs": ["run", "dev"], "port": 8765 + }, + { + "name": "cashbook-scratch", + "runtimeExecutable": "env", + "runtimeArgs": ["CASHBOOK_DATA=/tmp/claude-1000/-home-jimmy-projects-accounts/1e7ee5e8-8915-44fb-a3a5-3f2db7ea9ee9/scratchpad/data", + "HOME=/tmp/claude-1000/-home-jimmy-projects-accounts/1e7ee5e8-8915-44fb-a3a5-3f2db7ea9ee9/scratchpad/home", + "AUTH_ENABLED=false", ".venv/bin/uvicorn", "app:app", "--app-dir", "src", + "--host", "127.0.0.1", "--port", "8766"], + "port": 8766 } ] } diff --git a/Dockerfile b/Dockerfile index 377bb51..dafe275 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ FROM debian:bookworm-slim # LibreOffice (headless Calc) + its Python bridge for spreadsheet export, -# pdftotext for PDF statements +# pdftotext for PDF statements, tesseract + Pillow to read receipt photos RUN apt-get update \ && apt-get install -y --no-install-recommends \ python3 python3-venv python3-uno libreoffice-calc-nogui poppler-utils fonts-dejavu-core \ + tesseract-ocr tesseract-ocr-eng python3-pil \ && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/README.md b/README.md index a0d9712..2f1f9a6 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ administrators manage who has access. - **Export** a book as an Excel (.xlsx) or LibreOffice (.ods) cashbook spreadsheet, with formulas, GST and category columns and the summary sheet, or as a CSV of all transactions. +- **Receipts and invoices:** attach photos of receipts and PDF invoices to + transactions. The supplier, date, total, GST and line items are read from them on + your own server (Tesseract OCR), and they're matched to bank transactions by amount + and date (see [Receipts and invoices](#receipts-and-invoices)). - **Accounts:** sign-in with personal accounts. Administrators invite people, reset passwords and manage who's an administrator (see [Accounts](#accounts)). @@ -54,7 +58,8 @@ administrator account. Stop it with `docker compose down`, and check it with **What the compose file sets up** - **Data:** `./data` is mounted at `/data`. It holds the database (`cashbook.db`: - books and accounts) and `statements/`, where uploads go. Back up this folder. + books and accounts), `statements/`, where uploads go, and `attachments/`, the + receipt and invoice files. Back up this folder. - **Network:** published on `127.0.0.1:8765` only. - **Lockdown:** - read-only container filesystem, with scratch space only in `/tmp` (tmpfs); @@ -111,13 +116,16 @@ services: ### Without Docker -Needs Python 3, `pdftotext` for PDF statements, and LibreOffice's UNO bridge for -spreadsheet export and migration. On Debian/Ubuntu: +Needs Python 3, `pdftotext` for PDF statements, LibreOffice's UNO bridge for +spreadsheet export and migration, and Tesseract and Pillow to read receipts. On +Debian/Ubuntu: ```bash -sudo apt install libreoffice-calc python3-uno poppler-utils pipenv +sudo apt install libreoffice-calc python3-uno poppler-utils tesseract-ocr python3-pil pipenv ``` +Without Tesseract, receipts can still be attached, but their details have to be typed in. + The virtualenv needs the system site packages, so it can see LibreOffice's `uno`: ```bash @@ -251,6 +259,35 @@ authority, partnership or trust income, and no taxable property sales. Entered amounts are saved per book in the database. These figures are worked out from your records and aren't tax advice. +### Receipts and invoices + +Keep the paperwork behind each transaction, such as the tax invoices you need to claim GST. + +- **From a transaction:** click 📎 on a row of a month's sheet and add a photo or PDF. + It's linked to that transaction straight away. +- **In bulk:** open **Receipts**, then drop files, choose them, or **take a photo** on a + phone. Each one is read, then listed with the bank transactions it probably belongs to: + the same amount, within 7 days of the receipt's date, across all books. Click **Link** + to confirm one. Receipts can wait unlinked until their statement is imported. +- **Reading the details:** photos are straightened, evened out and read with Tesseract. + PDFs use their text, or are OCR'd if they're scans. The supplier, date, total, GST, + GST number and line items are filled in. Correct any of them in place, and the + suggested matches update. +- **Several pages:** a long receipt can take more than one photo, and an invoice can be + more than one PDF. Add several files at once and choose **One receipt**, or use + **+ Add page** on a receipt later. The pages are read together and the details are + updated, except anything you corrected by hand. *Read the details again* (under the + text read from the pages) also replaces your corrections. Remove a page with its ×. +- **Updating the transaction:** when a receipt is linked, the app offers to set the + transaction to *Incl GST* if the receipt shows GST. It also suggests a category from + earlier transactions with the same reference, or from the supplier's name. Only what + you tick is changed. + +JPEG, PNG, WebP and PDF files up to 20 MB are accepted. HEIC photos aren't: set an +iPhone's camera to *Most Compatible* or share the photo as JPEG. Files are stored in +`data/attachments`. Deleting a receipt removes its files. Deleting a transaction or book +leaves its receipts in the list, unlinked. + ### Sending invoices **Invoices** makes the invoices you send to customers, as PDFs. @@ -342,8 +379,8 @@ src/ main.py app setup, security middleware, startup config.py settings from the environment / .env deps.py current user, admin check, database connections - routers/ auth, users, books, statements, invoices, mileage and page routes - services/ accounts, statement files, invoice PDFs, LibreOffice + routers/ auth, users, books, statements, attachments, invoices, mileage and page routes + services/ accounts, statement files, receipt OCR, invoice PDFs, LibreOffice db/, schemas/ account tables (SQLAlchemy) and request/response models static/ the web pages (cashbook, sign-in, register, users), invoices.js, mileage.js ledger.py SQLite storage: books, transactions, GST and summaries @@ -359,7 +396,7 @@ src/ anz_import.py imports statements (CLI) template.ods blank cashbook template, used for exports tests/ pytest suite (pipenv run test) -data/ cashbook.db and statements (git-ignored) +data/ cashbook.db, statements and attachments (git-ignored) ``` `data/` is in `.gitignore`: books, backups and bank statements are never committed. diff --git a/src/cashbook_app/main.py b/src/cashbook_app/main.py index 4bf9696..8d3f065 100644 --- a/src/cashbook_app/main.py +++ b/src/cashbook_app/main.py @@ -11,12 +11,14 @@ import ledger from cashbook_app import config from cashbook_app.db.models import Base from cashbook_app.db.session import get_engine, get_session_factory +from cashbook_app.routers.attachments import router as attachments_router from cashbook_app.routers.auth_routes import router as auth_router from cashbook_app.routers.books import router as books_router from cashbook_app.routers.frontend import router as frontend_router from cashbook_app.routers.invoices import router as invoices_router from cashbook_app.routers.mileage import router as mileage_router from cashbook_app.routers.statements import router as statements_router +from cashbook_app.routers.suppliers import router as suppliers_router from cashbook_app.routers.users_admin import router as users_router from cashbook_app.services import accounts, spreadsheets @@ -121,8 +123,10 @@ def create_app() -> FastAPI: app.include_router(users_router) app.include_router(books_router) app.include_router(statements_router) + app.include_router(attachments_router) app.include_router(invoices_router) app.include_router(mileage_router) + app.include_router(suppliers_router) return app diff --git a/src/cashbook_app/routers/attachments.py b/src/cashbook_app/routers/attachments.py new file mode 100644 index 0000000..b69bb15 --- /dev/null +++ b/src/cashbook_app/routers/attachments.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from urllib.parse import quote + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field + +import ledger +from cashbook_app.deps import get_ledger, require_user +from cashbook_app.services import receipts +from cashbook_app.services.receipts import AttachmentError + +router = APIRouter(prefix="/api", tags=["attachments"], dependencies=[Depends(require_user)]) + + +class LinkRequest(BaseModel): + row: int + # None: just link, and return the GST/category the receipt suggests for the transaction + apply: dict[str, str] | None = None + + +class AttachmentUpdate(BaseModel): + kind: str | None = None + vendor: str | None = Field(None, max_length=200) + doc_date: str | None = None + total: float | None = None + gst: float | None = None + gst_number: str | None = Field(None, max_length=20) + supplier_id: int | None = None + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except ledger.NotFound as e: + raise HTTPException(status_code=404, detail=str(e)) + except (ValueError, AttachmentError) as e: + raise HTTPException(status_code=400, detail=str(e)) + + +def _with_suggestions(conn, a): + a = {**a, "suggestions": ledger.suggest_transactions(conn, a)} + if a["transaction_id"] is not None: + a["proposed"] = ledger.propose_changes(conn, a, a["transaction_id"]) + return a + + +async def _receive(request: Request, filename: str, conn): + """Store an uploaded photo/PDF and read it; returns (file details, OCR result).""" + if int(request.headers.get("content-length") or 0) > receipts.MAX_UPLOAD: + raise HTTPException(status_code=413, detail="the file is too large (20 MB at most)") + data = await request.body() + saved = _call(receipts.save_upload, filename, data) + if ledger.file_exists(conn, saved["sha256"]): + receipts.remove_file(saved["stored_name"]) + raise HTTPException(status_code=400, detail=f"{saved['filename']} has already been added") + ocr = await run_in_threadpool(receipts.read_document, receipts.file_path(saved["stored_name"]), saved["mime"]) + return saved, ocr + + +def _keep(saved, fn, *args, **kwargs): + """Record the stored file with fn, removing the file again if that fails.""" + try: + return _call(fn, *args, **kwargs) + except Exception: + receipts.remove_file(saved["stored_name"]) + raise + + +@router.post("/attachments") +async def upload(request: Request, filename: str, kind: str = "receipt", row: int | None = None, + conn=Depends(get_ledger)): + saved, ocr = await _receive(request, filename, conn) + a = _keep(saved, ledger.add_attachment, conn, file=saved, kind=kind, transaction_id=row, ocr=ocr) + return {**_with_suggestions(conn, a), "ocr_error": ocr.get("error")} + + +@router.post("/attachments/{attachment_id}/pages") +async def add_page(attachment_id: int, request: Request, filename: str, conn=Depends(get_ledger)): + """Another photo of the same receipt (e.g. the bottom half) or another page of an invoice. + The pages are read together, and details still missing are filled in.""" + _call(ledger.get_attachment, conn, attachment_id) + saved, ocr = await _receive(request, filename, conn) + _keep(saved, ledger.add_page, conn, attachment_id, saved, ocr.get("text")) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + a = ledger.apply_ocr(conn, attachment_id, found) + return {**_with_suggestions(conn, a), "ocr_error": ocr.get("error")} + + +@router.delete("/attachments/{attachment_id}/pages/{file_id}") +def remove_page(attachment_id: int, file_id: int, conn=Depends(get_ledger)): + receipts.remove_file(_call(ledger.remove_page, conn, attachment_id, file_id)) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + return _with_suggestions(conn, ledger.apply_ocr(conn, attachment_id, found)) + + +@router.post("/attachments/{attachment_id}/reread") +def reread(attachment_id: int, conn=Depends(get_ledger)): + """Replace the details with what's read from all the pages, undoing corrections.""" + _call(ledger.get_attachment, conn, attachment_id) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + return _with_suggestions(conn, ledger.apply_ocr(conn, attachment_id, found, overwrite=True)) + + +@router.get("/attachments") +def list_attachments(unmatched: bool = False, supplier: int | None = None, conn=Depends(get_ledger)): + return [_with_suggestions(conn, a) for a in ledger.list_attachments(conn, unmatched, supplier)] + + +@router.get("/attachments/{attachment_id}") +def get_attachment(attachment_id: int, conn=Depends(get_ledger)): + return _with_suggestions(conn, _call(ledger.get_attachment, conn, attachment_id)) + + +def _send(f): + path = _call(receipts.file_path, f["stored_name"]) + if not path.exists(): + raise HTTPException(status_code=404, detail="the file is missing from data/attachments") + return FileResponse(path, media_type=f["mime"], headers={ + "Content-Disposition": f"inline; filename*=UTF-8''{quote(f['filename'])}"}) + + +@router.get("/attachments/{attachment_id}/file") +def download(attachment_id: int, conn=Depends(get_ledger)): + """The first page.""" + return _send(_call(ledger.get_file, conn, attachment_id)) + + +@router.get("/attachments/{attachment_id}/files/{file_id}") +def download_page(attachment_id: int, file_id: int, conn=Depends(get_ledger)): + return _send(_call(ledger.get_file, conn, attachment_id, file_id)) + + +@router.patch("/attachments/{attachment_id}") +def update(attachment_id: int, body: AttachmentUpdate, conn=Depends(get_ledger)): + fields = body.model_dump(exclude_unset=True) + for k in ("total", "gst"): + if k in fields: + v = fields.pop(k) + fields[f"{k}_cents"] = ledger.cents(abs(v)) if v is not None else None + a = _call(ledger.update_attachment, conn, attachment_id, **fields) + return _with_suggestions(conn, a) + + +@router.post("/attachments/{attachment_id}/link") +def link(attachment_id: int, body: LinkRequest, conn=Depends(get_ledger)): + apply = body.apply or {} + if set(apply) - {"gst", "category"}: + raise HTTPException(status_code=400, detail="only gst and category can be applied") + a = _call(ledger.link_attachment, conn, attachment_id, body.row, + gst=apply.get("gst"), category=apply.get("category")) + return _with_suggestions(conn, a) + + +@router.post("/attachments/{attachment_id}/unlink") +def unlink(attachment_id: int, conn=Depends(get_ledger)): + return _with_suggestions(conn, _call(ledger.unlink_attachment, conn, attachment_id)) + + +@router.delete("/attachments/{attachment_id}") +def delete(attachment_id: int, conn=Depends(get_ledger)): + for name in _call(ledger.delete_attachment, conn, attachment_id): + receipts.remove_file(name) + return {"ok": True} + + +@router.get("/row/{row}/attachments") +def row_attachments(row: int, conn=Depends(get_ledger)): + return [_with_suggestions(conn, a) for a in _call(ledger.attachments_for, conn, row)] diff --git a/src/cashbook_app/routers/suppliers.py b/src/cashbook_app/routers/suppliers.py new file mode 100644 index 0000000..1bcacb0 --- /dev/null +++ b/src/cashbook_app/routers/suppliers.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +import ledger +import suppliers +from cashbook_app.deps import get_ledger, require_user + +router = APIRouter(prefix="/api", tags=["suppliers"], dependencies=[Depends(require_user)]) + + +class Supplier(BaseModel): + name: str = Field(min_length=1, max_length=200) + email: str = Field("", max_length=200) + address: str = Field("", max_length=500) + gst_number: str = Field("", max_length=20) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except ledger.NotFound as e: + raise HTTPException(status_code=404, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("/suppliers") +def list_suppliers(conn=Depends(get_ledger)): + return suppliers.list_suppliers(conn) + + +@router.post("/suppliers") +def create_supplier(body: Supplier, conn=Depends(get_ledger)): + return _call(suppliers.create_supplier, conn, **body.model_dump()) + + +@router.get("/suppliers/{supplier_id}") +def get_supplier(supplier_id: int, conn=Depends(get_ledger)): + return _call(suppliers.get_supplier, conn, supplier_id) + + +@router.put("/suppliers/{supplier_id}") +def update_supplier(supplier_id: int, body: Supplier, conn=Depends(get_ledger)): + return _call(suppliers.update_supplier, conn, supplier_id, **body.model_dump()) + + +@router.delete("/suppliers/{supplier_id}") +def delete_supplier(supplier_id: int, conn=Depends(get_ledger)): + _call(suppliers.delete_supplier, conn, supplier_id) + return {"ok": True} diff --git a/src/cashbook_app/services/receipts.py b/src/cashbook_app/services/receipts.py new file mode 100644 index 0000000..26021d8 --- /dev/null +++ b/src/cashbook_app/services/receipts.py @@ -0,0 +1,297 @@ +"""Receipt and invoice files: storing uploads, and reading their details with local OCR. + +Images are cleaned up with Pillow (if installed) and read by the `tesseract` command; PDFs use +their text layer (`pdftotext`), or are rendered with `pdftoppm` and OCR'd when they're scans. +Nothing leaves the machine. `parse_receipt` turns the text into vendor, date, total, GST etc. +""" +from __future__ import annotations + +import hashlib +import io +import os +import re +import shutil +import subprocess +import tempfile +import uuid +from datetime import date +from pathlib import Path + +from cashbook_app import config + +MAX_UPLOAD = 20 * 1024 * 1024 +OCR_TIMEOUT = 90 # seconds per page +MIME = {".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", + ".pdf": "application/pdf"} +EXT_OF = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp", "application/pdf": ".pdf"} + + +class AttachmentError(Exception): + pass + + +def attachments_dir() -> Path: + path = config.data_dir() / "attachments" + path.mkdir(parents=True, exist_ok=True) + return path + + +def file_path(stored_name: str) -> Path: + # stored names are ours (hex + extension), but don't trust them blindly + if not re.fullmatch(r"[0-9a-f-]+\.(jpg|png|webp|pdf)", stored_name): + raise AttachmentError("bad file name") + return attachments_dir() / stored_name + + +def sniff(data: bytes) -> str | None: + """The file's real type from its first bytes.""" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + if data.startswith(b"%PDF"): + return "application/pdf" + return None + + +def save_upload(filename: str, data: bytes) -> dict: + """Write an uploaded receipt/invoice into data/attachments; returns its details.""" + if not data: + raise AttachmentError("the file is empty") + if len(data) > MAX_UPLOAD: + raise AttachmentError("the file is too large (20 MB at most)") + mime = sniff(data) + if mime is None: + if data[4:12] in (b"ftypheic", b"ftypheix", b"ftypmif1", b"ftyphevc"): + raise AttachmentError("HEIC photos aren't supported; set your camera to JPEG (Most Compatible) " + "or export the photo as JPEG") + raise AttachmentError("only JPEG, PNG or WebP photos and PDF files can be attached") + sha = hashlib.sha256(data).hexdigest() + stored = f"{sha[:16]}-{uuid.uuid4().hex[:8]}{EXT_OF[mime]}" + dest = attachments_dir() / stored + tmp = dest.with_name(f".upload-{uuid.uuid4().hex}") + tmp.write_bytes(data) + os.replace(tmp, dest) + name = os.path.basename(filename.replace("\\", "/")).strip() or f"receipt{EXT_OF[mime]}" + return {"filename": name[:200], "stored_name": stored, "mime": mime, "sha256": sha, "size": len(data)} + + +def remove_file(stored_name: str) -> None: + try: + file_path(stored_name).unlink(missing_ok=True) + except AttachmentError: + pass + + +# --- OCR -------------------------------------------------------------------- + +def _run(cmd: list[str], data: bytes | None = None) -> str: + res = subprocess.run(cmd, input=data, capture_output=True, timeout=OCR_TIMEOUT, + env={**os.environ, "OMP_THREAD_LIMIT": "1"}) + if res.returncode != 0: + err = res.stderr.decode(errors="replace").strip().splitlines() + raise AttachmentError(f"{cmd[0]}: {err[-1] if err else 'failed'}") + return res.stdout.decode("utf-8", errors="replace") + + +def prepare_image(data: bytes) -> bytes: + """Make a phone photo easier to read: upright, grey, a sensible size, even lighting, high contrast.""" + try: + from PIL import Image, ImageChops, ImageFilter, ImageOps + except ImportError: + return data + img = ImageOps.exif_transpose(Image.open(io.BytesIO(data))).convert("L") + longest = max(img.size) + if longest < 1600 or longest > 3600: + scale = 2400 / longest + img = img.resize((round(img.width * scale), round(img.height * scale)), Image.LANCZOS) + # flatten shadows and uneven light: subtract the text from a blurred copy of the page + background = img.filter(ImageFilter.GaussianBlur(radius=max(img.size) // 60)) + img = ImageOps.invert(ImageChops.subtract(background, img)) + img = ImageOps.autocontrast(img, cutoff=1).filter(ImageFilter.SHARPEN) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def ocr_image(data: bytes) -> str: + if not shutil.which("tesseract"): + raise AttachmentError("tesseract isn't installed, so the text couldn't be read") + # --psm 4: a single column of text of variable sizes, which suits till receipts + return _run(["tesseract", "stdin", "stdout", "-l", "eng", "--psm", "4"], prepare_image(data)) + + +def pdf_text(path: Path) -> str: + text = _run(["pdftotext", "-layout", str(path), "-"]) if shutil.which("pdftotext") else "" + if len(text.strip()) >= 20: + return text + # a scanned PDF: render the first pages and OCR them + if not shutil.which("pdftoppm"): + raise AttachmentError("pdftoppm isn't installed, so the scanned PDF couldn't be read") + with tempfile.TemporaryDirectory() as tmp: + _run(["pdftoppm", "-r", "300", "-png", "-f", "1", "-l", "3", str(path), os.path.join(tmp, "p")]) + return "\n".join(ocr_image(p.read_bytes()) for p in sorted(Path(tmp).glob("p*.png"))) + + +def read_document(path: Path, mime: str) -> dict: + """Text and details of a stored receipt; never raises, so an upload is kept even if OCR fails.""" + try: + text = pdf_text(path) if mime == "application/pdf" else ocr_image(path.read_bytes()) + except (AttachmentError, OSError, subprocess.TimeoutExpired) as e: + return {"status": "failed", "error": str(e) if not isinstance(e, subprocess.TimeoutExpired) + else "reading the text took too long"} + except Exception as e: # e.g. Pillow can't decode the image + return {"status": "failed", "error": f"couldn't read the file: {e}"} + return {"status": "done", "text": text, **parse_receipt(text)} + + +# --- parsing ---------------------------------------------------------------- + +# amounts with two decimals; OCR often reads the point as a comma +MONEY = re.compile(r"(? list[int]: + """Money amounts on a line, in cents.""" + out = [] + for m in MONEY.finditer(line): + whole = int(m.group(1).replace(",", "")) + value = whole * 100 + int(m.group(2)) + out.append(-value if m.group(0).lstrip().startswith("-") else value) + return out + + +def _to_date(kind: str, g: tuple) -> date | None: + try: + if kind == "ymd": + y, m, d = int(g[0]), int(g[1]), int(g[2]) + elif kind == "dmy": + d, m, y = int(g[0]), int(g[1]), int(g[2]) + elif kind == "dMy": + d, m, y = int(g[0]), MONTHS.index(g[1].lower()[:3]) + 1, int(g[2]) + else: + m, d, y = MONTHS.index(g[0].lower()[:3]) + 1, int(g[1]), int(g[2]) + if y < 100: + y += 2000 + found = date(y, m, d) + except ValueError: + return None + return found if date(2000, 1, 1) <= found <= date.today().replace(year=date.today().year + 1) else None + + +def find_date(lines: list[str]) -> str | None: + """The document's date: one on a line saying "date" first, skipping due dates.""" + candidates = [] + for i, line in enumerate(lines): + if re.search(r"\bdue\b|expir", line, re.I): + continue + for pattern, kind in DATES: + for m in pattern.finditer(line): + d = _to_date(kind, m.groups()) + if d: + candidates.append((0 if re.search(r"date", line, re.I) else 1, i, d)) + return min(candidates)[2].isoformat() if candidates else None + + +def find_total(lines: list[str]) -> int | None: + totals, paid = [], [] + for i, line in enumerate(lines): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + found = amounts(line) or (amounts(nxt) if not re.search(r"[a-z]{3}", nxt, re.I) else []) + found = [a for a in found if a > 0] + if not found: + continue + if TOTAL_WORDS.search(line) and not NOT_TOTAL.search(line): + totals.append(found[-1]) + elif PAID_WORDS.search(line): + paid.append(found[-1]) + if totals: + return max(totals) # e.g. "Total" beats a smaller "Total excl" the filter missed + if paid: + return max(paid) + every = [a for line in lines for a in amounts(line) if a > 0] + return max(every) if every else None + + +def find_gst(lines: list[str], total: int | None) -> int | None: + """The GST amount: of the amounts on lines mentioning GST, the one nearest 3/23 of the total.""" + found = [] + for line in lines: + if re.search(r"\bg\.?s\.?t\b", line, re.I) and not find_gst_number(line): + found += [a for a in amounts(line) if a > 0] + if not found: + return None + if total: + expected = total * 3 / 23 + plausible = [a for a in found if a <= total * 0.2] + return min(plausible, key=lambda a: abs(a - expected)) if plausible else None + return min(found) + + +def find_gst_number(text: str) -> str | None: + for m in GST_NUMBER.finditer(text): + digits = re.sub(r"\D", "", m.group(1)) + if len(digits) in (8, 9) and not re.search(r"\d\.\d\d(?!\d)", m.group(1)): # not amounts + digits = digits.zfill(9) + return f"{digits[:3]}-{digits[3:6]}-{digits[6:]}" + return None + + +def find_vendor(lines: list[str]) -> str | None: + for line in lines[:8]: + clean = re.sub(r"[^\w&'.,\- ]", " ", line) + clean = re.sub(r"\s+", " ", clean).strip(" .,-") + letters = sum(c.isalpha() for c in clean) + if letters < 3 or letters < len(clean.replace(" ", "")) * 0.6 or HEADER_WORDS.search(clean): + continue + return clean[:80] + return None + + +def find_items(lines: list[str]) -> list[dict]: + items = [] + for line in lines: + if TOTAL_WORDS.search(line) and not NOT_TOTAL.search(line): + break # items come before the total + found = amounts(line) + if not found or NOT_ITEM.search(line): + continue + first = MONEY.search(line) + desc = line[:first.start()].strip(" .:$-*") + desc = re.sub(r"^\d+\s*[xX@]\s*", "", desc) + if sum(c.isalpha() for c in desc) < 2: + continue + items.append({"description": re.sub(r"\s+", " ", desc)[:100], "amount": found[-1] / 100}) + return items[:50] + + +def parse_receipt(text: str) -> dict: + lines = [line.strip() for line in text.splitlines() if line.strip()] + total = find_total(lines) + return { + "vendor": find_vendor(lines), + "date": find_date(lines), + "total_cents": total, + "gst_cents": find_gst(lines, total), + "gst_number": find_gst_number(text), + "items": find_items(lines), + } diff --git a/src/ledger.py b/src/ledger.py index 73ffaa2..eb4eebd 100644 --- a/src/ledger.py +++ b/src/ledger.py @@ -3,8 +3,11 @@ Amounts are stored as signed integer cents (positive = money in). GST is worked out the way the cashbook spreadsheet does it: 3/23 of the amount for "Incl GST" rows. """ +import json +import re import sqlite3 from collections import Counter +from contextlib import closing from datetime import date from statements import ORIGIN, classify, describe @@ -47,6 +50,45 @@ CREATE TABLE IF NOT EXISTS transactions ( created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS transactions_book_date ON transactions (book_id, date); +CREATE TABLE IF NOT EXISTS suppliers ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT NOT NULL DEFAULT '', + address TEXT NOT NULL DEFAULT '', + gst_number TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS attachments ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE SET NULL, -- NULL = not matched yet + supplier_id INTEGER REFERENCES suppliers(id), -- who the receipt or bill is from + kind TEXT NOT NULL DEFAULT 'receipt' CHECK (kind IN ('receipt', 'invoice')), + -- read from the document's pages (OCR), and correctable by hand + vendor TEXT, + doc_date TEXT, -- ISO date + total_cents INTEGER, -- positive + gst_cents INTEGER, + gst_number TEXT, + items_json TEXT, -- [{"description", "amount"}] + edited_json TEXT NOT NULL DEFAULT '[]', -- fields corrected by hand, which re-reading leaves alone + ocr_status TEXT NOT NULL DEFAULT 'skipped' CHECK (ocr_status IN ('done', 'failed', 'skipped')), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS attachments_transaction ON attachments (transaction_id); +-- the photos or PDFs of an attachment, e.g. the two halves of a long receipt +CREATE TABLE IF NOT EXISTS attachment_files ( + id INTEGER PRIMARY KEY, + attachment_id INTEGER NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + position INTEGER NOT NULL, -- page order, from 0 + filename TEXT NOT NULL, -- name it was uploaded as + stored_name TEXT NOT NULL UNIQUE, -- file in data/attachments + mime TEXT NOT NULL, + sha256 TEXT NOT NULL UNIQUE, + size INTEGER NOT NULL, + text TEXT, -- what OCR read from it; NULL = couldn't be read + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS attachment_files_attachment ON attachment_files (attachment_id, position); CREATE TABLE IF NOT EXISTS business ( -- the one business invoices are sent from id INTEGER PRIMARY KEY CHECK (id = 1), name TEXT NOT NULL DEFAULT '', @@ -137,7 +179,7 @@ class NotFound(Exception): pass -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 5 def connect(path): @@ -158,12 +200,47 @@ def _upgrade(conn): if "Zero-rated" not in sql: # version 1 -> 2: allow 'Zero-rated'. SQLite can't change a CHECK, so rebuild the table. conn.execute("PRAGMA foreign_keys = OFF") + # keep attachments' reference pointing at "transactions", not the renamed copy + conn.execute("PRAGMA legacy_alter_table = ON") with conn: conn.execute("ALTER TABLE transactions RENAME TO transactions_v1") conn.executescript(SCHEMA) conn.execute("INSERT INTO transactions SELECT * FROM transactions_v1") conn.execute("DROP TABLE transactions_v1") + conn.execute("PRAGMA legacy_alter_table = OFF") conn.execute("PRAGMA foreign_keys = ON") + if "stored_name" in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 2 -> 3: an attachment's file moves to attachment_files, so it can have several + path = conn.execute("PRAGMA database_list").fetchone()["file"] + if path: # keep a copy from before the change + with closing(sqlite3.connect(path + ".v2.bak")) as dest: + conn.backup(dest) + conn.execute("PRAGMA foreign_keys = OFF") + conn.execute("PRAGMA legacy_alter_table = ON") + with conn: + conn.execute("ALTER TABLE attachments RENAME TO attachments_v2") + conn.execute("DROP INDEX attachments_transaction") + conn.executescript(SCHEMA) + conn.execute( + "INSERT INTO attachments (id, transaction_id, kind, vendor, doc_date, total_cents, gst_cents, " + "gst_number, items_json, ocr_status, created_at) SELECT id, transaction_id, kind, vendor, doc_date, " + "total_cents, gst_cents, gst_number, items_json, ocr_status, created_at FROM attachments_v2") + conn.execute( + "INSERT INTO attachment_files (attachment_id, position, filename, stored_name, mime, sha256, size, " + "text, created_at) SELECT id, 0, filename, stored_name, mime, sha256, size, raw_text, created_at " + "FROM attachments_v2") + conn.execute("DROP TABLE attachments_v2") + conn.execute("PRAGMA legacy_alter_table = OFF") + conn.execute("PRAGMA foreign_keys = ON") + if "edited_json" not in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 3 -> 4: remember which details were corrected by hand + with conn: + conn.execute("ALTER TABLE attachments ADD COLUMN edited_json TEXT NOT NULL DEFAULT '[]'") + if "supplier_id" not in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 4 -> 5: receipts and bills can be linked to a supplier + with conn: + conn.execute("ALTER TABLE attachments ADD COLUMN supplier_id INTEGER REFERENCES suppliers(id)") + conn.execute("CREATE INDEX IF NOT EXISTS attachments_supplier ON attachments (supplier_id)") conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") @@ -314,7 +391,9 @@ def add_rows(conn, book, rows): # --- reading ---------------------------------------------------------------- def transactions(conn, book, start=None, end=None): - sql, args = "SELECT * FROM transactions WHERE book_id = ?", [book["id"]] + sql = ("SELECT t.*, (SELECT COUNT(*) FROM attachments a WHERE a.transaction_id = t.id) AS attachments " + "FROM transactions t WHERE book_id = ?") + args = [book["id"]] if start: sql, args = sql + " AND date >= ?", args + [start.isoformat()] if end: @@ -333,7 +412,7 @@ def _row(t): "income": amount / 100 if income else 0, "gst_in": gst if income else 0, "expense": -amount / 100 if not income else 0, "gst_out": gst if not income else 0, "category": t["category"] or "", "unallocated": 0 if t["category"] else net, - "net": net, + "net": net, "attachments": t["attachments"], } @@ -486,3 +565,291 @@ def _brief(t): return {"row": t["id"], "book": t["book"], "date": t["date"], "reference": t["reference"], "amount": t["amount_cents"] / 100, "gst": t["gst"], "category": t["category"] or "", "sheet": SHEETS[date.fromisoformat(t["date"]).month]} + + +# --- attachments (receipts and invoices) ------------------------------------ + +ATTACHMENT_KINDS = ("receipt", "invoice") +# fields read from the document that can be corrected by hand +ATTACHMENT_FIELDS = ("kind", "vendor", "doc_date", "total_cents", "gst_cents", "gst_number", "supplier_id") +MATCH_DAYS = 7 # how far a receipt's date may be from the bank transaction's + +# merchant names that point to an expense category (lower case, matched as substrings) +VENDOR_CATEGORIES = [ + ("Motor vehicle", ("z energy", "bp connect", "mobil", "caltex", "gull", "waitomo", "npd", "parking", + "repco", "supercheap", "tyre", "nzta", "waka kotahi")), + ("Telephone & Power", ("spark", "one nz", "vodafone", "2degrees", "skinny", "mercury", "genesis", + "contact energy", "meridian", "electric kiwi", "flick", "slingshot", "orcon")), + ("Postage & Stationery", ("nz post", "warehouse stationery", "officemax", "paper plus", "whitcoulls")), + ("ACC & Insurance", ("insurance", "accident compensation", "tower", "vero")), +] + + +def _files(conn, attachment_id): + return [dict(f) for f in conn.execute( + "SELECT * FROM attachment_files WHERE attachment_id = ? ORDER BY position, id", (attachment_id,))] + + +def _attachment(conn, a): + d = dict(a) + d["items"] = json.loads(d.pop("items_json") or "[]") + d["edited"] = json.loads(d.pop("edited_json") or "[]") + d["total"] = d["total_cents"] / 100 if d["total_cents"] is not None else None + d["gst"] = d["gst_cents"] / 100 if d["gst_cents"] is not None else None + files = _files(conn, a["id"]) + d["files"] = [{k: f[k] for k in ("id", "filename", "mime", "size")} for f in files] + d["filename"] = files[0]["filename"] if files else "" + d["raw_text"] = attachment_text(conn, a["id"], files) + d["transaction"] = None + d["supplier"] = None + if d["supplier_id"] is not None: + sup = conn.execute("SELECT id, name FROM suppliers WHERE id = ?", (d["supplier_id"],)).fetchone() + d["supplier"] = dict(sup) + return d + + +def attachment_text(conn, attachment_id, files=None): + """What OCR read from all the pages, in order.""" + files = files if files is not None else _files(conn, attachment_id) + return "\n".join(f["text"] for f in files if f["text"]) + + +def _with_transactions(conn, rows): + out = [_attachment(conn, a) for a in rows] + for a in out: + if a["transaction_id"] is not None: + a["transaction"] = _brief(_transaction_info(conn, a["transaction_id"])) + return out + + +def file_exists(conn, sha256): + return conn.execute("SELECT 1 FROM attachment_files WHERE sha256 = ?", (sha256,)).fetchone() is not None + + +def _insert_file(conn, attachment_id, position, file, text): + if file_exists(conn, file["sha256"]): + raise ValueError("that file has already been added") + conn.execute( + "INSERT INTO attachment_files (attachment_id, position, filename, stored_name, mime, sha256, size, text) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (attachment_id, position, file["filename"], file["stored_name"], file["mime"], file["sha256"], + file["size"], text)) + + +def add_attachment(conn, *, file, kind="receipt", transaction_id=None, ocr=None): + """Record an uploaded receipt (its first page); `file` is from services.receipts.save_upload and + `ocr` is what services.receipts.read_document read from it.""" + if kind not in ATTACHMENT_KINDS: + raise ValueError("kind must be receipt or invoice") + if transaction_id is not None: + _transaction_info(conn, transaction_id) + ocr = ocr or {} + supplier = find_supplier(conn, ocr.get("vendor"), ocr.get("gst_number")) + with conn: + cur = conn.execute( + "INSERT INTO attachments (transaction_id, supplier_id, kind, vendor, doc_date, total_cents, gst_cents, " + "gst_number, items_json, ocr_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (transaction_id, supplier, kind, ocr.get("vendor"), ocr.get("date"), ocr.get("total_cents"), + ocr.get("gst_cents"), ocr.get("gst_number"), json.dumps(ocr.get("items") or []), + ocr.get("status", "skipped"))) + _insert_file(conn, cur.lastrowid, 0, file, ocr.get("text")) + return get_attachment(conn, cur.lastrowid) + + +def add_page(conn, attachment_id, file, text): + """Add another photo/PDF to an attachment, after its other pages.""" + get_attachment(conn, attachment_id) + with conn: + last = conn.execute("SELECT MAX(position) FROM attachment_files WHERE attachment_id = ?", + (attachment_id,)).fetchone()[0] + _insert_file(conn, attachment_id, (last or 0) + 1, file, text) + return get_attachment(conn, attachment_id) + + +def get_file(conn, attachment_id, file_id=None): + """One page of an attachment (the first if file_id is None).""" + sql, args = "SELECT * FROM attachment_files WHERE attachment_id = ?", [attachment_id] + if file_id is not None: + sql, args = sql + " AND id = ?", args + [file_id] + f = conn.execute(sql + " ORDER BY position, id LIMIT 1", args).fetchone() + if f is None: + raise NotFound("no such file") + return f + + +def remove_page(conn, attachment_id, file_id): + """Take one page off an attachment; returns its stored file name so the caller can remove the file.""" + f = get_file(conn, attachment_id, file_id) + if len(_files(conn, attachment_id)) == 1: + raise ValueError("that's the only page; delete the receipt instead") + with conn: + conn.execute("DELETE FROM attachment_files WHERE id = ?", (file_id,)) + return f["stored_name"] + + +OCR_FIELDS = {"vendor": "vendor", "doc_date": "date", "total_cents": "total_cents", + "gst_cents": "gst_cents", "gst_number": "gst_number"} # column: parse_receipt key + + +def apply_ocr(conn, attachment_id, found, overwrite=False): + """Set an attachment's details to what was read from all its pages together + (services.receipts.parse_receipt), except those corrected by hand, unless `overwrite`.""" + a = get_attachment(conn, attachment_id) + keep = set() if overwrite else set(a["edited"]) + with conn: + for col, key in OCR_FIELDS.items(): + if col not in keep: + conn.execute(f"UPDATE attachments SET {col} = ? WHERE id = ?", (found.get(key), attachment_id)) + read = any(f["text"] for f in _files(conn, attachment_id)) + conn.execute("UPDATE attachments SET items_json = ?, ocr_status = ?, edited_json = ? WHERE id = ?", + (json.dumps(found.get("items") or []), "done" if read else "failed", + json.dumps(sorted(keep)), attachment_id)) + return get_attachment(conn, attachment_id) + + +def get_attachment(conn, attachment_id): + a = conn.execute("SELECT * FROM attachments WHERE id = ?", (attachment_id,)).fetchone() + if a is None: + raise NotFound("no such attachment") + return _with_transactions(conn, [a])[0] + + +def list_attachments(conn, unmatched=False, supplier_id=None): + where, args = [], [] + if unmatched: + where.append("transaction_id IS NULL") + if supplier_id is not None: + where.append("supplier_id = ?") + args.append(supplier_id) + sql = "SELECT * FROM attachments" + (" WHERE " + " AND ".join(where) if where else "") + return _with_transactions(conn, conn.execute(sql + " ORDER BY created_at DESC, id DESC", args).fetchall()) + + +def attachments_for(conn, row_id): + _transaction_info(conn, row_id) + return _with_transactions(conn, conn.execute( + "SELECT * FROM attachments WHERE transaction_id = ? ORDER BY id", (row_id,)).fetchall()) + + +def update_attachment(conn, attachment_id, **fields): + """Correct details by hand; corrected ones are then kept when the pages are read again.""" + a = get_attachment(conn, attachment_id) + if fields.get("kind") is not None and fields["kind"] not in ATTACHMENT_KINDS: + raise ValueError("kind must be receipt or invoice") + if fields.get("doc_date"): + date.fromisoformat(fields["doc_date"]) # ValueError if it isn't a date + if fields.get("supplier_id") is not None and not conn.execute( + "SELECT 1 FROM suppliers WHERE id = ?", (fields["supplier_id"],)).fetchone(): + raise NotFound("no such supplier") + edited = set(a["edited"]) + with conn: + for k, v in fields.items(): + if k not in ATTACHMENT_FIELDS: + raise ValueError(f"can't change {k}") + if k == "kind" and v is None: + continue + conn.execute(f"UPDATE attachments SET {k} = ? WHERE id = ?", (v if v != "" else None, attachment_id)) + if k in OCR_FIELDS: + edited.add(k) + conn.execute("UPDATE attachments SET edited_json = ? WHERE id = ?", (json.dumps(sorted(edited)), attachment_id)) + return get_attachment(conn, attachment_id) + + +def delete_attachment(conn, attachment_id): + """Forget an attachment; returns its stored file names so the caller can remove the files.""" + get_attachment(conn, attachment_id) + names = [f["stored_name"] for f in _files(conn, attachment_id)] + with conn: + conn.execute("DELETE FROM attachments WHERE id = ?", (attachment_id,)) # pages go too (cascade) + return names + + +def _company(name): + """A supplier name reduced for comparing: letters and digits only, without Ltd / Limited / NZ.""" + words = re.findall(r"[a-z0-9]+", (name or "").lower()) + return "".join(w for w in words if w not in ("ltd", "limited", "nz", "co", "the")) + + +def find_supplier(conn, vendor, gst_number=None): + """The id of the supplier a document is probably from: the same GST number, else the same name.""" + digits = re.sub(r"\D", "", gst_number or "") + rows = conn.execute("SELECT id, name, gst_number FROM suppliers ORDER BY id").fetchall() + if digits: + for r in rows: + if re.sub(r"\D", "", r["gst_number"]) == digits: + return r["id"] + name = _company(vendor) + if name: + for r in rows: + if _company(r["name"]) == name: + return r["id"] + return None + + +def _words(s): + return {w for w in re.findall(r"[a-z0-9]{3,}", (s or "").lower())} + + +def suggest_transactions(conn, attachment, limit=5): + """Bank transactions this receipt is probably for: the same amount, within MATCH_DAYS of its date + (any date if it has none), nearest first, then those whose reference shares words with the vendor.""" + total = attachment["total_cents"] + if not total: + return [] + rows = conn.execute( + "SELECT t.*, b.name AS book, (SELECT COUNT(*) FROM attachments a WHERE a.transaction_id = t.id) AS n " + "FROM transactions t JOIN books b ON b.id = t.book_id WHERE abs(t.amount_cents) = ?", (abs(total),)).fetchall() + when = date.fromisoformat(attachment["doc_date"]) if attachment["doc_date"] else None + vendor = _words(attachment["vendor"]) + scored = [] + for t in rows: + if t["id"] == attachment["transaction_id"]: + continue + gap = abs((date.fromisoformat(t["date"]) - when).days) if when else 0 + if gap > MATCH_DAYS: + continue + overlap = len(vendor & _words(t["reference"])) + scored.append(((gap, -overlap, t["n"], t["date"]), t)) + scored.sort(key=lambda s: s[0]) + return [{**_brief(t), "days_apart": key[0] if when else None, "attachments": t["n"]} + for key, t in scored[:limit]] + + +def propose_changes(conn, attachment, row_id): + """GST and category the transaction could take from this receipt; only what would change.""" + t = _transaction_info(conn, row_id) + out = {} + if (attachment["gst_cents"] or attachment["gst_number"]) and t["gst"] != "Incl GST": + out["gst"] = "Incl GST" + if not t["category"]: + allowed = INCOME_CATEGORIES if t["amount_cents"] > 0 else EXPENSE_CATEGORIES + same = conn.execute( + "SELECT category, COUNT(*) AS n FROM transactions WHERE reference = ? AND category IS NOT NULL " + "AND id != ? GROUP BY category ORDER BY n DESC LIMIT 1", (t["reference"], row_id)).fetchone() + if same and same["category"] in allowed: + out["category"] = same["category"] + elif t["amount_cents"] < 0: + text = f"{attachment['vendor'] or ''} {t['reference']}".lower() + category = next((c for c, words in VENDOR_CATEGORIES if any(w in text for w in words)), None) + if category: + out["category"] = category + return out + + +def link_attachment(conn, attachment_id, row_id, gst=None, category=None): + """Attach to a transaction, optionally setting its GST and category at the same time.""" + get_attachment(conn, attachment_id) + t = _transaction_info(conn, row_id) + if gst is not None or category is not None: + update_row(conn, conn.execute("SELECT * FROM books WHERE id = ?", (t["book_id"],)).fetchone(), + row_id, gst=gst, category=category) + with conn: + conn.execute("UPDATE attachments SET transaction_id = ? WHERE id = ?", (row_id, attachment_id)) + return get_attachment(conn, attachment_id) + + +def unlink_attachment(conn, attachment_id): + get_attachment(conn, attachment_id) + with conn: + conn.execute("UPDATE attachments SET transaction_id = NULL WHERE id = ?", (attachment_id,)) + return get_attachment(conn, attachment_id) diff --git a/src/static/index.html b/src/static/index.html index 7d0d15b..78f5a39 100644 --- a/src/static/index.html +++ b/src/static/index.html @@ -27,7 +27,13 @@ padding: 12px 20px; background: var(--panel); border-bottom: 1px solid var(--line); position: sticky; top: 0; z-index: 5; } header h1 { font-size: 17px; margin: 0 12px 0 0; letter-spacing: .2px; } - header .spacer { flex: 1; } + header { padding-top: 0; padding-bottom: 0; gap: 20px; } + header h1 { margin: 0; padding: 12px 0; } + header .tabs.primary { flex: 1; margin: 0; border: 0; align-self: stretch; } + header .tabs.primary button { padding: 0 14px; margin-bottom: -1px; } + .bookbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; padding: 8px 20px; + background: var(--bg); border-bottom: 1px solid var(--line); } + .bookbar[hidden] { display: none; } select, input, button { font: inherit; color: inherit; } select, input[type=text], input[type=number], input[type=date] { background: var(--panel); border: 1px solid var(--line); border-radius: 6px; padding: 5px 8px; } @@ -36,8 +42,8 @@ button:hover:not(:disabled) { border-color: var(--accent); } button.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); } button:disabled { opacity: .5; cursor: default; } - .layout { display: grid; grid-template-columns: 340px 1fr; gap: 16px; padding: 16px 20px; } - @media (max-width: 900px) { .layout { grid-template-columns: 1fr; padding: 12px 16px; } } + .layout { max-width: 1400px; margin: 0 auto; padding: 16px 20px; } + @media (max-width: 900px) { .layout { padding: 12px 16px; } } .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 14px; min-width: 0; } .panel h2 { font-size: 14px; margin: 0 0 10px; display: flex; align-items: center; gap: 8px; } .panel h2 .grow { flex: 1; } @@ -55,6 +61,12 @@ .tabs button { border-radius: 999px; padding: 3px 12px; } .tabs button.on { background: var(--accent-soft); border-color: var(--accent); color: var(--accent); } .tabs button.empty { color: var(--muted); } + .tabs .sep { width: 1px; background: var(--line); margin: 6px 8px; } + .tabs.primary { gap: 0; border-bottom: 1px solid var(--line); margin-bottom: 14px; } + .tabs.primary button { border: 0; border-bottom: 2px solid transparent; border-radius: 0; background: none; + padding: 8px 16px; font-size: 14px; color: var(--muted); } + .tabs.primary button.on { border-bottom-color: var(--accent); color: var(--accent); background: none; font-weight: 600; } + .tabs .count { background: var(--accent-soft); color: var(--accent); border-radius: 999px; padding: 0 7px; font-size: 12px; } .scroll { overflow-x: auto; } table { border-collapse: collapse; width: 100%; font-variant-numeric: tabular-nums; } th, td { padding: 6px 8px; border-bottom: 1px solid var(--line); text-align: left; white-space: nowrap; } @@ -130,20 +142,87 @@ .tax-table tr.tax-head td { font-weight: 700; padding-top: 14px; color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .4px; } .tax-input { width: 120px; text-align: right; padding: 3px 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 6px; } .row-toolbar { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; margin-bottom: 10px; } + .receipts { display: grid; gap: 12px; } + .receipt { display: grid; grid-template-columns: 120px 1fr; gap: 14px; border: 1px solid var(--line); border-radius: 10px; padding: 12px; } + @media (max-width: 600px) { .receipt { grid-template-columns: 1fr; } } + .receipt .thumb img { width: 120px; max-height: 170px; object-fit: cover; object-position: top; border-radius: 6px; border: 1px solid var(--line); display: block; } + .pages { display: flex; flex-wrap: wrap; gap: 4px; width: 120px; } + .page { position: relative; } + .receipt .thumb .pages.many img, .pages.many .pdf-thumb { width: 58px; height: 78px; max-height: none; font-size: 11px; } + .page .rm { position: absolute; top: 2px; right: 2px; padding: 0 5px; font-size: 11px; line-height: 16px; border-radius: 999px; opacity: .9; } + .add-page { width: 120px; margin-top: 6px; font-size: 12px; padding: 3px 6px; } + .pdf-thumb { display: flex; align-items: center; justify-content: center; width: 120px; height: 120px; border: 1px solid var(--line); + border-radius: 6px; color: var(--out); font-weight: 700; text-decoration: none; } .receipt-head { display: flex; gap: 8px; align-items: baseline; flex-wrap: wrap; margin-bottom: 8px; } .receipt-head .grow { flex: 1; min-width: 0; word-break: break-all; } .receipt-fields { display: grid; grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); gap: 6px 10px; } .receipt-fields label, dialog .receipt-fields label { font-size: 12px; color: var(--muted); display: flex; flex-direction: column; gap: 2px; margin: 0; } .receipt-fields input, .receipt-fields select { width: 100%; } .receipt-fields .wide { grid-column: span 2; } + .receipt details { margin-top: 8px; } + .receipt details summary { cursor: pointer; color: var(--muted); font-size: 12px; } + .receipt pre { white-space: pre-wrap; font-size: 12px; max-height: 240px; overflow: auto; background: var(--row-hover); padding: 8px; border-radius: 6px; } + .match { margin-top: 10px; padding: 8px 10px; border-radius: 8px; background: var(--row-hover); } + .match ul { list-style: none; margin: 4px 0 0; padding: 0; } + .match li { display: flex; gap: 8px; align-items: center; padding: 4px 0; flex-wrap: wrap; } + .match li .grow { flex: 1; min-width: 160px; } + .chip.linked { background: var(--ok-soft); color: var(--in); } + .chip.failed { background: var(--warn-soft); color: var(--warn); } + button.clip { padding: 1px 6px; font-size: 12px; opacity: .4; } + button.clip.has { opacity: 1; border-color: var(--accent); color: var(--accent); } + tr:hover button.clip { opacity: 1; } dialog.wide { width: min(780px, 94vw); } dialog .check { display: flex; gap: 8px; align-items: center; margin: 8px 0; font-size: 14px; color: var(--text); } dialog .check input { width: auto; } + + @media (max-width: 700px) { + body { font-size: 15px; } + header { padding: 10px 12px; gap: 8px; } + header h1 { font-size: 16px; margin-right: 4px; } + header .spacer { display: none; } + #bookInfo { flex-basis: 100%; order: 10; } + header h1 { flex-basis: 100%; margin: 0; padding: 0; } + header .tabs.primary { flex-basis: 100%; order: 5; margin: 0 -12px; padding: 0 12px; } + header .tabs.primary button { padding: 10px 14px; } + .bookbar { padding: 8px 12px; } + #book { flex: 1; min-width: 120px; } + .layout { padding: 10px 12px; gap: 12px; } + .panel { padding: 12px; } + /* 16px inputs stop iOS zooming on focus; larger tap targets */ + select, input[type=text], input[type=number], input[type=date], input[type=password], textarea { font-size: 16px; padding: 8px 10px; } + button, .menu summary { padding: 8px 14px; } + .tabs { flex-wrap: nowrap; overflow-x: auto; margin: 0 -12px 12px; padding: 0 12px 4px; scrollbar-width: none; } + .tabs::-webkit-scrollbar { display: none; } + .tabs button { flex: none; padding: 6px 14px; white-space: nowrap; } + .statements { max-height: 40vh; } + .stat { flex: 1 1 calc(50% - 10px); min-width: 0; } + .menu-items { min-width: 0; width: max-content; max-width: calc(100vw - 24px); } + .menu-items.right { right: 0; } + .menu-items a { padding: 10px 12px; } + .icon-btns { max-width: none; } + .tax-input { width: 100px; } + .receipt-fields .wide { grid-column: 1 / -1; } + .row-toolbar > * { flex: 1 1 auto; } + dialog { padding: 14px; max-height: 90vh; } + #toast { bottom: 12px; width: calc(100vw - 24px); text-align: center; } + .inv-lines input[type=text] { min-width: 160px; } + }
-

Cashbook

+

Accounts

+ + +
+
- - - +
- +
@@ -220,6 +278,44 @@ + +
+

Receipts and invoices

+ + +
+
+
+
+ + +
+

Update the transaction?

+
+
+
+ + +
+
+
+ + +
+

Add files

+
Are they photos of one receipt (e.g. the top and bottom of a long one) or pages of one invoice, or a different receipt each?
+
+ + + +
+
+
+ + + + +
Drop ANZ statements to add them
@@ -278,7 +374,6 @@ function localStorageGet(k) { try { return localStorage.getItem(k); } catch { re function localStorageSet(k, v) { try { localStorage.setItem(k, v); } catch {} } async function loadSummary() { - updatePreviewBtn(); if (!state.book) { state.summary = null; render(); return; } state.summary = await call(`/api/summary?book=${encodeURIComponent(state.book)}`); const s = state.summary; @@ -299,24 +394,6 @@ async function loadStatements() { state.statements = await call(`/api/statements${state.book ? "?book=" + encodeURIComponent(state.book) : ""}`); const known = new Set(state.statements.map(s => s.path)); for (const p of [...state.selected]) if (!known.has(p)) state.selected.delete(p); - renderStatements(); -} -function renderStatements() { - const ul = $("#statements"); - if (!state.statements.length) { ul.innerHTML = `
  • No statements found. Use “Add…”.
  • `; return; } - let html = "", folder = null; - const order = [...new Set(state.statements.map(s => s.folder))]; // server lists folders in priority order - const sorted = [...state.statements].sort((a, b) => (order.indexOf(a.folder) - order.indexOf(b.folder)) || ((b.first || "").localeCompare(a.first || ""))); - for (const s of sorted) { - if (s.folder !== folder) { folder = s.folder; html += `
  • ${esc(folder)}
  • `; } - const id = "st" + btoa(unescape(encodeURIComponent(s.path))).replace(/[^a-z0-9]/gi, ""); - const desc = s.error ? `Can't read: ${esc(s.error)}` - : `${s.count} transactions · ${fmtDate(s.first)} – ${fmtDate(s.last)}${s.periods.length > 1 ? ` · ${s.periods.length} statements` : ""}`; - html += `
  • -
  • `; - } - ul.innerHTML = html; - updatePreviewBtn(); } function importChip(s) { if (s.error || s.imported == null || !state.book) return ""; @@ -363,7 +440,6 @@ async function deleteStatement(path) { if (state.preview?.single && state.preview.statements[0] === path) { state.preview = null; state.view = "statements"; } await loadStatements(); render(); toast(`Removed ${st.name}`); } -function updatePreviewBtn() { $("#previewBtn").disabled = !state.selected.size || !state.book; } // ---- import ---- async function preview() { @@ -388,12 +464,14 @@ async function doImport() { // ---- rendering ---- function render() { const main = $("#main"); - if (state.view === "statements") { main.innerHTML = (state.summary ? tabsHtml() : "") + renderStatementsPage(); return; } - const pages = { invoices: renderInvoices, customers: renderCustomers, mileage: renderMileage }; - if (pages[state.view]) { main.innerHTML = (state.summary ? tabsHtml() : "") + pages[state.view](); return; } - if (state.view === "preview" && state.preview && !state.book) { main.innerHTML = renderPreview(); return; } + $("#nav").innerHTML = navHtml(); + $("#bookbar").hidden = !BOOK_VIEWS.includes(state.view); + const pages = { statements: renderStatementsPage, receipts: renderReceipts, invoices: renderInvoices, + customers: renderCustomers, suppliers: renderSuppliers, mileage: renderMileage }; + if (pages[state.view]) { main.innerHTML = tabsHtml() + pages[state.view](); return; } + if (state.view === "preview" && state.preview && !state.book) { main.innerHTML = tabsHtml() + renderPreview(); return; } if (!state.book) { - main.innerHTML = `
    No cashbook yet.

    `; + main.innerHTML = tabsHtml() + `
    No cashbook yet.

    `; return; } if (!state.summary) { main.innerHTML = ""; return; } @@ -404,16 +482,27 @@ function render() { else if (state.view === "month" && state.month) main.innerHTML = tabs + renderMonth(); else main.innerHTML = tabs + renderSummary(); } +// the cashbook (year, GST return, income tax and the months) is one tab with its own row beneath +const BOOK_VIEWS = ["summary", "gst", "tax", "month"]; +function navHtml() { + const inBook = BOOK_VIEWS.includes(state.view); + const tab = (view, label, on = state.view === view) => ``; + return `${tab("summary", "Cashbook", inBook)} + + ${tab("receipts", `Receipts${unmatchedCount() ? ` ${unmatchedCount()}` : ""}`)} + ${tab("invoices", "Invoices")}${tab("customers", "Customers")}${tab("suppliers", "Suppliers")} + + ${tab("mileage", "Mileage")}${tab("statements", "Statements")} + ${state.preview ? tab("preview", state.preview.single ? "Statement" : "Import preview") : ""}`; +} +// the row under the top bar: the cashbook's own sections and months function tabsHtml() { + if (!BOOK_VIEWS.includes(state.view) || !state.summary) return ""; + const tab = (view, label) => ``; return `
    - - - + ${tab("summary", "Year")}${tab("gst", "GST return")}${tab("tax", "Income tax")} + ${state.summary.months.map(m => ``).join("")} - - ${["invoices", "customers", "mileage"].map(v => ``).join("")} - - ${state.preview ? `` : ""}
    `; } @@ -617,15 +706,17 @@ function renderMonth() { ${m.rows.length ? `
    - + ${rows.map(r => ` - `).join("") || ``} - + + `).join("") || ``} +
    DateReferenceOriginInOutGSTGST amtCategory
    DateReferenceOriginInOutGSTGST amtCategory
    ${fmtDate(r.date)}${esc(r.reference)}${esc(r.origin)} ${money(r.income)}${money(r.expense)} ${money(r.gst_in || r.gst_out)}${catSelect(r)}
    Everything this month is allocated.
    ${m.rows.length} transactions${fmt(income)}${fmt(expense)}
    ${catSelect(r)}
    Everything this month is allocated.
    ${m.rows.length} transactions${fmt(income)}${fmt(expense)}
    ` : `
    No transactions this month.
    `}`; } @@ -661,6 +752,232 @@ function renderPreview() { `; } +// ---- receipts and invoices ---- +const unmatchedCount = () => (state.receipts || []).filter(a => !a.transaction).length; +async function loadReceipts(show) { + [state.receipts, state.suppliers] = await Promise.all([call("/api/attachments"), call("/api/suppliers")]); + if (show) state.view = "receipts"; + if (show || state.view === "receipts" || state.summary) render(); +} +// Upload receipt files: each as its own receipt, or all as pages of one (asPages), or as more pages +// of the existing receipt `into`. Returns the receipts as they are after the uploads. +async function uploadReceipts(files, { row = null, asPages = false, into = null } = {}) { + const added = [], errors = []; + let target = into; + for (const f of files) { + busyCount++; $("#busy").classList.add("on"); + try { + const name = `filename=${encodeURIComponent(f.name)}`; + const url = target ? `/api/attachments/${target}/pages?${name}` : `/api/attachments?${name}${row ? `&row=${row}` : ""}`; + const res = await fetch(url, { method: "POST", body: f }); + if (res.status === 401) { signInAgain(); break; } + const data = await res.json().catch(() => ({})); + if (res.ok) { + if (target && added.length) added[added.length - 1] = data; else added.push(data); + if (asPages || into) target = data.id; + if (data.ocr_error) errors.push(`${f.name} was saved, but ${data.ocr_error}. Fill in its details by hand.`); + } else errors.push(`${f.name}: ${errorText(data, res)}`); + } finally { if (--busyCount === 0) $("#busy").classList.remove("on"); } + } + const n = files.length - errors.filter(e => !e.includes(" was saved, ")).length; + if (errors.length) toast(errors.join(" · "), true); + else if (into) toast(`Added ${plural(n, "page")}`); + else if (added.length) toast((asPages ? `Added a receipt with ${plural(n, "page")}` : `Added ${plural(added.length, "file")}`) + + (row ? "" : added.some(a => a.suggestions.length) ? ". Check the suggested matches." : "")); + return added; +} +// several files at once: pages of one receipt, or a receipt each? (null = cancelled) +function askPages(n) { + $("#pagesTitle").textContent = `Add ${n} files`; + $("#pagesOne").textContent = `One receipt, ${n} pages`; + $("#pagesSeparate").textContent = `${n} separate receipts`; + const d = $("#pagesDialog"); + return new Promise(resolve => { + d.addEventListener("close", () => resolve(d.returnValue === "cancel" || !d.returnValue ? null : d.returnValue === "one"), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +// after linking, offer the GST type / category the receipt suggests; returns what was applied +function askApply(a) { + const p = a.proposed || {}, keys = Object.keys(p); + if (!keys.length) return Promise.resolve(null); + const t = a.transaction, label = { gst: "GST", category: "Category" }; + $("#applySub").textContent = `${fmtDate(t.date)} · ${t.reference} · ${fmt(t.amount)}`; + $("#applyBody").innerHTML = keys.map(k => ``).join("") + + `
    ${p.gst ? "The receipt shows GST, so it can be claimed. " : ""}You can change these later on the month's sheet.
    `; + const d = $("#applyDialog"); + return new Promise(resolve => { + d.addEventListener("close", () => resolve(d.returnValue === "ok" + ? Object.fromEntries([...d.querySelectorAll("[data-apply]:checked")].map(c => [c.dataset.apply, p[c.dataset.apply]])) : null), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +async function confirmChanges(a) { + const apply = await askApply(a); + if (apply && Object.keys(apply).length) { + a = await call(`/api/attachments/${a.id}/link`, { row: a.transaction.row, apply }); + toast("Transaction updated"); + } + return a; +} +async function refreshAfterChange() { + const jobs = [loadReceipts()]; + if (state.book) jobs.push(call(`/api/summary?book=${encodeURIComponent(state.book)}`).then(s => state.summary = s)); + if (state.view === "month" && state.sheet) + jobs.push(call(`/api/month?book=${encodeURIComponent(state.book)}&sheet=${state.sheet}`).then(m => state.month = m)); + await Promise.all(jobs); + render(); + if ($("#attDialog").open) await renderRowAttachments(); +} +async function gotoTransaction(book, sheet) { + $("#attDialog").close(); + if (book !== state.book) { state.book = book; $("#book").value = book; localStorageSet("book", book); updateExportLinks(); await loadSummary(); } + await loadMonth(sheet); +} +// buttons on receipt cards, in the Receipts view and the row dialog +async function receiptAction(b) { + const id = Number(b.dataset.id); + if (b.dataset.act === "att-link") { + const a = await call(`/api/attachments/${id}/link`, { row: Number(b.dataset.row) }); + toast(`Linked to ${a.transaction.reference}`); + await confirmChanges(a); + } else if (b.dataset.act === "att-unlink") { + await call(`/api/attachments/${id}/unlink`, {}); + } else if (b.dataset.act === "att-delete") { + const a = state.receipts.find(x => x.id === id) || state.rowAtt?.list.find(x => x.id === id); + const pages = a?.files.length || 1; + if (!confirm(`Delete ${a?.vendor || a?.filename || "this receipt"}${pages > 1 ? ` and its ${pages} pages` : ""}? ` + + `The ${pages > 1 ? "files are" : "file is"} removed from data/attachments.`)) return true; + await call(`/api/attachments/${id}`, undefined, "DELETE"); + } else if (b.dataset.act === "att-addpage") { + state.pageTarget = id; $("#pageInput").click(); return true; + } else if (b.dataset.act === "att-rmpage") { + if (!confirm("Remove this page? Its file is removed from data/attachments.")) return true; + await call(`/api/attachments/${id}/pages/${b.dataset.file}`, undefined, "DELETE"); + } else if (b.dataset.act === "att-reread") { + if (!confirm("Read the details again from the pages? Anything you corrected by hand is replaced.")) return true; + await call(`/api/attachments/${id}/reread`, {}); + } else if (b.dataset.act === "att-goto") { + await gotoTransaction(b.dataset.book, b.dataset.sheet); return true; + } else return false; + await refreshAfterChange(); + return true; +} +async function saveReceiptField(input) { + const f = input.dataset.attField, raw = input.value.trim(); + let value = ["total", "gst"].includes(f) ? (raw === "" ? null : Number(raw)) : (raw || null); + if (f === "supplier_id") { + if (raw === "new") { + const a = (state.receipts || []).find(x => x.id === Number(input.dataset.id)) || state.rowAtt?.list.find(x => x.id === Number(input.dataset.id)); + await askNewSupplier(a); + return refreshAfterChange(); + } else value = raw ? Number(raw) : null; + } + const focus = document.activeElement?.dataset; + const where = focus?.attField ? `[data-id="${focus.id}"][data-att-field="${focus.attField}"]` : null; + await call(`/api/attachments/${input.dataset.id}`, { [f]: value }, "PATCH"); + await refreshAfterChange(); + if (where) document.querySelector(where)?.focus(); // keep the cursor where the user tabbed to +} +function receiptCard(a, inDialog) { + const t = a.transaction, many = a.files.length > 1; + const page = (f, i) => { + const url = `/api/attachments/${a.id}/files/${f.id}`, title = many ? `Page ${i + 1}: ${f.filename}` : f.filename; + return `
    ${f.mime.startsWith("image/") + ? `${esc(title)}` + : `PDF`} + ${many ? `` : ""}
    `; + }; + const thumb = `
    ${a.files.map(page).join("")}
    + `; + const size = a.files.reduce((n, f) => n + f.size, 0); + const input = (k, label, val, type = "text", cls = "") => + ``; + const status = a.ocr_status === "done" ? "" : a.ocr_status === "failed" + ? `text not read` : ""; + let match; + if (t) { + match = `
    linked + + ${fmt(t.amount)} ${esc(t.book)} · ${esc(t.gst)}${t.category ? " · " + esc(t.category) : ""} +
    `; + } else if (a.suggestions.length) { + match = `
    Matching bank transactions
      ${a.suggestions.map(s => `
    • + ${fmtDate(s.date)} · ${esc(s.reference)} ${fmt(s.amount)} + ${esc(s.book)}${s.days_apart != null ? ` · ${s.days_apart ? plural(s.days_apart, "day") + " apart" : "same day"}` : ""}${s.attachments ? ` · already has ${plural(s.attachments, "file")}` : ""} +
    • `).join("")}
    `; + } else { + match = `
    ${a.total == null ? "Enter the total to find the bank transaction." : + `No bank transaction of ${fmt(a.total)} within 7 days${a.doc_date ? " of " + fmtDate(a.doc_date) : ""}. Check the total and date, or import the statement first.`}
    `; + } + return `
    +
    ${thumb}
    +
    +
    ${esc(a.vendor || a.filename)} + ${many ? `${a.files.length} pages` : esc(a.filename)} · ${fmtSize(size)} ${status} + ${inDialog ? `` : ""} +
    +
    + ${input("vendor", "Supplier", a.vendor, "text", "wide")} + ${input("doc_date", "Date", a.doc_date, "date")} + ${input("total", "Total (incl GST)", a.total?.toFixed(2), "number")} + ${input("gst", "GST", a.gst?.toFixed(2), "number")} + ${input("gst_number", "GST number", a.gst_number)} + + +
    + ${a.items.length ? `
    ${plural(a.items.length, "line item")} + ${a.items.map(i => ``).join("")}
    ${esc(i.description)}${fmt(i.amount)}
    ` : ""} + ${a.raw_text ? `
    Text read from the ${many ? "pages" : a.files[0].mime === "application/pdf" ? "PDF" : "photo"}
    ${esc(a.raw_text)}
    +
    ` : ""} + ${inDialog ? "" : match} +
    `; +} +function renderReceipts() { + const all = state.receipts || []; + const filter = state.receiptFilter || "unmatched"; + const sup = state.receiptSupplier || ""; + const list = (filter === "all" ? all : all.filter(a => !a.transaction)).filter(a => !sup || String(a.supplier_id) === sup); + return ` +
    Drop receipt photos or invoices here or + ·
    + JPEG, PNG or PDF. A long receipt can be several photos: add them together, or use “+ Add page” on it later. + The supplier, date, total and GST are read from the pages on this server, then matched to your bank transactions.
    +
    + Receipts and invoices +
    + + +
    + +
    + ${list.length ? `
    ${list.map(a => receiptCard(a)).join("")}
    ` + : `
    ${all.length ? "Every receipt is linked to a transaction." : "No receipts or invoices yet."}
    `}`; +} +// the attachments of one transaction, from the 📎 button on a month's sheet +async function openRowAttachments(row) { + state.rowAtt = { row: state.month.rows.find(r => r.row === row), list: [] }; + await renderRowAttachments(); + if (!$("#attDialog").open) $("#attDialog").showModal(); +} +async function renderRowAttachments() { + const r = state.rowAtt.row; + state.rowAtt.list = await call(`/api/row/${r.row}/attachments`); + $("#attSub").textContent = `${fmtDate(r.date)} · ${r.reference} · ${fmt(r.income || -r.expense)}`; + const list = state.rowAtt.list; + const unlinked = (state.receipts || []).filter(a => !a.transaction && a.suggestions.some(s => s.row === r.row)); + $("#attBody").innerHTML = (list.length ? list.map(a => receiptCard(a, true)).join("") + : `
    Nothing attached yet. Add a photo of the receipt or the invoice PDF.
    `) + + (unlinked.length ? `
    Unlinked receipts that match this transaction
      ${unlinked.map(a => `
    • + ${esc(a.vendor || a.filename)} ${a.doc_date ? fmtDate(a.doc_date) + " · " : ""}${fmt(a.total)} +
    • `).join("")}
    ` : ""); +} + // ---- new book ---- function openNewBook() { const sel = state.statements.filter(s => state.selected.has(s.path) && s.first); @@ -699,14 +1016,6 @@ document.addEventListener("click", e => { const menu = $("#exportMenu"); if (menu.open && (!menu.contains(e.target) || e.target.closest("a"))) menu.open = false; }); -$("#statements").addEventListener("change", e => { - const p = e.target.dataset.path; if (!p) return; - e.target.checked ? state.selected.add(p) : state.selected.delete(p); updatePreviewBtn(); -}); -$("#clearSel").addEventListener("click", () => { state.selected.clear(); renderStatements(); if (state.view === "statements") render(); }); -$("#manageBtn").addEventListener("click", () => { state.view = "statements"; render(); }); -$("#previewBtn").addEventListener("click", guard(preview)); -$("#uploadBtn").addEventListener("click", () => $("#uploadInput").click()); $("#uploadInput").addEventListener("change", guard(async e => { const files = [...e.target.files]; e.target.value = ""; await uploadFiles(files); @@ -714,21 +1023,65 @@ $("#uploadInput").addEventListener("change", guard(async e => { // drag and drop anywhere on the page let dragDepth = 0; const hasFiles = e => [...(e.dataTransfer?.types || [])].includes("Files"); -window.addEventListener("dragenter", e => { if (hasFiles(e)) { dragDepth++; document.body.classList.add("dragging"); } }); +const dropsReceipts = () => state.view === "receipts" || $("#attDialog").open; +window.addEventListener("dragenter", e => { + if (!hasFiles(e)) return; + dragDepth++; document.body.classList.add("dragging"); + $("#dropOverlay").textContent = dropsReceipts() ? "Drop receipts or invoices to add them" : "Drop ANZ statements to add them"; +}); window.addEventListener("dragleave", e => { if (hasFiles(e) && --dragDepth <= 0) { dragDepth = 0; document.body.classList.remove("dragging"); } }); window.addEventListener("dragover", e => { if (hasFiles(e)) e.preventDefault(); }); window.addEventListener("drop", guard(async e => { if (!hasFiles(e)) return; e.preventDefault(); dragDepth = 0; document.body.classList.remove("dragging"); + if (dropsReceipts()) return addReceipts([...e.dataTransfer.files], $("#attDialog").open ? state.rowAtt.row.row : null); await uploadFiles([...e.dataTransfer.files]); })); -$("#main").addEventListener("click", guard(async e => { +// receipts: from the Receipts view (row = null) or a transaction's 📎 dialog +async function addReceipts(files, row) { + if (!files.length) return; + const asPages = files.length > 1 ? await askPages(files.length) : false; + if (asPages === null) return; + const added = await uploadReceipts(files, { row, asPages }); + if (!added.length) return; + await refreshAfterChange(); + const last = added[added.length - 1]; + if (row && await confirmChanges(last) !== last) await refreshAfterChange(); + await offerSuppliers(added); + await refreshAfterChange(); +} +for (const id of ["#receiptInput", "#cameraInput"]) + $(id).addEventListener("change", guard(async e => { + const files = [...e.target.files]; e.target.value = ""; + await addReceipts(files, $("#attDialog").open ? state.rowAtt.row.row : null); + })); +$("#pageInput").addEventListener("change", guard(async e => { + const files = [...e.target.files]; e.target.value = ""; + if (!files.length || !state.pageTarget) return; + const added = await uploadReceipts(files, { into: state.pageTarget }); + if (added.length) await refreshAfterChange(); +})); +$("#attAdd").addEventListener("click", () => $("#receiptInput").click()); +$("#attClose").addEventListener("click", () => $("#attDialog").close()); +$("#attDialog").addEventListener("click", guard(async e => { + const b = e.target.closest("[data-act]"); + if (b && !b.disabled) await receiptAction(b); +})); +$("#attDialog").addEventListener("change", guard(async e => { if (e.target.dataset.attField) await saveReceiptField(e.target); })); +const navClick = guard(async e => { const copy = e.target.closest("[data-copy]"); if (copy) { await navigator.clipboard?.writeText(copy.dataset.copy).catch(() => {}); return toast(`Copied ${copy.dataset.copy}`); } - const b = e.target.closest("[data-sheet],[data-view],[data-act],[data-inv],[data-mi],#importBtn,#cancelPreview,#chooseFiles,#previewBtn2"); + const b = e.target.closest("[data-sheet],[data-view],[data-act],[data-receipts],[data-inv],[data-mi],#importBtn,#cancelPreview,#chooseFiles,#previewBtn2,#chooseReceipts,#takePhoto"); if (!b || b.disabled) return; if (b.id === "chooseFiles") return $("#uploadInput").click(); + if (b.id === "chooseReceipts") return $("#receiptInput").click(); + if (b.id === "takePhoto") return $("#cameraInput").click(); + if (b.dataset.receipts) { state.receiptFilter = b.dataset.receipts; return render(); } + if (b.dataset.act === "row-att") return openRowAttachments(Number(b.dataset.row)); + if (b.dataset.act?.startsWith("att-")) return receiptAction(b); + if (b.dataset.view === "receipts") return loadReceipts(true); if (b.dataset.view === "customers") return loadCustomers(); + if (b.dataset.view === "suppliers") return loadSuppliers(); if (b.dataset.inv) return invoiceAction(b); if (b.dataset.mi) return mileageAction(b); if (b.dataset.view === "invoices") return loadInvoices(); @@ -743,16 +1096,20 @@ $("#main").addEventListener("click", guard(async e => { if (b.dataset.view === "tax") return loadTax(); if (b.dataset.view) { state.view = b.dataset.view; return render(); } if (b.dataset.sheet) return loadMonth(b.dataset.sheet); -})); +}); +$("#main").addEventListener("click", navClick); +$("#nav").addEventListener("click", navClick); $("#main").addEventListener("change", guard(async e => { + if (e.target.id === "receiptSupplier") { state.receiptSupplier = e.target.value; return render(); } if (e.target.id === "invoiceCustomer") { state.invoiceCustomer = e.target.value; return render(); } if (e.target.id === "onlyUnalloc") { state.onlyUnalloc = e.target.checked; return render(); } if (e.target.dataset.taxBox) return saveTaxInput(e.target); if (e.target.dataset.mi) return mileageChange(e.target); + if (e.target.dataset.attField) return saveReceiptField(e.target); if (e.target.id === "gstFreq") { state.gstFreq = e.target.value; localStorageSet("gstFreq", state.gstFreq); return loadGst(); } if (e.target.dataset.select) { e.target.checked ? state.selected.add(e.target.dataset.select) : state.selected.delete(e.target.dataset.select); - renderStatements(); return render(); + return render(); } const { row, field } = e.target.dataset; if (!row) return; state.month = await call("/api/row", { book: state.book, sheet: state.sheet, row: Number(row), [field]: e.target.value }); @@ -796,10 +1153,11 @@ document.addEventListener("click", e => { guard(async () => { await loadMe(); await loadBooks(); - await loadStatements(); + await Promise.all([loadStatements(), loadReceipts()]); })(); + diff --git a/src/static/suppliers.js b/src/static/suppliers.js new file mode 100644 index 0000000..2ae62bb --- /dev/null +++ b/src/static/suppliers.js @@ -0,0 +1,96 @@ +// ---- suppliers: who receipts and bills are from ---- +// Uses $, state, call, esc, plural, fmt, guard and render from index.html. +document.body.insertAdjacentHTML("beforeend", ` + +
    +

    New supplier?

    +
    + + + + +
    + + +
    +
    +
    `); + +// Offer to make a supplier from a receipt or invoice that matched none; links it if the user agrees. +// Returns the updated attachment, or null if declined. +function askNewSupplier(a) { + $("#newSupSub").textContent = `${a.kind === "invoice" ? "Invoice" : "Receipt"} ${a.filename} isn't linked to a supplier. Add one so its receipts and bills are kept together.`; + $("#newSupName").value = a.vendor || ""; $("#newSupGst").value = a.gst_number || ""; + const d = $("#newSupDialog"); + return new Promise(resolve => { + d.addEventListener("close", guard(async () => { + if (d.returnValue !== "ok") return resolve(null); + const sup = await call("/api/suppliers", { name: $("#newSupName").value, gst_number: $("#newSupGst").value }); + const linked = await call(`/api/attachments/${a.id}`, { supplier_id: sup.id }, "PATCH"); + toast(`Added supplier ${sup.name}`); + resolve(linked); + }), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +// after an upload: ask for each document with no supplier; one answer covers the others from the same vendor +async function offerSuppliers(added) { + const same = (x, y) => (x.gst_number && x.gst_number === y.gst_number) || + (x.vendor && y.vendor && x.vendor.trim().toLowerCase() === y.vendor.trim().toLowerCase()); + const done = new Set(); + for (const a of added) { + if (a.supplier_id != null || done.has(a.id)) continue; + const linked = await askNewSupplier(a); + if (!linked) continue; + for (const b of added) + if (b.id !== a.id && b.supplier_id == null && same(a, b)) { + await call(`/api/attachments/${b.id}`, { supplier_id: linked.supplier_id }, "PATCH"); done.add(b.id); + } + } +} + +async function loadSuppliers() { + state.suppliers = await call("/api/suppliers"); + state.view = "suppliers"; render(); +} + +function renderSuppliers() { + const list = state.suppliers || []; + const cell = (s, k) => ``; + return ` +
    Suppliers
    + ${list.length ? `
    + ${list.map(s => `${["name", "gst_number", "email"].map(k => cell(s, k)).join("")} + + + `).join("")} +
    NameGST numberEmailAddressSpent
    ${s.documents ? `
    ${plural(s.documents, "receipt")}
    ` : ""}
    ${s.documents ? "" : ``}
    +
    Changes are saved as you leave each box. Receipts and bills are linked to a supplier automatically when the GST number or name matches; otherwise pick one on the receipt.
    ` + : `
    No suppliers yet. Add one below, or choose “New supplier” on a receipt.
    `} +
    + + + + + +
    `; +} + +$("#main").addEventListener("change", guard(async e => { + const k = e.target.dataset.supField; if (!k) return; + const s = state.suppliers.find(x => x.id === Number(e.target.dataset.id)); + await call(`/api/suppliers/${s.id}`, { name: s.name, email: s.email, address: s.address, gst_number: s.gst_number, [k]: e.target.value }, "PUT"); + state.suppliers = await call("/api/suppliers"); + toast("Saved"); +})); +$("#main").addEventListener("click", guard(async e => { + const del = e.target.dataset.supDelete, show = e.target.dataset.supShow; + if (show) { state.receiptSupplier = show; state.receiptFilter = "all"; await loadReceipts(true); return; } + if (del) { await call(`/api/suppliers/${del}`, undefined, "DELETE"); await loadSuppliers(); } +})); +$("#main").addEventListener("submit", guard(async e => { + if (e.target.id !== "supAdd") return; + e.preventDefault(); + await call("/api/suppliers", { name: $("#supName").value, gst_number: $("#supGst").value, email: $("#supEmail").value, address: $("#supAddress").value }); + await loadSuppliers(); +})); diff --git a/src/suppliers.py b/src/suppliers.py new file mode 100644 index 0000000..6469843 --- /dev/null +++ b/src/suppliers.py @@ -0,0 +1,60 @@ +"""Suppliers: who receipts and bills are from. A document is linked to a supplier by id +(attachments.supplier_id); uploads are linked automatically when the document's GST number or +name matches a supplier (see ledger.find_supplier).""" +import ledger + +SUPPLIER_FIELDS = ("name", "email", "address", "gst_number") + + +def _supplier(conn, s): + d = dict(s) + row = conn.execute("SELECT COUNT(*) AS n, COALESCE(SUM(total_cents), 0) AS total FROM attachments " + "WHERE supplier_id = ?", (s["id"],)).fetchone() + d["documents"], d["total"] = row["n"], row["total"] / 100 + return d + + +def list_suppliers(conn): + return [_supplier(conn, s) for s in conn.execute("SELECT * FROM suppliers ORDER BY name COLLATE NOCASE")] + + +def get_supplier(conn, supplier_id): + s = conn.execute("SELECT * FROM suppliers WHERE id = ?", (supplier_id,)).fetchone() + if s is None: + raise ledger.NotFound("no such supplier") + return _supplier(conn, s) + + +def _fields(fields): + out = {} + for k, v in fields.items(): + if k not in SUPPLIER_FIELDS: + raise ValueError(f"can't change {k}") + out[k] = (v or "").strip() + if "name" in out and not out["name"]: + raise ValueError("the supplier needs a name") + return out + + +def create_supplier(conn, name, email="", address="", gst_number=""): + f = _fields({"name": name, "email": email, "address": address, "gst_number": gst_number}) + with conn: + cur = conn.execute("INSERT INTO suppliers (name, email, address, gst_number) VALUES (?, ?, ?, ?)", + (f["name"], f["email"], f["address"], f["gst_number"])) + return get_supplier(conn, cur.lastrowid) + + +def update_supplier(conn, supplier_id, **fields): + get_supplier(conn, supplier_id) + for k, v in _fields(fields).items(): + with conn: + conn.execute(f"UPDATE suppliers SET {k} = ? WHERE id = ?", (v, supplier_id)) + return get_supplier(conn, supplier_id) + + +def delete_supplier(conn, supplier_id): + get_supplier(conn, supplier_id) + if conn.execute("SELECT 1 FROM attachments WHERE supplier_id = ?", (supplier_id,)).fetchone(): + raise ValueError("this supplier has receipts or bills, so it can't be deleted") + with conn: + conn.execute("DELETE FROM suppliers WHERE id = ?", (supplier_id,)) diff --git a/tests/test_receipts.py b/tests/test_receipts.py new file mode 100644 index 0000000..cffaf35 --- /dev/null +++ b/tests/test_receipts.py @@ -0,0 +1,321 @@ +import os +import shutil +import sqlite3 + +import pytest + +import ledger +from cashbook_app.services import receipts +from cashbook_app.services.receipts import parse_receipt +from conftest import SAMPLE_CSV + +TILL_RECEIPT = """ + BUNNINGS WAREHOUSE + Mt Wellington +TAX INVOICE GST No: 123-456-789 +Date: 14/05/2026 Time 10:42 +2 x Timber 90x45 24.50 +Screws Galv 8g 12,99 +Paint Brush 50mm 9.90 +SUBTOTAL 47.39 +TOTAL $47.39 +GST INCLUDED 6.18 +EFTPOS $47.39 +CHANGE 0.00 +""" + +INVOICE = """Acme Consulting Ltd +Invoice INV-0042 +Invoice Date: 3 Jun 2026 +Due Date: 20 Jun 2026 +GST Reg 98-765-432 +Web design services 1,000.00 +Subtotal (excl GST) 1,000.00 +GST 15% 150.00 +Total NZD 1,150.00 +""" + + +def test_parse_till_receipt(): + r = parse_receipt(TILL_RECEIPT) + assert r["vendor"] == "BUNNINGS WAREHOUSE" + assert r["date"] == "2026-05-14" + assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (4739, 618, "123-456-789") + assert r["items"] == [{"description": "Timber 90x45", "amount": 24.5}, + {"description": "Screws Galv 8g", "amount": 12.99}, # OCR read the point as a comma + {"description": "Paint Brush 50mm", "amount": 9.9}] + + +def test_parse_invoice(): + r = parse_receipt(INVOICE) + assert r["vendor"] == "Acme Consulting Ltd" + assert r["date"] == "2026-06-03" # the invoice date, not the due date + assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (115000, 15000, "098-765-432") + assert r["items"] == [{"description": "Web design services", "amount": 1000.0}] + + +def test_parse_fallbacks(): + # no "total" line: the card payment is the total; GST far from 3/23 of it is ignored + r = parse_receipt("Corner Dairy\n12 Mar 2026\nMilk 3.50\nBread 4.20\nEFTPOS 7.70\nGST 5.00\n") + assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-03-12", 770, None) + assert parse_receipt("")["total_cents"] is None + # OCR noise inside the GST number + assert parse_receipt("TAX INVOICE GST No: 45-6/78-912\nTOTAL 10.00")["gst_number"] == "045-678-912" + assert parse_receipt("GST 150.00 150.00\nTOTAL 1150.00")["gst_cents"] == 15000 + + +def png(seed: bytes) -> bytes: + return b"\x89PNG\r\n\x1a\n" + seed * 20 # only the header is checked; OCR is faked below + + +@pytest.fixture +def books(admin, monkeypatch): + """A book with SAMPLE_CSV imported, and OCR replaced by whatever `fake_ocr` holds.""" + admin.post("/api/books", json={"name": "b", "year": 2026}) + path = admin.post("/api/upload?filename=s.csv", content=SAMPLE_CSV.encode()).json()["path"] + admin.post("/api/import", json={"book": "b", "statements": [path]}) + admin.fake_ocr = {"status": "done", "text": "", "items": []} + monkeypatch.setattr(receipts, "read_document", lambda p, mime: dict(admin.fake_ocr)) + return admin + + +def rows(client, sheet): + return client.get(f"/api/month?book=b&sheet={sheet}").json()["rows"] + + +def test_upload_suggest_and_link(books, tmp_path): + books.fake_ocr.update(vendor="Jaycar Electronics", date="2026-04-12", total_cents=11500, gst_cents=1500, + items=[{"description": "Soldering iron", "amount": 115.0}]) + r = books.post("/api/attachments?filename=jaycar.png", content=png(b"j")) + assert r.status_code == 200, r.text + a = r.json() + assert (a["vendor"], a["total"], a["gst"], a["transaction"]) == ("Jaycar Electronics", 115.0, 15.0, None) + assert a["items"][0]["description"] == "Soldering iron" + [s] = a["suggestions"] + assert (s["reference"], s["amount"], s["days_apart"], s["sheet"]) == ("Jaycar Chris", -115.0, 1, "APR") + + # the same file again is refused, and its copy isn't kept + assert books.post("/api/attachments?filename=again.png", content=png(b"j")).status_code == 400 + assert len(os.listdir(tmp_path / "attachments")) == 1 + + unmatched = books.get("/api/attachments?unmatched=1").json() + assert [x["id"] for x in unmatched] == [a["id"]] + + linked = books.post(f"/api/attachments/{a['id']}/link", json={"row": s["row"]}).json() + assert linked["transaction"]["reference"] == "Jaycar Chris" + assert linked["proposed"] == {} # already Incl GST, and nothing to go on for a category + assert books.get("/api/attachments?unmatched=1").json() == [] + assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 1 + assert [x["id"] for x in books.get(f"/api/row/{s['row']}/attachments").json()] == [a["id"]] + + f = books.get(f"/api/attachments/{a['id']}/file") + assert f.status_code == 200 and f.content == png(b"j") and f.headers["content-type"] == "image/png" + + books.post(f"/api/attachments/{a['id']}/unlink") + assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 0 + + assert books.delete(f"/api/attachments/{a['id']}").json() == {"ok": True} + assert os.listdir(tmp_path / "attachments") == [] + assert books.get(f"/api/attachments/{a['id']}").status_code == 404 + + +def test_link_offers_gst_and_category(books): + hetzner = next(r for r in rows(books, "MAY") if "Hetzner" in r["reference"]) + assert hetzner["gst"] == "Excl GST" + # uploaded from the transaction's row: linked straight away, with changes to offer + books.fake_ocr.update(vendor="Hetzner Online", total_cents=553, gst_cents=72, gst_number="123-456-789") + a = books.post(f"/api/attachments?filename=h.pdf&kind=invoice&row={hetzner['row']}", + content=b"%PDF-1.4 hetzner").json() + assert (a["kind"], a["files"][0]["mime"], a["transaction"]["row"]) == ("invoice", "application/pdf", hetzner["row"]) + assert a["proposed"] == {"gst": "Incl GST"} + + a = books.post(f"/api/attachments/{a['id']}/link", + json={"row": hetzner["row"], "apply": {"gst": "Incl GST", "category": "Purchases"}}).json() + assert a["proposed"] == {} + hetzner = next(r for r in rows(books, "MAY") if r["row"] == hetzner["row"]) + assert (hetzner["gst"], hetzner["category"], hetzner["attachments"]) == ("Incl GST", "Purchases", 1) + + r = books.post(f"/api/attachments/{a['id']}/link", json={"row": hetzner["row"], "apply": {"amount": "1"}}) + assert r.status_code == 400 + + +def test_category_from_vendor_and_history(books, tmp_path): + con = sqlite3.connect(tmp_path / "cashbook.db") + with con: + con.execute("UPDATE transactions SET category = NULL") + fee = next(r for r in rows(books, "APR") if r["reference"] == "Monthly A/C Fee") + books.fake_ocr.update(vendor="Spark NZ", total_cents=850) + a = books.post(f"/api/attachments?filename=spark.png&row={fee['row']}", content=png(b"s")).json() + assert a["proposed"] == {"category": "Telephone & Power"} + + +def test_edit_details_updates_suggestions(books): + books.fake_ocr.update(status="failed", error="tesseract isn't installed") + r = books.post("/api/attachments?filename=blurry.jpg", content=b"\xff\xd8\xff" + b"x" * 50).json() + assert (r["ocr_status"], r["ocr_error"], r["suggestions"]) == ("failed", "tesseract isn't installed", []) + + a = books.patch(f"/api/attachments/{r['id']}", json={"total": 1150, "doc_date": "2026-04-21", + "vendor": "Client Ltd"}).json() + assert (a["total_cents"], a["vendor"]) == (115000, "Client Ltd") + assert [s["reference"] for s in a["suggestions"]] == ["Client Ltd (Inv 101)"] + # too far from the transaction's date + a = books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "2026-05-21"}).json() + assert a["suggestions"] == [] + assert books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "21/05/2026"}).status_code == 400 + assert books.patch(f"/api/attachments/{r['id']}", json={"kind": "bill"}).status_code == 400 + + +def test_several_photos_of_one_receipt(books, tmp_path, monkeypatch): + # a long receipt in two photos: the top has the shop and date, the bottom the total + pages = {b"top": "JAYCAR ELECTRONICS\nDate: 12/04/2026\nSoldering Station 89.00\n", + b"bottom": "Solder 26.00\nTOTAL $115.00\nGST INCLUDED 15.00\nGST No: 45-678-912\n"} + def read(p, mime): + text = pages[p.read_bytes()[8:]] + return {"status": "done", "text": text, **parse_receipt(text)} + monkeypatch.setattr(receipts, "read_document", read) + + def photo(name): + return b"\x89PNG\r\n\x1a\n" + name + + top = books.post("/api/attachments?filename=top.png", content=photo(b"top")).json() + # with no TOTAL line, the top alone gives a wrong guess at the total + assert (top["total"], top["suggestions"], len(top["files"])) == (89.0, [], 1) + books.patch(f"/api/attachments/{top['id']}", json={"vendor": "Jaycar"}) # a correction by hand + + a = books.post(f"/api/attachments/{top['id']}/pages?filename=bottom.png", content=photo(b"bottom")).json() + assert [f["filename"] for f in a["files"]] == ["top.png", "bottom.png"] + # the details come from both pages read together, except the correction, which is kept + assert (a["vendor"], a["doc_date"], a["total"], a["gst"], a["gst_number"], a["edited"]) == \ + ("Jaycar", "2026-04-12", 115.0, 15.0, "045-678-912", ["vendor"]) + assert [i["description"] for i in a["items"]] == ["Soldering Station", "Solder"] + assert a["raw_text"].startswith("JAYCAR") and "TOTAL" in a["raw_text"] + assert [s["reference"] for s in a["suggestions"]] == ["Jaycar Chris"] + assert len(books.get("/api/attachments").json()) == 1 + + bottom_id = a["files"][1]["id"] + f = books.get(f"/api/attachments/{a['id']}/files/{bottom_id}") + assert f.status_code == 200 and f.content == photo(b"bottom") + assert books.get(f"/api/attachments/{a['id']}/file").content == photo(b"top") + assert books.get(f"/api/attachments/999/files/{bottom_id}").status_code == 404 + + # the same photo can't go on twice, here or on another receipt + r = books.post(f"/api/attachments/{a['id']}/pages?filename=again.png", content=photo(b"bottom")) + assert r.status_code == 400 + assert books.post(f"/api/attachments/999/pages?filename=x.png", content=photo(b"x")).status_code == 404 + assert len(os.listdir(tmp_path / "attachments")) == 2 + + a = books.post(f"/api/attachments/{a['id']}/reread").json() + assert (a["vendor"], a["edited"]) == ("JAYCAR ELECTRONICS", []) # re-reading replaces corrections + + # without the bottom, it's back to what the top says + books.patch(f"/api/attachments/{a['id']}", json={"gst": 15}) + a = books.delete(f"/api/attachments/{a['id']}/pages/{bottom_id}").json() + assert ([f["filename"] for f in a["files"]], a["total"], a["gst"], a["gst_number"]) == \ + (["top.png"], 89.0, 15.0, None) + assert len(os.listdir(tmp_path / "attachments")) == 1 + r = books.delete(f"/api/attachments/{a['id']}/pages/{a['files'][0]['id']}") + assert r.status_code == 400 and "only page" in r.json()["detail"] + + books.post(f"/api/attachments/{a['id']}/pages?filename=bottom.png", content=photo(b"bottom")) + books.delete(f"/api/attachments/{a['id']}") + assert os.listdir(tmp_path / "attachments") == [] + + +def test_rejects_other_files(books, tmp_path): + r = books.post("/api/attachments?filename=a.txt", content=b"hello") + assert r.status_code == 400 and "JPEG" in r.json()["detail"] + r = books.post("/api/attachments?filename=a.heic", content=b"\0\0\0\x18ftypheic" + b"\0" * 20) + assert r.status_code == 400 and "HEIC" in r.json()["detail"] + assert books.post("/api/attachments?filename=a.png", content=b"").status_code == 400 + assert books.post("/api/attachments?filename=a.png&row=9999", content=png(b"z")).status_code == 404 + assert os.listdir(tmp_path / "attachments") == [] + + +def test_needs_sign_in(client): + assert client.get("/api/attachments").status_code == 401 + assert client.post("/api/attachments?filename=a.png", content=png(b"a")).status_code == 401 + + +def test_upgrade_keeps_attachment_reference(tmp_path): + """The v1 -> v2 table rebuild mustn't leave attachments pointing at the old table.""" + path = tmp_path / "old.db" + con = sqlite3.connect(path) + con.executescript(ledger.SCHEMA.replace("'Zero-rated', ", "")) + con.close() + conn = ledger.connect(path) + sql = conn.execute("SELECT sql FROM sqlite_master WHERE name = 'attachments'").fetchone()[0] + assert "transactions_v1" not in sql + assert "Zero-rated" in conn.execute("SELECT sql FROM sqlite_master WHERE name = 'transactions'").fetchone()[0] + + +V2_ATTACHMENTS = """CREATE TABLE attachments ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE SET NULL, + kind TEXT NOT NULL DEFAULT 'receipt' CHECK (kind IN ('receipt', 'invoice')), + filename TEXT NOT NULL, stored_name TEXT NOT NULL UNIQUE, mime TEXT NOT NULL, + sha256 TEXT NOT NULL UNIQUE, size INTEGER NOT NULL, + vendor TEXT, doc_date TEXT, total_cents INTEGER, gst_cents INTEGER, gst_number TEXT, + items_json TEXT, raw_text TEXT, + ocr_status TEXT NOT NULL DEFAULT 'skipped' CHECK (ocr_status IN ('done', 'failed', 'skipped')), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE INDEX attachments_transaction ON attachments (transaction_id);""" + + +def test_upgrade_moves_files_to_pages(tmp_path): + """Version 2 databases had one file per attachment, in the attachments table.""" + path = tmp_path / "v2.db" + con = sqlite3.connect(path) + con.executescript("""CREATE TABLE books (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, + start_year INTEGER NOT NULL, opening_cents INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + CREATE TABLE transactions (id INTEGER PRIMARY KEY, + book_id INTEGER NOT NULL REFERENCES books(id) ON DELETE CASCADE, date TEXT NOT NULL, + reference TEXT NOT NULL DEFAULT '', origin TEXT NOT NULL DEFAULT '', amount_cents INTEGER NOT NULL, + gst TEXT NOT NULL CHECK (gst IN ('Incl GST', 'Zero-rated', 'Excl GST')), category TEXT, source TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);""" + V2_ATTACHMENTS + """ + INSERT INTO books (id, name, start_year) VALUES (1, 'b', 2026); + INSERT INTO transactions (id, book_id, date, reference, amount_cents, gst) VALUES (5, 1, '2026-04-13', 'Jaycar', -11500, 'Incl GST'); + INSERT INTO attachments (id, transaction_id, filename, stored_name, mime, sha256, size, vendor, total_cents, + items_json, raw_text, ocr_status) VALUES (7, 5, 'r.jpg', 'abc-1.jpg', 'image/jpeg', 'abc', 10, 'Jaycar', + 11500, '[]', 'JAYCAR TOTAL 115.00', 'done'); + PRAGMA user_version = 2;""") + con.close() + + conn = ledger.connect(path) + assert conn.execute("PRAGMA user_version").fetchone()[0] == ledger.SCHEMA_VERSION + a = ledger.get_attachment(conn, 7) + assert a["edited"] == [] + assert (a["transaction"]["row"], a["vendor"], a["total"], a["raw_text"], a["filename"]) == \ + (5, "Jaycar", 115.0, "JAYCAR TOTAL 115.00", "r.jpg") + assert [(f["filename"], f["mime"], f["size"]) for f in a["files"]] == [("r.jpg", "image/jpeg", 10)] + assert ledger.get_file(conn, 7)["stored_name"] == "abc-1.jpg" + assert conn.execute("SELECT name FROM sqlite_master WHERE name = 'attachments_transaction'").fetchone() + assert "attachments_v2" not in conn.execute( + "SELECT sql FROM sqlite_master WHERE name = 'attachment_files'").fetchone()[0] + assert (tmp_path / "v2.db.v2.bak").exists() + # pages go with their attachment, and the link still works both ways + conn.execute("DELETE FROM transactions WHERE id = 5") + assert ledger.get_attachment(conn, 7)["transaction_id"] is None + ledger.delete_attachment(conn, 7) + assert conn.execute("SELECT COUNT(*) FROM attachment_files").fetchone()[0] == 0 + conn.close() + ledger.connect(path).close() # and connecting again changes nothing + + +@pytest.mark.skipif(not shutil.which("tesseract"), reason="tesseract isn't installed") +def test_ocr_round_trip(): + from PIL import Image, ImageDraw, ImageFont + img = Image.new("RGB", (900, 700), "white") + draw = ImageDraw.Draw(img) + try: + font = ImageFont.truetype("DejaVuSansMono.ttf", 34) + except OSError: + font = ImageFont.load_default(size=34) + for i, line in enumerate(TILL_RECEIPT.strip().splitlines()): + draw.text((40, 30 + i * 50), line.strip(), fill="black", font=font) + import io + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=90) + r = parse_receipt(receipts.ocr_image(buf.getvalue())) + assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-05-14", 4739, 618) + assert "BUNNINGS" in r["vendor"] diff --git a/tests/test_suppliers.py b/tests/test_suppliers.py new file mode 100644 index 0000000..538271d --- /dev/null +++ b/tests/test_suppliers.py @@ -0,0 +1,84 @@ +import sqlite3 + +import pytest + +import ledger +from cashbook_app.services import receipts +from test_receipts import png + + +@pytest.fixture +def ocr(admin, monkeypatch): + """OCR replaced by whatever `fake_ocr` holds.""" + admin.fake_ocr = {"status": "done", "text": "", "items": []} + monkeypatch.setattr(receipts, "read_document", lambda p, mime: dict(admin.fake_ocr)) + return admin + + +def test_supplier_crud(admin): + s = admin.post("/api/suppliers", json={"name": " Bunnings ", "gst_number": "123-456-789"}).json() + assert (s["name"], s["gst_number"], s["documents"], s["total"]) == ("Bunnings", "123-456-789", 0, 0) + assert admin.post("/api/suppliers", json={"name": ""}).status_code == 422 + s = admin.put(f"/api/suppliers/{s['id']}", json={"name": "Bunnings Ltd", "email": "a@b.nz"}).json() + assert (s["name"], s["email"], s["gst_number"]) == ("Bunnings Ltd", "a@b.nz", "") + assert [x["name"] for x in admin.get("/api/suppliers").json()] == ["Bunnings Ltd"] + assert admin.delete(f"/api/suppliers/{s['id']}").json() == {"ok": True} + assert admin.get(f"/api/suppliers/{s['id']}").status_code == 404 + + +def test_receipts_link_to_suppliers(ocr): + sup = ocr.post("/api/suppliers", json={"name": "Jaycar Electronics Ltd", "gst_number": "098-765-432"}).json() + + # the same name (ignoring Ltd and case) links on upload + ocr.fake_ocr.update(vendor="JAYCAR ELECTRONICS", total_cents=11500) + a = ocr.post("/api/attachments?filename=a.png", content=png(b"a")).json() + assert a["supplier"] == {"id": sup["id"], "name": "Jaycar Electronics Ltd"} + + # so does the GST number, whatever the name says + ocr.fake_ocr.update(vendor="J'car", gst_number="098-765-432", total_cents=500) + b = ocr.post("/api/attachments?filename=b.png", content=png(b"b")).json() + assert b["supplier_id"] == sup["id"] + + # an unknown vendor is left for the user + ocr.fake_ocr.update(vendor="Somewhere Else", gst_number=None) + c = ocr.post("/api/attachments?filename=c.png", content=png(b"c")).json() + assert c["supplier"] is None + + s = ocr.get(f"/api/suppliers/{sup['id']}").json() + assert (s["documents"], s["total"]) == (2, 120.0) + assert {x["id"] for x in ocr.get(f"/api/attachments?supplier={sup['id']}").json()} == {a["id"], b["id"]} + + # link by hand, and unlink + c = ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": sup["id"]}).json() + assert c["supplier"]["name"] == "Jaycar Electronics Ltd" + assert c["vendor"] == "Somewhere Else" # what was read is kept + assert ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": None}).json()["supplier"] is None + assert ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": 999}).status_code == 404 + + # a supplier with documents can't be deleted until they're unlinked + assert ocr.delete(f"/api/suppliers/{sup['id']}").status_code == 400 + + +def test_customer_invoices_filter(admin): + admin.post("/api/books", json={"name": "b", "year": 2026}) + ids = [admin.post("/api/customers", json={"name": n}).json()["id"] for n in ("A", "B")] + for cid in (ids[0], ids[1], ids[1]): + r = admin.post("/api/invoices", json={"customer_id": cid, "issue_date": "2026-04-10", + "lines": [{"description": "x", "unit": 10}]}) + assert r.status_code == 200, r.text + assert len(admin.get(f"/api/invoices?customer={ids[1]}").json()) == 2 + assert len(admin.get("/api/invoices").json()) == 3 + + +def test_upgrade_adds_supplier_link(tmp_path): + """Databases from before suppliers get the new column and keep their receipts.""" + path = tmp_path / "old.db" + con = sqlite3.connect(path) + con.executescript(ledger.SCHEMA.replace(" supplier_id INTEGER REFERENCES suppliers(id), -- who the receipt or bill is from\n", "")) + con.execute("INSERT INTO attachments (vendor) VALUES ('Old receipt')") + con.execute("PRAGMA user_version = 4") + con.commit() + con.close() + conn = ledger.connect(path) + assert "supplier_id" in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")} + assert ledger.list_attachments(conn)[0]["supplier"] is None