diff --git a/.claude/launch.json b/.claude/launch.json index a7e66c2..2976145 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -6,6 +6,15 @@ "runtimeExecutable": "pipenv", "runtimeArgs": ["run", "dev"], "port": 8765 + }, + { + "name": "cashbook-scratch", + "runtimeExecutable": "env", + "runtimeArgs": ["CASHBOOK_DATA=/tmp/claude-1000/-home-jimmy-projects-accounts/1e7ee5e8-8915-44fb-a3a5-3f2db7ea9ee9/scratchpad/data", + "HOME=/tmp/claude-1000/-home-jimmy-projects-accounts/1e7ee5e8-8915-44fb-a3a5-3f2db7ea9ee9/scratchpad/home", + "AUTH_ENABLED=false", ".venv/bin/uvicorn", "app:app", "--app-dir", "src", + "--host", "127.0.0.1", "--port", "8766"], + "port": 8766 } ] } diff --git a/Dockerfile b/Dockerfile index 377bb51..dafe275 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ FROM debian:bookworm-slim # LibreOffice (headless Calc) + its Python bridge for spreadsheet export, -# pdftotext for PDF statements +# pdftotext for PDF statements, tesseract + Pillow to read receipt photos RUN apt-get update \ && apt-get install -y --no-install-recommends \ python3 python3-venv python3-uno libreoffice-calc-nogui poppler-utils fonts-dejavu-core \ + tesseract-ocr tesseract-ocr-eng python3-pil \ && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/README.md b/README.md index a0d9712..2f1f9a6 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ administrators manage who has access. - **Export** a book as an Excel (.xlsx) or LibreOffice (.ods) cashbook spreadsheet, with formulas, GST and category columns and the summary sheet, or as a CSV of all transactions. +- **Receipts and invoices:** attach photos of receipts and PDF invoices to + transactions. The supplier, date, total, GST and line items are read from them on + your own server (Tesseract OCR), and they're matched to bank transactions by amount + and date (see [Receipts and invoices](#receipts-and-invoices)). - **Accounts:** sign-in with personal accounts. Administrators invite people, reset passwords and manage who's an administrator (see [Accounts](#accounts)). @@ -54,7 +58,8 @@ administrator account. Stop it with `docker compose down`, and check it with **What the compose file sets up** - **Data:** `./data` is mounted at `/data`. It holds the database (`cashbook.db`: - books and accounts) and `statements/`, where uploads go. Back up this folder. + books and accounts), `statements/`, where uploads go, and `attachments/`, the + receipt and invoice files. Back up this folder. - **Network:** published on `127.0.0.1:8765` only. - **Lockdown:** - read-only container filesystem, with scratch space only in `/tmp` (tmpfs); @@ -111,13 +116,16 @@ services: ### Without Docker -Needs Python 3, `pdftotext` for PDF statements, and LibreOffice's UNO bridge for -spreadsheet export and migration. On Debian/Ubuntu: +Needs Python 3, `pdftotext` for PDF statements, LibreOffice's UNO bridge for +spreadsheet export and migration, and Tesseract and Pillow to read receipts. On +Debian/Ubuntu: ```bash -sudo apt install libreoffice-calc python3-uno poppler-utils pipenv +sudo apt install libreoffice-calc python3-uno poppler-utils tesseract-ocr python3-pil pipenv ``` +Without Tesseract, receipts can still be attached, but their details have to be typed in. + The virtualenv needs the system site packages, so it can see LibreOffice's `uno`: ```bash @@ -251,6 +259,35 @@ authority, partnership or trust income, and no taxable property sales. Entered amounts are saved per book in the database. These figures are worked out from your records and aren't tax advice. +### Receipts and invoices + +Keep the paperwork behind each transaction, such as the tax invoices you need to claim GST. + +- **From a transaction:** click 📎 on a row of a month's sheet and add a photo or PDF. + It's linked to that transaction straight away. +- **In bulk:** open **Receipts**, then drop files, choose them, or **take a photo** on a + phone. Each one is read, then listed with the bank transactions it probably belongs to: + the same amount, within 7 days of the receipt's date, across all books. Click **Link** + to confirm one. Receipts can wait unlinked until their statement is imported. +- **Reading the details:** photos are straightened, evened out and read with Tesseract. + PDFs use their text, or are OCR'd if they're scans. The supplier, date, total, GST, + GST number and line items are filled in. Correct any of them in place, and the + suggested matches update. +- **Several pages:** a long receipt can take more than one photo, and an invoice can be + more than one PDF. Add several files at once and choose **One receipt**, or use + **+ Add page** on a receipt later. The pages are read together and the details are + updated, except anything you corrected by hand. *Read the details again* (under the + text read from the pages) also replaces your corrections. Remove a page with its ×. +- **Updating the transaction:** when a receipt is linked, the app offers to set the + transaction to *Incl GST* if the receipt shows GST. It also suggests a category from + earlier transactions with the same reference, or from the supplier's name. Only what + you tick is changed. + +JPEG, PNG, WebP and PDF files up to 20 MB are accepted. HEIC photos aren't: set an +iPhone's camera to *Most Compatible* or share the photo as JPEG. Files are stored in +`data/attachments`. Deleting a receipt removes its files. Deleting a transaction or book +leaves its receipts in the list, unlinked. + ### Sending invoices **Invoices** makes the invoices you send to customers, as PDFs. @@ -342,8 +379,8 @@ src/ main.py app setup, security middleware, startup config.py settings from the environment / .env deps.py current user, admin check, database connections - routers/ auth, users, books, statements, invoices, mileage and page routes - services/ accounts, statement files, invoice PDFs, LibreOffice + routers/ auth, users, books, statements, attachments, invoices, mileage and page routes + services/ accounts, statement files, receipt OCR, invoice PDFs, LibreOffice db/, schemas/ account tables (SQLAlchemy) and request/response models static/ the web pages (cashbook, sign-in, register, users), invoices.js, mileage.js ledger.py SQLite storage: books, transactions, GST and summaries @@ -359,7 +396,7 @@ src/ anz_import.py imports statements (CLI) template.ods blank cashbook template, used for exports tests/ pytest suite (pipenv run test) -data/ cashbook.db and statements (git-ignored) +data/ cashbook.db, statements and attachments (git-ignored) ``` `data/` is in `.gitignore`: books, backups and bank statements are never committed. diff --git a/src/cashbook_app/main.py b/src/cashbook_app/main.py index 4bf9696..8d3f065 100644 --- a/src/cashbook_app/main.py +++ b/src/cashbook_app/main.py @@ -11,12 +11,14 @@ import ledger from cashbook_app import config from cashbook_app.db.models import Base from cashbook_app.db.session import get_engine, get_session_factory +from cashbook_app.routers.attachments import router as attachments_router from cashbook_app.routers.auth_routes import router as auth_router from cashbook_app.routers.books import router as books_router from cashbook_app.routers.frontend import router as frontend_router from cashbook_app.routers.invoices import router as invoices_router from cashbook_app.routers.mileage import router as mileage_router from cashbook_app.routers.statements import router as statements_router +from cashbook_app.routers.suppliers import router as suppliers_router from cashbook_app.routers.users_admin import router as users_router from cashbook_app.services import accounts, spreadsheets @@ -121,8 +123,10 @@ def create_app() -> FastAPI: app.include_router(users_router) app.include_router(books_router) app.include_router(statements_router) + app.include_router(attachments_router) app.include_router(invoices_router) app.include_router(mileage_router) + app.include_router(suppliers_router) return app diff --git a/src/cashbook_app/routers/attachments.py b/src/cashbook_app/routers/attachments.py new file mode 100644 index 0000000..b69bb15 --- /dev/null +++ b/src/cashbook_app/routers/attachments.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from urllib.parse import quote + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field + +import ledger +from cashbook_app.deps import get_ledger, require_user +from cashbook_app.services import receipts +from cashbook_app.services.receipts import AttachmentError + +router = APIRouter(prefix="/api", tags=["attachments"], dependencies=[Depends(require_user)]) + + +class LinkRequest(BaseModel): + row: int + # None: just link, and return the GST/category the receipt suggests for the transaction + apply: dict[str, str] | None = None + + +class AttachmentUpdate(BaseModel): + kind: str | None = None + vendor: str | None = Field(None, max_length=200) + doc_date: str | None = None + total: float | None = None + gst: float | None = None + gst_number: str | None = Field(None, max_length=20) + supplier_id: int | None = None + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except ledger.NotFound as e: + raise HTTPException(status_code=404, detail=str(e)) + except (ValueError, AttachmentError) as e: + raise HTTPException(status_code=400, detail=str(e)) + + +def _with_suggestions(conn, a): + a = {**a, "suggestions": ledger.suggest_transactions(conn, a)} + if a["transaction_id"] is not None: + a["proposed"] = ledger.propose_changes(conn, a, a["transaction_id"]) + return a + + +async def _receive(request: Request, filename: str, conn): + """Store an uploaded photo/PDF and read it; returns (file details, OCR result).""" + if int(request.headers.get("content-length") or 0) > receipts.MAX_UPLOAD: + raise HTTPException(status_code=413, detail="the file is too large (20 MB at most)") + data = await request.body() + saved = _call(receipts.save_upload, filename, data) + if ledger.file_exists(conn, saved["sha256"]): + receipts.remove_file(saved["stored_name"]) + raise HTTPException(status_code=400, detail=f"{saved['filename']} has already been added") + ocr = await run_in_threadpool(receipts.read_document, receipts.file_path(saved["stored_name"]), saved["mime"]) + return saved, ocr + + +def _keep(saved, fn, *args, **kwargs): + """Record the stored file with fn, removing the file again if that fails.""" + try: + return _call(fn, *args, **kwargs) + except Exception: + receipts.remove_file(saved["stored_name"]) + raise + + +@router.post("/attachments") +async def upload(request: Request, filename: str, kind: str = "receipt", row: int | None = None, + conn=Depends(get_ledger)): + saved, ocr = await _receive(request, filename, conn) + a = _keep(saved, ledger.add_attachment, conn, file=saved, kind=kind, transaction_id=row, ocr=ocr) + return {**_with_suggestions(conn, a), "ocr_error": ocr.get("error")} + + +@router.post("/attachments/{attachment_id}/pages") +async def add_page(attachment_id: int, request: Request, filename: str, conn=Depends(get_ledger)): + """Another photo of the same receipt (e.g. the bottom half) or another page of an invoice. + The pages are read together, and details still missing are filled in.""" + _call(ledger.get_attachment, conn, attachment_id) + saved, ocr = await _receive(request, filename, conn) + _keep(saved, ledger.add_page, conn, attachment_id, saved, ocr.get("text")) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + a = ledger.apply_ocr(conn, attachment_id, found) + return {**_with_suggestions(conn, a), "ocr_error": ocr.get("error")} + + +@router.delete("/attachments/{attachment_id}/pages/{file_id}") +def remove_page(attachment_id: int, file_id: int, conn=Depends(get_ledger)): + receipts.remove_file(_call(ledger.remove_page, conn, attachment_id, file_id)) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + return _with_suggestions(conn, ledger.apply_ocr(conn, attachment_id, found)) + + +@router.post("/attachments/{attachment_id}/reread") +def reread(attachment_id: int, conn=Depends(get_ledger)): + """Replace the details with what's read from all the pages, undoing corrections.""" + _call(ledger.get_attachment, conn, attachment_id) + found = receipts.parse_receipt(ledger.attachment_text(conn, attachment_id)) + return _with_suggestions(conn, ledger.apply_ocr(conn, attachment_id, found, overwrite=True)) + + +@router.get("/attachments") +def list_attachments(unmatched: bool = False, supplier: int | None = None, conn=Depends(get_ledger)): + return [_with_suggestions(conn, a) for a in ledger.list_attachments(conn, unmatched, supplier)] + + +@router.get("/attachments/{attachment_id}") +def get_attachment(attachment_id: int, conn=Depends(get_ledger)): + return _with_suggestions(conn, _call(ledger.get_attachment, conn, attachment_id)) + + +def _send(f): + path = _call(receipts.file_path, f["stored_name"]) + if not path.exists(): + raise HTTPException(status_code=404, detail="the file is missing from data/attachments") + return FileResponse(path, media_type=f["mime"], headers={ + "Content-Disposition": f"inline; filename*=UTF-8''{quote(f['filename'])}"}) + + +@router.get("/attachments/{attachment_id}/file") +def download(attachment_id: int, conn=Depends(get_ledger)): + """The first page.""" + return _send(_call(ledger.get_file, conn, attachment_id)) + + +@router.get("/attachments/{attachment_id}/files/{file_id}") +def download_page(attachment_id: int, file_id: int, conn=Depends(get_ledger)): + return _send(_call(ledger.get_file, conn, attachment_id, file_id)) + + +@router.patch("/attachments/{attachment_id}") +def update(attachment_id: int, body: AttachmentUpdate, conn=Depends(get_ledger)): + fields = body.model_dump(exclude_unset=True) + for k in ("total", "gst"): + if k in fields: + v = fields.pop(k) + fields[f"{k}_cents"] = ledger.cents(abs(v)) if v is not None else None + a = _call(ledger.update_attachment, conn, attachment_id, **fields) + return _with_suggestions(conn, a) + + +@router.post("/attachments/{attachment_id}/link") +def link(attachment_id: int, body: LinkRequest, conn=Depends(get_ledger)): + apply = body.apply or {} + if set(apply) - {"gst", "category"}: + raise HTTPException(status_code=400, detail="only gst and category can be applied") + a = _call(ledger.link_attachment, conn, attachment_id, body.row, + gst=apply.get("gst"), category=apply.get("category")) + return _with_suggestions(conn, a) + + +@router.post("/attachments/{attachment_id}/unlink") +def unlink(attachment_id: int, conn=Depends(get_ledger)): + return _with_suggestions(conn, _call(ledger.unlink_attachment, conn, attachment_id)) + + +@router.delete("/attachments/{attachment_id}") +def delete(attachment_id: int, conn=Depends(get_ledger)): + for name in _call(ledger.delete_attachment, conn, attachment_id): + receipts.remove_file(name) + return {"ok": True} + + +@router.get("/row/{row}/attachments") +def row_attachments(row: int, conn=Depends(get_ledger)): + return [_with_suggestions(conn, a) for a in _call(ledger.attachments_for, conn, row)] diff --git a/src/cashbook_app/routers/suppliers.py b/src/cashbook_app/routers/suppliers.py new file mode 100644 index 0000000..1bcacb0 --- /dev/null +++ b/src/cashbook_app/routers/suppliers.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +import ledger +import suppliers +from cashbook_app.deps import get_ledger, require_user + +router = APIRouter(prefix="/api", tags=["suppliers"], dependencies=[Depends(require_user)]) + + +class Supplier(BaseModel): + name: str = Field(min_length=1, max_length=200) + email: str = Field("", max_length=200) + address: str = Field("", max_length=500) + gst_number: str = Field("", max_length=20) + + +def _call(fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except ledger.NotFound as e: + raise HTTPException(status_code=404, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("/suppliers") +def list_suppliers(conn=Depends(get_ledger)): + return suppliers.list_suppliers(conn) + + +@router.post("/suppliers") +def create_supplier(body: Supplier, conn=Depends(get_ledger)): + return _call(suppliers.create_supplier, conn, **body.model_dump()) + + +@router.get("/suppliers/{supplier_id}") +def get_supplier(supplier_id: int, conn=Depends(get_ledger)): + return _call(suppliers.get_supplier, conn, supplier_id) + + +@router.put("/suppliers/{supplier_id}") +def update_supplier(supplier_id: int, body: Supplier, conn=Depends(get_ledger)): + return _call(suppliers.update_supplier, conn, supplier_id, **body.model_dump()) + + +@router.delete("/suppliers/{supplier_id}") +def delete_supplier(supplier_id: int, conn=Depends(get_ledger)): + _call(suppliers.delete_supplier, conn, supplier_id) + return {"ok": True} diff --git a/src/cashbook_app/services/receipts.py b/src/cashbook_app/services/receipts.py new file mode 100644 index 0000000..26021d8 --- /dev/null +++ b/src/cashbook_app/services/receipts.py @@ -0,0 +1,297 @@ +"""Receipt and invoice files: storing uploads, and reading their details with local OCR. + +Images are cleaned up with Pillow (if installed) and read by the `tesseract` command; PDFs use +their text layer (`pdftotext`), or are rendered with `pdftoppm` and OCR'd when they're scans. +Nothing leaves the machine. `parse_receipt` turns the text into vendor, date, total, GST etc. +""" +from __future__ import annotations + +import hashlib +import io +import os +import re +import shutil +import subprocess +import tempfile +import uuid +from datetime import date +from pathlib import Path + +from cashbook_app import config + +MAX_UPLOAD = 20 * 1024 * 1024 +OCR_TIMEOUT = 90 # seconds per page +MIME = {".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", + ".pdf": "application/pdf"} +EXT_OF = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp", "application/pdf": ".pdf"} + + +class AttachmentError(Exception): + pass + + +def attachments_dir() -> Path: + path = config.data_dir() / "attachments" + path.mkdir(parents=True, exist_ok=True) + return path + + +def file_path(stored_name: str) -> Path: + # stored names are ours (hex + extension), but don't trust them blindly + if not re.fullmatch(r"[0-9a-f-]+\.(jpg|png|webp|pdf)", stored_name): + raise AttachmentError("bad file name") + return attachments_dir() / stored_name + + +def sniff(data: bytes) -> str | None: + """The file's real type from its first bytes.""" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + if data.startswith(b"%PDF"): + return "application/pdf" + return None + + +def save_upload(filename: str, data: bytes) -> dict: + """Write an uploaded receipt/invoice into data/attachments; returns its details.""" + if not data: + raise AttachmentError("the file is empty") + if len(data) > MAX_UPLOAD: + raise AttachmentError("the file is too large (20 MB at most)") + mime = sniff(data) + if mime is None: + if data[4:12] in (b"ftypheic", b"ftypheix", b"ftypmif1", b"ftyphevc"): + raise AttachmentError("HEIC photos aren't supported; set your camera to JPEG (Most Compatible) " + "or export the photo as JPEG") + raise AttachmentError("only JPEG, PNG or WebP photos and PDF files can be attached") + sha = hashlib.sha256(data).hexdigest() + stored = f"{sha[:16]}-{uuid.uuid4().hex[:8]}{EXT_OF[mime]}" + dest = attachments_dir() / stored + tmp = dest.with_name(f".upload-{uuid.uuid4().hex}") + tmp.write_bytes(data) + os.replace(tmp, dest) + name = os.path.basename(filename.replace("\\", "/")).strip() or f"receipt{EXT_OF[mime]}" + return {"filename": name[:200], "stored_name": stored, "mime": mime, "sha256": sha, "size": len(data)} + + +def remove_file(stored_name: str) -> None: + try: + file_path(stored_name).unlink(missing_ok=True) + except AttachmentError: + pass + + +# --- OCR -------------------------------------------------------------------- + +def _run(cmd: list[str], data: bytes | None = None) -> str: + res = subprocess.run(cmd, input=data, capture_output=True, timeout=OCR_TIMEOUT, + env={**os.environ, "OMP_THREAD_LIMIT": "1"}) + if res.returncode != 0: + err = res.stderr.decode(errors="replace").strip().splitlines() + raise AttachmentError(f"{cmd[0]}: {err[-1] if err else 'failed'}") + return res.stdout.decode("utf-8", errors="replace") + + +def prepare_image(data: bytes) -> bytes: + """Make a phone photo easier to read: upright, grey, a sensible size, even lighting, high contrast.""" + try: + from PIL import Image, ImageChops, ImageFilter, ImageOps + except ImportError: + return data + img = ImageOps.exif_transpose(Image.open(io.BytesIO(data))).convert("L") + longest = max(img.size) + if longest < 1600 or longest > 3600: + scale = 2400 / longest + img = img.resize((round(img.width * scale), round(img.height * scale)), Image.LANCZOS) + # flatten shadows and uneven light: subtract the text from a blurred copy of the page + background = img.filter(ImageFilter.GaussianBlur(radius=max(img.size) // 60)) + img = ImageOps.invert(ImageChops.subtract(background, img)) + img = ImageOps.autocontrast(img, cutoff=1).filter(ImageFilter.SHARPEN) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def ocr_image(data: bytes) -> str: + if not shutil.which("tesseract"): + raise AttachmentError("tesseract isn't installed, so the text couldn't be read") + # --psm 4: a single column of text of variable sizes, which suits till receipts + return _run(["tesseract", "stdin", "stdout", "-l", "eng", "--psm", "4"], prepare_image(data)) + + +def pdf_text(path: Path) -> str: + text = _run(["pdftotext", "-layout", str(path), "-"]) if shutil.which("pdftotext") else "" + if len(text.strip()) >= 20: + return text + # a scanned PDF: render the first pages and OCR them + if not shutil.which("pdftoppm"): + raise AttachmentError("pdftoppm isn't installed, so the scanned PDF couldn't be read") + with tempfile.TemporaryDirectory() as tmp: + _run(["pdftoppm", "-r", "300", "-png", "-f", "1", "-l", "3", str(path), os.path.join(tmp, "p")]) + return "\n".join(ocr_image(p.read_bytes()) for p in sorted(Path(tmp).glob("p*.png"))) + + +def read_document(path: Path, mime: str) -> dict: + """Text and details of a stored receipt; never raises, so an upload is kept even if OCR fails.""" + try: + text = pdf_text(path) if mime == "application/pdf" else ocr_image(path.read_bytes()) + except (AttachmentError, OSError, subprocess.TimeoutExpired) as e: + return {"status": "failed", "error": str(e) if not isinstance(e, subprocess.TimeoutExpired) + else "reading the text took too long"} + except Exception as e: # e.g. Pillow can't decode the image + return {"status": "failed", "error": f"couldn't read the file: {e}"} + return {"status": "done", "text": text, **parse_receipt(text)} + + +# --- parsing ---------------------------------------------------------------- + +# amounts with two decimals; OCR often reads the point as a comma +MONEY = re.compile(r"(? list[int]: + """Money amounts on a line, in cents.""" + out = [] + for m in MONEY.finditer(line): + whole = int(m.group(1).replace(",", "")) + value = whole * 100 + int(m.group(2)) + out.append(-value if m.group(0).lstrip().startswith("-") else value) + return out + + +def _to_date(kind: str, g: tuple) -> date | None: + try: + if kind == "ymd": + y, m, d = int(g[0]), int(g[1]), int(g[2]) + elif kind == "dmy": + d, m, y = int(g[0]), int(g[1]), int(g[2]) + elif kind == "dMy": + d, m, y = int(g[0]), MONTHS.index(g[1].lower()[:3]) + 1, int(g[2]) + else: + m, d, y = MONTHS.index(g[0].lower()[:3]) + 1, int(g[1]), int(g[2]) + if y < 100: + y += 2000 + found = date(y, m, d) + except ValueError: + return None + return found if date(2000, 1, 1) <= found <= date.today().replace(year=date.today().year + 1) else None + + +def find_date(lines: list[str]) -> str | None: + """The document's date: one on a line saying "date" first, skipping due dates.""" + candidates = [] + for i, line in enumerate(lines): + if re.search(r"\bdue\b|expir", line, re.I): + continue + for pattern, kind in DATES: + for m in pattern.finditer(line): + d = _to_date(kind, m.groups()) + if d: + candidates.append((0 if re.search(r"date", line, re.I) else 1, i, d)) + return min(candidates)[2].isoformat() if candidates else None + + +def find_total(lines: list[str]) -> int | None: + totals, paid = [], [] + for i, line in enumerate(lines): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + found = amounts(line) or (amounts(nxt) if not re.search(r"[a-z]{3}", nxt, re.I) else []) + found = [a for a in found if a > 0] + if not found: + continue + if TOTAL_WORDS.search(line) and not NOT_TOTAL.search(line): + totals.append(found[-1]) + elif PAID_WORDS.search(line): + paid.append(found[-1]) + if totals: + return max(totals) # e.g. "Total" beats a smaller "Total excl" the filter missed + if paid: + return max(paid) + every = [a for line in lines for a in amounts(line) if a > 0] + return max(every) if every else None + + +def find_gst(lines: list[str], total: int | None) -> int | None: + """The GST amount: of the amounts on lines mentioning GST, the one nearest 3/23 of the total.""" + found = [] + for line in lines: + if re.search(r"\bg\.?s\.?t\b", line, re.I) and not find_gst_number(line): + found += [a for a in amounts(line) if a > 0] + if not found: + return None + if total: + expected = total * 3 / 23 + plausible = [a for a in found if a <= total * 0.2] + return min(plausible, key=lambda a: abs(a - expected)) if plausible else None + return min(found) + + +def find_gst_number(text: str) -> str | None: + for m in GST_NUMBER.finditer(text): + digits = re.sub(r"\D", "", m.group(1)) + if len(digits) in (8, 9) and not re.search(r"\d\.\d\d(?!\d)", m.group(1)): # not amounts + digits = digits.zfill(9) + return f"{digits[:3]}-{digits[3:6]}-{digits[6:]}" + return None + + +def find_vendor(lines: list[str]) -> str | None: + for line in lines[:8]: + clean = re.sub(r"[^\w&'.,\- ]", " ", line) + clean = re.sub(r"\s+", " ", clean).strip(" .,-") + letters = sum(c.isalpha() for c in clean) + if letters < 3 or letters < len(clean.replace(" ", "")) * 0.6 or HEADER_WORDS.search(clean): + continue + return clean[:80] + return None + + +def find_items(lines: list[str]) -> list[dict]: + items = [] + for line in lines: + if TOTAL_WORDS.search(line) and not NOT_TOTAL.search(line): + break # items come before the total + found = amounts(line) + if not found or NOT_ITEM.search(line): + continue + first = MONEY.search(line) + desc = line[:first.start()].strip(" .:$-*") + desc = re.sub(r"^\d+\s*[xX@]\s*", "", desc) + if sum(c.isalpha() for c in desc) < 2: + continue + items.append({"description": re.sub(r"\s+", " ", desc)[:100], "amount": found[-1] / 100}) + return items[:50] + + +def parse_receipt(text: str) -> dict: + lines = [line.strip() for line in text.splitlines() if line.strip()] + total = find_total(lines) + return { + "vendor": find_vendor(lines), + "date": find_date(lines), + "total_cents": total, + "gst_cents": find_gst(lines, total), + "gst_number": find_gst_number(text), + "items": find_items(lines), + } diff --git a/src/ledger.py b/src/ledger.py index 73ffaa2..eb4eebd 100644 --- a/src/ledger.py +++ b/src/ledger.py @@ -3,8 +3,11 @@ Amounts are stored as signed integer cents (positive = money in). GST is worked out the way the cashbook spreadsheet does it: 3/23 of the amount for "Incl GST" rows. """ +import json +import re import sqlite3 from collections import Counter +from contextlib import closing from datetime import date from statements import ORIGIN, classify, describe @@ -47,6 +50,45 @@ CREATE TABLE IF NOT EXISTS transactions ( created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS transactions_book_date ON transactions (book_id, date); +CREATE TABLE IF NOT EXISTS suppliers ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT NOT NULL DEFAULT '', + address TEXT NOT NULL DEFAULT '', + gst_number TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS attachments ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE SET NULL, -- NULL = not matched yet + supplier_id INTEGER REFERENCES suppliers(id), -- who the receipt or bill is from + kind TEXT NOT NULL DEFAULT 'receipt' CHECK (kind IN ('receipt', 'invoice')), + -- read from the document's pages (OCR), and correctable by hand + vendor TEXT, + doc_date TEXT, -- ISO date + total_cents INTEGER, -- positive + gst_cents INTEGER, + gst_number TEXT, + items_json TEXT, -- [{"description", "amount"}] + edited_json TEXT NOT NULL DEFAULT '[]', -- fields corrected by hand, which re-reading leaves alone + ocr_status TEXT NOT NULL DEFAULT 'skipped' CHECK (ocr_status IN ('done', 'failed', 'skipped')), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS attachments_transaction ON attachments (transaction_id); +-- the photos or PDFs of an attachment, e.g. the two halves of a long receipt +CREATE TABLE IF NOT EXISTS attachment_files ( + id INTEGER PRIMARY KEY, + attachment_id INTEGER NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + position INTEGER NOT NULL, -- page order, from 0 + filename TEXT NOT NULL, -- name it was uploaded as + stored_name TEXT NOT NULL UNIQUE, -- file in data/attachments + mime TEXT NOT NULL, + sha256 TEXT NOT NULL UNIQUE, + size INTEGER NOT NULL, + text TEXT, -- what OCR read from it; NULL = couldn't be read + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS attachment_files_attachment ON attachment_files (attachment_id, position); CREATE TABLE IF NOT EXISTS business ( -- the one business invoices are sent from id INTEGER PRIMARY KEY CHECK (id = 1), name TEXT NOT NULL DEFAULT '', @@ -137,7 +179,7 @@ class NotFound(Exception): pass -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 5 def connect(path): @@ -158,12 +200,47 @@ def _upgrade(conn): if "Zero-rated" not in sql: # version 1 -> 2: allow 'Zero-rated'. SQLite can't change a CHECK, so rebuild the table. conn.execute("PRAGMA foreign_keys = OFF") + # keep attachments' reference pointing at "transactions", not the renamed copy + conn.execute("PRAGMA legacy_alter_table = ON") with conn: conn.execute("ALTER TABLE transactions RENAME TO transactions_v1") conn.executescript(SCHEMA) conn.execute("INSERT INTO transactions SELECT * FROM transactions_v1") conn.execute("DROP TABLE transactions_v1") + conn.execute("PRAGMA legacy_alter_table = OFF") conn.execute("PRAGMA foreign_keys = ON") + if "stored_name" in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 2 -> 3: an attachment's file moves to attachment_files, so it can have several + path = conn.execute("PRAGMA database_list").fetchone()["file"] + if path: # keep a copy from before the change + with closing(sqlite3.connect(path + ".v2.bak")) as dest: + conn.backup(dest) + conn.execute("PRAGMA foreign_keys = OFF") + conn.execute("PRAGMA legacy_alter_table = ON") + with conn: + conn.execute("ALTER TABLE attachments RENAME TO attachments_v2") + conn.execute("DROP INDEX attachments_transaction") + conn.executescript(SCHEMA) + conn.execute( + "INSERT INTO attachments (id, transaction_id, kind, vendor, doc_date, total_cents, gst_cents, " + "gst_number, items_json, ocr_status, created_at) SELECT id, transaction_id, kind, vendor, doc_date, " + "total_cents, gst_cents, gst_number, items_json, ocr_status, created_at FROM attachments_v2") + conn.execute( + "INSERT INTO attachment_files (attachment_id, position, filename, stored_name, mime, sha256, size, " + "text, created_at) SELECT id, 0, filename, stored_name, mime, sha256, size, raw_text, created_at " + "FROM attachments_v2") + conn.execute("DROP TABLE attachments_v2") + conn.execute("PRAGMA legacy_alter_table = OFF") + conn.execute("PRAGMA foreign_keys = ON") + if "edited_json" not in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 3 -> 4: remember which details were corrected by hand + with conn: + conn.execute("ALTER TABLE attachments ADD COLUMN edited_json TEXT NOT NULL DEFAULT '[]'") + if "supplier_id" not in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")}: + # version 4 -> 5: receipts and bills can be linked to a supplier + with conn: + conn.execute("ALTER TABLE attachments ADD COLUMN supplier_id INTEGER REFERENCES suppliers(id)") + conn.execute("CREATE INDEX IF NOT EXISTS attachments_supplier ON attachments (supplier_id)") conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") @@ -314,7 +391,9 @@ def add_rows(conn, book, rows): # --- reading ---------------------------------------------------------------- def transactions(conn, book, start=None, end=None): - sql, args = "SELECT * FROM transactions WHERE book_id = ?", [book["id"]] + sql = ("SELECT t.*, (SELECT COUNT(*) FROM attachments a WHERE a.transaction_id = t.id) AS attachments " + "FROM transactions t WHERE book_id = ?") + args = [book["id"]] if start: sql, args = sql + " AND date >= ?", args + [start.isoformat()] if end: @@ -333,7 +412,7 @@ def _row(t): "income": amount / 100 if income else 0, "gst_in": gst if income else 0, "expense": -amount / 100 if not income else 0, "gst_out": gst if not income else 0, "category": t["category"] or "", "unallocated": 0 if t["category"] else net, - "net": net, + "net": net, "attachments": t["attachments"], } @@ -486,3 +565,291 @@ def _brief(t): return {"row": t["id"], "book": t["book"], "date": t["date"], "reference": t["reference"], "amount": t["amount_cents"] / 100, "gst": t["gst"], "category": t["category"] or "", "sheet": SHEETS[date.fromisoformat(t["date"]).month]} + + +# --- attachments (receipts and invoices) ------------------------------------ + +ATTACHMENT_KINDS = ("receipt", "invoice") +# fields read from the document that can be corrected by hand +ATTACHMENT_FIELDS = ("kind", "vendor", "doc_date", "total_cents", "gst_cents", "gst_number", "supplier_id") +MATCH_DAYS = 7 # how far a receipt's date may be from the bank transaction's + +# merchant names that point to an expense category (lower case, matched as substrings) +VENDOR_CATEGORIES = [ + ("Motor vehicle", ("z energy", "bp connect", "mobil", "caltex", "gull", "waitomo", "npd", "parking", + "repco", "supercheap", "tyre", "nzta", "waka kotahi")), + ("Telephone & Power", ("spark", "one nz", "vodafone", "2degrees", "skinny", "mercury", "genesis", + "contact energy", "meridian", "electric kiwi", "flick", "slingshot", "orcon")), + ("Postage & Stationery", ("nz post", "warehouse stationery", "officemax", "paper plus", "whitcoulls")), + ("ACC & Insurance", ("insurance", "accident compensation", "tower", "vero")), +] + + +def _files(conn, attachment_id): + return [dict(f) for f in conn.execute( + "SELECT * FROM attachment_files WHERE attachment_id = ? ORDER BY position, id", (attachment_id,))] + + +def _attachment(conn, a): + d = dict(a) + d["items"] = json.loads(d.pop("items_json") or "[]") + d["edited"] = json.loads(d.pop("edited_json") or "[]") + d["total"] = d["total_cents"] / 100 if d["total_cents"] is not None else None + d["gst"] = d["gst_cents"] / 100 if d["gst_cents"] is not None else None + files = _files(conn, a["id"]) + d["files"] = [{k: f[k] for k in ("id", "filename", "mime", "size")} for f in files] + d["filename"] = files[0]["filename"] if files else "" + d["raw_text"] = attachment_text(conn, a["id"], files) + d["transaction"] = None + d["supplier"] = None + if d["supplier_id"] is not None: + sup = conn.execute("SELECT id, name FROM suppliers WHERE id = ?", (d["supplier_id"],)).fetchone() + d["supplier"] = dict(sup) + return d + + +def attachment_text(conn, attachment_id, files=None): + """What OCR read from all the pages, in order.""" + files = files if files is not None else _files(conn, attachment_id) + return "\n".join(f["text"] for f in files if f["text"]) + + +def _with_transactions(conn, rows): + out = [_attachment(conn, a) for a in rows] + for a in out: + if a["transaction_id"] is not None: + a["transaction"] = _brief(_transaction_info(conn, a["transaction_id"])) + return out + + +def file_exists(conn, sha256): + return conn.execute("SELECT 1 FROM attachment_files WHERE sha256 = ?", (sha256,)).fetchone() is not None + + +def _insert_file(conn, attachment_id, position, file, text): + if file_exists(conn, file["sha256"]): + raise ValueError("that file has already been added") + conn.execute( + "INSERT INTO attachment_files (attachment_id, position, filename, stored_name, mime, sha256, size, text) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (attachment_id, position, file["filename"], file["stored_name"], file["mime"], file["sha256"], + file["size"], text)) + + +def add_attachment(conn, *, file, kind="receipt", transaction_id=None, ocr=None): + """Record an uploaded receipt (its first page); `file` is from services.receipts.save_upload and + `ocr` is what services.receipts.read_document read from it.""" + if kind not in ATTACHMENT_KINDS: + raise ValueError("kind must be receipt or invoice") + if transaction_id is not None: + _transaction_info(conn, transaction_id) + ocr = ocr or {} + supplier = find_supplier(conn, ocr.get("vendor"), ocr.get("gst_number")) + with conn: + cur = conn.execute( + "INSERT INTO attachments (transaction_id, supplier_id, kind, vendor, doc_date, total_cents, gst_cents, " + "gst_number, items_json, ocr_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (transaction_id, supplier, kind, ocr.get("vendor"), ocr.get("date"), ocr.get("total_cents"), + ocr.get("gst_cents"), ocr.get("gst_number"), json.dumps(ocr.get("items") or []), + ocr.get("status", "skipped"))) + _insert_file(conn, cur.lastrowid, 0, file, ocr.get("text")) + return get_attachment(conn, cur.lastrowid) + + +def add_page(conn, attachment_id, file, text): + """Add another photo/PDF to an attachment, after its other pages.""" + get_attachment(conn, attachment_id) + with conn: + last = conn.execute("SELECT MAX(position) FROM attachment_files WHERE attachment_id = ?", + (attachment_id,)).fetchone()[0] + _insert_file(conn, attachment_id, (last or 0) + 1, file, text) + return get_attachment(conn, attachment_id) + + +def get_file(conn, attachment_id, file_id=None): + """One page of an attachment (the first if file_id is None).""" + sql, args = "SELECT * FROM attachment_files WHERE attachment_id = ?", [attachment_id] + if file_id is not None: + sql, args = sql + " AND id = ?", args + [file_id] + f = conn.execute(sql + " ORDER BY position, id LIMIT 1", args).fetchone() + if f is None: + raise NotFound("no such file") + return f + + +def remove_page(conn, attachment_id, file_id): + """Take one page off an attachment; returns its stored file name so the caller can remove the file.""" + f = get_file(conn, attachment_id, file_id) + if len(_files(conn, attachment_id)) == 1: + raise ValueError("that's the only page; delete the receipt instead") + with conn: + conn.execute("DELETE FROM attachment_files WHERE id = ?", (file_id,)) + return f["stored_name"] + + +OCR_FIELDS = {"vendor": "vendor", "doc_date": "date", "total_cents": "total_cents", + "gst_cents": "gst_cents", "gst_number": "gst_number"} # column: parse_receipt key + + +def apply_ocr(conn, attachment_id, found, overwrite=False): + """Set an attachment's details to what was read from all its pages together + (services.receipts.parse_receipt), except those corrected by hand, unless `overwrite`.""" + a = get_attachment(conn, attachment_id) + keep = set() if overwrite else set(a["edited"]) + with conn: + for col, key in OCR_FIELDS.items(): + if col not in keep: + conn.execute(f"UPDATE attachments SET {col} = ? WHERE id = ?", (found.get(key), attachment_id)) + read = any(f["text"] for f in _files(conn, attachment_id)) + conn.execute("UPDATE attachments SET items_json = ?, ocr_status = ?, edited_json = ? WHERE id = ?", + (json.dumps(found.get("items") or []), "done" if read else "failed", + json.dumps(sorted(keep)), attachment_id)) + return get_attachment(conn, attachment_id) + + +def get_attachment(conn, attachment_id): + a = conn.execute("SELECT * FROM attachments WHERE id = ?", (attachment_id,)).fetchone() + if a is None: + raise NotFound("no such attachment") + return _with_transactions(conn, [a])[0] + + +def list_attachments(conn, unmatched=False, supplier_id=None): + where, args = [], [] + if unmatched: + where.append("transaction_id IS NULL") + if supplier_id is not None: + where.append("supplier_id = ?") + args.append(supplier_id) + sql = "SELECT * FROM attachments" + (" WHERE " + " AND ".join(where) if where else "") + return _with_transactions(conn, conn.execute(sql + " ORDER BY created_at DESC, id DESC", args).fetchall()) + + +def attachments_for(conn, row_id): + _transaction_info(conn, row_id) + return _with_transactions(conn, conn.execute( + "SELECT * FROM attachments WHERE transaction_id = ? ORDER BY id", (row_id,)).fetchall()) + + +def update_attachment(conn, attachment_id, **fields): + """Correct details by hand; corrected ones are then kept when the pages are read again.""" + a = get_attachment(conn, attachment_id) + if fields.get("kind") is not None and fields["kind"] not in ATTACHMENT_KINDS: + raise ValueError("kind must be receipt or invoice") + if fields.get("doc_date"): + date.fromisoformat(fields["doc_date"]) # ValueError if it isn't a date + if fields.get("supplier_id") is not None and not conn.execute( + "SELECT 1 FROM suppliers WHERE id = ?", (fields["supplier_id"],)).fetchone(): + raise NotFound("no such supplier") + edited = set(a["edited"]) + with conn: + for k, v in fields.items(): + if k not in ATTACHMENT_FIELDS: + raise ValueError(f"can't change {k}") + if k == "kind" and v is None: + continue + conn.execute(f"UPDATE attachments SET {k} = ? WHERE id = ?", (v if v != "" else None, attachment_id)) + if k in OCR_FIELDS: + edited.add(k) + conn.execute("UPDATE attachments SET edited_json = ? WHERE id = ?", (json.dumps(sorted(edited)), attachment_id)) + return get_attachment(conn, attachment_id) + + +def delete_attachment(conn, attachment_id): + """Forget an attachment; returns its stored file names so the caller can remove the files.""" + get_attachment(conn, attachment_id) + names = [f["stored_name"] for f in _files(conn, attachment_id)] + with conn: + conn.execute("DELETE FROM attachments WHERE id = ?", (attachment_id,)) # pages go too (cascade) + return names + + +def _company(name): + """A supplier name reduced for comparing: letters and digits only, without Ltd / Limited / NZ.""" + words = re.findall(r"[a-z0-9]+", (name or "").lower()) + return "".join(w for w in words if w not in ("ltd", "limited", "nz", "co", "the")) + + +def find_supplier(conn, vendor, gst_number=None): + """The id of the supplier a document is probably from: the same GST number, else the same name.""" + digits = re.sub(r"\D", "", gst_number or "") + rows = conn.execute("SELECT id, name, gst_number FROM suppliers ORDER BY id").fetchall() + if digits: + for r in rows: + if re.sub(r"\D", "", r["gst_number"]) == digits: + return r["id"] + name = _company(vendor) + if name: + for r in rows: + if _company(r["name"]) == name: + return r["id"] + return None + + +def _words(s): + return {w for w in re.findall(r"[a-z0-9]{3,}", (s or "").lower())} + + +def suggest_transactions(conn, attachment, limit=5): + """Bank transactions this receipt is probably for: the same amount, within MATCH_DAYS of its date + (any date if it has none), nearest first, then those whose reference shares words with the vendor.""" + total = attachment["total_cents"] + if not total: + return [] + rows = conn.execute( + "SELECT t.*, b.name AS book, (SELECT COUNT(*) FROM attachments a WHERE a.transaction_id = t.id) AS n " + "FROM transactions t JOIN books b ON b.id = t.book_id WHERE abs(t.amount_cents) = ?", (abs(total),)).fetchall() + when = date.fromisoformat(attachment["doc_date"]) if attachment["doc_date"] else None + vendor = _words(attachment["vendor"]) + scored = [] + for t in rows: + if t["id"] == attachment["transaction_id"]: + continue + gap = abs((date.fromisoformat(t["date"]) - when).days) if when else 0 + if gap > MATCH_DAYS: + continue + overlap = len(vendor & _words(t["reference"])) + scored.append(((gap, -overlap, t["n"], t["date"]), t)) + scored.sort(key=lambda s: s[0]) + return [{**_brief(t), "days_apart": key[0] if when else None, "attachments": t["n"]} + for key, t in scored[:limit]] + + +def propose_changes(conn, attachment, row_id): + """GST and category the transaction could take from this receipt; only what would change.""" + t = _transaction_info(conn, row_id) + out = {} + if (attachment["gst_cents"] or attachment["gst_number"]) and t["gst"] != "Incl GST": + out["gst"] = "Incl GST" + if not t["category"]: + allowed = INCOME_CATEGORIES if t["amount_cents"] > 0 else EXPENSE_CATEGORIES + same = conn.execute( + "SELECT category, COUNT(*) AS n FROM transactions WHERE reference = ? AND category IS NOT NULL " + "AND id != ? GROUP BY category ORDER BY n DESC LIMIT 1", (t["reference"], row_id)).fetchone() + if same and same["category"] in allowed: + out["category"] = same["category"] + elif t["amount_cents"] < 0: + text = f"{attachment['vendor'] or ''} {t['reference']}".lower() + category = next((c for c, words in VENDOR_CATEGORIES if any(w in text for w in words)), None) + if category: + out["category"] = category + return out + + +def link_attachment(conn, attachment_id, row_id, gst=None, category=None): + """Attach to a transaction, optionally setting its GST and category at the same time.""" + get_attachment(conn, attachment_id) + t = _transaction_info(conn, row_id) + if gst is not None or category is not None: + update_row(conn, conn.execute("SELECT * FROM books WHERE id = ?", (t["book_id"],)).fetchone(), + row_id, gst=gst, category=category) + with conn: + conn.execute("UPDATE attachments SET transaction_id = ? WHERE id = ?", (row_id, attachment_id)) + return get_attachment(conn, attachment_id) + + +def unlink_attachment(conn, attachment_id): + get_attachment(conn, attachment_id) + with conn: + conn.execute("UPDATE attachments SET transaction_id = NULL WHERE id = ?", (attachment_id,)) + return get_attachment(conn, attachment_id) diff --git a/src/static/index.html b/src/static/index.html index 7d0d15b..78f5a39 100644 --- a/src/static/index.html +++ b/src/static/index.html @@ -27,7 +27,13 @@ padding: 12px 20px; background: var(--panel); border-bottom: 1px solid var(--line); position: sticky; top: 0; z-index: 5; } header h1 { font-size: 17px; margin: 0 12px 0 0; letter-spacing: .2px; } - header .spacer { flex: 1; } + header { padding-top: 0; padding-bottom: 0; gap: 20px; } + header h1 { margin: 0; padding: 12px 0; } + header .tabs.primary { flex: 1; margin: 0; border: 0; align-self: stretch; } + header .tabs.primary button { padding: 0 14px; margin-bottom: -1px; } + .bookbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; padding: 8px 20px; + background: var(--bg); border-bottom: 1px solid var(--line); } + .bookbar[hidden] { display: none; } select, input, button { font: inherit; color: inherit; } select, input[type=text], input[type=number], input[type=date] { background: var(--panel); border: 1px solid var(--line); border-radius: 6px; padding: 5px 8px; } @@ -36,8 +42,8 @@ button:hover:not(:disabled) { border-color: var(--accent); } button.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); } button:disabled { opacity: .5; cursor: default; } - .layout { display: grid; grid-template-columns: 340px 1fr; gap: 16px; padding: 16px 20px; } - @media (max-width: 900px) { .layout { grid-template-columns: 1fr; padding: 12px 16px; } } + .layout { max-width: 1400px; margin: 0 auto; padding: 16px 20px; } + @media (max-width: 900px) { .layout { padding: 12px 16px; } } .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 14px; min-width: 0; } .panel h2 { font-size: 14px; margin: 0 0 10px; display: flex; align-items: center; gap: 8px; } .panel h2 .grow { flex: 1; } @@ -55,6 +61,12 @@ .tabs button { border-radius: 999px; padding: 3px 12px; } .tabs button.on { background: var(--accent-soft); border-color: var(--accent); color: var(--accent); } .tabs button.empty { color: var(--muted); } + .tabs .sep { width: 1px; background: var(--line); margin: 6px 8px; } + .tabs.primary { gap: 0; border-bottom: 1px solid var(--line); margin-bottom: 14px; } + .tabs.primary button { border: 0; border-bottom: 2px solid transparent; border-radius: 0; background: none; + padding: 8px 16px; font-size: 14px; color: var(--muted); } + .tabs.primary button.on { border-bottom-color: var(--accent); color: var(--accent); background: none; font-weight: 600; } + .tabs .count { background: var(--accent-soft); color: var(--accent); border-radius: 999px; padding: 0 7px; font-size: 12px; } .scroll { overflow-x: auto; } table { border-collapse: collapse; width: 100%; font-variant-numeric: tabular-nums; } th, td { padding: 6px 8px; border-bottom: 1px solid var(--line); text-align: left; white-space: nowrap; } @@ -130,20 +142,87 @@ .tax-table tr.tax-head td { font-weight: 700; padding-top: 14px; color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .4px; } .tax-input { width: 120px; text-align: right; padding: 3px 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 6px; } .row-toolbar { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; margin-bottom: 10px; } + .receipts { display: grid; gap: 12px; } + .receipt { display: grid; grid-template-columns: 120px 1fr; gap: 14px; border: 1px solid var(--line); border-radius: 10px; padding: 12px; } + @media (max-width: 600px) { .receipt { grid-template-columns: 1fr; } } + .receipt .thumb img { width: 120px; max-height: 170px; object-fit: cover; object-position: top; border-radius: 6px; border: 1px solid var(--line); display: block; } + .pages { display: flex; flex-wrap: wrap; gap: 4px; width: 120px; } + .page { position: relative; } + .receipt .thumb .pages.many img, .pages.many .pdf-thumb { width: 58px; height: 78px; max-height: none; font-size: 11px; } + .page .rm { position: absolute; top: 2px; right: 2px; padding: 0 5px; font-size: 11px; line-height: 16px; border-radius: 999px; opacity: .9; } + .add-page { width: 120px; margin-top: 6px; font-size: 12px; padding: 3px 6px; } + .pdf-thumb { display: flex; align-items: center; justify-content: center; width: 120px; height: 120px; border: 1px solid var(--line); + border-radius: 6px; color: var(--out); font-weight: 700; text-decoration: none; } .receipt-head { display: flex; gap: 8px; align-items: baseline; flex-wrap: wrap; margin-bottom: 8px; } .receipt-head .grow { flex: 1; min-width: 0; word-break: break-all; } .receipt-fields { display: grid; grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); gap: 6px 10px; } .receipt-fields label, dialog .receipt-fields label { font-size: 12px; color: var(--muted); display: flex; flex-direction: column; gap: 2px; margin: 0; } .receipt-fields input, .receipt-fields select { width: 100%; } .receipt-fields .wide { grid-column: span 2; } + .receipt details { margin-top: 8px; } + .receipt details summary { cursor: pointer; color: var(--muted); font-size: 12px; } + .receipt pre { white-space: pre-wrap; font-size: 12px; max-height: 240px; overflow: auto; background: var(--row-hover); padding: 8px; border-radius: 6px; } + .match { margin-top: 10px; padding: 8px 10px; border-radius: 8px; background: var(--row-hover); } + .match ul { list-style: none; margin: 4px 0 0; padding: 0; } + .match li { display: flex; gap: 8px; align-items: center; padding: 4px 0; flex-wrap: wrap; } + .match li .grow { flex: 1; min-width: 160px; } + .chip.linked { background: var(--ok-soft); color: var(--in); } + .chip.failed { background: var(--warn-soft); color: var(--warn); } + button.clip { padding: 1px 6px; font-size: 12px; opacity: .4; } + button.clip.has { opacity: 1; border-color: var(--accent); color: var(--accent); } + tr:hover button.clip { opacity: 1; } dialog.wide { width: min(780px, 94vw); } dialog .check { display: flex; gap: 8px; align-items: center; margin: 8px 0; font-size: 14px; color: var(--text); } dialog .check input { width: auto; } + + @media (max-width: 700px) { + body { font-size: 15px; } + header { padding: 10px 12px; gap: 8px; } + header h1 { font-size: 16px; margin-right: 4px; } + header .spacer { display: none; } + #bookInfo { flex-basis: 100%; order: 10; } + header h1 { flex-basis: 100%; margin: 0; padding: 0; } + header .tabs.primary { flex-basis: 100%; order: 5; margin: 0 -12px; padding: 0 12px; } + header .tabs.primary button { padding: 10px 14px; } + .bookbar { padding: 8px 12px; } + #book { flex: 1; min-width: 120px; } + .layout { padding: 10px 12px; gap: 12px; } + .panel { padding: 12px; } + /* 16px inputs stop iOS zooming on focus; larger tap targets */ + select, input[type=text], input[type=number], input[type=date], input[type=password], textarea { font-size: 16px; padding: 8px 10px; } + button, .menu summary { padding: 8px 14px; } + .tabs { flex-wrap: nowrap; overflow-x: auto; margin: 0 -12px 12px; padding: 0 12px 4px; scrollbar-width: none; } + .tabs::-webkit-scrollbar { display: none; } + .tabs button { flex: none; padding: 6px 14px; white-space: nowrap; } + .statements { max-height: 40vh; } + .stat { flex: 1 1 calc(50% - 10px); min-width: 0; } + .menu-items { min-width: 0; width: max-content; max-width: calc(100vw - 24px); } + .menu-items.right { right: 0; } + .menu-items a { padding: 10px 12px; } + .icon-btns { max-width: none; } + .tax-input { width: 100px; } + .receipt-fields .wide { grid-column: 1 / -1; } + .row-toolbar > * { flex: 1 1 auto; } + dialog { padding: 14px; max-height: 90vh; } + #toast { bottom: 12px; width: calc(100vw - 24px); text-align: center; } + .inv-lines input[type=text] { min-width: 160px; } + }
-

Cashbook

+

Accounts

+ + +
+
- - - +
- +
@@ -220,6 +278,44 @@ + +
+

Receipts and invoices

+ + +
+
+
+
+ + +
+

Update the transaction?

+
+
+
+ + +
+
+
+ + +
+

Add files

+
Are they photos of one receipt (e.g. the top and bottom of a long one) or pages of one invoice, or a different receipt each?
+
+ + + +
+
+
+ + + + +
Drop ANZ statements to add them
@@ -278,7 +374,6 @@ function localStorageGet(k) { try { return localStorage.getItem(k); } catch { re function localStorageSet(k, v) { try { localStorage.setItem(k, v); } catch {} } async function loadSummary() { - updatePreviewBtn(); if (!state.book) { state.summary = null; render(); return; } state.summary = await call(`/api/summary?book=${encodeURIComponent(state.book)}`); const s = state.summary; @@ -299,24 +394,6 @@ async function loadStatements() { state.statements = await call(`/api/statements${state.book ? "?book=" + encodeURIComponent(state.book) : ""}`); const known = new Set(state.statements.map(s => s.path)); for (const p of [...state.selected]) if (!known.has(p)) state.selected.delete(p); - renderStatements(); -} -function renderStatements() { - const ul = $("#statements"); - if (!state.statements.length) { ul.innerHTML = `
  • No statements found. Use “Add…”.
  • `; return; } - let html = "", folder = null; - const order = [...new Set(state.statements.map(s => s.folder))]; // server lists folders in priority order - const sorted = [...state.statements].sort((a, b) => (order.indexOf(a.folder) - order.indexOf(b.folder)) || ((b.first || "").localeCompare(a.first || ""))); - for (const s of sorted) { - if (s.folder !== folder) { folder = s.folder; html += `
  • ${esc(folder)}
  • `; } - const id = "st" + btoa(unescape(encodeURIComponent(s.path))).replace(/[^a-z0-9]/gi, ""); - const desc = s.error ? `Can't read: ${esc(s.error)}` - : `${s.count} transactions · ${fmtDate(s.first)} – ${fmtDate(s.last)}${s.periods.length > 1 ? ` · ${s.periods.length} statements` : ""}`; - html += `
  • -
  • `; - } - ul.innerHTML = html; - updatePreviewBtn(); } function importChip(s) { if (s.error || s.imported == null || !state.book) return ""; @@ -363,7 +440,6 @@ async function deleteStatement(path) { if (state.preview?.single && state.preview.statements[0] === path) { state.preview = null; state.view = "statements"; } await loadStatements(); render(); toast(`Removed ${st.name}`); } -function updatePreviewBtn() { $("#previewBtn").disabled = !state.selected.size || !state.book; } // ---- import ---- async function preview() { @@ -388,12 +464,14 @@ async function doImport() { // ---- rendering ---- function render() { const main = $("#main"); - if (state.view === "statements") { main.innerHTML = (state.summary ? tabsHtml() : "") + renderStatementsPage(); return; } - const pages = { invoices: renderInvoices, customers: renderCustomers, mileage: renderMileage }; - if (pages[state.view]) { main.innerHTML = (state.summary ? tabsHtml() : "") + pages[state.view](); return; } - if (state.view === "preview" && state.preview && !state.book) { main.innerHTML = renderPreview(); return; } + $("#nav").innerHTML = navHtml(); + $("#bookbar").hidden = !BOOK_VIEWS.includes(state.view); + const pages = { statements: renderStatementsPage, receipts: renderReceipts, invoices: renderInvoices, + customers: renderCustomers, suppliers: renderSuppliers, mileage: renderMileage }; + if (pages[state.view]) { main.innerHTML = tabsHtml() + pages[state.view](); return; } + if (state.view === "preview" && state.preview && !state.book) { main.innerHTML = tabsHtml() + renderPreview(); return; } if (!state.book) { - main.innerHTML = `
    No cashbook yet.

    `; + main.innerHTML = tabsHtml() + `
    No cashbook yet.

    `; return; } if (!state.summary) { main.innerHTML = ""; return; } @@ -404,16 +482,27 @@ function render() { else if (state.view === "month" && state.month) main.innerHTML = tabs + renderMonth(); else main.innerHTML = tabs + renderSummary(); } +// the cashbook (year, GST return, income tax and the months) is one tab with its own row beneath +const BOOK_VIEWS = ["summary", "gst", "tax", "month"]; +function navHtml() { + const inBook = BOOK_VIEWS.includes(state.view); + const tab = (view, label, on = state.view === view) => ``; + return `${tab("summary", "Cashbook", inBook)} + + ${tab("receipts", `Receipts${unmatchedCount() ? ` ${unmatchedCount()}` : ""}`)} + ${tab("invoices", "Invoices")}${tab("customers", "Customers")}${tab("suppliers", "Suppliers")} + + ${tab("mileage", "Mileage")}${tab("statements", "Statements")} + ${state.preview ? tab("preview", state.preview.single ? "Statement" : "Import preview") : ""}`; +} +// the row under the top bar: the cashbook's own sections and months function tabsHtml() { + if (!BOOK_VIEWS.includes(state.view) || !state.summary) return ""; + const tab = (view, label) => ``; return `
    - - - + ${tab("summary", "Year")}${tab("gst", "GST return")}${tab("tax", "Income tax")} + ${state.summary.months.map(m => ``).join("")} - - ${["invoices", "customers", "mileage"].map(v => ``).join("")} - - ${state.preview ? `` : ""}
    `; } @@ -617,15 +706,17 @@ function renderMonth() { ${m.rows.length ? `
    - + ${rows.map(r => ` - `).join("") || ``} - + + `).join("") || ``} +
    DateReferenceOriginInOutGSTGST amtCategory
    DateReferenceOriginInOutGSTGST amtCategory
    ${fmtDate(r.date)}${esc(r.reference)}${esc(r.origin)} ${money(r.income)}${money(r.expense)} ${money(r.gst_in || r.gst_out)}${catSelect(r)}
    Everything this month is allocated.
    ${m.rows.length} transactions${fmt(income)}${fmt(expense)}
    ${catSelect(r)}
    Everything this month is allocated.
    ${m.rows.length} transactions${fmt(income)}${fmt(expense)}
    ` : `
    No transactions this month.
    `}`; } @@ -661,6 +752,232 @@ function renderPreview() { `; } +// ---- receipts and invoices ---- +const unmatchedCount = () => (state.receipts || []).filter(a => !a.transaction).length; +async function loadReceipts(show) { + [state.receipts, state.suppliers] = await Promise.all([call("/api/attachments"), call("/api/suppliers")]); + if (show) state.view = "receipts"; + if (show || state.view === "receipts" || state.summary) render(); +} +// Upload receipt files: each as its own receipt, or all as pages of one (asPages), or as more pages +// of the existing receipt `into`. Returns the receipts as they are after the uploads. +async function uploadReceipts(files, { row = null, asPages = false, into = null } = {}) { + const added = [], errors = []; + let target = into; + for (const f of files) { + busyCount++; $("#busy").classList.add("on"); + try { + const name = `filename=${encodeURIComponent(f.name)}`; + const url = target ? `/api/attachments/${target}/pages?${name}` : `/api/attachments?${name}${row ? `&row=${row}` : ""}`; + const res = await fetch(url, { method: "POST", body: f }); + if (res.status === 401) { signInAgain(); break; } + const data = await res.json().catch(() => ({})); + if (res.ok) { + if (target && added.length) added[added.length - 1] = data; else added.push(data); + if (asPages || into) target = data.id; + if (data.ocr_error) errors.push(`${f.name} was saved, but ${data.ocr_error}. Fill in its details by hand.`); + } else errors.push(`${f.name}: ${errorText(data, res)}`); + } finally { if (--busyCount === 0) $("#busy").classList.remove("on"); } + } + const n = files.length - errors.filter(e => !e.includes(" was saved, ")).length; + if (errors.length) toast(errors.join(" · "), true); + else if (into) toast(`Added ${plural(n, "page")}`); + else if (added.length) toast((asPages ? `Added a receipt with ${plural(n, "page")}` : `Added ${plural(added.length, "file")}`) + + (row ? "" : added.some(a => a.suggestions.length) ? ". Check the suggested matches." : "")); + return added; +} +// several files at once: pages of one receipt, or a receipt each? (null = cancelled) +function askPages(n) { + $("#pagesTitle").textContent = `Add ${n} files`; + $("#pagesOne").textContent = `One receipt, ${n} pages`; + $("#pagesSeparate").textContent = `${n} separate receipts`; + const d = $("#pagesDialog"); + return new Promise(resolve => { + d.addEventListener("close", () => resolve(d.returnValue === "cancel" || !d.returnValue ? null : d.returnValue === "one"), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +// after linking, offer the GST type / category the receipt suggests; returns what was applied +function askApply(a) { + const p = a.proposed || {}, keys = Object.keys(p); + if (!keys.length) return Promise.resolve(null); + const t = a.transaction, label = { gst: "GST", category: "Category" }; + $("#applySub").textContent = `${fmtDate(t.date)} · ${t.reference} · ${fmt(t.amount)}`; + $("#applyBody").innerHTML = keys.map(k => ``).join("") + + `
    ${p.gst ? "The receipt shows GST, so it can be claimed. " : ""}You can change these later on the month's sheet.
    `; + const d = $("#applyDialog"); + return new Promise(resolve => { + d.addEventListener("close", () => resolve(d.returnValue === "ok" + ? Object.fromEntries([...d.querySelectorAll("[data-apply]:checked")].map(c => [c.dataset.apply, p[c.dataset.apply]])) : null), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +async function confirmChanges(a) { + const apply = await askApply(a); + if (apply && Object.keys(apply).length) { + a = await call(`/api/attachments/${a.id}/link`, { row: a.transaction.row, apply }); + toast("Transaction updated"); + } + return a; +} +async function refreshAfterChange() { + const jobs = [loadReceipts()]; + if (state.book) jobs.push(call(`/api/summary?book=${encodeURIComponent(state.book)}`).then(s => state.summary = s)); + if (state.view === "month" && state.sheet) + jobs.push(call(`/api/month?book=${encodeURIComponent(state.book)}&sheet=${state.sheet}`).then(m => state.month = m)); + await Promise.all(jobs); + render(); + if ($("#attDialog").open) await renderRowAttachments(); +} +async function gotoTransaction(book, sheet) { + $("#attDialog").close(); + if (book !== state.book) { state.book = book; $("#book").value = book; localStorageSet("book", book); updateExportLinks(); await loadSummary(); } + await loadMonth(sheet); +} +// buttons on receipt cards, in the Receipts view and the row dialog +async function receiptAction(b) { + const id = Number(b.dataset.id); + if (b.dataset.act === "att-link") { + const a = await call(`/api/attachments/${id}/link`, { row: Number(b.dataset.row) }); + toast(`Linked to ${a.transaction.reference}`); + await confirmChanges(a); + } else if (b.dataset.act === "att-unlink") { + await call(`/api/attachments/${id}/unlink`, {}); + } else if (b.dataset.act === "att-delete") { + const a = state.receipts.find(x => x.id === id) || state.rowAtt?.list.find(x => x.id === id); + const pages = a?.files.length || 1; + if (!confirm(`Delete ${a?.vendor || a?.filename || "this receipt"}${pages > 1 ? ` and its ${pages} pages` : ""}? ` + + `The ${pages > 1 ? "files are" : "file is"} removed from data/attachments.`)) return true; + await call(`/api/attachments/${id}`, undefined, "DELETE"); + } else if (b.dataset.act === "att-addpage") { + state.pageTarget = id; $("#pageInput").click(); return true; + } else if (b.dataset.act === "att-rmpage") { + if (!confirm("Remove this page? Its file is removed from data/attachments.")) return true; + await call(`/api/attachments/${id}/pages/${b.dataset.file}`, undefined, "DELETE"); + } else if (b.dataset.act === "att-reread") { + if (!confirm("Read the details again from the pages? Anything you corrected by hand is replaced.")) return true; + await call(`/api/attachments/${id}/reread`, {}); + } else if (b.dataset.act === "att-goto") { + await gotoTransaction(b.dataset.book, b.dataset.sheet); return true; + } else return false; + await refreshAfterChange(); + return true; +} +async function saveReceiptField(input) { + const f = input.dataset.attField, raw = input.value.trim(); + let value = ["total", "gst"].includes(f) ? (raw === "" ? null : Number(raw)) : (raw || null); + if (f === "supplier_id") { + if (raw === "new") { + const a = (state.receipts || []).find(x => x.id === Number(input.dataset.id)) || state.rowAtt?.list.find(x => x.id === Number(input.dataset.id)); + await askNewSupplier(a); + return refreshAfterChange(); + } else value = raw ? Number(raw) : null; + } + const focus = document.activeElement?.dataset; + const where = focus?.attField ? `[data-id="${focus.id}"][data-att-field="${focus.attField}"]` : null; + await call(`/api/attachments/${input.dataset.id}`, { [f]: value }, "PATCH"); + await refreshAfterChange(); + if (where) document.querySelector(where)?.focus(); // keep the cursor where the user tabbed to +} +function receiptCard(a, inDialog) { + const t = a.transaction, many = a.files.length > 1; + const page = (f, i) => { + const url = `/api/attachments/${a.id}/files/${f.id}`, title = many ? `Page ${i + 1}: ${f.filename}` : f.filename; + return `
    ${f.mime.startsWith("image/") + ? `${esc(title)}` + : `PDF`} + ${many ? `` : ""}
    `; + }; + const thumb = `
    ${a.files.map(page).join("")}
    + `; + const size = a.files.reduce((n, f) => n + f.size, 0); + const input = (k, label, val, type = "text", cls = "") => + ``; + const status = a.ocr_status === "done" ? "" : a.ocr_status === "failed" + ? `text not read` : ""; + let match; + if (t) { + match = `
    linked + + ${fmt(t.amount)} ${esc(t.book)} · ${esc(t.gst)}${t.category ? " · " + esc(t.category) : ""} +
    `; + } else if (a.suggestions.length) { + match = `
    Matching bank transactions
    `; + } else { + match = `
    ${a.total == null ? "Enter the total to find the bank transaction." : + `No bank transaction of ${fmt(a.total)} within 7 days${a.doc_date ? " of " + fmtDate(a.doc_date) : ""}. Check the total and date, or import the statement first.`}
    `; + } + return `
    +
    ${thumb}
    +
    +
    ${esc(a.vendor || a.filename)} + ${many ? `${a.files.length} pages` : esc(a.filename)} · ${fmtSize(size)} ${status} + ${inDialog ? `` : ""} +
    +
    + ${input("vendor", "Supplier", a.vendor, "text", "wide")} + ${input("doc_date", "Date", a.doc_date, "date")} + ${input("total", "Total (incl GST)", a.total?.toFixed(2), "number")} + ${input("gst", "GST", a.gst?.toFixed(2), "number")} + ${input("gst_number", "GST number", a.gst_number)} + + +
    + ${a.items.length ? `
    ${plural(a.items.length, "line item")} + ${a.items.map(i => ``).join("")}
    ${esc(i.description)}${fmt(i.amount)}
    ` : ""} + ${a.raw_text ? `
    Text read from the ${many ? "pages" : a.files[0].mime === "application/pdf" ? "PDF" : "photo"}
    ${esc(a.raw_text)}
    +
    ` : ""} + ${inDialog ? "" : match} +
    `; +} +function renderReceipts() { + const all = state.receipts || []; + const filter = state.receiptFilter || "unmatched"; + const sup = state.receiptSupplier || ""; + const list = (filter === "all" ? all : all.filter(a => !a.transaction)).filter(a => !sup || String(a.supplier_id) === sup); + return ` +
    Drop receipt photos or invoices here or + ·
    + JPEG, PNG or PDF. A long receipt can be several photos: add them together, or use “+ Add page” on it later. + The supplier, date, total and GST are read from the pages on this server, then matched to your bank transactions.
    +
    + Receipts and invoices +
    + + +
    + +
    + ${list.length ? `
    ${list.map(a => receiptCard(a)).join("")}
    ` + : `
    ${all.length ? "Every receipt is linked to a transaction." : "No receipts or invoices yet."}
    `}`; +} +// the attachments of one transaction, from the 📎 button on a month's sheet +async function openRowAttachments(row) { + state.rowAtt = { row: state.month.rows.find(r => r.row === row), list: [] }; + await renderRowAttachments(); + if (!$("#attDialog").open) $("#attDialog").showModal(); +} +async function renderRowAttachments() { + const r = state.rowAtt.row; + state.rowAtt.list = await call(`/api/row/${r.row}/attachments`); + $("#attSub").textContent = `${fmtDate(r.date)} · ${r.reference} · ${fmt(r.income || -r.expense)}`; + const list = state.rowAtt.list; + const unlinked = (state.receipts || []).filter(a => !a.transaction && a.suggestions.some(s => s.row === r.row)); + $("#attBody").innerHTML = (list.length ? list.map(a => receiptCard(a, true)).join("") + : `
    Nothing attached yet. Add a photo of the receipt or the invoice PDF.
    `) + + (unlinked.length ? `
    Unlinked receipts that match this transaction
    ` : ""); +} + // ---- new book ---- function openNewBook() { const sel = state.statements.filter(s => state.selected.has(s.path) && s.first); @@ -699,14 +1016,6 @@ document.addEventListener("click", e => { const menu = $("#exportMenu"); if (menu.open && (!menu.contains(e.target) || e.target.closest("a"))) menu.open = false; }); -$("#statements").addEventListener("change", e => { - const p = e.target.dataset.path; if (!p) return; - e.target.checked ? state.selected.add(p) : state.selected.delete(p); updatePreviewBtn(); -}); -$("#clearSel").addEventListener("click", () => { state.selected.clear(); renderStatements(); if (state.view === "statements") render(); }); -$("#manageBtn").addEventListener("click", () => { state.view = "statements"; render(); }); -$("#previewBtn").addEventListener("click", guard(preview)); -$("#uploadBtn").addEventListener("click", () => $("#uploadInput").click()); $("#uploadInput").addEventListener("change", guard(async e => { const files = [...e.target.files]; e.target.value = ""; await uploadFiles(files); @@ -714,21 +1023,65 @@ $("#uploadInput").addEventListener("change", guard(async e => { // drag and drop anywhere on the page let dragDepth = 0; const hasFiles = e => [...(e.dataTransfer?.types || [])].includes("Files"); -window.addEventListener("dragenter", e => { if (hasFiles(e)) { dragDepth++; document.body.classList.add("dragging"); } }); +const dropsReceipts = () => state.view === "receipts" || $("#attDialog").open; +window.addEventListener("dragenter", e => { + if (!hasFiles(e)) return; + dragDepth++; document.body.classList.add("dragging"); + $("#dropOverlay").textContent = dropsReceipts() ? "Drop receipts or invoices to add them" : "Drop ANZ statements to add them"; +}); window.addEventListener("dragleave", e => { if (hasFiles(e) && --dragDepth <= 0) { dragDepth = 0; document.body.classList.remove("dragging"); } }); window.addEventListener("dragover", e => { if (hasFiles(e)) e.preventDefault(); }); window.addEventListener("drop", guard(async e => { if (!hasFiles(e)) return; e.preventDefault(); dragDepth = 0; document.body.classList.remove("dragging"); + if (dropsReceipts()) return addReceipts([...e.dataTransfer.files], $("#attDialog").open ? state.rowAtt.row.row : null); await uploadFiles([...e.dataTransfer.files]); })); -$("#main").addEventListener("click", guard(async e => { +// receipts: from the Receipts view (row = null) or a transaction's 📎 dialog +async function addReceipts(files, row) { + if (!files.length) return; + const asPages = files.length > 1 ? await askPages(files.length) : false; + if (asPages === null) return; + const added = await uploadReceipts(files, { row, asPages }); + if (!added.length) return; + await refreshAfterChange(); + const last = added[added.length - 1]; + if (row && await confirmChanges(last) !== last) await refreshAfterChange(); + await offerSuppliers(added); + await refreshAfterChange(); +} +for (const id of ["#receiptInput", "#cameraInput"]) + $(id).addEventListener("change", guard(async e => { + const files = [...e.target.files]; e.target.value = ""; + await addReceipts(files, $("#attDialog").open ? state.rowAtt.row.row : null); + })); +$("#pageInput").addEventListener("change", guard(async e => { + const files = [...e.target.files]; e.target.value = ""; + if (!files.length || !state.pageTarget) return; + const added = await uploadReceipts(files, { into: state.pageTarget }); + if (added.length) await refreshAfterChange(); +})); +$("#attAdd").addEventListener("click", () => $("#receiptInput").click()); +$("#attClose").addEventListener("click", () => $("#attDialog").close()); +$("#attDialog").addEventListener("click", guard(async e => { + const b = e.target.closest("[data-act]"); + if (b && !b.disabled) await receiptAction(b); +})); +$("#attDialog").addEventListener("change", guard(async e => { if (e.target.dataset.attField) await saveReceiptField(e.target); })); +const navClick = guard(async e => { const copy = e.target.closest("[data-copy]"); if (copy) { await navigator.clipboard?.writeText(copy.dataset.copy).catch(() => {}); return toast(`Copied ${copy.dataset.copy}`); } - const b = e.target.closest("[data-sheet],[data-view],[data-act],[data-inv],[data-mi],#importBtn,#cancelPreview,#chooseFiles,#previewBtn2"); + const b = e.target.closest("[data-sheet],[data-view],[data-act],[data-receipts],[data-inv],[data-mi],#importBtn,#cancelPreview,#chooseFiles,#previewBtn2,#chooseReceipts,#takePhoto"); if (!b || b.disabled) return; if (b.id === "chooseFiles") return $("#uploadInput").click(); + if (b.id === "chooseReceipts") return $("#receiptInput").click(); + if (b.id === "takePhoto") return $("#cameraInput").click(); + if (b.dataset.receipts) { state.receiptFilter = b.dataset.receipts; return render(); } + if (b.dataset.act === "row-att") return openRowAttachments(Number(b.dataset.row)); + if (b.dataset.act?.startsWith("att-")) return receiptAction(b); + if (b.dataset.view === "receipts") return loadReceipts(true); if (b.dataset.view === "customers") return loadCustomers(); + if (b.dataset.view === "suppliers") return loadSuppliers(); if (b.dataset.inv) return invoiceAction(b); if (b.dataset.mi) return mileageAction(b); if (b.dataset.view === "invoices") return loadInvoices(); @@ -743,16 +1096,20 @@ $("#main").addEventListener("click", guard(async e => { if (b.dataset.view === "tax") return loadTax(); if (b.dataset.view) { state.view = b.dataset.view; return render(); } if (b.dataset.sheet) return loadMonth(b.dataset.sheet); -})); +}); +$("#main").addEventListener("click", navClick); +$("#nav").addEventListener("click", navClick); $("#main").addEventListener("change", guard(async e => { + if (e.target.id === "receiptSupplier") { state.receiptSupplier = e.target.value; return render(); } if (e.target.id === "invoiceCustomer") { state.invoiceCustomer = e.target.value; return render(); } if (e.target.id === "onlyUnalloc") { state.onlyUnalloc = e.target.checked; return render(); } if (e.target.dataset.taxBox) return saveTaxInput(e.target); if (e.target.dataset.mi) return mileageChange(e.target); + if (e.target.dataset.attField) return saveReceiptField(e.target); if (e.target.id === "gstFreq") { state.gstFreq = e.target.value; localStorageSet("gstFreq", state.gstFreq); return loadGst(); } if (e.target.dataset.select) { e.target.checked ? state.selected.add(e.target.dataset.select) : state.selected.delete(e.target.dataset.select); - renderStatements(); return render(); + return render(); } const { row, field } = e.target.dataset; if (!row) return; state.month = await call("/api/row", { book: state.book, sheet: state.sheet, row: Number(row), [field]: e.target.value }); @@ -796,10 +1153,11 @@ document.addEventListener("click", e => { guard(async () => { await loadMe(); await loadBooks(); - await loadStatements(); + await Promise.all([loadStatements(), loadReceipts()]); })(); + diff --git a/src/static/suppliers.js b/src/static/suppliers.js new file mode 100644 index 0000000..2ae62bb --- /dev/null +++ b/src/static/suppliers.js @@ -0,0 +1,96 @@ +// ---- suppliers: who receipts and bills are from ---- +// Uses $, state, call, esc, plural, fmt, guard and render from index.html. +document.body.insertAdjacentHTML("beforeend", ` + +
    +

    New supplier?

    +
    + + + + +
    + + +
    +
    +
    `); + +// Offer to make a supplier from a receipt or invoice that matched none; links it if the user agrees. +// Returns the updated attachment, or null if declined. +function askNewSupplier(a) { + $("#newSupSub").textContent = `${a.kind === "invoice" ? "Invoice" : "Receipt"} ${a.filename} isn't linked to a supplier. Add one so its receipts and bills are kept together.`; + $("#newSupName").value = a.vendor || ""; $("#newSupGst").value = a.gst_number || ""; + const d = $("#newSupDialog"); + return new Promise(resolve => { + d.addEventListener("close", guard(async () => { + if (d.returnValue !== "ok") return resolve(null); + const sup = await call("/api/suppliers", { name: $("#newSupName").value, gst_number: $("#newSupGst").value }); + const linked = await call(`/api/attachments/${a.id}`, { supplier_id: sup.id }, "PATCH"); + toast(`Added supplier ${sup.name}`); + resolve(linked); + }), { once: true }); + d.returnValue = ""; d.showModal(); + }); +} +// after an upload: ask for each document with no supplier; one answer covers the others from the same vendor +async function offerSuppliers(added) { + const same = (x, y) => (x.gst_number && x.gst_number === y.gst_number) || + (x.vendor && y.vendor && x.vendor.trim().toLowerCase() === y.vendor.trim().toLowerCase()); + const done = new Set(); + for (const a of added) { + if (a.supplier_id != null || done.has(a.id)) continue; + const linked = await askNewSupplier(a); + if (!linked) continue; + for (const b of added) + if (b.id !== a.id && b.supplier_id == null && same(a, b)) { + await call(`/api/attachments/${b.id}`, { supplier_id: linked.supplier_id }, "PATCH"); done.add(b.id); + } + } +} + +async function loadSuppliers() { + state.suppliers = await call("/api/suppliers"); + state.view = "suppliers"; render(); +} + +function renderSuppliers() { + const list = state.suppliers || []; + const cell = (s, k) => ``; + return ` +
    Suppliers
    + ${list.length ? `
    + ${list.map(s => `${["name", "gst_number", "email"].map(k => cell(s, k)).join("")} + + + `).join("")} +
    NameGST numberEmailAddressSpent
    ${s.documents ? `
    ${plural(s.documents, "receipt")}
    ` : ""}
    ${s.documents ? "" : ``}
    +
    Changes are saved as you leave each box. Receipts and bills are linked to a supplier automatically when the GST number or name matches; otherwise pick one on the receipt.
    ` + : `
    No suppliers yet. Add one below, or choose “New supplier” on a receipt.
    `} +
    + + + + + +
    `; +} + +$("#main").addEventListener("change", guard(async e => { + const k = e.target.dataset.supField; if (!k) return; + const s = state.suppliers.find(x => x.id === Number(e.target.dataset.id)); + await call(`/api/suppliers/${s.id}`, { name: s.name, email: s.email, address: s.address, gst_number: s.gst_number, [k]: e.target.value }, "PUT"); + state.suppliers = await call("/api/suppliers"); + toast("Saved"); +})); +$("#main").addEventListener("click", guard(async e => { + const del = e.target.dataset.supDelete, show = e.target.dataset.supShow; + if (show) { state.receiptSupplier = show; state.receiptFilter = "all"; await loadReceipts(true); return; } + if (del) { await call(`/api/suppliers/${del}`, undefined, "DELETE"); await loadSuppliers(); } +})); +$("#main").addEventListener("submit", guard(async e => { + if (e.target.id !== "supAdd") return; + e.preventDefault(); + await call("/api/suppliers", { name: $("#supName").value, gst_number: $("#supGst").value, email: $("#supEmail").value, address: $("#supAddress").value }); + await loadSuppliers(); +})); diff --git a/src/suppliers.py b/src/suppliers.py new file mode 100644 index 0000000..6469843 --- /dev/null +++ b/src/suppliers.py @@ -0,0 +1,60 @@ +"""Suppliers: who receipts and bills are from. A document is linked to a supplier by id +(attachments.supplier_id); uploads are linked automatically when the document's GST number or +name matches a supplier (see ledger.find_supplier).""" +import ledger + +SUPPLIER_FIELDS = ("name", "email", "address", "gst_number") + + +def _supplier(conn, s): + d = dict(s) + row = conn.execute("SELECT COUNT(*) AS n, COALESCE(SUM(total_cents), 0) AS total FROM attachments " + "WHERE supplier_id = ?", (s["id"],)).fetchone() + d["documents"], d["total"] = row["n"], row["total"] / 100 + return d + + +def list_suppliers(conn): + return [_supplier(conn, s) for s in conn.execute("SELECT * FROM suppliers ORDER BY name COLLATE NOCASE")] + + +def get_supplier(conn, supplier_id): + s = conn.execute("SELECT * FROM suppliers WHERE id = ?", (supplier_id,)).fetchone() + if s is None: + raise ledger.NotFound("no such supplier") + return _supplier(conn, s) + + +def _fields(fields): + out = {} + for k, v in fields.items(): + if k not in SUPPLIER_FIELDS: + raise ValueError(f"can't change {k}") + out[k] = (v or "").strip() + if "name" in out and not out["name"]: + raise ValueError("the supplier needs a name") + return out + + +def create_supplier(conn, name, email="", address="", gst_number=""): + f = _fields({"name": name, "email": email, "address": address, "gst_number": gst_number}) + with conn: + cur = conn.execute("INSERT INTO suppliers (name, email, address, gst_number) VALUES (?, ?, ?, ?)", + (f["name"], f["email"], f["address"], f["gst_number"])) + return get_supplier(conn, cur.lastrowid) + + +def update_supplier(conn, supplier_id, **fields): + get_supplier(conn, supplier_id) + for k, v in _fields(fields).items(): + with conn: + conn.execute(f"UPDATE suppliers SET {k} = ? WHERE id = ?", (v, supplier_id)) + return get_supplier(conn, supplier_id) + + +def delete_supplier(conn, supplier_id): + get_supplier(conn, supplier_id) + if conn.execute("SELECT 1 FROM attachments WHERE supplier_id = ?", (supplier_id,)).fetchone(): + raise ValueError("this supplier has receipts or bills, so it can't be deleted") + with conn: + conn.execute("DELETE FROM suppliers WHERE id = ?", (supplier_id,)) diff --git a/tests/test_receipts.py b/tests/test_receipts.py new file mode 100644 index 0000000..cffaf35 --- /dev/null +++ b/tests/test_receipts.py @@ -0,0 +1,321 @@ +import os +import shutil +import sqlite3 + +import pytest + +import ledger +from cashbook_app.services import receipts +from cashbook_app.services.receipts import parse_receipt +from conftest import SAMPLE_CSV + +TILL_RECEIPT = """ + BUNNINGS WAREHOUSE + Mt Wellington +TAX INVOICE GST No: 123-456-789 +Date: 14/05/2026 Time 10:42 +2 x Timber 90x45 24.50 +Screws Galv 8g 12,99 +Paint Brush 50mm 9.90 +SUBTOTAL 47.39 +TOTAL $47.39 +GST INCLUDED 6.18 +EFTPOS $47.39 +CHANGE 0.00 +""" + +INVOICE = """Acme Consulting Ltd +Invoice INV-0042 +Invoice Date: 3 Jun 2026 +Due Date: 20 Jun 2026 +GST Reg 98-765-432 +Web design services 1,000.00 +Subtotal (excl GST) 1,000.00 +GST 15% 150.00 +Total NZD 1,150.00 +""" + + +def test_parse_till_receipt(): + r = parse_receipt(TILL_RECEIPT) + assert r["vendor"] == "BUNNINGS WAREHOUSE" + assert r["date"] == "2026-05-14" + assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (4739, 618, "123-456-789") + assert r["items"] == [{"description": "Timber 90x45", "amount": 24.5}, + {"description": "Screws Galv 8g", "amount": 12.99}, # OCR read the point as a comma + {"description": "Paint Brush 50mm", "amount": 9.9}] + + +def test_parse_invoice(): + r = parse_receipt(INVOICE) + assert r["vendor"] == "Acme Consulting Ltd" + assert r["date"] == "2026-06-03" # the invoice date, not the due date + assert (r["total_cents"], r["gst_cents"], r["gst_number"]) == (115000, 15000, "098-765-432") + assert r["items"] == [{"description": "Web design services", "amount": 1000.0}] + + +def test_parse_fallbacks(): + # no "total" line: the card payment is the total; GST far from 3/23 of it is ignored + r = parse_receipt("Corner Dairy\n12 Mar 2026\nMilk 3.50\nBread 4.20\nEFTPOS 7.70\nGST 5.00\n") + assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-03-12", 770, None) + assert parse_receipt("")["total_cents"] is None + # OCR noise inside the GST number + assert parse_receipt("TAX INVOICE GST No: 45-6/78-912\nTOTAL 10.00")["gst_number"] == "045-678-912" + assert parse_receipt("GST 150.00 150.00\nTOTAL 1150.00")["gst_cents"] == 15000 + + +def png(seed: bytes) -> bytes: + return b"\x89PNG\r\n\x1a\n" + seed * 20 # only the header is checked; OCR is faked below + + +@pytest.fixture +def books(admin, monkeypatch): + """A book with SAMPLE_CSV imported, and OCR replaced by whatever `fake_ocr` holds.""" + admin.post("/api/books", json={"name": "b", "year": 2026}) + path = admin.post("/api/upload?filename=s.csv", content=SAMPLE_CSV.encode()).json()["path"] + admin.post("/api/import", json={"book": "b", "statements": [path]}) + admin.fake_ocr = {"status": "done", "text": "", "items": []} + monkeypatch.setattr(receipts, "read_document", lambda p, mime: dict(admin.fake_ocr)) + return admin + + +def rows(client, sheet): + return client.get(f"/api/month?book=b&sheet={sheet}").json()["rows"] + + +def test_upload_suggest_and_link(books, tmp_path): + books.fake_ocr.update(vendor="Jaycar Electronics", date="2026-04-12", total_cents=11500, gst_cents=1500, + items=[{"description": "Soldering iron", "amount": 115.0}]) + r = books.post("/api/attachments?filename=jaycar.png", content=png(b"j")) + assert r.status_code == 200, r.text + a = r.json() + assert (a["vendor"], a["total"], a["gst"], a["transaction"]) == ("Jaycar Electronics", 115.0, 15.0, None) + assert a["items"][0]["description"] == "Soldering iron" + [s] = a["suggestions"] + assert (s["reference"], s["amount"], s["days_apart"], s["sheet"]) == ("Jaycar Chris", -115.0, 1, "APR") + + # the same file again is refused, and its copy isn't kept + assert books.post("/api/attachments?filename=again.png", content=png(b"j")).status_code == 400 + assert len(os.listdir(tmp_path / "attachments")) == 1 + + unmatched = books.get("/api/attachments?unmatched=1").json() + assert [x["id"] for x in unmatched] == [a["id"]] + + linked = books.post(f"/api/attachments/{a['id']}/link", json={"row": s["row"]}).json() + assert linked["transaction"]["reference"] == "Jaycar Chris" + assert linked["proposed"] == {} # already Incl GST, and nothing to go on for a category + assert books.get("/api/attachments?unmatched=1").json() == [] + assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 1 + assert [x["id"] for x in books.get(f"/api/row/{s['row']}/attachments").json()] == [a["id"]] + + f = books.get(f"/api/attachments/{a['id']}/file") + assert f.status_code == 200 and f.content == png(b"j") and f.headers["content-type"] == "image/png" + + books.post(f"/api/attachments/{a['id']}/unlink") + assert next(r for r in rows(books, "APR") if r["row"] == s["row"])["attachments"] == 0 + + assert books.delete(f"/api/attachments/{a['id']}").json() == {"ok": True} + assert os.listdir(tmp_path / "attachments") == [] + assert books.get(f"/api/attachments/{a['id']}").status_code == 404 + + +def test_link_offers_gst_and_category(books): + hetzner = next(r for r in rows(books, "MAY") if "Hetzner" in r["reference"]) + assert hetzner["gst"] == "Excl GST" + # uploaded from the transaction's row: linked straight away, with changes to offer + books.fake_ocr.update(vendor="Hetzner Online", total_cents=553, gst_cents=72, gst_number="123-456-789") + a = books.post(f"/api/attachments?filename=h.pdf&kind=invoice&row={hetzner['row']}", + content=b"%PDF-1.4 hetzner").json() + assert (a["kind"], a["files"][0]["mime"], a["transaction"]["row"]) == ("invoice", "application/pdf", hetzner["row"]) + assert a["proposed"] == {"gst": "Incl GST"} + + a = books.post(f"/api/attachments/{a['id']}/link", + json={"row": hetzner["row"], "apply": {"gst": "Incl GST", "category": "Purchases"}}).json() + assert a["proposed"] == {} + hetzner = next(r for r in rows(books, "MAY") if r["row"] == hetzner["row"]) + assert (hetzner["gst"], hetzner["category"], hetzner["attachments"]) == ("Incl GST", "Purchases", 1) + + r = books.post(f"/api/attachments/{a['id']}/link", json={"row": hetzner["row"], "apply": {"amount": "1"}}) + assert r.status_code == 400 + + +def test_category_from_vendor_and_history(books, tmp_path): + con = sqlite3.connect(tmp_path / "cashbook.db") + with con: + con.execute("UPDATE transactions SET category = NULL") + fee = next(r for r in rows(books, "APR") if r["reference"] == "Monthly A/C Fee") + books.fake_ocr.update(vendor="Spark NZ", total_cents=850) + a = books.post(f"/api/attachments?filename=spark.png&row={fee['row']}", content=png(b"s")).json() + assert a["proposed"] == {"category": "Telephone & Power"} + + +def test_edit_details_updates_suggestions(books): + books.fake_ocr.update(status="failed", error="tesseract isn't installed") + r = books.post("/api/attachments?filename=blurry.jpg", content=b"\xff\xd8\xff" + b"x" * 50).json() + assert (r["ocr_status"], r["ocr_error"], r["suggestions"]) == ("failed", "tesseract isn't installed", []) + + a = books.patch(f"/api/attachments/{r['id']}", json={"total": 1150, "doc_date": "2026-04-21", + "vendor": "Client Ltd"}).json() + assert (a["total_cents"], a["vendor"]) == (115000, "Client Ltd") + assert [s["reference"] for s in a["suggestions"]] == ["Client Ltd (Inv 101)"] + # too far from the transaction's date + a = books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "2026-05-21"}).json() + assert a["suggestions"] == [] + assert books.patch(f"/api/attachments/{r['id']}", json={"doc_date": "21/05/2026"}).status_code == 400 + assert books.patch(f"/api/attachments/{r['id']}", json={"kind": "bill"}).status_code == 400 + + +def test_several_photos_of_one_receipt(books, tmp_path, monkeypatch): + # a long receipt in two photos: the top has the shop and date, the bottom the total + pages = {b"top": "JAYCAR ELECTRONICS\nDate: 12/04/2026\nSoldering Station 89.00\n", + b"bottom": "Solder 26.00\nTOTAL $115.00\nGST INCLUDED 15.00\nGST No: 45-678-912\n"} + def read(p, mime): + text = pages[p.read_bytes()[8:]] + return {"status": "done", "text": text, **parse_receipt(text)} + monkeypatch.setattr(receipts, "read_document", read) + + def photo(name): + return b"\x89PNG\r\n\x1a\n" + name + + top = books.post("/api/attachments?filename=top.png", content=photo(b"top")).json() + # with no TOTAL line, the top alone gives a wrong guess at the total + assert (top["total"], top["suggestions"], len(top["files"])) == (89.0, [], 1) + books.patch(f"/api/attachments/{top['id']}", json={"vendor": "Jaycar"}) # a correction by hand + + a = books.post(f"/api/attachments/{top['id']}/pages?filename=bottom.png", content=photo(b"bottom")).json() + assert [f["filename"] for f in a["files"]] == ["top.png", "bottom.png"] + # the details come from both pages read together, except the correction, which is kept + assert (a["vendor"], a["doc_date"], a["total"], a["gst"], a["gst_number"], a["edited"]) == \ + ("Jaycar", "2026-04-12", 115.0, 15.0, "045-678-912", ["vendor"]) + assert [i["description"] for i in a["items"]] == ["Soldering Station", "Solder"] + assert a["raw_text"].startswith("JAYCAR") and "TOTAL" in a["raw_text"] + assert [s["reference"] for s in a["suggestions"]] == ["Jaycar Chris"] + assert len(books.get("/api/attachments").json()) == 1 + + bottom_id = a["files"][1]["id"] + f = books.get(f"/api/attachments/{a['id']}/files/{bottom_id}") + assert f.status_code == 200 and f.content == photo(b"bottom") + assert books.get(f"/api/attachments/{a['id']}/file").content == photo(b"top") + assert books.get(f"/api/attachments/999/files/{bottom_id}").status_code == 404 + + # the same photo can't go on twice, here or on another receipt + r = books.post(f"/api/attachments/{a['id']}/pages?filename=again.png", content=photo(b"bottom")) + assert r.status_code == 400 + assert books.post(f"/api/attachments/999/pages?filename=x.png", content=photo(b"x")).status_code == 404 + assert len(os.listdir(tmp_path / "attachments")) == 2 + + a = books.post(f"/api/attachments/{a['id']}/reread").json() + assert (a["vendor"], a["edited"]) == ("JAYCAR ELECTRONICS", []) # re-reading replaces corrections + + # without the bottom, it's back to what the top says + books.patch(f"/api/attachments/{a['id']}", json={"gst": 15}) + a = books.delete(f"/api/attachments/{a['id']}/pages/{bottom_id}").json() + assert ([f["filename"] for f in a["files"]], a["total"], a["gst"], a["gst_number"]) == \ + (["top.png"], 89.0, 15.0, None) + assert len(os.listdir(tmp_path / "attachments")) == 1 + r = books.delete(f"/api/attachments/{a['id']}/pages/{a['files'][0]['id']}") + assert r.status_code == 400 and "only page" in r.json()["detail"] + + books.post(f"/api/attachments/{a['id']}/pages?filename=bottom.png", content=photo(b"bottom")) + books.delete(f"/api/attachments/{a['id']}") + assert os.listdir(tmp_path / "attachments") == [] + + +def test_rejects_other_files(books, tmp_path): + r = books.post("/api/attachments?filename=a.txt", content=b"hello") + assert r.status_code == 400 and "JPEG" in r.json()["detail"] + r = books.post("/api/attachments?filename=a.heic", content=b"\0\0\0\x18ftypheic" + b"\0" * 20) + assert r.status_code == 400 and "HEIC" in r.json()["detail"] + assert books.post("/api/attachments?filename=a.png", content=b"").status_code == 400 + assert books.post("/api/attachments?filename=a.png&row=9999", content=png(b"z")).status_code == 404 + assert os.listdir(tmp_path / "attachments") == [] + + +def test_needs_sign_in(client): + assert client.get("/api/attachments").status_code == 401 + assert client.post("/api/attachments?filename=a.png", content=png(b"a")).status_code == 401 + + +def test_upgrade_keeps_attachment_reference(tmp_path): + """The v1 -> v2 table rebuild mustn't leave attachments pointing at the old table.""" + path = tmp_path / "old.db" + con = sqlite3.connect(path) + con.executescript(ledger.SCHEMA.replace("'Zero-rated', ", "")) + con.close() + conn = ledger.connect(path) + sql = conn.execute("SELECT sql FROM sqlite_master WHERE name = 'attachments'").fetchone()[0] + assert "transactions_v1" not in sql + assert "Zero-rated" in conn.execute("SELECT sql FROM sqlite_master WHERE name = 'transactions'").fetchone()[0] + + +V2_ATTACHMENTS = """CREATE TABLE attachments ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER REFERENCES transactions(id) ON DELETE SET NULL, + kind TEXT NOT NULL DEFAULT 'receipt' CHECK (kind IN ('receipt', 'invoice')), + filename TEXT NOT NULL, stored_name TEXT NOT NULL UNIQUE, mime TEXT NOT NULL, + sha256 TEXT NOT NULL UNIQUE, size INTEGER NOT NULL, + vendor TEXT, doc_date TEXT, total_cents INTEGER, gst_cents INTEGER, gst_number TEXT, + items_json TEXT, raw_text TEXT, + ocr_status TEXT NOT NULL DEFAULT 'skipped' CHECK (ocr_status IN ('done', 'failed', 'skipped')), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +CREATE INDEX attachments_transaction ON attachments (transaction_id);""" + + +def test_upgrade_moves_files_to_pages(tmp_path): + """Version 2 databases had one file per attachment, in the attachments table.""" + path = tmp_path / "v2.db" + con = sqlite3.connect(path) + con.executescript("""CREATE TABLE books (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, + start_year INTEGER NOT NULL, opening_cents INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + CREATE TABLE transactions (id INTEGER PRIMARY KEY, + book_id INTEGER NOT NULL REFERENCES books(id) ON DELETE CASCADE, date TEXT NOT NULL, + reference TEXT NOT NULL DEFAULT '', origin TEXT NOT NULL DEFAULT '', amount_cents INTEGER NOT NULL, + gst TEXT NOT NULL CHECK (gst IN ('Incl GST', 'Zero-rated', 'Excl GST')), category TEXT, source TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);""" + V2_ATTACHMENTS + """ + INSERT INTO books (id, name, start_year) VALUES (1, 'b', 2026); + INSERT INTO transactions (id, book_id, date, reference, amount_cents, gst) VALUES (5, 1, '2026-04-13', 'Jaycar', -11500, 'Incl GST'); + INSERT INTO attachments (id, transaction_id, filename, stored_name, mime, sha256, size, vendor, total_cents, + items_json, raw_text, ocr_status) VALUES (7, 5, 'r.jpg', 'abc-1.jpg', 'image/jpeg', 'abc', 10, 'Jaycar', + 11500, '[]', 'JAYCAR TOTAL 115.00', 'done'); + PRAGMA user_version = 2;""") + con.close() + + conn = ledger.connect(path) + assert conn.execute("PRAGMA user_version").fetchone()[0] == ledger.SCHEMA_VERSION + a = ledger.get_attachment(conn, 7) + assert a["edited"] == [] + assert (a["transaction"]["row"], a["vendor"], a["total"], a["raw_text"], a["filename"]) == \ + (5, "Jaycar", 115.0, "JAYCAR TOTAL 115.00", "r.jpg") + assert [(f["filename"], f["mime"], f["size"]) for f in a["files"]] == [("r.jpg", "image/jpeg", 10)] + assert ledger.get_file(conn, 7)["stored_name"] == "abc-1.jpg" + assert conn.execute("SELECT name FROM sqlite_master WHERE name = 'attachments_transaction'").fetchone() + assert "attachments_v2" not in conn.execute( + "SELECT sql FROM sqlite_master WHERE name = 'attachment_files'").fetchone()[0] + assert (tmp_path / "v2.db.v2.bak").exists() + # pages go with their attachment, and the link still works both ways + conn.execute("DELETE FROM transactions WHERE id = 5") + assert ledger.get_attachment(conn, 7)["transaction_id"] is None + ledger.delete_attachment(conn, 7) + assert conn.execute("SELECT COUNT(*) FROM attachment_files").fetchone()[0] == 0 + conn.close() + ledger.connect(path).close() # and connecting again changes nothing + + +@pytest.mark.skipif(not shutil.which("tesseract"), reason="tesseract isn't installed") +def test_ocr_round_trip(): + from PIL import Image, ImageDraw, ImageFont + img = Image.new("RGB", (900, 700), "white") + draw = ImageDraw.Draw(img) + try: + font = ImageFont.truetype("DejaVuSansMono.ttf", 34) + except OSError: + font = ImageFont.load_default(size=34) + for i, line in enumerate(TILL_RECEIPT.strip().splitlines()): + draw.text((40, 30 + i * 50), line.strip(), fill="black", font=font) + import io + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=90) + r = parse_receipt(receipts.ocr_image(buf.getvalue())) + assert (r["date"], r["total_cents"], r["gst_cents"]) == ("2026-05-14", 4739, 618) + assert "BUNNINGS" in r["vendor"] diff --git a/tests/test_suppliers.py b/tests/test_suppliers.py new file mode 100644 index 0000000..538271d --- /dev/null +++ b/tests/test_suppliers.py @@ -0,0 +1,84 @@ +import sqlite3 + +import pytest + +import ledger +from cashbook_app.services import receipts +from test_receipts import png + + +@pytest.fixture +def ocr(admin, monkeypatch): + """OCR replaced by whatever `fake_ocr` holds.""" + admin.fake_ocr = {"status": "done", "text": "", "items": []} + monkeypatch.setattr(receipts, "read_document", lambda p, mime: dict(admin.fake_ocr)) + return admin + + +def test_supplier_crud(admin): + s = admin.post("/api/suppliers", json={"name": " Bunnings ", "gst_number": "123-456-789"}).json() + assert (s["name"], s["gst_number"], s["documents"], s["total"]) == ("Bunnings", "123-456-789", 0, 0) + assert admin.post("/api/suppliers", json={"name": ""}).status_code == 422 + s = admin.put(f"/api/suppliers/{s['id']}", json={"name": "Bunnings Ltd", "email": "a@b.nz"}).json() + assert (s["name"], s["email"], s["gst_number"]) == ("Bunnings Ltd", "a@b.nz", "") + assert [x["name"] for x in admin.get("/api/suppliers").json()] == ["Bunnings Ltd"] + assert admin.delete(f"/api/suppliers/{s['id']}").json() == {"ok": True} + assert admin.get(f"/api/suppliers/{s['id']}").status_code == 404 + + +def test_receipts_link_to_suppliers(ocr): + sup = ocr.post("/api/suppliers", json={"name": "Jaycar Electronics Ltd", "gst_number": "098-765-432"}).json() + + # the same name (ignoring Ltd and case) links on upload + ocr.fake_ocr.update(vendor="JAYCAR ELECTRONICS", total_cents=11500) + a = ocr.post("/api/attachments?filename=a.png", content=png(b"a")).json() + assert a["supplier"] == {"id": sup["id"], "name": "Jaycar Electronics Ltd"} + + # so does the GST number, whatever the name says + ocr.fake_ocr.update(vendor="J'car", gst_number="098-765-432", total_cents=500) + b = ocr.post("/api/attachments?filename=b.png", content=png(b"b")).json() + assert b["supplier_id"] == sup["id"] + + # an unknown vendor is left for the user + ocr.fake_ocr.update(vendor="Somewhere Else", gst_number=None) + c = ocr.post("/api/attachments?filename=c.png", content=png(b"c")).json() + assert c["supplier"] is None + + s = ocr.get(f"/api/suppliers/{sup['id']}").json() + assert (s["documents"], s["total"]) == (2, 120.0) + assert {x["id"] for x in ocr.get(f"/api/attachments?supplier={sup['id']}").json()} == {a["id"], b["id"]} + + # link by hand, and unlink + c = ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": sup["id"]}).json() + assert c["supplier"]["name"] == "Jaycar Electronics Ltd" + assert c["vendor"] == "Somewhere Else" # what was read is kept + assert ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": None}).json()["supplier"] is None + assert ocr.patch(f"/api/attachments/{c['id']}", json={"supplier_id": 999}).status_code == 404 + + # a supplier with documents can't be deleted until they're unlinked + assert ocr.delete(f"/api/suppliers/{sup['id']}").status_code == 400 + + +def test_customer_invoices_filter(admin): + admin.post("/api/books", json={"name": "b", "year": 2026}) + ids = [admin.post("/api/customers", json={"name": n}).json()["id"] for n in ("A", "B")] + for cid in (ids[0], ids[1], ids[1]): + r = admin.post("/api/invoices", json={"customer_id": cid, "issue_date": "2026-04-10", + "lines": [{"description": "x", "unit": 10}]}) + assert r.status_code == 200, r.text + assert len(admin.get(f"/api/invoices?customer={ids[1]}").json()) == 2 + assert len(admin.get("/api/invoices").json()) == 3 + + +def test_upgrade_adds_supplier_link(tmp_path): + """Databases from before suppliers get the new column and keep their receipts.""" + path = tmp_path / "old.db" + con = sqlite3.connect(path) + con.executescript(ledger.SCHEMA.replace(" supplier_id INTEGER REFERENCES suppliers(id), -- who the receipt or bill is from\n", "")) + con.execute("INSERT INTO attachments (vendor) VALUES ('Old receipt')") + con.execute("PRAGMA user_version = 4") + con.commit() + con.close() + conn = ledger.connect(path) + assert "supplier_id" in {c["name"] for c in conn.execute("PRAGMA table_info(attachments)")} + assert ledger.list_attachments(conn)[0]["supplier"] is None