export_data.py

#!/usr/bin/env python3
"""Emit web/static/data/app.json for the Svelte PWA (served at /data/app.json, fetched at runtime).

Thin exporter: reuses dashboard.load() + insights.compute() so all extraction,
categorization, and leak logic stays in the frozen parse.py/insights.py. Adds only
a `committed` block (recurring monthly obligations) for the headroom hero.

FIELD NAME NOTE: insights.compute() recs use:
  - r["type"]      (not "kind") to distinguish sub/creep/oneoff
  - r["rmMonthly"] (not "monthly") for per-sub monthly price
  - r["stale"]     (bool) for staleness flag
  installments[] use:
  - p["monthly"]   for monthly charge
  - p["name"]      for display name
"""
import csv
import json
import os
import re
from collections import Counter

import dashboard
import insights

OUT = os.environ.get("STMT_OUT", os.path.join("web", "static", "data", "app.json"))


def build_committed(insights_out):
    """Sum recurring monthly obligations: active subs + installments.

    Uses "type" (not "kind") and "rmMonthly" (not "monthly") to match the
    real insights.compute() output shape. The exported items use "kind" as
    the field name so Svelte consumers get a consistent, clean contract.
    """
    inst = insights_out.get("installments", []) or []
    active_inst = [p for p in inst if not p.get("ended", False)]
    inst_total = round(sum(float(p.get("monthly", 0) or 0) for p in active_inst), 2)
    items = [{"name": p.get("name", "?"), "monthly": round(float(p.get("monthly", 0) or 0), 2),
              "kind": "installment"} for p in active_inst]

    subs_total = 0.0
    sub_cats: list[str] = []
    for r in insights_out.get("recs", []) or []:
        # insights.py uses r["type"] == "sub" (not "kind")
        # and r["rmMonthly"] for the per-sub monthly price (not "monthly")
        if r.get("type") == "sub" and not r.get("stale"):
            m = round(float(r.get("rmMonthly", 0) or 0), 2)
            subs_total += m
            items.append(
                {
                    "name": r.get("merchant", r.get("name", "?")),
                    "monthly": m,
                    "kind": "sub",
                }
            )
            cat = r.get("cat")
            if cat and cat not in sub_cats:
                sub_cats.append(cat)
    subs_total = round(subs_total, 2)
    sub_cats.sort()

    return {
        "monthly": round(inst_total + subs_total, 2),
        "subs": subs_total,
        "installments": inst_total,
        "subCats": sub_cats,
        "items": items,
    }


def build_cycles(csv_path="transactions.csv"):
    """Map each card -> its statement closing day (1-31).

    Day is the mode of statement_date day-of-month per card; ties break to the
    day of the most recent statement_date. Keyed "<bank>·<last4>" to match rows[].c.
    """
    dates: dict[str, list[str]] = {}
    with open(csv_path, encoding="utf-8-sig") as fh:
        for r in csv.DictReader(fh):
            key = f"{r['bank']}·{r['card_last4']}"
            sd = (r.get("statement_date") or "").strip()
            if sd:
                dates.setdefault(key, []).append(sd)
    cycles: dict[str, int] = {}
    for key, ds in dates.items():
        days = [int(d.split("-")[2]) for d in ds]
        cnt = Counter(days)
        top = max(cnt.values())
        tied = {d for d, c in cnt.items() if c == top}
        # tie-break: day of the latest statement_date among tied days
        latest = max(d for d in ds if int(d.split("-")[2]) in tied)
        cycles[key] = int(latest.split("-")[2])
    return cycles


def build_bills(csv_path="reconciliation.csv"):
    """Newest non-duplicate statement per bank -> bill {bank, statement_month,
    current_balance, payment_due_date, minimum_payment}. Bill identity is the
    bank (multi-card banks issue one statement with one due date + summed balance)."""
    newest = {}
    with open(csv_path, encoding="utf-8-sig") as fh:
        for r in csv.DictReader(fh):
            if r.get("status") in ("DUPLICATE", "ERROR"):
                continue
            sm = r.get("smonth") or ""
            # Guard: only YYYY-MM smonths participate in the newest-per-bank
            # string-compare. An UNKNOWN-smonth NO_BALANCE row would otherwise
            # sort above "2026-NN" and win with a null balance.
            if not re.fullmatch(r"\d{4}-\d{2}", sm):
                continue
            bank = r["bank"]
            if bank not in newest or sm > newest[bank]["smonth"]:
                newest[bank] = r
    bills = []
    for bank, r in sorted(newest.items()):
        cur = r.get("cur")
        bills.append({
            "bank": bank,
            "statement_month": r["smonth"],
            "current_balance": float(cur) if cur not in (None, "") else None,
            "payment_due_date": r.get("due") or None,
            "minimum_payment": None,   # not extracted yet (Plan 1 scope)
        })
    return bills


def build_card_numbers(csv_path="transactions.csv"):
    """Map each card -> its full PAN digits, but ONLY where the statement printed one
    unmasked (SC prints `5520-40XX-XXXX-XXXX`, so those cards simply have no entry and
    the UI shows no copy affordance). Keyed "<bank>·<last4>" to match rows[].c.

    A missing/older transactions.csv (no card_number column) is no numbers, not an error.
    """
    nums: dict[str, str] = {}
    with open(csv_path, encoding="utf-8-sig") as fh:
        for r in csv.DictReader(fh):
            n = (r.get("card_number") or "").strip()
            if n:
                nums.setdefault(f"{r['bank']}·{r['card_last4']}", n)
    return nums


def build_other(rows, limit=50):
    """Merchants that fell through CATS into `Other` — what the confirmation UI asks
    about (#82).

    Grouped by insights.norm_merchant so the key is EXACTLY what parse.py applies an
    override with; a different grouping here means a confirmed merchant reappears as
    unknown under a slightly different tail. `rm` is netted the way every other total in
    this app is (debit +, credit -), so a refunded one-off does not read as spend.
    Biggest first, because that is the order worth answering in."""
    agg: dict = {}
    for r in rows:
        if r["g"] != "Other":
            continue
        k = insights.norm_merchant(r["d"])
        if not k:
            continue
        a = agg.setdefault(k, {"m": k, "n": 0, "rm": 0.0})
        a["n"] += 1
        a["rm"] += r["a"] if r["t"] == 0 else -r["a"]
    out = sorted(agg.values(), key=lambda a: -a["rm"])[:limit]
    for a in out:
        a["rm"] = round(a["rm"], 2)
    return out


def build_payload(rows, insights_out, cycles=None, bills=None, card_numbers=None):
    """Assemble the full app.json payload from transaction rows + insights output."""
    months = sorted({r["m"] for r in rows})
    cards = sorted({r["c"] for r in rows})
    # Only include cats that actually appear in the data, in COLORS order
    cats = [c for c in dashboard.COLORS if any(r["g"] == c for r in rows)]
    # dashboard.NON_SPEND is a list; pass it through as-is
    return {
        # strip the monthly ":NN/MM" installment ratio for display (insights already
        # consumed it for exact progress) — keeps merchant charts/tables from fragmenting
        "rows": dashboard.display_rows(rows),
        "months": months,
        "cards": cards,
        "cats": cats,
        "nonSpend": dashboard.NON_SPEND,  # a list (Svelte: use .includes())
        "colors": dashboard.COLORS,
        "catIcon": dashboard.CAT_ICON,
        "icons": dashboard.MDI,
        "range": f"{months[0]} → {months[-1]}" if months else "",
        "recs": insights_out.get("recs", []),
        "installments": insights_out.get("installments", []),
        "transfers": insights_out.get("transfers", []),
        "committed": build_committed(insights_out),
        "cycles": cycles or {},
        "bills": bills or [],
        # full PAN digits per card, only where the statement printed one unmasked.
        # The UI keeps showing the masked last4; this is only what a copy button copies.
        "cardNumbers": card_numbers or {},
        # #82: what the categoriser could not place, and the full taxonomy to place it
        # into. `cats` above is only what APPEARS in the data — a dropdown built from
        # that could never offer a category nothing has been filed under yet.
        "other": build_other(rows),
        "allCats": [c for c in dashboard.COLORS if c != "Other"],
    }


def main():
    rows = dashboard.load()
    insights_out = insights.compute(rows)
    cycles = build_cycles()
    bills = build_bills()
    card_numbers = build_card_numbers()
    payload = build_payload(rows, insights_out, cycles, bills, card_numbers)
    os.makedirs(os.path.dirname(OUT) or ".", exist_ok=True)
    tmp = OUT + ".tmp"
    with open(tmp, "w", encoding="utf-8") as fh:
        json.dump(payload, fh, separators=(",", ":"))
    os.replace(tmp, OUT)  # atomic on POSIX; last-good app.json survives a crash
    print(
        f"wrote {OUT}: {len(rows)} txns, "
        f"committed RM{payload['committed']['monthly']}/mo "
        f"(subs RM{payload['committed']['subs']}, "
        f"installments RM{payload['committed']['installments']})"
    )


if __name__ == "__main__":
    main()