Files
brokeless/src/lib/statement.ts
T
MangaValkandClaude Opus 5.5 33a8a237ea Statement import: ABN AMRO's TXT/TAB download
It has no header row, so nothing came in. Recognize its eight columns, and take
the other party's name out of ABN's coded text (/NAME/, card payments, Naam:).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 16:40:42 +01:00

276 lines
15 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Bank statement import: CSV, OFX and QIF to transactions. Pure, no "@/" imports, so statement.test.ts runs with plain node.
// Output uses the app's sign: positive = money out.
export type Row = { date: string; amount: number; name: string; key: string; category?: string };
export type Table = { header: string[]; rows: string[][] };
export type DateOrder = "dmy" | "mdy" | "ymd";
// Column indexes; -1 = not used. A single signed amount, or separate debit (out) and credit (in) columns.
// sign: a column that says which way the money went, for banks whose amount is always positive (ING's "Af Bij").
export type Mapping = { date: number; name: number; amount: number; debit: number; credit: number; order: DateOrder; outPositive: boolean; category?: number; sign?: number };
export type Parsed = { kind: "csv"; table: Table; mapping: Mapping } | { kind: "ofx" | "qif"; rows: Row[] };
const MONTHS = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
const pad = (n: number) => String(n).padStart(2, "0");
// "1/10/2026", "2026/10/01", "01-10-26", "1 Oct 2026", "Oct 1, 2026", "20261001", "2026-10-01 09:30", "1/10'26" (QIF).
export function parseDate(raw: string, order: DateOrder): string | null {
const s = raw.trim().replace(/'/g, "/").replace(/[T\s]\d{1,2}:\d{2}.*$/, "");
let y: number, m: number, d: number;
let p: RegExpMatchArray | null;
if ((p = s.match(/^(\d{4})(\d{2})(\d{2})/))) [y, m, d] = [+p[1], +p[2], +p[3]];
else if ((p = s.match(/^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})$/))) [y, m, d] = [+p[1], +p[2], +p[3]];
else if ((p = s.match(/^(\d{1,2})[-/.](\d{1,2})[-/.](\d{2}|\d{4})$/))) {
[d, m] = order === "mdy" ? [+p[2], +p[1]] : [+p[1], +p[2]];
y = +p[3];
} else if ((p = s.match(/^(\d{1,2})[-\s/]([a-z]{3})[a-z]*[-\s/,]+(\d{2}|\d{4})$/i))) [d, m, y] = [+p[1], MONTHS.indexOf(p[2].toLowerCase()) + 1, +p[3]];
else if ((p = s.match(/^([a-z]{3})[a-z]*\s+(\d{1,2}),?\s+(\d{4})$/i))) [m, d, y] = [MONTHS.indexOf(p[1].toLowerCase()) + 1, +p[2], +p[3]];
else return null;
if (y < 100) y += 2000;
if (m < 1 || m > 12 || d < 1 || d > 31) return null;
const iso = `${y}-${pad(m)}-${pad(d)}`;
return new Date(iso + "T00:00:00Z").getUTCDate() === d ? iso : null; // rejects 31 Feb
}
const NUMERIC_DATE = /^(\d{1,2})[-/.](\d{1,2})[-/.]\d{2,4}/;
// Which way round dd/mm and mm/dd go: the data decides when a day is over 12, else the country's own date format.
export function dateOrder(values: string[], locale: string): DateOrder {
if (values.some((v) => /^\s*\d{4}[-/.]?\d{2}/.test(v))) return "ymd";
let dmy = false, mdy = false;
for (const v of values) {
const p = v.trim().match(NUMERIC_DATE);
if (!p) continue;
if (+p[1] > 12) dmy = true;
if (+p[2] > 12) mdy = true;
}
if (dmy !== mdy) return dmy ? "dmy" : "mdy";
const parts = new Intl.DateTimeFormat(locale).formatToParts(new Date(Date.UTC(2020, 11, 31)));
const i = (t: string) => parts.findIndex((x) => x.type === t);
return i("month") < i("day") ? "mdy" : "dmy";
}
// "1,234.56", "1.234,56", "1,25,000.00", "-$45.20", "(12.00)", "12.00 DR", "£3.50". Empty or "-" = null.
export function parseAmount(raw: string, locale = "en-US"): number | null {
let s = raw.trim();
if (!s || s === "-") return null;
let neg = /^\(.*\)$/.test(s) || /-/.test(s.replace(/\d[-]\d/g, "")) || /\bDR\.?$/i.test(s);
s = s.replace(/[^\d.,]/g, "");
if (!/\d/.test(s)) return null;
const last = Math.max(s.lastIndexOf("."), s.lastIndexOf(","));
const sep = s[last];
// Both marks: the last is the decimal. One mark used once: decimal unless 3 digits follow, then the country decides.
const isDecimal = last >= 0 && (
(s.includes(".") && s.includes(","))
|| (s.split(sep).length === 2 && (s.length - last - 1 !== 3 || sep === new Intl.NumberFormat(locale).format(1.5)[1]))
);
const n = isDecimal ? Number(s.slice(0, last).replace(/[.,]/g, "") + "." + s.slice(last + 1)) : Number(s.replace(/[.,]/g, ""));
if (!Number.isFinite(n)) return null;
if (/^-?0+(\.0+)?$/.test(String(n))) neg = false;
return neg ? -n : n;
}
// RFC 4180-ish: quoted fields, "" escapes, newlines inside quotes.
function splitRows(text: string, delim: string): string[][] {
const rows: string[][] = [];
let row: string[] = [], cell = "", q = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (q) {
if (c === '"' && text[i + 1] === '"') {
cell += c;
i++;
}
else if (c === '"') q = false;
else cell += c;
} else if (c === '"') q = true;
else if (c === delim) {
row.push(cell);
cell = "";
}
else if (c === "\n" || c === "\r") {
if (c === "\r" && text[i + 1] === "\n") i++;
row.push(cell);
rows.push(row);
row = [];
cell = "";
} else cell += c;
}
if (cell || row.length) rows.push([...row, cell]);
return rows.map((r) => r.map((x) => x.trim())).filter((r) => r.some(Boolean));
}
const looksDate = (v: string) => parseDate(v, "dmy") !== null || parseDate(v, "mdy") !== null;
// Delimiter: the one giving the most rows with the same (2+) field count. Header: the row just above the first
// row with a date in it, when it has no dates itself (skips bank preambles like ASB's account lines).
export function readTable(text: string): Table {
const clean = text.replace(/^/, "");
let best: string[][] = [];
let bestScore = -1;
for (const d of [",", ";", "\t", "|"]) {
const rows = splitRows(clean, d);
const counts = new Map<number, number>();
for (const r of rows) if (r.length > 1) counts.set(r.length, (counts.get(r.length) ?? 0) + 1);
const score = Math.max(0, ...counts.values());
if (score > bestScore) {
bestScore = score;
best = rows;
}
}
const first = best.findIndex((r) => r.length > 1 && r.some(looksDate));
if (first < 0) return { header: [], rows: [] };
const width = best[first].length;
const h = best[first - 1];
const hasHeader = !!h && h.length >= width - 1 && !h.some(looksDate);
const header = hasHeader ? Array.from({ length: width }, (_, i) => h[i] || `Column ${i + 1}`) : Array.from({ length: width }, (_, i) => `Column ${i + 1}`);
return { header, rows: best.slice(first).filter((r) => r.length >= Math.min(width, 2)) };
}
const find = (header: string[], tests: RegExp[], skip: RegExp = /$^/) => {
for (const t of tests) {
const i = header.findIndex((h) => t.test(h) && !skip.test(h));
if (i >= 0) return i;
}
return -1;
};
// ABN AMRO's TXT/TAB download has no header: account, currency, date, opening balance, closing balance, value date,
// signed amount, description. Two of the other columns are amounts too, so it can't be guessed from the values.
const isAbnAmro = (t: Table, col: (i: number) => string[]) =>
t.header.length === 8 && t.header[0] === "Column 1" && col(1).every((v) => /^[A-Z]{3}$/.test(v)) && col(2).every((v) => /^\d{8}$/.test(v));
export function guessMapping(t: Table, locale: string): Mapping {
const h = t.header.map((x) => x.toLowerCase().trim());
const col = (i: number) => t.rows.slice(0, 50).map((r) => r[i] ?? "");
if (isAbnAmro(t, col)) return { date: 2, name: 7, amount: 6, debit: -1, credit: -1, order: "ymd", outPositive: false, category: -1 };
let date = find(h, [/^(transaction |txn |posted |posting )?date$/, /date/], /value|process|created/);
if (date < 0) date = h.findIndex((_, i) => col(i).filter(looksDate).length > col(i).length / 2);
let name = find(h, [/payee/, /counter ?party/, /^name$/, /^details$/, /narration/, /description/, /merchant/, /memo/, /particulars/, /reference/]);
if (name < 0) name = h.findIndex((_, i) => i !== date && col(i).some((v) => /[a-z]{3}/i.test(v) && parseAmount(v) === null));
const amount = find(h, [/^amount$/, /^amount\b/, /amount/, /betrag|montant|importe|importo|bedrag|belopp|monto|valor/], /balance|foreign|local|credit|debit|\bcr\b|\bdr\b/);
const debit = amount >= 0 ? -1 : find(h, [/withdrawal/, /debit/, /money out/, /paid out/, /^dr$/, /\bout\b/]);
const credit = amount >= 0 ? -1 : find(h, [/deposit/, /credit/, /money in/, /paid in/, /^cr$/, /\bin\b/]);
const category = find(h, [/^category$/, /categor/]);
const sign = amount < 0 ? -1 : find(h, [/^af ?[/-]? ?bij$/, /^(debit|credit) ?[/-]? ?(credit|debit)$/, /^d ?[/-]? ?c$/, /^soll ?[/-]? ?haben$/]);
return { date, name, amount, debit, credit, order: dateOrder(date >= 0 ? col(date) : [], locale), outPositive: false, category, ...(sign >= 0 && { sign }) };
}
// Stable dedupe keys: the same date, amount and name gets 1, 2, 3... in file order, so re-importing adds nothing.
export function withKeys(rows: Omit<Row, "key">[], ids?: (string | undefined)[]): Row[] {
const seen = new Map<string, number>();
return rows.map((r, i) => {
const base = `${r.date}|${r.amount.toFixed(2)}|${r.name.toLowerCase()}`;
const n = (seen.get(base) ?? 0) + 1;
seen.set(base, n);
return { ...r, key: ids?.[i] ? `id:${ids[i]}` : `${base}|${n}` };
});
}
// Dutch banks (ABN AMRO above all) pack the other party's name into coded text. Pull the name out; any other text
// stays as the bank wrote it.
export function cleanName(raw: string) {
const s = raw.replace(/\s+/g, " ").trim();
// SEPA transfer or direct debit: /TRTP/SEPA OVERBOEKING/IBAN/.../NAME/Werkgever BV/REMI/...
const sepa = s.match(/\/NAME\/([^/]+)/);
if (sepa) return sepa[1].trim();
// Card payment or cash: "BEA, Apple Pay ALBERT HEIJN 1234,PAS123 NR:..." or "BEA NR:XX 14.11.26/18.02 ALBERT HEIJN,PAS123"
const card = s.match(/^(?:BEA|GEA|eCom)\b[,:]?\s*(?:(?:Apple|Google) Pay|Betaalpas)?\s*(.+?),\s*PAS\d+/i);
if (card) return card[1].replace(/^NR:\S+\s+\d{2}\.\d{2}\.\d{2}\/\d{2}[.:]\d{2}\s+/, "").trim();
// Older SEPA text: "SEPA Incasso ... Naam: Netflix International B.V. Machtiging: ..."
const naam = s.match(/\bNaam:\s*(.+?)(?=\s+(?:Machtiging|Omschrijving|IBAN|Kenmerk|BIC|Incassant|Betalingskenm\.?)[:\s]|$)/i);
if (naam) return naam[1].trim();
return s;
}
export function applyMapping(t: Table, m: Mapping, locale: string): { rows: Row[]; skipped: number } {
const out: Omit<Row, "key">[] = [];
let skipped = 0;
for (const r of t.rows) {
const date = m.date >= 0 ? parseDate(r[m.date] ?? "", m.order) : null;
let amount: number | null = null;
if (m.amount >= 0) {
const a = parseAmount(r[m.amount] ?? "", locale);
// Most banks export money out as negative; the app stores it as positive.
if (a !== null) amount = m.outPositive ? a : -a;
// A direction column wins over the sign: "Af"/"Debit" is money out, "Bij"/"Credit" is money in.
const way = (m.sign ?? -1) >= 0 ? (r[m.sign!] ?? "").trim().toLowerCase() : "";
if (a !== null && /^(af|debit|d|dr|s|soll)$/.test(way)) amount = Math.abs(a);
else if (a !== null && /^(bij|credit|c|cr|h|haben)$/.test(way)) amount = -Math.abs(a);
} else {
const dr = m.debit >= 0 ? parseAmount(r[m.debit] ?? "", locale) : null;
const cr = m.credit >= 0 ? parseAmount(r[m.credit] ?? "", locale) : null;
if (dr !== null || cr !== null) amount = Math.abs(dr ?? 0) - Math.abs(cr ?? 0);
}
const name = cleanName(m.name >= 0 ? r[m.name] ?? "" : "");
if (!date || amount === null || !amount) skipped++;
else {
const category = (m.category ?? -1) >= 0 ? (r[m.category!] ?? "").trim() : "";
out.push({ date, amount: Math.round(amount * 100) / 100, name: name || "Imported transaction", ...(category && { category }) });
}
}
return { rows: withKeys(out), skipped };
}
// OFX 1.x (SGML, unclosed tags) and 2.x (XML). TRNAMT is negative for money out. FITID is the bank's own id.
function parseOfx(text: string): Row[] {
const tag = (block: string, t: string) => block.match(new RegExp(`<${t}>([^<\\r\\n]*)`, "i"))?.[1].trim() ?? "";
const rows: Omit<Row, "key">[] = [];
const ids: string[] = [];
for (const b of text.split(/<STMTTRN>/i).slice(1)) {
const date = parseDate(tag(b, "DTPOSTED"), "ymd");
const amt = parseAmount(tag(b, "TRNAMT"));
if (!date || !amt) continue;
rows.push({ date, amount: -amt, name: (tag(b, "NAME") || tag(b, "MEMO") || "Imported transaction").replace(/&amp;/g, "&") });
ids.push(tag(b, "FITID"));
}
return withKeys(rows, ids);
}
// QIF: one field per line (D date, T amount, P payee, M memo), records end with "^". Negative T = money out.
function parseQif(text: string, order: DateOrder, locale: string): Row[] {
const rows: Omit<Row, "key">[] = [];
for (const rec of text.split(/^\^\s*$/m)) {
const f: Record<string, string> = {};
for (const line of rec.split(/\r?\n/)) if (/^[A-Z]/.test(line) && !line.startsWith("!")) f[line[0]] ??= line.slice(1).trim();
const date = f.D ? parseDate(f.D, order) : null;
const amt = f.T ? parseAmount(f.T, locale) : null;
if (date && amt) rows.push({ date, amount: -amt, name: f.P || f.M || "Imported transaction" });
}
return withKeys(rows);
}
export function parseStatement(text: string, filename: string, locale: string): Parsed {
if (/<OFX>|OFXHEADER/i.test(text) || /\.(ofx|qfx)$/i.test(filename)) return { kind: "ofx", rows: parseOfx(text) };
if (/^\s*!Type:/i.test(text) || /\.qif$/i.test(filename)) {
const dates = [...text.matchAll(/^D(.+)$/gm)].map((m) => m[1].replace(/'/g, "/"));
return { kind: "qif", rows: parseQif(text, dateOrder(dates, locale), locale) };
}
const table = readTable(text);
return { kind: "csv", table, mapping: guessMapping(table, locale) };
}
// A bank's own category name (Chase, Amex, Monzo exports) to broke's category key. null = leave it unsorted.
const CATEGORY_WORDS: [RegExp, string][] = [
[/payroll|salary|paycheck|wage|income|deposit|refund/, "INCOME"],
[/grocer|supermarket/, "GROCERIES"],
[/restaurant|dining|food|drink|coffee|cafe|takeaway|eating/, "FOOD_AND_DRINK"],
[/rent|mortgage|bill|utilit|electric|internet|phone|mobile|insurance/, "RENT_AND_UTILITIES"],
[/gas|fuel|auto|transport|car|uber|taxi|parking|transit/, "TRANSPORTATION"],
[/travel|flight|airline|hotel|holiday/, "TRAVEL"],
[/entertain|subscription|stream|movie|music|game/, "ENTERTAINMENT"],
[/health|medical|pharmac|doctor|dental|fitness|gym/, "MEDICAL"],
[/personal|beauty|hair/, "PERSONAL_CARE"],
[/shop|merchandise|retail|clothing/, "GENERAL_MERCHANDISE"],
[/home|garden|hardware/, "HOME_IMPROVEMENT"],
[/transfer|saving/, "TRANSFER_OUT"],
[/fee|interest|charge/, "BANK_FEES"],
[/tax|gift|donat|charit/, "GOVERNMENT_AND_NON_PROFIT"],
[/loan/, "LOAN_PAYMENTS"],
[/service/, "GENERAL_SERVICES"],
];
export function categoryKey(text: string | undefined, labels: Record<string, string> = {}): string | null {
const t = (text ?? "").toLowerCase().trim();
if (!t) return null;
for (const [key, label] of Object.entries(labels)) if (label.toLowerCase() === t) return key;
return CATEGORY_WORDS.find(([re]) => re.test(t))?.[1] ?? null;
}