From 1cf12f4619dd43f5be5b270f0ed09d8447a412ec Mon Sep 17 00:00:00 2001 From: minimus7 Date: Wed, 11 Nov 2026 10:19:53 -0600 Subject: [PATCH] Recurring: broke finds repeating charges and pay itself, for every account Recurring charges only came from Plaid's detection, so imported statements, bank feeds and hand-added accounts had none (Coming up stayed empty), and a Plaid bill dropped off Coming up once its predicted date passed. - findRecurring (pure, tested): groups by cleaned merchant name and direction, finds a weekly, 2-weekly, monthly, quarterly or yearly rhythm (3 hits, 2 for yearly, most gaps on the rhythm). Bills may vary 40% (electricity); everyday spending (groceries, dining, gas) counts only at the same amount. Transfers skipped. Next date keeps the day of the month. Stopped ones are ended. - saveFoundRecurring runs after every sync and after a statement import. Plaid's own stream wins for the same merchant. Updates keep the category you picked and anything you hid. Plaid streams with a past next date roll forward. Tested: the sample CSV alone finds Geico, Planet Fitness, Spotify, Netflix, Verizon, rent, PG&E and Comcast, and Home's Coming up fills in. Co-Authored-By: Claude Opus 5.5 --- src/app/api/plaid/sync/route.ts | 3 + src/app/api/recurring/find/route.ts | 10 +++ src/components/ImportStatement.tsx | 1 + src/lib/recurring-find.test.ts | 47 +++++++++++++ src/lib/recurring-find.ts | 101 ++++++++++++++++++++++++++++ src/lib/recurring-save.ts | 56 +++++++++++++++ 6 files changed, 218 insertions(+) create mode 100644 src/app/api/recurring/find/route.ts create mode 100644 src/lib/recurring-find.test.ts create mode 100644 src/lib/recurring-find.ts create mode 100644 src/lib/recurring-save.ts diff --git a/src/app/api/plaid/sync/route.ts b/src/app/api/plaid/sync/route.ts index db45b8f..71d5e33 100644 --- a/src/app/api/plaid/sync/route.ts +++ b/src/app/api/plaid/sync/route.ts @@ -1,3 +1,4 @@ +import { saveFoundRecurring } from "@/lib/recurring-save"; import { NextResponse } from "next/server"; import { plaidError } from "@/lib/plaid"; import { ITEM_COLS, syncItem } from "@/lib/sync"; @@ -34,5 +35,7 @@ export async function POST() { results.push({ id: item.id, added: 0, modified: 0, removed: 0, error: err.code }); } } + // broke's own recurring finder, over every account (Plaid's misses, feeds, imported statements). + await saveFoundRecurring(user.id).catch((e) => console.error("[broke] recurring:", e instanceof Error ? e.message : e)); return NextResponse.json({ ok: true, results }); } diff --git a/src/app/api/recurring/find/route.ts b/src/app/api/recurring/find/route.ts new file mode 100644 index 0000000..4811298 --- /dev/null +++ b/src/app/api/recurring/find/route.ts @@ -0,0 +1,10 @@ +import { NextResponse } from "next/server"; +import { saveFoundRecurring } from "@/lib/recurring-save"; +import { createClient } from "@/utils/supabase/server"; + +// Finds charges and pay that repeat in the signed-in user's own transactions. Called after a statement import. +export async function POST() { + const { data: { user } } = await (await createClient()).auth.getUser(); + if (!user) return NextResponse.json({ error: "Not signed in" }, { status: 401 }); + return NextResponse.json(await saveFoundRecurring(user.id)); +} diff --git a/src/components/ImportStatement.tsx b/src/components/ImportStatement.tsx index 32c7075..5a94e99 100644 --- a/src/components/ImportStatement.tsx +++ b/src/components/ImportStatement.tsx @@ -132,6 +132,7 @@ export function ImportStatement({ accounts, locale, currency, link = false, trig } added += data?.length ?? 0; } + if (added) await fetch("/api/recurring/find", { method: "POST" }).catch(() => null); // repeating charges in what came in setBusy(false); const dupes = rows.length - added; toast(`${added} added${dupes ? `, ${dupes} already there` : ""}`); diff --git a/src/lib/recurring-find.test.ts b/src/lib/recurring-find.test.ts new file mode 100644 index 0000000..c02a248 --- /dev/null +++ b/src/lib/recurring-find.test.ts @@ -0,0 +1,47 @@ +// Run: node src/lib/recurring-find.test.ts +import assert from "node:assert/strict"; +import { cleanName, findRecurring, nextAfter } from "./recurring-find.ts"; + +const today = "2026-11-11"; +const monthly = (name: string, amounts: number[], day = "05", category: string | null = "RENT_AND_UTILITIES") => + amounts.map((amount, i) => ({ name, date: `2026-${String(6 + i).padStart(2, "0")}-${day}`, amount, category })); + +// A bill that varies (electricity) is found; next date is after today. +let f = findRecurring(monthly("PG&E Electric", [62, 90, 118, 84, 75, 101]), today); +assert.equal(f.length, 1); +assert.equal(f[0].frequency, "MONTHLY"); +assert.equal(f[0].next_date, "2026-12-05"); +assert.equal(f[0].status, "MATURE"); +assert.equal(f[0].is_income, false); + +// Pay every 2 weeks, money in. +const pay = [0, 14, 28, 42, 56].map((d) => ({ name: "Northwind Labs Payroll", date: new Date(Date.UTC(2026, 8, 4 + d)).toISOString().slice(0, 10), amount: -2150, category: "INCOME" })); +f = findRecurring(pay, today); +assert.equal(f[0].frequency, "BIWEEKLY"); +assert.equal(f[0].is_income, true); + +// Weekly groceries that vary aren't a bill; the same amount every week (a meal kit) is. +const weekly = (amount: (i: number) => number) => [0, 7, 14, 21, 28].map((d, i) => ({ name: "Whole Foods Market", date: new Date(Date.UTC(2026, 9, 3 + d)).toISOString().slice(0, 10), amount: amount(i), category: "GROCERIES" })); +assert.equal(findRecurring(weekly((i) => 55 + i * 17), today).length, 0); +assert.equal(findRecurring(weekly(() => 69.99), today).length, 1); + +// Transfers are skipped; one-offs and random-day spending are not patterns. +assert.equal(findRecurring(monthly("Transfer to Savings", [300, 300, 300, 300], "22", "TRANSFER_OUT"), today).length, 0); +assert.equal(findRecurring([{ name: "Delta", date: "2026-07-03", amount: 386, category: "TRAVEL" }], today).length, 0); +assert.equal(findRecurring(["2026-10-01", "2026-10-03", "2026-10-10", "2026-10-11", "2026-10-25"].map((date) => ({ name: "Chipotle", date, amount: 14, category: "FOOD_AND_DRINK" })), today).length, 0); + +// Names with store numbers group together. +assert.equal(cleanName("SPOTIFY USA #1234 NY"), cleanName("Spotify USA #9876 NY")); + +// A subscription that stopped months ago is marked ended, not due. +f = findRecurring(monthly("Hulu", [17.99, 17.99, 17.99], "10", "ENTERTAINMENT").map((r) => ({ ...r, date: r.date.replace("2026-0", "2026-0").replace(/2026-(\d\d)/, (_, m) => `2026-${String(Number(m) - 4).padStart(2, "0")}`) })), today); +assert.equal(f[0].status, "TOMBSTONED"); + +// Yearly needs only 2. +assert.equal(findRecurring([{ name: "Amazon Prime", date: "2025-03-02", amount: 139, category: "GENERAL_MERCHANDISE" }, { name: "Amazon Prime", date: "2026-03-02", amount: 139, category: "GENERAL_MERCHANDISE" }], today)[0].frequency, "ANNUALLY"); + +assert.equal(nextAfter("2026-10-01", "MONTHLY", "2026-11-11"), "2026-12-01"); +assert.equal(nextAfter("2026-01-31", "MONTHLY", "2026-02-10"), "2026-02-28", "short month"); +assert.equal(nextAfter("2026-11-04", "BIWEEKLY", "2026-11-11"), "2026-11-18"); +assert.equal(nextAfter("2025-03-02", "ANNUALLY", "2026-11-11"), "2027-03-02"); +console.log("recurring-find: ok"); diff --git a/src/lib/recurring-find.ts b/src/lib/recurring-find.ts new file mode 100644 index 0000000..c57b899 --- /dev/null +++ b/src/lib/recurring-find.ts @@ -0,0 +1,101 @@ +// Charges and pay that repeat, found from your own transactions: works for imported statements, bank feeds and hand-added +// accounts, which Plaid's recurring detection never sees, and catches what Plaid misses. Pure, no "@/" imports: +// `node src/lib/recurring-find.test.ts` runs it. + +export type FindRow = { name: string; date: string; amount: number; category: string | null }; // amount > 0 = money out +export type Found = { + key: string; // direction + cleaned name: one stream per merchant and direction + merchant: string; + frequency: "WEEKLY" | "BIWEEKLY" | "MONTHLY" | "QUARTERLY" | "ANNUALLY"; + average: number; + last_amount: number; + last_date: string; + next_date: string; + is_income: boolean; + status: "MATURE" | "EARLY_DETECTION" | "TOMBSTONED"; + category: string | null; +}; + +const EVERY = [ + { frequency: "WEEKLY", days: 7, slack: 2 }, + { frequency: "BIWEEKLY", days: 14, slack: 3 }, + { frequency: "MONTHLY", days: 30.4, slack: 4 }, + { frequency: "QUARTERLY", days: 91, slack: 10 }, + { frequency: "ANNUALLY", days: 365, slack: 15 }, +] as const; +// Everyday spending repeats without being a bill (the same grocery store each Saturday). Those count only when the +// amount is the same every time, like a subscription billed through a store. +const EVERYDAY = new Set(["FOOD_AND_DRINK", "GROCERIES", "GENERAL_MERCHANDISE", "TRANSPORTATION", "TRAVEL", "PERSONAL_CARE"]); +const SKIP = new Set(["TRANSFER_IN", "TRANSFER_OUT"]); + +const mid = (ns: number[]) => { const s = [...ns].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; }; +const day = (d: string) => Date.parse(`${d}T00:00:00Z`) / 864e5; +const ymd = (n: number) => new Date(Math.round(n) * 864e5).toISOString().slice(0, 10); +/** "SPOTIFY USA #1234 NY" and "Spotify" become the same merchant. */ +export const cleanName = (s: string) => s.toLowerCase().replace(/[#*]\s*\w*\d\w*/g, " ").replace(/\d{3,}/g, " ").replace(/[^a-z&' ]+/g, " ").replace(/\s+/g, " ").trim(); + +const MONTHS: Record = { MONTHLY: 1, QUARTERLY: 3, ANNUALLY: 12 }; +/** Next date on or after today on this rhythm, from the last charge. Month-based ones keep their day of the month. */ +export function nextAfter(last: string, frequency: Found["frequency"], today: string) { + const months = MONTHS[frequency]; + const [y, m, d] = last.split("-").map(Number); + for (let i = 1; ; i++) { + let next: string; + if (months) { + const t = new Date(Date.UTC(y, m - 1 + i * months, 1)); + const dim = new Date(Date.UTC(t.getUTCFullYear(), t.getUTCMonth() + 1, 0)).getUTCDate(); + next = ymd(day(t.toISOString().slice(0, 10)) + Math.min(d, dim) - 1); // the 31st becomes the 30th in a short month + } else next = ymd(day(last) + i * (frequency === "WEEKLY" ? 7 : 14)); + if (next >= today) return next; + } +} + +export function findRecurring(rows: FindRow[], today: string): Found[] { + const groups = new Map(); + for (const r of rows) { + if (!r.amount || (r.category && SKIP.has(r.category))) continue; + const name = cleanName(r.name); + if (!name) continue; + const key = `${r.amount < 0 ? "in" : "out"}:${name}`; + groups.set(key, [...(groups.get(key) ?? []), r]); + } + const out: Found[] = []; + for (const [key, rs] of groups) { + const sorted = [...rs].sort((a, b) => a.date.localeCompare(b.date)); + // Two charges on one day count once (a split payment), so the gaps stay true. + const dates = [...new Set(sorted.map((r) => r.date))]; + if (dates.length < 2) continue; + const gaps = dates.slice(1).map((d, i) => day(d) - day(dates[i])); + const gap = mid(gaps); + const every = EVERY.find((e) => Math.abs(gap - e.days) <= e.slack); + if (!every) continue; + // Enough of a pattern: 3 for weekly to quarterly, 2 for yearly. Most gaps on the rhythm, not just the middle one. + if (dates.length < (every.frequency === "ANNUALLY" ? 2 : 3)) continue; + if (gaps.filter((g) => Math.abs(g - every.days) <= every.slack).length < Math.ceil(gaps.length * 0.7)) continue; + const amounts = sorted.map((r) => Math.abs(r.amount)); + const typical = mid(amounts); + const near = (pct: number) => amounts.filter((a) => Math.abs(a - typical) <= typical * pct).length >= Math.ceil(amounts.length * 0.8); + // The category most of its charges are filed under. + const counts = new Map(); + for (const r of sorted) counts.set(r.category, (counts.get(r.category) ?? 0) + 1); + const category = [...counts].sort((a, b) => b[1] - a[1])[0][0]; + // A bill can vary (electricity); everyday spending has to be the same amount to count. + if (!(category && EVERYDAY.has(category) ? near(0.03) : near(0.4))) continue; + const last = sorted[sorted.length - 1]; + const overdue = day(today) - day(last.date); + const status = overdue > every.days * 2 + every.slack ? "TOMBSTONED" : dates.length >= 3 ? "MATURE" : "EARLY_DETECTION"; + out.push({ + key, + merchant: last.name.trim(), + frequency: every.frequency, + average: Math.round((amounts.reduce((s, a) => s + a, 0) / amounts.length) * 100) / 100, + last_amount: Math.abs(last.amount), + last_date: last.date, + next_date: nextAfter(last.date, every.frequency, today), + is_income: last.amount < 0, + status, + category, + }); + } + return out.sort((a, b) => a.next_date.localeCompare(b.next_date)); +} diff --git a/src/lib/recurring-save.ts b/src/lib/recurring-save.ts new file mode 100644 index 0000000..56e7d27 --- /dev/null +++ b/src/lib/recurring-save.ts @@ -0,0 +1,56 @@ +// Runs broke's own recurring finder over a user's transactions (every account: Plaid, feeds, imported statements, +// hand-added) and saves what it finds next to Plaid's streams. Plaid's own stream wins for the same merchant. +// Also moves a stream's next date forward once it has passed, so bills don't drop off "Coming up". +import { allRows } from "@/lib/data"; +import { cleanName, findRecurring, nextAfter, type Found } from "@/lib/recurring-find"; +import { createAdminClient } from "@/utils/supabase/admin"; + +const OURS = "broke:"; +const FREQ: Record = { WEEKLY: "WEEKLY", BIWEEKLY: "BIWEEKLY", SEMI_MONTHLY: "BIWEEKLY", MONTHLY: "MONTHLY", QUARTERLY: "QUARTERLY", ANNUALLY: "ANNUALLY" }; + +type Txn = { name: string; merchant_name: string | null; amount: number; date: string; account_id: string; category: { key: string; id: string } | null }; +type Stream = { id: string; plaid_stream_id: string; merchant_name: string; is_income: boolean; frequency: string; next_date: string | null; status: string | null }; + +export async function saveFoundRecurring(userId: string, today = new Date().toISOString().slice(0, 10)) { + const db = createAdminClient(); + const since = new Date(Date.parse(today) - 400 * 864e5).toISOString().slice(0, 10); + const [txns, { data: streams }, { data: cats }] = await Promise.all([ + allRows((f, t) => db.from("transactions").select("name, merchant_name, amount, date, account_id, category:categories(key, id)") + .eq("user_id", userId).eq("ignored", false).gte("date", since).order("id").range(f, t)), + db.from("recurring").select("id, plaid_stream_id, merchant_name, is_income, frequency, next_date, status").eq("user_id", userId), + db.from("categories").select("id, key").is("user_id", null), + ]); + const all = (streams ?? []) as Stream[]; + const bank = all.filter((s) => !s.plaid_stream_id.startsWith(OURS)); + const ours = new Map(all.filter((s) => s.plaid_stream_id.startsWith(OURS)).map((s) => [s.plaid_stream_id, s])); + const keyOf = (income: boolean, name: string) => `${income ? "in" : "out"}:${cleanName(name)}`; + const covered = new Set(bank.map((s) => keyOf(s.is_income, s.merchant_name))); + + // Plaid's predicted date can be in the past until the next charge posts: roll it forward on its own rhythm. + for (const s of bank) { + const f = FREQ[s.frequency]; + if (s.next_date && s.next_date < today && f && s.status !== "TOMBSTONED") + await db.from("recurring").update({ next_date: nextAfter(s.next_date, f, today) }).eq("id", s.id); + } + + const lastAccount = new Map(); + for (const t of [...txns].sort((a, b) => a.date.localeCompare(b.date))) lastAccount.set(keyOf(t.amount < 0, t.merchant_name || t.name), t.account_id); + const catId = new Map((cats ?? []).map((c) => [c.key as string, c.id as string])); + const found = findRecurring(txns.map((t) => ({ name: t.merchant_name || t.name, date: t.date, amount: Number(t.amount), category: t.category?.key ?? null })), today) + .filter((f) => !covered.has(f.key)); + + const seen = new Set(); + for (const f of found) { + const id = `${OURS}${userId}:${f.key}`; + seen.add(id); + const fields = { merchant_name: f.merchant, frequency: f.frequency, average_amount: f.average, last_amount: f.last_amount, last_date: f.last_date, + next_date: f.next_date, is_income: f.is_income, status: f.status, account_id: lastAccount.get(f.key) ?? null }; + // Existing: only the numbers change, so a category you picked or a stream you hid stays as you left it. + if (ours.has(id)) await db.from("recurring").update(fields).eq("id", ours.get(id)!.id); + else await db.from("recurring").insert({ user_id: userId, plaid_stream_id: id, ...fields, category_id: f.category ? catId.get(f.category) ?? null : null }); + } + // Found before, not now (the charges stopped, or were deleted): ended, not deleted, so your choices on it stay. + const gone = [...ours.keys()].filter((id) => !seen.has(id)); + if (gone.length) await db.from("recurring").update({ status: "TOMBSTONED" }).in("plaid_stream_id", gone); + return { found: found.length }; +}