diff --git a/src/app/api/bank/connect/[id]/callback/route.ts b/src/app/api/bank/connect/[id]/callback/route.ts index b53b6b6..a383a6c 100644 --- a/src/app/api/bank/connect/[id]/callback/route.ts +++ b/src/app/api/bank/connect/[id]/callback/route.ts @@ -40,7 +40,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id: } return back("connect=done"); } catch (e) { - console.error(`[broke] ${id} connect:`, e instanceof Error ? e.message : e); + console.error(`[broke] ${id} connect:`, e instanceof Error ? e.message : JSON.stringify(e)); return back("connect=failed"); } } diff --git a/src/lib/plaid.ts b/src/lib/plaid.ts index d00a387..cf3a5ca 100644 --- a/src/lib/plaid.ts +++ b/src/lib/plaid.ts @@ -42,7 +42,9 @@ export function plaidError(e: unknown): { code: string; message: string } { if (d?.error_code) return { code: d.error_code, message: d.display_message || d.error_message || d.error_code }; const code = (e as { code?: unknown })?.code; // other providers (src/lib/providers) throw { code: "AKAHU_401", message } if (typeof code === "string" && /^(AKAHU_|CONSENT_EXPIRED$|FEED_CONSENT$)/.test(code) && e instanceof Error) return { code, message: e.message }; - return { code: "ERROR", message: e instanceof Error ? e.message : "Plaid request failed" }; + // A database error is a plain object with a message, not an Error: keep its message so the log and Accounts say what broke. + const message = e instanceof Error ? e.message : typeof (e as { message?: unknown })?.message === "string" ? (e as { message: string }).message : "Plaid request failed"; + return { code: "ERROR", message }; } /** Revokes a bank connection at Plaid. Akahu tokens belong to the server owner, and a test connection can't be revoked diff --git a/src/lib/sync.ts b/src/lib/sync.ts index aad6efc..7ccc273 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -420,30 +420,41 @@ export async function linkFeed(userId: string, id: FeedId, ref: string, institut return result; } -/** New-looking rows that match a saved row of this provider (same account, day, amount, name) whose id this fetch didn't - * return: move the saved row to the new id. Returns the new ids that now exist. */ -async function adoptRenamed(item: Item, unknown: FeedTxn[], fetched: Set, accountId: Map) { - if (!unknown.length || !item.provider) return []; +/** This provider's saved rows on these accounts between the first and last day of the fetch, read in pages. Read by + * account and date, not by a list of ids: bank ids can be 150 characters, and hundreds of them make a URL the database + * gateway turns away. */ +async function savedInWindow(item: Item, txns: FeedTxn[], accountId: Map) { + const rows: (SavedRow & { plaid_transaction_id: string })[] = []; + if (!txns.length) return rows; const db = createAdminClient(); - const fresh = unknown.map((t) => ({ ...t, account_id: accountId.get(t.account_ref)! })); - const dates = fresh.map((t) => t.date).sort(); - const saved: SavedRow[] = []; + const dates = txns.map((t) => t.date).sort(); + const accounts = [...new Set(txns.map((t) => accountId.get(t.account_ref)!))]; for (let from = 0; ; from += 1000) { - const { data, error } = await db.from("transactions").select("id, plaid_transaction_id, account_id, date, amount, name, ignored") - .eq("user_id", item.user_id).in("account_id", [...new Set(fresh.map((t) => t.account_id))]) - .gte("date", dates[0]).lte("date", dates[dates.length - 1]).like("plaid_transaction_id", `${item.provider}:%`) - .order("id").range(from, from + 999); + let q = db.from("transactions").select("id, plaid_transaction_id, account_id, date, amount, name, ignored") + .eq("user_id", item.user_id).in("account_id", accounts).gte("date", dates[0]).lte("date", dates[dates.length - 1]); + if (item.provider) q = q.like("plaid_transaction_id", `${item.provider}:%`); + const { data, error } = await q.order("id").range(from, from + 999); if (error) throw error; - for (const r of data ?? []) if (!fetched.has(r.plaid_transaction_id)) saved.push(r); + rows.push(...((data ?? []) as typeof rows)); if ((data ?? []).length < 1000) break; } - const moves = [...matchRenamed(fresh, saved)]; + return rows; +} + +/** New-looking rows that match a saved row (same account, day, amount, name) whose id this fetch didn't return: the + * saved row moves to the new id, keeping its edits. Returns the new ids that now exist. */ +async function adoptRenamed(unknown: FeedTxn[], unmatched: SavedRow[], accountId: Map) { + if (!unknown.length || !unmatched.length) return []; + const db = createAdminClient(); + const moves = [...matchRenamed(unknown.map((t) => ({ ...t, account_id: accountId.get(t.account_ref)! })), unmatched)]; + const moved: string[] = []; for (let i = 0; i < moves.length; i += 50) await Promise.all(moves.slice(i, i + 50).map(async ([to, id]) => { const { error } = await db.from("transactions").update({ plaid_transaction_id: to }).eq("id", id); - if (error) throw error; + if (!error) moved.push(to); + else if (error.code !== "23505") throw error; // the new id is already saved elsewhere: leave this row be })); - return moves.map(([to]) => to); + return moved; } // Writes any provider's accounts and rows the same way: new rows get the user's rules, seen rows keep their edits. @@ -459,31 +470,32 @@ async function saveFeed(item: Item, accounts: FeedAccount[] | ReturnType ({ ...t, user_id: item.user_id, account_id: accountId.get(account_ref)! }); const chunks = (xs: T[]) => Array.from({ length: Math.ceil(xs.length / 500) }, (_, i) => xs.slice(i * 500, i * 500 + 500)); - const known = new Set(); - for (const c of chunks(txns.map((t) => t.plaid_transaction_id))) { - const { data } = await db.from("transactions").select("plaid_transaction_id").eq("user_id", item.user_id).in("plaid_transaction_id", c); - for (const r of data ?? []) known.add(r.plaid_transaction_id); - } + const fetched = new Set(txns.map((t) => t.plaid_transaction_id)); + const stored = await savedInWindow(item, txns, accountId); + const known = new Set(stored.map((r) => r.plaid_transaction_id).filter((id) => fetched.has(id))); // Rows the bank sent again under a new id: the saved row takes the new id, so it updates instead of doubling. - const renamed = await adoptRenamed(item, txns.filter((t) => !known.has(t.plaid_transaction_id)), new Set(txns.map((t) => t.plaid_transaction_id)), accountId); - for (const id of renamed) known.add(id); + const unmatched = stored.filter((r) => !fetched.has(r.plaid_transaction_id)); + for (const id of await adoptRenamed(txns.filter((t) => !known.has(t.plaid_transaction_id)), unmatched, accountId)) known.add(id); const fresh = txns.filter((t) => !known.has(t.plaid_transaction_id)); const seen = txns.filter((t) => known.has(t.plaid_transaction_id)); const { data: rules } = await db.from("rules").select(RULE_COLS).eq("user_id", item.user_id).order("created_at", { ascending: false }); const { data: other } = await db.from("categories").select("id").is("user_id", null).eq("key", "OTHER").maybeSingle(); - for (const c of chunks(fresh)) { - const { error } = await db.from("transactions").insert(c.map((t) => { + // A new row whose id is saved after all (outside the date window, say) is skipped, never a failed sync: the saved + // one keeps its edits. The same id twice in one fetch is one row. + const unique = [...new Map(fresh.map((t) => [t.plaid_transaction_id, t])).values()]; + for (const c of chunks(unique)) { + const { error } = await db.from("transactions").upsert(c.map((t) => { const r = applyRules((rules ?? []) as Rule[], row(t)); // No category we can map with confidence: money out is Other, money in stays blank (counts as income). return { ...row(t), category_id: r.category_id ?? (t.amount > 0 ? other?.id ?? null : null), custom_name: r.custom_name ?? null, ignored: r.ignored ?? false, tags: r.tags ?? [] }; - })); + }), { onConflict: "plaid_transaction_id", ignoreDuplicates: true }); if (error) throw error; } // Seen rows: refresh what the provider owns; the user's category, name, note and tags stay. - for (const c of chunks(seen)) { + for (const c of chunks([...new Map(seen.map((t) => [t.plaid_transaction_id, t])).values()])) { const { error } = await db.from("transactions").upsert(c.map(row), { onConflict: "plaid_transaction_id" }); if (error) throw error; } await db.from("plaid_items").update({ cursor: new Date().toISOString(), last_synced_at: new Date().toISOString(), status: "ok", error: null }).eq("id", item.id); - return { added: fresh.length, modified: seen.length, removed: 0 }; + return { added: unique.length, modified: seen.length, removed: 0 }; }