Bank feeds: sync no longer fails on long transaction ids
Enable Banking ids are about 150 characters, so looking up 500 of them in one request made a URL the database gateway turned away. The error was ignored, every row looked new, and inserting them failed on the saved ones (23505), so the connection showed "Needs you to sign in again" and linking again failed too. - saved rows are read by account and date range, in pages, and errors stop the sync - a new row whose id is saved after all is skipped; the same id twice in one fetch is one row - database errors keep their message in the log and on Accounts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0d399b77c3
commit
fbdeea630a
3 files changed
+43
-29
No files matched your search
@@ -40,7 +40,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:
|
||||
}
|
||||
return back("connect=done");
|
||||
} catch (e) {
|
||||
console.error(`[broke] ${id} connect:`, e instanceof Error ? e.message : e);
|
||||
console.error(`[broke] ${id} connect:`, e instanceof Error ? e.message : JSON.stringify(e));
|
||||
return back("connect=failed");
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -42,7 +42,9 @@ export function plaidError(e: unknown): { code: string; message: string } {
|
||||
if (d?.error_code) return { code: d.error_code, message: d.display_message || d.error_message || d.error_code };
|
||||
const code = (e as { code?: unknown })?.code; // other providers (src/lib/providers) throw { code: "AKAHU_401", message }
|
||||
if (typeof code === "string" && /^(AKAHU_|CONSENT_EXPIRED$|FEED_CONSENT$)/.test(code) && e instanceof Error) return { code, message: e.message };
|
||||
return { code: "ERROR", message: e instanceof Error ? e.message : "Plaid request failed" };
|
||||
// A database error is a plain object with a message, not an Error: keep its message so the log and Accounts say what broke.
|
||||
const message = e instanceof Error ? e.message : typeof (e as { message?: unknown })?.message === "string" ? (e as { message: string }).message : "Plaid request failed";
|
||||
return { code: "ERROR", message };
|
||||
}
|
||||
|
||||
/** Revokes a bank connection at Plaid. Akahu tokens belong to the server owner, and a test connection can't be revoked
|
||||
|
||||
+39
-27
@@ -420,30 +420,41 @@ export async function linkFeed(userId: string, id: FeedId, ref: string, institut
|
||||
return result;
|
||||
}
|
||||
|
||||
/** New-looking rows that match a saved row of this provider (same account, day, amount, name) whose id this fetch didn't
|
||||
* return: move the saved row to the new id. Returns the new ids that now exist. */
|
||||
async function adoptRenamed(item: Item, unknown: FeedTxn[], fetched: Set<string>, accountId: Map<string, string>) {
|
||||
if (!unknown.length || !item.provider) return [];
|
||||
/** This provider's saved rows on these accounts between the first and last day of the fetch, read in pages. Read by
|
||||
* account and date, not by a list of ids: bank ids can be 150 characters, and hundreds of them make a URL the database
|
||||
* gateway turns away. */
|
||||
async function savedInWindow(item: Item, txns: FeedTxn[], accountId: Map<string, string>) {
|
||||
const rows: (SavedRow & { plaid_transaction_id: string })[] = [];
|
||||
if (!txns.length) return rows;
|
||||
const db = createAdminClient();
|
||||
const fresh = unknown.map((t) => ({ ...t, account_id: accountId.get(t.account_ref)! }));
|
||||
const dates = fresh.map((t) => t.date).sort();
|
||||
const saved: SavedRow[] = [];
|
||||
const dates = txns.map((t) => t.date).sort();
|
||||
const accounts = [...new Set(txns.map((t) => accountId.get(t.account_ref)!))];
|
||||
for (let from = 0; ; from += 1000) {
|
||||
const { data, error } = await db.from("transactions").select("id, plaid_transaction_id, account_id, date, amount, name, ignored")
|
||||
.eq("user_id", item.user_id).in("account_id", [...new Set(fresh.map((t) => t.account_id))])
|
||||
.gte("date", dates[0]).lte("date", dates[dates.length - 1]).like("plaid_transaction_id", `${item.provider}:%`)
|
||||
.order("id").range(from, from + 999);
|
||||
let q = db.from("transactions").select("id, plaid_transaction_id, account_id, date, amount, name, ignored")
|
||||
.eq("user_id", item.user_id).in("account_id", accounts).gte("date", dates[0]).lte("date", dates[dates.length - 1]);
|
||||
if (item.provider) q = q.like("plaid_transaction_id", `${item.provider}:%`);
|
||||
const { data, error } = await q.order("id").range(from, from + 999);
|
||||
if (error) throw error;
|
||||
for (const r of data ?? []) if (!fetched.has(r.plaid_transaction_id)) saved.push(r);
|
||||
rows.push(...((data ?? []) as typeof rows));
|
||||
if ((data ?? []).length < 1000) break;
|
||||
}
|
||||
const moves = [...matchRenamed(fresh, saved)];
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** New-looking rows that match a saved row (same account, day, amount, name) whose id this fetch didn't return: the
|
||||
* saved row moves to the new id, keeping its edits. Returns the new ids that now exist. */
|
||||
async function adoptRenamed(unknown: FeedTxn[], unmatched: SavedRow[], accountId: Map<string, string>) {
|
||||
if (!unknown.length || !unmatched.length) return [];
|
||||
const db = createAdminClient();
|
||||
const moves = [...matchRenamed(unknown.map((t) => ({ ...t, account_id: accountId.get(t.account_ref)! })), unmatched)];
|
||||
const moved: string[] = [];
|
||||
for (let i = 0; i < moves.length; i += 50)
|
||||
await Promise.all(moves.slice(i, i + 50).map(async ([to, id]) => {
|
||||
const { error } = await db.from("transactions").update({ plaid_transaction_id: to }).eq("id", id);
|
||||
if (error) throw error;
|
||||
if (!error) moved.push(to);
|
||||
else if (error.code !== "23505") throw error; // the new id is already saved elsewhere: leave this row be
|
||||
}));
|
||||
return moves.map(([to]) => to);
|
||||
return moved;
|
||||
}
|
||||
|
||||
// Writes any provider's accounts and rows the same way: new rows get the user's rules, seen rows keep their edits.
|
||||
@@ -459,31 +470,32 @@ async function saveFeed(item: Item, accounts: FeedAccount[] | ReturnType<typeof
|
||||
const row = ({ account_ref, ...t }: FeedTxn) => ({ ...t, user_id: item.user_id, account_id: accountId.get(account_ref)! });
|
||||
|
||||
const chunks = <T,>(xs: T[]) => Array.from({ length: Math.ceil(xs.length / 500) }, (_, i) => xs.slice(i * 500, i * 500 + 500));
|
||||
const known = new Set<string>();
|
||||
for (const c of chunks(txns.map((t) => t.plaid_transaction_id))) {
|
||||
const { data } = await db.from("transactions").select("plaid_transaction_id").eq("user_id", item.user_id).in("plaid_transaction_id", c);
|
||||
for (const r of data ?? []) known.add(r.plaid_transaction_id);
|
||||
}
|
||||
const fetched = new Set(txns.map((t) => t.plaid_transaction_id));
|
||||
const stored = await savedInWindow(item, txns, accountId);
|
||||
const known = new Set(stored.map((r) => r.plaid_transaction_id).filter((id) => fetched.has(id)));
|
||||
// Rows the bank sent again under a new id: the saved row takes the new id, so it updates instead of doubling.
|
||||
const renamed = await adoptRenamed(item, txns.filter((t) => !known.has(t.plaid_transaction_id)), new Set(txns.map((t) => t.plaid_transaction_id)), accountId);
|
||||
for (const id of renamed) known.add(id);
|
||||
const unmatched = stored.filter((r) => !fetched.has(r.plaid_transaction_id));
|
||||
for (const id of await adoptRenamed(txns.filter((t) => !known.has(t.plaid_transaction_id)), unmatched, accountId)) known.add(id);
|
||||
const fresh = txns.filter((t) => !known.has(t.plaid_transaction_id));
|
||||
const seen = txns.filter((t) => known.has(t.plaid_transaction_id));
|
||||
const { data: rules } = await db.from("rules").select(RULE_COLS).eq("user_id", item.user_id).order("created_at", { ascending: false });
|
||||
const { data: other } = await db.from("categories").select("id").is("user_id", null).eq("key", "OTHER").maybeSingle();
|
||||
for (const c of chunks(fresh)) {
|
||||
const { error } = await db.from("transactions").insert(c.map((t) => {
|
||||
// A new row whose id is saved after all (outside the date window, say) is skipped, never a failed sync: the saved
|
||||
// one keeps its edits. The same id twice in one fetch is one row.
|
||||
const unique = [...new Map(fresh.map((t) => [t.plaid_transaction_id, t])).values()];
|
||||
for (const c of chunks(unique)) {
|
||||
const { error } = await db.from("transactions").upsert(c.map((t) => {
|
||||
const r = applyRules((rules ?? []) as Rule[], row(t));
|
||||
// No category we can map with confidence: money out is Other, money in stays blank (counts as income).
|
||||
return { ...row(t), category_id: r.category_id ?? (t.amount > 0 ? other?.id ?? null : null), custom_name: r.custom_name ?? null, ignored: r.ignored ?? false, tags: r.tags ?? [] };
|
||||
}));
|
||||
}), { onConflict: "plaid_transaction_id", ignoreDuplicates: true });
|
||||
if (error) throw error;
|
||||
}
|
||||
// Seen rows: refresh what the provider owns; the user's category, name, note and tags stay.
|
||||
for (const c of chunks(seen)) {
|
||||
for (const c of chunks([...new Map(seen.map((t) => [t.plaid_transaction_id, t])).values()])) {
|
||||
const { error } = await db.from("transactions").upsert(c.map(row), { onConflict: "plaid_transaction_id" });
|
||||
if (error) throw error;
|
||||
}
|
||||
await db.from("plaid_items").update({ cursor: new Date().toISOString(), last_synced_at: new Date().toISOString(), status: "ok", error: null }).eq("id", item.id);
|
||||
return { added: fresh.length, modified: seen.length, removed: 0 };
|
||||
return { added: unique.length, modified: seen.length, removed: 0 };
|
||||
}
|
||||
Reference in new issue
Block a user