From 03979a9b4893303c795886b18c0a1a1a06360f53 Mon Sep 17 00:00:00 2001 From: dlawler489 <104159223@student.swin.edu.au> Date: Sat, 13 Jun 2026 15:16:15 +1000 Subject: [PATCH] Phase 2: sync Etsy payment ledger into expenses (exact fees) - syncLedgerEntries pulls payment-account ledger entries (chunked by 90d over required min_created/max_created), classifies debits into fee categories, and inserts them as idempotent expenses (reference etsy-ledger-). Amounts are integer minor units -> /100. - Conservative classifier: only genuine costs become expenses; sale credits, disbursements, and refunds are skipped. Unclassified debits are reported back so rules can be refined. - Folded into POST /api/etsy/sync alongside orders; response includes ledger result and a legacyEtsyExpenses count (pre-ledger CSV fees). - DELETE /api/etsy/legacy-fees removes CSV-imported Etsy fee expenses to avoid double counting; Settings surfaces the count with a one-click remove, plus a list of skipped/unknown ledger charge types. Co-Authored-By: Claude Fable 5 --- client/src/pages/Settings.tsx | 63 +++++++++++++++- server/src/routes/etsy.ts | 33 ++++++++- server/src/services/etsyApi.ts | 132 +++++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 3 deletions(-) diff --git a/client/src/pages/Settings.tsx b/client/src/pages/Settings.tsx index 8b9ce3a..dd79975 100644 --- a/client/src/pages/Settings.tsx +++ b/client/src/pages/Settings.tsx @@ -42,6 +42,8 @@ const Settings = () => { const [isSyncing, setIsSyncing] = useState(false); const [unmatchedItems, setUnmatchedItems] = useState([]); const [showResolveModal, setShowResolveModal] = useState(false); + const [unknownDebits, setUnknownDebits] = useState([]); + const [legacyEtsyExpenses, setLegacyEtsyExpenses] = useState(0); useEffect(() => { updateStorageSummary(); @@ -122,10 +124,16 @@ const Settings = () => { const handleEtsySync = async () => { setIsSyncing(true); setUnmatchedItems([]); + setUnknownDebits([]); try { const res = await api.post('/etsy/sync'); - const { created, updated, unmatchedItems: unmatched, receiptsSeen } = res.data; + const { created, updated, unmatchedItems: unmatched, receiptsSeen, ledger, legacyEtsyExpenses: legacy } = res.data; toast.success(`Synced ${receiptsSeen} Etsy orders: ${created} new, ${updated} updated`); + if (ledger) { + toast.success(`Fees from Etsy ledger: ${ledger.feesCreated} new expense(s) imported`); + setUnknownDebits(ledger.unknownDebits || []); + } + setLegacyEtsyExpenses(legacy || 0); if (unmatched?.length > 0) { setUnmatchedItems(unmatched); } @@ -140,6 +148,20 @@ const Settings = () => { } }; + const handleRemoveLegacyFees = async () => { + if (!window.confirm( + 'Remove Etsy fee expenses that were imported from the statement CSV? ' + + 'The Etsy ledger sync replaces them. This avoids double-counting fees.' + )) return; + try { + const res = await api.delete('/etsy/legacy-fees'); + toast.success(`Removed ${res.data.deleted} CSV-imported Etsy fee expense(s)`); + setLegacyEtsyExpenses(0); + } catch { + toast.error('Failed to remove legacy Etsy fees'); + } + }; + const handleEtsyDisconnect = async () => { if (!window.confirm('Disconnect your Etsy shop? Synced data stays; only the connection is removed.')) return; try { @@ -256,6 +278,45 @@ const Settings = () => {

)} + + {legacyEtsyExpenses > 0 && ( +
+
+
+

+ {legacyEtsyExpenses} Etsy fee expense(s) from the old statement-CSV import are still present. +

+

+ The ledger sync now pulls fees directly from Etsy. Remove the old CSV-imported + ones so fees aren't counted twice. (You can stop importing the Etsy statement CSV.) +

+
+ +
+
+ )} + + {unknownDebits.length > 0 && ( +
+

+ {unknownDebits.length} ledger charge type(s) weren't auto-categorised as fees and were skipped: +

+
    + {unknownDebits.map((d, idx) => ( +
  • {d}
    • + ))} +
+

+ If any of these are real business costs, let me know and I'll add them to the fee rules. +

+
+ )} ) : (
diff --git a/server/src/routes/etsy.ts b/server/src/routes/etsy.ts index 9a031d4..047fd7b 100644 --- a/server/src/routes/etsy.ts +++ b/server/src/routes/etsy.ts @@ -3,12 +3,14 @@ import crypto from 'crypto'; import { authenticate, AuthRequest } from '../middleware/authenticate'; import EtsyConnection from '../models/EtsyConnection'; import EtsySettings from '../models/EtsySettings'; +import Expense from '../models/Expense'; import { buildAuthUrl, exchangeCode, fetchIdentity, generatePkce, syncReceipts, + syncLedgerEntries, } from '../services/etsyApi'; const router = Router(); @@ -213,14 +215,41 @@ router.post('/sync', authenticate, async (req: AuthRequest, res: Response) => { const config = await loadConfig(req.userId!); if (!config) return res.status(400).json({ message: 'Etsy API key is not configured' }); - const result = await syncReceipts(toCredentials(config), connection); - res.json(result); + const creds = toCredentials(config); + const result = await syncReceipts(creds, connection); + + // Also pull the payment-account ledger into expenses (exact fees from Etsy) + const ledger = await syncLedgerEntries(creds, connection); + + // Warn if pre-ledger CSV-imported Etsy fees still exist (would double-count) + const legacyEtsyExpenses = await Expense.countDocuments({ + userId: req.userId, + vendor: 'Etsy', + reference: { $not: /^etsy-ledger-/ }, + }); + + res.json({ ...result, ledger, legacyEtsyExpenses }); } catch (err: any) { console.error('Etsy sync failed:', err); res.status(500).json({ message: err.message || 'Etsy sync failed' }); } }); +// Remove fee expenses imported from the statement CSV before ledger sync, +// so the ledger becomes the single source of Etsy fees (no double counting) +router.delete('/legacy-fees', authenticate, async (req: AuthRequest, res: Response) => { + try { + const result = await Expense.deleteMany({ + userId: req.userId, + vendor: 'Etsy', + reference: { $not: /^etsy-ledger-/ }, + }); + res.json({ deleted: result.deletedCount }); + } catch (err) { + res.status(500).json({ message: 'Failed to remove legacy Etsy fees', error: err }); + } +}); + router.delete('/connection', authenticate, async (req: AuthRequest, res: Response) => { try { await EtsyConnection.findOneAndDelete({ userId: req.userId }); diff --git a/server/src/services/etsyApi.ts b/server/src/services/etsyApi.ts index 76aa6d8..c6da893 100644 --- a/server/src/services/etsyApi.ts +++ b/server/src/services/etsyApi.ts @@ -2,6 +2,7 @@ import crypto from 'crypto'; import { IEtsyConnection } from '../models/EtsyConnection'; import Order from '../models/Order'; import Product, { IProduct } from '../models/Product'; +import Expense from '../models/Expense'; const ETSY_API_BASE = 'https://api.etsy.com/v3'; const ETSY_TOKEN_URL = 'https://api.etsy.com/v3/public/oauth/token'; @@ -319,3 +320,134 @@ const upsertOrderFromReceipt = async ( result.created++; } }; + +// ---------- Payment ledger → Expense sync ---------- + +export interface LedgerSyncResult { + feesCreated: number; + feesDuplicate: number; + feesByCategory: Record; + entriesSeen: number; + // Debit entries we couldn't confidently classify as a fee — surfaced so the + // category rules can be refined rather than silently miscategorising money + unknownDebits: string[]; +} + +// Etsy ledger amounts are integers in the entry currency's minor unit (cents); +// there is no per-entry divisor, so /100 (all Etsy currencies are 2-decimal) +const ledgerAmount = (entry: any): number => (entry.amount || 0) / 100; + +// Map a debit ledger entry to an expense category, or null to skip it. +// Conservative: only genuine business costs become expenses. Credits (sales), +// disbursements to bank, and refunds (revenue reversals) are never expenses. +export const classifyLedgerEntry = (entry: any): string | null => { + if ((entry.amount || 0) >= 0) return null; // credits: sale payments, misc credits + + const text = `${entry.description || ''} ${entry.ledger_type || ''} ${entry.reference_type || ''}`.toLowerCase(); + + // Transfers and revenue reversals are not expenses + if (/disburse|deposit|withdraw|payout/.test(text)) return null; + if (/refund|reversal|return/.test(text)) return null; + + if (/listing/.test(text)) return 'Listing Fees'; + if (/transaction/.test(text)) return 'Transaction Fees'; + if (/processing|payment fee/.test(text)) return 'Payment Processing Fees'; + if (/ads|advertis|marketing|offsite/.test(text)) return 'Marketing & Advertising'; + if (/shipping|postage|label/.test(text)) return 'Shipping & Postage'; + if (/vat|gst|\btax\b/.test(text)) return 'Taxes & GST'; + if (/subscription|etsy plus/.test(text)) return 'Subscriptions'; + if (/regulatory|operating fee|fee|bill/.test(text)) return 'Other Etsy Fees'; + + return null; // unknown debit — reported, not guessed +}; + +export const syncLedgerEntries = async ( + creds: EtsyCredentials, + connection: IEtsyConnection, + // Default lookback covers the shop's full selling history; chunked to avoid + // any range limits on the endpoint + startDate: Date = new Date(Date.now() - 5 * 365 * 24 * 60 * 60 * 1000) +): Promise => { + const result: LedgerSyncResult = { + feesCreated: 0, + feesDuplicate: 0, + feesByCategory: {}, + entriesSeen: 0, + unknownDebits: [], + }; + + const windowSeconds = 90 * 24 * 60 * 60; + const nowSec = Math.floor(Date.now() / 1000); + let windowStart = Math.floor(startDate.getTime() / 1000); + + const toCreate: any[] = []; + + while (windowStart < nowSec) { + const windowEnd = Math.min(windowStart + windowSeconds, nowSec); + let offset = 0; + + while (true) { + const page = await etsyGet( + creds, + connection, + `/application/shops/${connection.shopId}/payment-account/ledger-entries` + + `?min_created=${windowStart}&max_created=${windowEnd}&limit=100&offset=${offset}` + ); + const entries: any[] = page.results || []; + if (entries.length === 0) break; + + for (const entry of entries) { + result.entriesSeen++; + const category = classifyLedgerEntry(entry); + if (!category) { + // Collect unclassified debits for reporting (capped) + if ((entry.amount || 0) < 0 && result.unknownDebits.length < 25) { + const desc = String(entry.description || entry.ledger_type || 'unknown').trim(); + if (!result.unknownDebits.includes(desc)) result.unknownDebits.push(desc); + } + continue; + } + + const when = new Date((entry.created_timestamp || entry.create_date) * 1000); + toCreate.push({ + category, + description: String(entry.description || category), + amount: Math.abs(ledgerAmount(entry)), + date: when, + taxDeductible: true, + vendor: 'Etsy', + // Stable per-entry reference makes re-syncs idempotent via the unique + // {reference, vendor, amount, date} index + reference: `etsy-ledger-${entry.entry_id}`, + notes: entry.reference_id ? `Etsy ${entry.reference_type || 'ref'} ${entry.reference_id}` : undefined, + userId: connection.userId, + }); + result.feesByCategory[category] = (result.feesByCategory[category] || 0) + 1; + } + + offset += entries.length; + if (offset >= (page.count || 0)) break; + } + + windowStart = windowEnd; + } + + // Idempotent insert: the unique index rejects entries already imported + if (toCreate.length > 0) { + try { + const inserted = await Expense.insertMany(toCreate, { ordered: false, rawResult: true } as any); + result.feesCreated = (inserted as any).insertedCount ?? toCreate.length; + } catch (bulkError: any) { + if (bulkError.writeErrors) { + for (const e of bulkError.writeErrors) { + if (e.code === 11000) result.feesDuplicate++; + } + result.feesCreated = bulkError.result?.insertedCount ?? (toCreate.length - bulkError.writeErrors.length); + } else { + throw bulkError; + } + } + } + + return result; +};