import crypto from 'crypto'; import { IEtsyConnection } from '../models/EtsyConnection'; import Order from '../models/Order'; import Product, { IProduct } from '../models/Product'; import Expense from '../models/Expense'; const ETSY_API_BASE = 'https://api.etsy.com/v3'; const ETSY_TOKEN_URL = 'https://api.etsy.com/v3/public/oauth/token'; const ETSY_CONNECT_URL = 'https://www.etsy.com/oauth/connect'; // Read scopes: receipts/transactions, listings, shop info export const ETSY_SCOPES = 'transactions_r listings_r shops_r'; // Etsy splits its credentials: the keystring is the OAuth client id, but API // resource calls want the app's shared secret in the x-api-key header export interface EtsyCredentials { clientId: string; apiKeyHeader: string; } // ---------- OAuth (authorization code + PKCE) ---------- export const generatePkce = () => { const verifier = crypto.randomBytes(32).toString('base64url'); const challenge = crypto.createHash('sha256').update(verifier).digest('base64url'); return { verifier, challenge }; }; export const buildAuthUrl = (apiKey: string, redirectUri: string, state: string, codeChallenge: string): string => { const params = new URLSearchParams({ response_type: 'code', client_id: apiKey, redirect_uri: redirectUri, scope: ETSY_SCOPES, state, code_challenge: codeChallenge, code_challenge_method: 'S256', }); return `${ETSY_CONNECT_URL}?${params.toString()}`; }; interface TokenResponse { access_token: string; refresh_token: string; expires_in: number; } const requestToken = async (body: Record): Promise => { // OAuth 2.0 token endpoints take form-encoded bodies (RFC 6749) const res = await fetch(ETSY_TOKEN_URL, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams(body).toString(), }); if (!res.ok) { throw new Error(`Etsy token request failed (${res.status}): ${await res.text()}`); } return res.json() as Promise; }; export const exchangeCode = (apiKey: string, code: string, codeVerifier: string, redirectUri: string) => requestToken({ grant_type: 'authorization_code', client_id: apiKey, redirect_uri: redirectUri, code, code_verifier: codeVerifier, }); const refreshTokens = (apiKey: string, refreshToken: string) => requestToken({ grant_type: 'refresh_token', client_id: apiKey, refresh_token: refreshToken, }); // Refresh the connection's access token if it expires within the next minute const ensureFreshToken = async (creds: EtsyCredentials, connection: IEtsyConnection): Promise => { if (connection.tokenExpiresAt.getTime() > Date.now() + 60_000) return; const tokens = await refreshTokens(creds.clientId, connection.refreshToken); connection.accessToken = tokens.access_token; connection.refreshToken = tokens.refresh_token; connection.tokenExpiresAt = new Date(Date.now() + tokens.expires_in * 1000); await connection.save(); }; // ---------- API client ---------- const etsyGet = async (creds: EtsyCredentials, connection: IEtsyConnection, path: string): Promise => { await ensureFreshToken(creds, connection); const res = await fetch(`${ETSY_API_BASE}${path}`, { headers: { 'x-api-key': creds.apiKeyHeader, Authorization: `Bearer ${connection.accessToken}`, }, }); if (!res.ok) { throw new Error(`Etsy API ${path} failed (${res.status}): ${await res.text()}`); } return res.json(); }; export interface EtsyIdentity { etsyUserId: string; shopId: string; shopName?: string; } // Resolve the connected user's shop from a fresh access token export const fetchIdentity = async (creds: EtsyCredentials, accessToken: string): Promise => { const res = await fetch(`${ETSY_API_BASE}/application/users/me`, { headers: { 'x-api-key': creds.apiKeyHeader, Authorization: `Bearer ${accessToken}` }, }); if (!res.ok) { throw new Error(`Etsy users/me failed (${res.status}): ${await res.text()}`); } const me: any = await res.json(); if (!me.shop_id) { throw new Error('The connected Etsy account has no shop'); } let shopName: string | undefined; try { const shopRes = await fetch(`${ETSY_API_BASE}/application/shops/${me.shop_id}`, { headers: { 'x-api-key': creds.apiKeyHeader, Authorization: `Bearer ${accessToken}` }, }); if (shopRes.ok) { const shop: any = await shopRes.json(); shopName = shop.shop_name; } } catch { // Shop name is cosmetic; ignore failures } return { etsyUserId: String(me.user_id), shopId: String(me.shop_id), shopName }; }; // ---------- Receipt → Order sync ---------- // Etsy money objects are { amount, divisor, currency_code } const money = (m: any): number => (m && m.divisor ? m.amount / m.divisor : 0); const normalizeTitle = (title: string): string => title.replace(/\s+/g, ' ').trim().toLowerCase(); const sizeToken = (text: string): string => { const lower = text.toLowerCase(); if (lower.includes('large') && !lower.includes('small')) return 'large'; if (lower.includes('small') && !lower.includes('large')) return 'small'; if (lower.includes('med')) return 'med'; return ''; }; // Build the variant suffix for a transaction, e.g. "Size: Large" export const transactionVariantString = (transaction: any): string => (transaction.variations || []) .map((v: any) => `${v.formatted_name}: ${v.formatted_value}`) .join(', '); // Match an Etsy transaction to a catalog product: // 1. exact title/alias match — "title variant" first so a size-specific // product beats a generic one when listings have size variants // 2. listing id match, disambiguated by size when several products share it const matchProduct = ( products: IProduct[], transaction: any ): IProduct | undefined => { const variantStr = transactionVariantString(transaction); const candidatesByTitle = [ variantStr ? `${transaction.title} ${variantStr}` : '', transaction.title || '', ].filter(Boolean).map(normalizeTitle); for (const candidate of candidatesByTitle) { const product = products.find(p => [p.title, ...(p.aliases || [])].map(normalizeTitle).includes(candidate) ); if (product) return product; } if (transaction.listing_id) { const listingId = String(transaction.listing_id); const byListing = products.filter(p => p.etsyListingId === listingId); if (byListing.length === 1) return byListing[0]; if (byListing.length > 1) { const size = sizeToken(`${transaction.title || ''} ${variantStr}`); if (size) { const bySize = byListing.filter(p => sizeToken(p.title) === size); if (bySize.length === 1) return bySize[0]; } } } return undefined; }; const mapStatus = (receipt: any): string => { const status = String(receipt.status || '').toLowerCase(); if (status === 'canceled') return 'cancelled'; if (status === 'completed') return 'delivered'; if (receipt.is_shipped) return 'shipped'; return 'processing'; }; export interface SyncResult { created: number; updated: number; unmatchedItems: string[]; receiptsSeen: number; } export const syncReceipts = async (creds: EtsyCredentials, connection: IEtsyConnection): Promise => { const products = await Product.find({ userId: connection.userId }); const result: SyncResult = { created: 0, updated: 0, unmatchedItems: [], receiptsSeen: 0 }; const limit = 100; let offset = 0; while (true) { const page = await etsyGet( creds, connection, `/application/shops/${connection.shopId}/receipts?limit=${limit}&offset=${offset}` ); const receipts: any[] = page.results || []; if (receipts.length === 0) break; for (const receipt of receipts) { result.receiptsSeen++; await upsertOrderFromReceipt(connection, receipt, products, result); } offset += receipts.length; if (offset >= (page.count || 0)) break; } connection.lastSyncedAt = new Date(); await connection.save(); return result; }; const upsertOrderFromReceipt = async ( connection: IEtsyConnection, receipt: any, products: IProduct[], result: SyncResult ): Promise => { const orderNumber = String(receipt.receipt_id); // Snapshot catalog costs onto items at sync time (same model as PDF import) let allMatched = true; const items = (receipt.transactions || []).map((transaction: any) => { const variantStr = transactionVariantString(transaction); const product = matchProduct(products, transaction); if (!product) { allMatched = false; // Report with the variant suffix so each size resolves to its own // product; this exact string is what the matcher checks on re-sync const unmatchedKey = variantStr ? `${transaction.title} ${variantStr}` : transaction.title; if (!result.unmatchedItems.includes(unmatchedKey)) { result.unmatchedItems.push(unmatchedKey); } } else if (!product.etsyListingId && transaction.listing_id) { // Learn the listing id for future deterministic matching product.etsyListingId = String(transaction.listing_id); product.save().catch(() => {}); } return { title: transaction.title, quantity: transaction.quantity || 1, price: money(transaction.price), sku: transaction.sku || undefined, variant: variantStr || undefined, productId: product?._id, printingCost: product?.printingCost || 0, costOfGoods: product?.costOfGoods || 0, }; }); const shipment = (receipt.shipments || [])[0]; const orderData: any = { orderNumber, etsyOrderId: orderNumber, total: money(receipt.grandtotal), subtotal: money(receipt.subtotal), shipping: money(receipt.total_shipping_cost), tax: money(receipt.total_tax_cost), status: mapStatus(receipt), paymentStatus: receipt.is_paid ? 'paid' : 'pending', dateOrdered: new Date((receipt.created_timestamp || receipt.create_timestamp) * 1000), customer: { name: receipt.name || 'Etsy Customer', email: '' }, shippingAddress: { name: receipt.name, street1: receipt.first_line, street2: receipt.second_line, city: receipt.city, state: receipt.state, zipCode: receipt.zip, country: receipt.country_iso, }, trackingNumber: shipment?.tracking_code, shippingCarrier: shipment?.carrier_name, }; const existing = await Order.findOne({ orderNumber, userId: connection.userId }); if (existing) { // Keep richer existing items (e.g. from packing-slip import with costs) // unless every synced item matched a catalog product const keepExistingItems = !allMatched && existing.items && existing.items.length > 0; if (!keepExistingItems) orderData.items = items; await Order.findByIdAndUpdate(existing._id, orderData, { runValidators: true }); result.updated++; } else { orderData.items = items; await Order.create({ ...orderData, userId: connection.userId }); result.created++; } }; // ---------- Payment ledger → Expense sync ---------- export interface LedgerSyncResult { feesCreated: number; feesDuplicate: number; feesByCategory: Record; entriesSeen: number; // Debit entries we couldn't confidently classify as a fee — surfaced so the // category rules can be refined rather than silently miscategorising money unknownDebits: string[]; } // Etsy ledger amounts are integers in the entry currency's minor unit (cents); // there is no per-entry divisor, so /100 (all Etsy currencies are 2-decimal) const ledgerAmount = (entry: any): number => (entry.amount || 0) / 100; // Map a debit ledger entry to an expense category, or null to skip it. // Conservative: only genuine business costs become expenses. Credits (sales), // disbursements to bank, and refunds (revenue reversals) are never expenses. export const classifyLedgerEntry = (entry: any): string | null => { if ((entry.amount || 0) >= 0) return null; // credits: sale payments, misc credits const text = `${entry.description || ''} ${entry.ledger_type || ''} ${entry.reference_type || ''}`.toLowerCase(); // Transfers and revenue reversals are not expenses if (/disburse|deposit|withdraw|payout/.test(text)) return null; if (/refund|reversal|return/.test(text)) return null; if (/listing/.test(text)) return 'Listing Fees'; if (/transaction/.test(text)) return 'Transaction Fees'; if (/processing|payment fee/.test(text)) return 'Payment Processing Fees'; if (/ads|advertis|marketing|offsite/.test(text)) return 'Marketing & Advertising'; if (/shipping|postage|label/.test(text)) return 'Shipping & Postage'; if (/vat|gst|\btax\b/.test(text)) return 'Taxes & GST'; if (/subscription|etsy plus/.test(text)) return 'Subscriptions'; if (/regulatory|operating fee|fee|bill/.test(text)) return 'Other Etsy Fees'; return null; // unknown debit — reported, not guessed }; export const syncLedgerEntries = async ( creds: EtsyCredentials, connection: IEtsyConnection, // Default lookback covers the shop's full selling history; chunked to avoid // any range limits on the endpoint startDate: Date = new Date(Date.now() - 5 * 365 * 24 * 60 * 60 * 1000) ): Promise => { const result: LedgerSyncResult = { feesCreated: 0, feesDuplicate: 0, feesByCategory: {}, entriesSeen: 0, unknownDebits: [], }; const windowSeconds = 90 * 24 * 60 * 60; const nowSec = Math.floor(Date.now() / 1000); let windowStart = Math.floor(startDate.getTime() / 1000); const toCreate: any[] = []; while (windowStart < nowSec) { const windowEnd = Math.min(windowStart + windowSeconds, nowSec); let offset = 0; while (true) { const page = await etsyGet( creds, connection, `/application/shops/${connection.shopId}/payment-account/ledger-entries` + `?min_created=${windowStart}&max_created=${windowEnd}&limit=100&offset=${offset}` ); const entries: any[] = page.results || []; if (entries.length === 0) break; for (const entry of entries) { result.entriesSeen++; const category = classifyLedgerEntry(entry); if (!category) { // Collect unclassified debits for reporting (capped) if ((entry.amount || 0) < 0 && result.unknownDebits.length < 25) { const desc = String(entry.description || entry.ledger_type || 'unknown').trim(); if (!result.unknownDebits.includes(desc)) result.unknownDebits.push(desc); } continue; } const when = new Date((entry.created_timestamp || entry.create_date) * 1000); toCreate.push({ category, description: String(entry.description || category), amount: Math.abs(ledgerAmount(entry)), date: when, taxDeductible: true, vendor: 'Etsy', // Stable per-entry reference makes re-syncs idempotent via the unique // {reference, vendor, amount, date} index reference: `etsy-ledger-${entry.entry_id}`, notes: entry.reference_id ? `Etsy ${entry.reference_type || 'ref'} ${entry.reference_id}` : undefined, userId: connection.userId, }); result.feesByCategory[category] = (result.feesByCategory[category] || 0) + 1; } offset += entries.length; if (offset >= (page.count || 0)) break; } windowStart = windowEnd; } // Idempotent insert: the unique index rejects entries already imported if (toCreate.length > 0) { try { const inserted = await Expense.insertMany(toCreate, { ordered: false, rawResult: true } as any); result.feesCreated = (inserted as any).insertedCount ?? toCreate.length; } catch (bulkError: any) { if (bulkError.writeErrors) { for (const e of bulkError.writeErrors) { if (e.code === 11000) result.feesDuplicate++; } result.feesCreated = bulkError.result?.insertedCount ?? (toCreate.length - bulkError.writeErrors.length); } else { throw bulkError; } } } return result; };