etsy-finance-tracker/server/src/services/etsyApi.ts

import crypto from 'crypto';
import { IEtsyConnection } from '../models/EtsyConnection';
import Order from '../models/Order';
import Product, { IProduct } from '../models/Product';
import Expense from '../models/Expense';

const ETSY_API_BASE = 'https://api.etsy.com/v3';
const ETSY_TOKEN_URL = 'https://api.etsy.com/v3/public/oauth/token';
const ETSY_CONNECT_URL = 'https://www.etsy.com/oauth/connect';

// Read scopes: receipts/transactions, listings, shop info
export const ETSY_SCOPES = 'transactions_r listings_r shops_r';

// Etsy splits its credentials: the keystring is the OAuth client id, but API
// resource calls want the app's shared secret in the x-api-key header
export interface EtsyCredentials {
  clientId: string;
  apiKeyHeader: string;
}

// ---------- OAuth (authorization code + PKCE) ----------

export const generatePkce = () => {
  const verifier = crypto.randomBytes(32).toString('base64url');
  const challenge = crypto.createHash('sha256').update(verifier).digest('base64url');
  return { verifier, challenge };
};

export const buildAuthUrl = (apiKey: string, redirectUri: string, state: string, codeChallenge: string): string => {
  const params = new URLSearchParams({
    response_type: 'code',
    client_id: apiKey,
    redirect_uri: redirectUri,
    scope: ETSY_SCOPES,
    state,
    code_challenge: codeChallenge,
    code_challenge_method: 'S256',
  });
  return `${ETSY_CONNECT_URL}?${params.toString()}`;
};

interface TokenResponse {
  access_token: string;
  refresh_token: string;
  expires_in: number;
}

const requestToken = async (body: Record<string, string>): Promise<TokenResponse> => {
  // OAuth 2.0 token endpoints take form-encoded bodies (RFC 6749)
  const res = await fetch(ETSY_TOKEN_URL, {
    method: 'POST',
    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
    body: new URLSearchParams(body).toString(),
  });
  if (!res.ok) {
    throw new Error(`Etsy token request failed (${res.status}): ${await res.text()}`);
  }
  return res.json() as Promise<TokenResponse>;
};

export const exchangeCode = (apiKey: string, code: string, codeVerifier: string, redirectUri: string) =>
  requestToken({
    grant_type: 'authorization_code',
    client_id: apiKey,
    redirect_uri: redirectUri,
    code,
    code_verifier: codeVerifier,
  });

const refreshTokens = (apiKey: string, refreshToken: string) =>
  requestToken({
    grant_type: 'refresh_token',
    client_id: apiKey,
    refresh_token: refreshToken,
  });

// Refresh the connection's access token if it expires within the next minute
const ensureFreshToken = async (creds: EtsyCredentials, connection: IEtsyConnection): Promise<void> => {
  if (connection.tokenExpiresAt.getTime() > Date.now() + 60_000) return;

  const tokens = await refreshTokens(creds.clientId, connection.refreshToken);
  connection.accessToken = tokens.access_token;
  connection.refreshToken = tokens.refresh_token;
  connection.tokenExpiresAt = new Date(Date.now() + tokens.expires_in * 1000);
  await connection.save();
};

// ---------- API client ----------

const etsyGet = async (creds: EtsyCredentials, connection: IEtsyConnection, path: string): Promise<any> => {
  await ensureFreshToken(creds, connection);
  const res = await fetch(`${ETSY_API_BASE}${path}`, {
    headers: {
      'x-api-key': creds.apiKeyHeader,
      Authorization: `Bearer ${connection.accessToken}`,
    },
  });
  if (!res.ok) {
    throw new Error(`Etsy API ${path} failed (${res.status}): ${await res.text()}`);
  }
  return res.json();
};

export interface EtsyIdentity {
  etsyUserId: string;
  shopId: string;
  shopName?: string;
}

// Resolve the connected user's shop from a fresh access token
export const fetchIdentity = async (creds: EtsyCredentials, accessToken: string): Promise<EtsyIdentity> => {
  const res = await fetch(`${ETSY_API_BASE}/application/users/me`, {
    headers: { 'x-api-key': creds.apiKeyHeader, Authorization: `Bearer ${accessToken}` },
  });
  if (!res.ok) {
    throw new Error(`Etsy users/me failed (${res.status}): ${await res.text()}`);
  }
  const me: any = await res.json();
  if (!me.shop_id) {
    throw new Error('The connected Etsy account has no shop');
  }

  let shopName: string | undefined;
  try {
    const shopRes = await fetch(`${ETSY_API_BASE}/application/shops/${me.shop_id}`, {
      headers: { 'x-api-key': creds.apiKeyHeader, Authorization: `Bearer ${accessToken}` },
    });
    if (shopRes.ok) {
      const shop: any = await shopRes.json();
      shopName = shop.shop_name;
    }
  } catch {
    // Shop name is cosmetic; ignore failures
  }

  return { etsyUserId: String(me.user_id), shopId: String(me.shop_id), shopName };
};

// ---------- Receipt → Order sync ----------

// Etsy money objects are { amount, divisor, currency_code }
const money = (m: any): number => (m && m.divisor ? m.amount / m.divisor : 0);

const normalizeTitle = (title: string): string =>
  title.replace(/\s+/g, ' ').trim().toLowerCase();

const sizeToken = (text: string): string => {
  const lower = text.toLowerCase();
  if (lower.includes('large') && !lower.includes('small')) return 'large';
  if (lower.includes('small') && !lower.includes('large')) return 'small';
  if (lower.includes('med')) return 'med';
  return '';
};

// Build the variant suffix for a transaction, e.g. "Size: Large"
export const transactionVariantString = (transaction: any): string =>
  (transaction.variations || [])
    .map((v: any) => `${v.formatted_name}: ${v.formatted_value}`)
    .join(', ');

// Match an Etsy transaction to a catalog product:
// 1. exact title/alias match — "title variant" first so a size-specific
//    product beats a generic one when listings have size variants
// 2. listing id match, disambiguated by size when several products share it
const matchProduct = (
  products: IProduct[],
  transaction: any
): IProduct | undefined => {
  const variantStr = transactionVariantString(transaction);
  const candidatesByTitle = [
    variantStr ? `${transaction.title} ${variantStr}` : '',
    transaction.title || '',
  ].filter(Boolean).map(normalizeTitle);

  for (const candidate of candidatesByTitle) {
    const product = products.find(p =>
      [p.title, ...(p.aliases || [])].map(normalizeTitle).includes(candidate)
    );
    if (product) return product;
  }

  if (transaction.listing_id) {
    const listingId = String(transaction.listing_id);
    const byListing = products.filter(p => p.etsyListingId === listingId);
    if (byListing.length === 1) return byListing[0];
    if (byListing.length > 1) {
      const size = sizeToken(`${transaction.title || ''} ${variantStr}`);
      if (size) {
        const bySize = byListing.filter(p => sizeToken(p.title) === size);
        if (bySize.length === 1) return bySize[0];
      }
    }
  }

  return undefined;
};

const mapStatus = (receipt: any): string => {
  const status = String(receipt.status || '').toLowerCase();
  if (status === 'canceled') return 'cancelled';
  if (status === 'completed') return 'delivered';
  if (receipt.is_shipped) return 'shipped';
  return 'processing';
};

export interface SyncResult {
  created: number;
  updated: number;
  unmatchedItems: string[];
  receiptsSeen: number;
}

export const syncReceipts = async (creds: EtsyCredentials, connection: IEtsyConnection): Promise<SyncResult> => {
  const products = await Product.find({ userId: connection.userId });
  const result: SyncResult = { created: 0, updated: 0, unmatchedItems: [], receiptsSeen: 0 };

  const limit = 100;
  let offset = 0;

  while (true) {
    const page = await etsyGet(
      creds,
      connection,
      `/application/shops/${connection.shopId}/receipts?limit=${limit}&offset=${offset}`
    );
    const receipts: any[] = page.results || [];
    if (receipts.length === 0) break;

    for (const receipt of receipts) {
      result.receiptsSeen++;
      await upsertOrderFromReceipt(connection, receipt, products, result);
    }

    offset += receipts.length;
    if (offset >= (page.count || 0)) break;
  }

  connection.lastSyncedAt = new Date();
  await connection.save();
  return result;
};

const upsertOrderFromReceipt = async (
  connection: IEtsyConnection,
  receipt: any,
  products: IProduct[],
  result: SyncResult
): Promise<void> => {
  const orderNumber = String(receipt.receipt_id);

  // Snapshot catalog costs onto items at sync time (same model as PDF import)
  let allMatched = true;
  const items = (receipt.transactions || []).map((transaction: any) => {
    const variantStr = transactionVariantString(transaction);
    const product = matchProduct(products, transaction);
    if (!product) {
      allMatched = false;
      // Report with the variant suffix so each size resolves to its own
      // product; this exact string is what the matcher checks on re-sync
      const unmatchedKey = variantStr ? `${transaction.title} ${variantStr}` : transaction.title;
      if (!result.unmatchedItems.includes(unmatchedKey)) {
        result.unmatchedItems.push(unmatchedKey);
      }
    } else if (!product.etsyListingId && transaction.listing_id) {
      // Learn the listing id for future deterministic matching
      product.etsyListingId = String(transaction.listing_id);
      product.save().catch(() => {});
    }

    return {
      title: transaction.title,
      quantity: transaction.quantity || 1,
      price: money(transaction.price),
      sku: transaction.sku || undefined,
      variant: variantStr || undefined,
      productId: product?._id,
      printingCost: product?.printingCost || 0,
      costOfGoods: product?.costOfGoods || 0,
    };
  });

  const shipment = (receipt.shipments || [])[0];

  const orderData: any = {
    orderNumber,
    etsyOrderId: orderNumber,
    total: money(receipt.grandtotal),
    subtotal: money(receipt.subtotal),
    shipping: money(receipt.total_shipping_cost),
    tax: money(receipt.total_tax_cost),
    status: mapStatus(receipt),
    paymentStatus: receipt.is_paid ? 'paid' : 'pending',
    dateOrdered: new Date((receipt.created_timestamp || receipt.create_timestamp) * 1000),
    customer: { name: receipt.name || 'Etsy Customer', email: '' },
    shippingAddress: {
      name: receipt.name,
      street1: receipt.first_line,
      street2: receipt.second_line,
      city: receipt.city,
      state: receipt.state,
      zipCode: receipt.zip,
      country: receipt.country_iso,
    },
    trackingNumber: shipment?.tracking_code,
    shippingCarrier: shipment?.carrier_name,
  };

  const existing = await Order.findOne({ orderNumber, userId: connection.userId });
  if (existing) {
    // Keep richer existing items (e.g. from packing-slip import with costs)
    // unless every synced item matched a catalog product
    const keepExistingItems = !allMatched && existing.items && existing.items.length > 0;
    if (!keepExistingItems) orderData.items = items;

    await Order.findByIdAndUpdate(existing._id, orderData, { runValidators: true });
    result.updated++;
  } else {
    orderData.items = items;
    await Order.create({ ...orderData, userId: connection.userId });
    result.created++;
  }
};

// ---------- Payment ledger → Expense sync ----------

export interface LedgerSyncResult {
  feesCreated: number;
  feesDuplicate: number;
  feesByCategory: Record<string, number>;
  entriesSeen: number;
  // Debit entries we couldn't confidently classify as a fee — surfaced so the
  // category rules can be refined rather than silently miscategorising money
  unknownDebits: string[];
}

// Etsy ledger amounts are integers in the entry currency's minor unit (cents);
// there is no per-entry divisor, so /100 (all Etsy currencies are 2-decimal)
const ledgerAmount = (entry: any): number => (entry.amount || 0) / 100;

// Map a debit ledger entry to an expense category, or null to skip it.
// Conservative: only genuine business costs become expenses. Credits (sales),
// disbursements to bank, and refunds (revenue reversals) are never expenses.
export const classifyLedgerEntry = (entry: any): string | null => {
  if ((entry.amount || 0) >= 0) return null; // credits: sale payments, misc credits

  const text = `${entry.description || ''} ${entry.ledger_type || ''} ${entry.reference_type || ''}`.toLowerCase();

  // Transfers and revenue reversals are not expenses
  if (/disburse|deposit|withdraw|payout/.test(text)) return null;
  if (/refund|reversal|return/.test(text)) return null;

  if (/listing/.test(text)) return 'Listing Fees';
  if (/transaction/.test(text)) return 'Transaction Fees';
  if (/processing|payment fee/.test(text)) return 'Payment Processing Fees';
  if (/ads|advertis|marketing|offsite/.test(text)) return 'Marketing & Advertising';
  if (/shipping|postage|label/.test(text)) return 'Shipping & Postage';
  if (/vat|gst|\btax\b/.test(text)) return 'Taxes & GST';
  if (/subscription|etsy plus/.test(text)) return 'Subscriptions';
  if (/regulatory|operating fee|fee|bill/.test(text)) return 'Other Etsy Fees';

  return null; // unknown debit — reported, not guessed
};

export const syncLedgerEntries = async (
  creds: EtsyCredentials,
  connection: IEtsyConnection,
  // Default lookback covers the shop's full selling history; chunked to avoid
  // any range limits on the endpoint
  startDate: Date = new Date(Date.now() - 5 * 365 * 24 * 60 * 60 * 1000)
): Promise<LedgerSyncResult> => {
  const result: LedgerSyncResult = {
    feesCreated: 0,
    feesDuplicate: 0,
    feesByCategory: {},
    entriesSeen: 0,
    unknownDebits: [],
  };

  const windowSeconds = 90 * 24 * 60 * 60;
  const nowSec = Math.floor(Date.now() / 1000);
  let windowStart = Math.floor(startDate.getTime() / 1000);

  const toCreate: any[] = [];

  while (windowStart < nowSec) {
    const windowEnd = Math.min(windowStart + windowSeconds, nowSec);
    let offset = 0;

    while (true) {
      const page = await etsyGet(
        creds,
        connection,
        `/application/shops/${connection.shopId}/payment-account/ledger-entries` +
          `?min_created=${windowStart}&max_created=${windowEnd}&limit=100&offset=${offset}`
      );
      const entries: any[] = page.results || [];
      if (entries.length === 0) break;

      for (const entry of entries) {
        result.entriesSeen++;
        const category = classifyLedgerEntry(entry);
        if (!category) {
          // Collect unclassified debits for reporting (capped)
          if ((entry.amount || 0) < 0 && result.unknownDebits.length < 25) {
            const desc = String(entry.description || entry.ledger_type || 'unknown').trim();
            if (!result.unknownDebits.includes(desc)) result.unknownDebits.push(desc);
          }
          continue;
        }

        const when = new Date((entry.created_timestamp || entry.create_date) * 1000);
        toCreate.push({
          category,
          description: String(entry.description || category),
          amount: Math.abs(ledgerAmount(entry)),
          date: when,
          taxDeductible: true,
          vendor: 'Etsy',
          // Stable per-entry reference makes re-syncs idempotent via the unique
          // {reference, vendor, amount, date} index
          reference: `etsy-ledger-${entry.entry_id}`,
          notes: entry.reference_id ? `Etsy ${entry.reference_type || 'ref'} ${entry.reference_id}` : undefined,
          userId: connection.userId,
        });
        result.feesByCategory[category] = (result.feesByCategory[category] || 0) + 1;
      }

      offset += entries.length;
      if (offset >= (page.count || 0)) break;
    }

    windowStart = windowEnd;
  }

  // Idempotent insert: the unique index rejects entries already imported
  if (toCreate.length > 0) {
    try {
      const inserted = await Expense.insertMany(toCreate, { ordered: false, rawResult: true } as any);
      result.feesCreated = (inserted as any).insertedCount ?? toCreate.length;
    } catch (bulkError: any) {
      if (bulkError.writeErrors) {
        for (const e of bulkError.writeErrors) {
          if (e.code === 11000) result.feesDuplicate++;
        }
        result.feesCreated = bulkError.result?.insertedCount ?? (toCreate.length - bulkError.writeErrors.length);
      } else {
        throw bulkError;
      }
    }
  }

  return result;
};