diff --git a/client/src/pages/DataImport.tsx b/client/src/pages/DataImport.tsx index 18d9f34..9efd882 100644 --- a/client/src/pages/DataImport.tsx +++ b/client/src/pages/DataImport.tsx @@ -7,7 +7,6 @@ import { DataManager } from '../utils/dataManager'; import { useDispatch, useSelector } from 'react-redux'; import { RootState } from '../store'; import { addOrder, updateOrder, setOrders } from '../store/slices/orderSlice'; -import { addExpenses } from '../store/slices/expenseSlice'; import { Upload, FileText, Package, Truck, Trash2 } from 'lucide-react'; import toast from 'react-hot-toast'; import { dateTestResults } from '../utils/testDateParsing'; @@ -54,7 +53,6 @@ export default function DataImport() { const [etsyFile, setEtsyFile] = useState(null); const [shippingFile, setShippingFile] = useState(null); const [isLoading, setIsLoading] = useState(false); - const [isProcessingExpenses, setIsProcessingExpenses] = useState(false); const [results, setResults] = useState(null); // PDF Import State @@ -68,11 +66,115 @@ export default function DataImport() { // UI State const [activeTab, setActiveTab] = useState<'csv' | 'pdf'>('csv'); const [error, setError] = useState(''); + const [backgroundProcessing, setBackgroundProcessing] = useState(''); const etsyFileRef = useRef(null); const shippingFileRef = useRef(null); const pdfFileRef = useRef(null); + // Background expense processing + const processExpensesInBackground = async (etsyFile: File | null, shippingFile: File | null) => { + if (!etsyFile && !shippingFile) return; + + try { + setBackgroundProcessing('Processing expenses from uploaded files...'); + + const existingExpenses = await api.get('/expenses').then(res => res.data); + const expensesToCreate: any[] = []; + + if (etsyFile) { + const etsyContent = await readFileAsText(etsyFile); + const etsyFees = csvImportService.parseEtsyFees(etsyContent); + + // Add Etsy fees (check for duplicates) + etsyFees.forEach((fee: EtsyFeeRecord) => { + const isDuplicate = existingExpenses.some((expense: any) => + expense.description === fee.description && + Math.abs(expense.amount - fee.amount) < 0.01 && + new Date(expense.date).toDateString() === new Date(fee.date).toDateString() && + expense.vendor === fee.vendor + ); + + if (!isDuplicate) { + expensesToCreate.push({ + description: fee.description, + amount: fee.amount, + category: fee.category, + date: fee.date, + taxDeductible: fee.taxDeductible, + vendor: fee.vendor, + reference: fee.reference + }); + } + }); + } + + if (shippingFile) { + const shippingContent = await readFileAsText(shippingFile); + const shippingRecords = csvImportService.parseAustraliaPostShipping(shippingContent); + + // Add shipping expenses (check for duplicates) + shippingRecords.forEach((shipping: ParsedShippingRecord) => { + if (shipping.totalCost > 0) { + const isDuplicate = existingExpenses.some((expense: any) => + expense.reference === shipping.trackingNumber && + expense.vendor === 'Australia Post' && + Math.abs(expense.amount - shipping.totalCost) < 0.01 && + new Date(expense.date).toDateString() === new Date(shipping.date).toDateString() + ); + + if (!isDuplicate) { + expensesToCreate.push({ + description: `Australia Post - ${shipping.trackingNumber}`, + amount: shipping.totalCost, + category: 'Shipping & Postage', + date: shipping.date, + taxDeductible: true, + vendor: 'Australia Post', + reference: shipping.trackingNumber + }); + } + } + }); + } + + // Process expenses with very conservative rate limiting + if (expensesToCreate.length > 0) { + const DELAY_MS = 3000; // 3 seconds between each expense + let created = 0; + + console.log(`Background processing: Creating ${expensesToCreate.length} expenses one at a time...`); + + for (let i = 0; i < expensesToCreate.length; i++) { + try { + setBackgroundProcessing(`Processing expense ${i + 1}/${expensesToCreate.length}...`); + await api.post('/expenses', expensesToCreate[i]); + created++; + + // Wait before next expense (except for the last one) + if (i < expensesToCreate.length - 1) { + await new Promise(resolve => setTimeout(resolve, DELAY_MS)); + } + } catch (error: any) { + if (error.response?.status === 11000 || error.response?.data?.message?.includes('duplicate')) { + console.log(`Skipping duplicate expense: ${expensesToCreate[i].description}`); + } else { + console.error(`Failed to create expense: ${expensesToCreate[i].description}`, error); + } + } + } + + console.log(`Background processing completed: ${created} expenses created`); + toast.success(`${created} expenses processed in background`); + } + } catch (error) { + console.error('Error in background expense processing:', error); + toast.error('Error processing expenses in background'); + } finally { + setBackgroundProcessing(''); + } + }; + const handleFileChange = ( event: React.ChangeEvent, type: 'etsy' | 'shipping' | 'pdf' @@ -91,12 +193,22 @@ export default function DataImport() { } else { const file = files[0]; if (file && file.type === 'text/csv') { + let newEtsyFile = etsyFile; + let newShippingFile = shippingFile; + if (type === 'etsy') { setEtsyFile(file); + newEtsyFile = file; } else { setShippingFile(file); + newShippingFile = file; } setError(''); + + // Start background processing immediately when files are uploaded + if (newEtsyFile || newShippingFile) { + processExpensesInBackground(newEtsyFile, newShippingFile); + } } else { setError('Please select a valid CSV file'); } @@ -112,151 +224,6 @@ export default function DataImport() { }); }; - // Create expenses from CSV import data - const createExpensesFromCsvData = async ( - shippingRecords: ParsedShippingRecord[], - etsyFees: EtsyFeeRecord[] - ) => { - try { - setIsProcessingExpenses(true); - - // Get existing expenses to avoid duplicates - const existingExpensesRes = await api.get('/expenses?limit=1000'); - const existingExpenses = existingExpensesRes.data.expenses || []; - - const expensesToCreate: any[] = []; - - // Create granular Etsy fee expenses with proper categorization - etsyFees.forEach(fee => { - const isDuplicate = existingExpenses.some((expense: any) => - expense.vendor === fee.vendor && - expense.category === fee.category && - expense.description === fee.description && - Math.abs(expense.amount - fee.amount) < 0.01 && - new Date(expense.date).toDateString() === new Date(fee.date).toDateString() - ); - - if (!isDuplicate) { - expensesToCreate.push({ - description: fee.description, - amount: fee.amount, - category: fee.category, - date: fee.date, - taxDeductible: fee.taxDeductible, - vendor: fee.vendor, - reference: fee.reference - }); - } else { - console.log(`Skipping duplicate Etsy fee: ${fee.description} - $${fee.amount} on ${fee.date}`); - } - }); - - // Create shipping expenses (check for duplicates by tracking number, date, and amount) - shippingRecords.forEach(shipping => { - if (shipping.totalCost > 0) { - const isDuplicate = existingExpenses.some((expense: any) => - expense.reference === shipping.trackingNumber && - expense.vendor === 'Australia Post' && - Math.abs(expense.amount - shipping.totalCost) < 0.01 && // Compare amounts (allow for floating point precision) - new Date(expense.date).toDateString() === new Date(shipping.date).toDateString() // Compare dates - ); - - if (!isDuplicate) { - expensesToCreate.push({ - description: `Australia Post - ${shipping.trackingNumber}`, - amount: shipping.totalCost, - category: 'Shipping & Postage', - date: shipping.date, - taxDeductible: true, - vendor: 'Australia Post', - reference: shipping.trackingNumber - }); - } else { - console.log(`Skipping duplicate shipping expense: ${shipping.trackingNumber} - $${shipping.totalCost} on ${shipping.date}`); - } - } - }); - - // Save expenses to database with rate limiting - if (expensesToCreate.length > 0) { - const savedExpenses: any[] = []; - let created = 0; - let skippedDuplicates = 0; - - // Process expenses with very conservative rate limiting to prevent 429 errors - const BATCH_SIZE = 2; // Further reduced to 2 expenses per batch - const DELAY_MS = 2000; // Increased to 2 seconds between batches - const totalBatches = Math.ceil(expensesToCreate.length / BATCH_SIZE); - - console.log(`Creating ${expensesToCreate.length} expenses in ${totalBatches} batches to avoid rate limiting...`); - - for (let i = 0; i < expensesToCreate.length; i += BATCH_SIZE) { - const batch = expensesToCreate.slice(i, i + BATCH_SIZE); - - // Process batch concurrently - const batchPromises = batch.map(async (expense) => { - try { - const res = await api.post('/expenses', expense); - return { success: true, data: res.data }; - } catch (error: any) { - // Check if it's a duplicate error (MongoDB duplicate key error code 11000) - if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) { - console.log(`Skipping duplicate expense at database level:`, expense.description); - return { success: false, isDuplicate: true }; - } else { - console.error('Failed to create expense:', expense, error); - return { success: false, isDuplicate: false, error }; - } - } - }); - - const batchResults = await Promise.all(batchPromises); - - // Process results - batchResults.forEach(result => { - if (result.success) { - savedExpenses.push(result.data); - created++; - } else if (result.isDuplicate) { - skippedDuplicates++; - } - }); - - // Add delay between batches (except for the last batch) - if (i + BATCH_SIZE < expensesToCreate.length) { - const batchNum = Math.floor(i / BATCH_SIZE) + 1; - console.log(`Processed batch ${batchNum}/${totalBatches} (${created + skippedDuplicates}/${expensesToCreate.length} expenses), waiting ${DELAY_MS}ms...`); - await new Promise(resolve => setTimeout(resolve, DELAY_MS)); - } - } - - console.log(`Expense creation completed: ${created} created, ${skippedDuplicates} duplicates skipped`); - - if (savedExpenses.length > 0) { - dispatch(addExpenses(savedExpenses)); - } - - // Provide informative feedback - if (created > 0 && skippedDuplicates > 0) { - toast.success(`Created ${created} new expenses. Skipped ${skippedDuplicates} duplicates.`); - } else if (created > 0) { - toast.success(`Created ${created} expenses from CSV data (listing fees, ads, shipping, transaction fees)`); - } else if (skippedDuplicates > 0) { - toast.success(`All ${skippedDuplicates} expenses already exist - no duplicates created`); - } else { - toast.error('No expenses were created. Check console for errors.'); - } - } else { - toast.success('No new expenses to create from CSV data'); - } - } catch (error) { - console.error('Error creating expenses:', error); - toast.error('Failed to create expenses from CSV data'); - } finally { - setIsProcessingExpenses(false); - } - }; - const processCsvFiles = async () => { if (!etsyFile) { setError('Please select an Etsy statement CSV file'); @@ -269,9 +236,6 @@ export default function DataImport() { try { const etsyContent = await readFileAsText(etsyFile); const etsyOrders = csvImportService.parseEtsyStatement(etsyContent); - - // Parse individual Etsy fees for granular expense tracking - const etsyFees = csvImportService.parseEtsyFees(etsyContent); let shippingRecords: ParsedShippingRecord[] = []; if (shippingFile) { @@ -330,9 +294,6 @@ export default function DataImport() { const ordersRes = await api.get('/orders?limit=1000'); dispatch(setOrders(ordersRes.data.orders)); - // Create expenses from CSV data - await createExpensesFromCsvData(shippingRecords, etsyFees); - toast.success(`CSV imported! Created ${res.data.created} new orders and updated ${res.data.updated} existing orders.`); } catch { toast.error('Failed to save orders to database'); @@ -812,6 +773,16 @@ export default function DataImport() { + {/* Background Processing Status */} + {backgroundProcessing && ( +
+
+
+ {backgroundProcessing} +
+
+ )} + {/* Automatic Expense Creation Notice */}
@@ -843,10 +814,10 @@ export default function DataImport() {
@@ -978,8 +949,7 @@ export default function DataImport() {