diff --git a/client/src/pages/DataImport.tsx b/client/src/pages/DataImport.tsx index 9729571..c87b6b4 100644 --- a/client/src/pages/DataImport.tsx +++ b/client/src/pages/DataImport.tsx @@ -9,7 +9,6 @@ import { RootState } from '../store'; import { addOrder, updateOrder, setOrders } from '../store/slices/orderSlice'; import { Upload, FileText, Package, Truck, Trash2 } from 'lucide-react'; import toast from 'react-hot-toast'; -import { dateTestResults } from '../utils/testDateParsing'; import api from '../utils/api'; interface ImportResults { @@ -41,10 +40,6 @@ interface ImportResults { } export default function DataImport() { - // Test date parsing immediately - console.log('=== DATE PARSING TEST RESULTS ==='); - console.log('Date test results:', dateTestResults); - const dispatch = useDispatch(); const orders = useSelector((state: RootState) => state.orders.orders); const products = useSelector((state: RootState) => state.products.products); @@ -154,34 +149,33 @@ export default function DataImport() { }); } - // Process expenses with very conservative rate limiting + // Process expenses with bulk API for much faster processing if (expensesToCreate.length > 0) { - const DELAY_MS = 3000; // 3 seconds between each expense - let created = 0; - - console.log(`Background processing: Creating ${expensesToCreate.length} expenses one at a time...`); + console.log(`Creating ${expensesToCreate.length} expenses via bulk API...`); + setBackgroundProcessing(`Creating ${expensesToCreate.length} expenses...`); - for (let i = 0; i < expensesToCreate.length; i++) { - try { - setBackgroundProcessing(`Processing expense ${i + 1}/${expensesToCreate.length}...`); - await api.post('/expenses', expensesToCreate[i]); - created++; - - // Wait before next expense (except for the last one) - if (i < expensesToCreate.length - 1) { - await new Promise(resolve => setTimeout(resolve, DELAY_MS)); - } - } catch (error: any) { - if (error.response?.status === 11000 || error.response?.data?.message?.includes('duplicate')) { - console.log(`Skipping duplicate expense: ${expensesToCreate[i].description}`); - } else { - console.error(`Failed to create expense: ${expensesToCreate[i].description}`, error); - } + try { + const response = await api.post('/expenses/bulk', expensesToCreate); + const { created, duplicates, errors, message } = response.data; + + console.log(`Bulk expense creation completed: ${created} created, ${duplicates} duplicates, ${errors} errors`); + + if (created > 0 || duplicates > 0) { + toast.success(`✅ ${message}`); + } else if (errors > 0) { + toast.error(`❌ Failed to create expenses: ${message}`); + } + + } catch (error: any) { + console.error('Bulk expense creation failed:', error); + if (error.response?.status === 429) { + toast.error('⏳ Server busy - expenses will be processed shortly'); + } else { + toast.error('❌ Failed to create expenses from CSV data'); } } - - console.log(`Background processing completed: ${created} expenses created`); - toast.success(`${created} expenses processed in background`); + } else { + toast.success('✅ No new expenses to create from CSV data'); } } catch (error) { console.error('Error in background expense processing:', error); diff --git a/server/src/routes/expenses.ts b/server/src/routes/expenses.ts index 2cf1ba1..138cb4b 100644 --- a/server/src/routes/expenses.ts +++ b/server/src/routes/expenses.ts @@ -149,6 +149,85 @@ router.post('/cleanup-duplicates', async (req: AuthRequest, res: Response) => { } }); +// Bulk create expenses endpoint for CSV imports +router.post('/bulk', expenseCreateLimiter, async (req: AuthRequest, res: Response) => { + try { + const expenses = req.body; + + if (!Array.isArray(expenses)) { + return res.status(400).json({ message: 'Request body must be an array of expenses' }); + } + + if (expenses.length === 0) { + return res.json({ created: 0, duplicates: 0, errors: 0, expenses: [] }); + } + + // Add userId to all expenses + const expensesWithUserId = expenses.map(expense => ({ + ...expense, + userId: req.userId + })); + + const results = { + created: 0, + duplicates: 0, + errors: 0, + expenses: [] as any[], + errorDetails: [] as any[] + }; + + // Process expenses with insertMany using ordered: false to continue on duplicates + try { + const savedExpenses = await Expense.insertMany(expensesWithUserId, { + ordered: false, // Continue processing even if some fail + rawResult: true // Get detailed results + }); + + results.created = savedExpenses.insertedCount || 0; + results.expenses = savedExpenses.insertedIds ? Object.values(savedExpenses.insertedIds) : []; + } catch (bulkError: any) { + // Handle bulk write errors (duplicates, validation errors, etc.) + if (bulkError.writeErrors) { + bulkError.writeErrors.forEach((error: any) => { + if (error.code === 11000) { + results.duplicates++; + } else { + results.errors++; + results.errorDetails.push({ + index: error.index, + message: error.errmsg, + expense: expensesWithUserId[error.index] + }); + } + }); + + // Count successful insertions + if (bulkError.result && bulkError.result.insertedCount) { + results.created = bulkError.result.insertedCount; + } + } else { + // Non-bulk write error + results.errors = expenses.length; + results.errorDetails.push({ message: bulkError.message }); + } + } + + const message = `Bulk operation completed: ${results.created} created, ${results.duplicates} duplicates skipped, ${results.errors} errors`; + + res.status(200).json({ + ...results, + message + }); + + } catch (err: any) { + console.error('Bulk expense creation error:', err); + res.status(500).json({ + message: 'Failed to create expenses in bulk', + error: err.message + }); + } +}); + router.delete('/:id', async (req: AuthRequest, res: Response) => { try { const expense = await Expense.findOneAndDelete({ _id: req.params.id, userId: req.userId });