diff --git a/client/src/pages/DataImport.tsx b/client/src/pages/DataImport.tsx index a81ea96..14331da 100644 --- a/client/src/pages/DataImport.tsx +++ b/client/src/pages/DataImport.tsx @@ -174,28 +174,61 @@ export default function DataImport() { } }); - // Save expenses to database + // Save expenses to database with rate limiting if (expensesToCreate.length > 0) { - const savedExpenses = []; + const savedExpenses: any[] = []; let created = 0; let skippedDuplicates = 0; - for (const expense of expensesToCreate) { - try { - const res = await api.post('/expenses', expense); - savedExpenses.push(res.data); - created++; - } catch (error: any) { - // Check if it's a duplicate error (MongoDB duplicate key error code 11000) - if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) { - console.log(`Skipping duplicate expense at database level:`, expense.description); - skippedDuplicates++; - } else { - console.error('Failed to create expense:', expense, error); + // Process expenses with rate limiting (conservative approach to avoid 429 errors) + const BATCH_SIZE = 3; // Reduced from 5 to be more conservative + const DELAY_MS = 1500; // Increased delay to 1.5 seconds + const totalBatches = Math.ceil(expensesToCreate.length / BATCH_SIZE); + + console.log(`Creating ${expensesToCreate.length} expenses in ${totalBatches} batches to avoid rate limiting...`); + + for (let i = 0; i < expensesToCreate.length; i += BATCH_SIZE) { + const batch = expensesToCreate.slice(i, i + BATCH_SIZE); + + // Process batch concurrently + const batchPromises = batch.map(async (expense) => { + try { + const res = await api.post('/expenses', expense); + return { success: true, data: res.data }; + } catch (error: any) { + // Check if it's a duplicate error (MongoDB duplicate key error code 11000) + if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) { + console.log(`Skipping duplicate expense at database level:`, expense.description); + return { success: false, isDuplicate: true }; + } else { + console.error('Failed to create expense:', expense, error); + return { success: false, isDuplicate: false, error }; + } } + }); + + const batchResults = await Promise.all(batchPromises); + + // Process results + batchResults.forEach(result => { + if (result.success) { + savedExpenses.push(result.data); + created++; + } else if (result.isDuplicate) { + skippedDuplicates++; + } + }); + + // Add delay between batches (except for the last batch) + if (i + BATCH_SIZE < expensesToCreate.length) { + const batchNum = Math.floor(i / BATCH_SIZE) + 1; + console.log(`Processed batch ${batchNum}/${totalBatches} (${created + skippedDuplicates}/${expensesToCreate.length} expenses), waiting ${DELAY_MS}ms...`); + await new Promise(resolve => setTimeout(resolve, DELAY_MS)); } } + console.log(`Expense creation completed: ${created} created, ${skippedDuplicates} duplicates skipped`); + if (savedExpenses.length > 0) { dispatch(addExpenses(savedExpenses)); } diff --git a/server/src/routes/expenses.ts b/server/src/routes/expenses.ts index e568462..2cf1ba1 100644 --- a/server/src/routes/expenses.ts +++ b/server/src/routes/expenses.ts @@ -1,9 +1,22 @@ import { Router, Response } from 'express'; +import rateLimit from 'express-rate-limit'; import Expense from '../models/Expense'; import { AuthRequest } from '../middleware/authenticate'; const router = Router(); +// Specific rate limit for expense creation (more permissive for bulk imports) +const expenseCreateLimiter = rateLimit({ + windowMs: 1 * 60 * 1000, // 1 minute window + max: 50, // Max 50 expense creations per minute per IP + message: { + error: 'Too many expense creation requests. Please wait a moment before trying again.', + retryAfter: '60 seconds' + }, + standardHeaders: true, + legacyHeaders: false +}); + router.get('/', async (req: AuthRequest, res: Response) => { try { const { page = 1, limit = 20, category } = req.query; @@ -22,7 +35,7 @@ router.get('/', async (req: AuthRequest, res: Response) => { } }); -router.post('/', async (req: AuthRequest, res: Response) => { +router.post('/', expenseCreateLimiter, async (req: AuthRequest, res: Response) => { try { const expense = new Expense({ ...req.body, userId: req.userId }); await expense.save();