From b8d0416a79073b09d0645015b62c71e1f547b686 Mon Sep 17 00:00:00 2001
From: dlawler489 <104159223@student.swin.edu.au>
Date: Tue, 5 May 2026 19:31:46 +1000
Subject: [PATCH] Fix CSV import rate limiting to prevent HTTP 429 errors

Frontend improvements:
- Add batch processing for expense creation (3 expenses per batch)
- Implement 1.5 second delays between batches to avoid overwhelming server
- Better progress logging and user feedback during batch processing
- Handle rate limit errors gracefully with proper error categorization

Backend improvements:
- Add specific rate limiter for expense creation endpoint (50 per minute)
- More informative error messages for rate limit violations
- Separate rate limiting for expense creation vs general API usage

This prevents the HTTP 429 'Too Many Requests' errors when importing large CSV files with many individual expense records (listing fees, ads, GST entries, etc.).
---
 client/src/pages/DataImport.tsx | 61 +++++++++++++++++++++++++--------
 server/src/routes/expenses.ts   | 15 +++++++-
 2 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/client/src/pages/DataImport.tsx b/client/src/pages/DataImport.tsx
index a81ea96..14331da 100644
--- a/client/src/pages/DataImport.tsx
+++ b/client/src/pages/DataImport.tsx
@@ -174,28 +174,61 @@ export default function DataImport() {
         }
       });
       
-      // Save expenses to database
+      // Save expenses to database with rate limiting
       if (expensesToCreate.length > 0) {
-        const savedExpenses = [];
+        const savedExpenses: any[] = [];
         let created = 0;
         let skippedDuplicates = 0;
         
-        for (const expense of expensesToCreate) {
-          try {
-            const res = await api.post('/expenses', expense);
-            savedExpenses.push(res.data);
-            created++;
-          } catch (error: any) {
-            // Check if it's a duplicate error (MongoDB duplicate key error code 11000)
-            if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) {
-              console.log(`Skipping duplicate expense at database level:`, expense.description);
-              skippedDuplicates++;
-            } else {
-              console.error('Failed to create expense:', expense, error);
+        // Process expenses with rate limiting (conservative approach to avoid 429 errors)
+        const BATCH_SIZE = 3; // Reduced from 5 to be more conservative  
+        const DELAY_MS = 1500; // Increased delay to 1.5 seconds
+        const totalBatches = Math.ceil(expensesToCreate.length / BATCH_SIZE);
+        
+        console.log(`Creating ${expensesToCreate.length} expenses in ${totalBatches} batches to avoid rate limiting...`);
+        
+        for (let i = 0; i < expensesToCreate.length; i += BATCH_SIZE) {
+          const batch = expensesToCreate.slice(i, i + BATCH_SIZE);
+          
+          // Process batch concurrently
+          const batchPromises = batch.map(async (expense) => {
+            try {
+              const res = await api.post('/expenses', expense);
+              return { success: true, data: res.data };
+            } catch (error: any) {
+              // Check if it's a duplicate error (MongoDB duplicate key error code 11000)
+              if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) {
+                console.log(`Skipping duplicate expense at database level:`, expense.description);
+                return { success: false, isDuplicate: true };
+              } else {
+                console.error('Failed to create expense:', expense, error);
+                return { success: false, isDuplicate: false, error };
+              }
             }
+          });
+          
+          const batchResults = await Promise.all(batchPromises);
+          
+          // Process results
+          batchResults.forEach(result => {
+            if (result.success) {
+              savedExpenses.push(result.data);
+              created++;
+            } else if (result.isDuplicate) {
+              skippedDuplicates++;
+            }
+          });
+          
+          // Add delay between batches (except for the last batch)
+          if (i + BATCH_SIZE < expensesToCreate.length) {
+            const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+            console.log(`Processed batch ${batchNum}/${totalBatches} (${created + skippedDuplicates}/${expensesToCreate.length} expenses), waiting ${DELAY_MS}ms...`);
+            await new Promise(resolve => setTimeout(resolve, DELAY_MS));
           }
         }
         
+        console.log(`Expense creation completed: ${created} created, ${skippedDuplicates} duplicates skipped`);
+        
         if (savedExpenses.length > 0) {
           dispatch(addExpenses(savedExpenses));
         }
diff --git a/server/src/routes/expenses.ts b/server/src/routes/expenses.ts
index e568462..2cf1ba1 100644
--- a/server/src/routes/expenses.ts
+++ b/server/src/routes/expenses.ts
@@ -1,9 +1,22 @@
 import { Router, Response } from 'express';
+import rateLimit from 'express-rate-limit';
 import Expense from '../models/Expense';
 import { AuthRequest } from '../middleware/authenticate';
 
 const router = Router();
 
+// Specific rate limit for expense creation (more permissive for bulk imports)
+const expenseCreateLimiter = rateLimit({
+  windowMs: 1 * 60 * 1000, // 1 minute window
+  max: 50, // Max 50 expense creations per minute per IP
+  message: {
+    error: 'Too many expense creation requests. Please wait a moment before trying again.',
+    retryAfter: '60 seconds'
+  },
+  standardHeaders: true,
+  legacyHeaders: false
+});
+
 router.get('/', async (req: AuthRequest, res: Response) => {
   try {
     const { page = 1, limit = 20, category } = req.query;
@@ -22,7 +35,7 @@ router.get('/', async (req: AuthRequest, res: Response) => {
   }
 });
 
-router.post('/', async (req: AuthRequest, res: Response) => {
+router.post('/', expenseCreateLimiter, async (req: AuthRequest, res: Response) => {
   try {
     const expense = new Expense({ ...req.body, userId: req.userId });
     await expense.save();