Fix Australia Post CSV duplicate imports with comprehensive duplicate prevention

Multi-layer duplicate prevention system:
- Enhanced frontend duplicate detection with tracking number, amount, and date comparison
- Added MongoDB compound index to prevent database-level duplicates
- Improved backend error handling for duplicate key violations
- Added cleanup endpoint to remove existing duplicates
- Enhanced user feedback for import operations

Frontend changes:
- Stricter duplicate detection comparing tracking number, vendor, amount, and date
- Better error handling and user feedback for duplicate scenarios
- Added 'Clean Duplicates' button to remove existing duplicates

Backend changes:
- Database compound index on reference, vendor, userId, amount, date
- Enhanced error responses with duplicate detection flags
- New POST /expenses/cleanup-duplicates endpoint
- Improved duplicate key error handling

This should eliminate the double Australia Post expense entries.

client/src/pages/DataImport.tsx

@@ -146,12 +146,14 @@ export default function DataImport() {
         }
       });
       
-      // Create shipping expenses (check for duplicates by tracking number)
+      // Create shipping expenses (check for duplicates by tracking number, date, and amount)
       shippingRecords.forEach(shipping => {
         if (shipping.totalCost > 0) {
           const isDuplicate = existingExpenses.some((expense: any) => 
             expense.reference === shipping.trackingNumber && 
-            expense.vendor === 'Australia Post'
+            expense.vendor === 'Australia Post' &&
+            Math.abs(expense.amount - shipping.totalCost) < 0.01 && // Compare amounts (allow for floating point precision)
+            new Date(expense.date).toDateString() === new Date(shipping.date).toDateString() // Compare dates
           );
           
           if (!isDuplicate) {
@@ -164,6 +166,8 @@ export default function DataImport() {
               vendor: 'Australia Post',
               reference: shipping.trackingNumber
             });
+          } else {
+            console.log(`Skipping duplicate shipping expense: ${shipping.trackingNumber} - $${shipping.totalCost} on ${shipping.date}`);
           }
         }
       });
@@ -172,22 +176,37 @@ export default function DataImport() {
       if (expensesToCreate.length > 0) {
         const savedExpenses = [];
         let created = 0;
+        let skippedDuplicates = 0;
         
         for (const expense of expensesToCreate) {
           try {
             const res = await api.post('/expenses', expense);
             savedExpenses.push(res.data);
             created++;
-          } catch (error) {
-            console.error('Failed to create expense:', expense, error);
+          } catch (error: any) {
+            // Check if it's a duplicate error (MongoDB duplicate key error code 11000)
+            if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) {
+              console.log(`Skipping duplicate expense at database level:`, expense.description);
+              skippedDuplicates++;
+            } else {
+              console.error('Failed to create expense:', expense, error);
+            }
           }
         }
         
         if (savedExpenses.length > 0) {
           dispatch(addExpenses(savedExpenses));
+        }
+        
+        // Provide informative feedback
+        if (created > 0 && skippedDuplicates > 0) {
+          toast.success(`Created ${created} new expenses. Skipped ${skippedDuplicates} duplicates.`);
+        } else if (created > 0) {
           toast.success(`Created ${created} expenses from CSV data (Etsy fees + shipping costs)`);
-        } else if (expensesToCreate.length === 0) {
-          toast.success('All expenses from CSV data already exist - no duplicates created');
+        } else if (skippedDuplicates > 0) {
+          toast.success(`All ${skippedDuplicates} expenses already exist - no duplicates created`);
+        } else {
+          toast.error('No expenses were created. Check console for errors.');
         }
       } else {
         toast.success('No new expenses to create from CSV data');

client/src/pages/Expenses.tsx

@@ -285,6 +285,34 @@ const Expenses = () => {
     });
   };
 
+  const handleCleanupDuplicates = async () => {
+    if (!confirm('This will remove duplicate Australia Post shipping expenses, keeping only the earliest created record for each tracking number. This action cannot be undone. Continue?')) {
+      return;
+    }
+
+    try {
+      const response = await api.post('/expenses/cleanup-duplicates');
+      const { deletedCount, duplicateGroups, details } = response.data;
+      
+      if (deletedCount > 0) {
+        toast.success(`Successfully removed ${deletedCount} duplicate expenses from ${duplicateGroups} tracking numbers`);
+        
+        // Refresh expenses from server
+        const updatedResponse = await api.get('/expenses?limit=1000');
+        if (updatedResponse.data.expenses) {
+          dispatch(addExpenses(updatedResponse.data.expenses));
+        }
+        
+        console.log('Cleanup details:', details);
+      } else {
+        toast.success('No duplicate Australia Post expenses found');
+      }
+    } catch (error) {
+      console.error('Cleanup failed:', error);
+      toast.error('Failed to cleanup duplicate expenses');
+    }
+  };
+
   const filteredExpenses = expenses
     .filter(expense => {
       const matchesSearch = expense.description.toLowerCase().includes(searchTerm.toLowerCase()) ||
@@ -363,6 +391,13 @@ const Expenses = () => {
             <Download className="w-4 h-4" />
             Export CSV
           </button>
+          <button
+            onClick={handleCleanupDuplicates}
+            className="flex items-center gap-2 px-4 py-2 bg-red-600 text-white rounded-lg hover:bg-red-700"
+          >
+            <Trash2 className="w-4 h-4" />
+            Clean Duplicates
+          </button>
           <button
             onClick={() => setShowAddForm(true)}
             className="flex items-center gap-2 px-4 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700"

server/src/models/Expense.ts

@@ -35,4 +35,19 @@ ExpenseSchema.pre('save', function (next) {
   next();
 });
 
+// Create compound index to prevent duplicate expenses with same reference, vendor, and user
+ExpenseSchema.index({ 
+  reference: 1, 
+  vendor: 1, 
+  userId: 1, 
+  amount: 1, 
+  date: 1 
+}, { 
+  unique: true, 
+  partialFilterExpression: { 
+    reference: { $exists: true, $ne: null },
+    vendor: { $exists: true, $ne: null }
+  } 
+});
+
 export default mongoose.model<IExpense>('Expense', ExpenseSchema);

server/src/routes/expenses.ts

@@ -27,8 +27,20 @@ router.post('/', async (req: AuthRequest, res: Response) => {
     const expense = new Expense({ ...req.body, userId: req.userId });
     await expense.save();
     res.status(201).json(expense);
-  } catch (err) {
-    res.status(400).json({ message: 'Failed to create expense', error: err });
+  } catch (err: any) {
+    // Handle duplicate key error (MongoDB error code 11000)
+    if (err.code === 11000) {
+      const duplicateField = Object.keys(err.keyPattern || {})[0];
+      res.status(400).json({ 
+        message: `Expense with this ${duplicateField || 'reference'} already exists (duplicate prevented)`,
+        isDuplicate: true,
+        error: err 
+      });
+    } else if (err.name === 'ValidationError') {
+      res.status(400).json({ message: 'Validation failed', error: err.message });
+    } else {
+      res.status(400).json({ message: 'Failed to create expense', error: err });
+    }
   }
 });
 
@@ -56,6 +68,74 @@ router.put('/:id', async (req: AuthRequest, res: Response) => {
   }
 });
 
+// Clean up duplicate Australia Post expenses
+router.post('/cleanup-duplicates', async (req: AuthRequest, res: Response) => {
+  try {
+    // Find all Australia Post expenses for this user
+    const expenses = await Expense.find({ 
+      userId: req.userId,
+      vendor: 'Australia Post'
+    }).sort({ dateCreated: 1 }); // Sort by creation date, keep earliest
+
+    // Group by tracking number (reference field)
+    const groupedByTracking = new Map();
+    expenses.forEach(expense => {
+      if (expense.reference) {
+        if (!groupedByTracking.has(expense.reference)) {
+          groupedByTracking.set(expense.reference, []);
+        }
+        groupedByTracking.get(expense.reference).push(expense);
+      }
+    });
+
+    // Find duplicates and delete all but the first (earliest) one
+    let deletedCount = 0;
+    const duplicateGroups = [];
+
+    for (const [trackingNumber, expenseGroup] of groupedByTracking) {
+      if (expenseGroup.length > 1) {
+        // Keep the first (earliest created) and delete the rest
+        const toKeep = expenseGroup[0];
+        const toDelete = expenseGroup.slice(1);
+        
+        duplicateGroups.push({
+          trackingNumber,
+          total: expenseGroup.length,
+          keeping: {
+            id: toKeep._id,
+            date: toKeep.date,
+            amount: toKeep.amount,
+            created: toKeep.dateCreated
+          },
+          deleting: toDelete.map((exp: any) => ({
+            id: exp._id,
+            date: exp.date,
+            amount: exp.amount,
+            created: exp.dateCreated
+          }))
+        });
+
+        // Delete the duplicates
+        await Expense.deleteMany({
+          _id: { $in: toDelete.map((exp: any) => exp._id) },
+          userId: req.userId
+        });
+        
+        deletedCount += toDelete.length;
+      }
+    }
+
+    res.json({
+      message: `Cleaned up ${deletedCount} duplicate Australia Post expenses`,
+      deletedCount,
+      duplicateGroups: duplicateGroups.length,
+      details: duplicateGroups
+    });
+  } catch (err: any) {
+    res.status(500).json({ message: 'Failed to cleanup duplicates', error: err });
+  }
+});
+
 router.delete('/:id', async (req: AuthRequest, res: Response) => {
   try {
     const expense = await Expense.findOneAndDelete({ _id: req.params.id, userId: req.userId });