feat: move expense processing to background during CSV upload
- Process expenses immediately when CSV files are uploaded, not when user clicks analyze - Use much more conservative rate limiting (3 seconds between expenses) to prevent 429 errors - Remove bulk batch processing that was overwhelming the rate limiter - Add background processing status indicator with spinner animation - Clean up unused state variables and functions - Improve user experience by eliminating long wait times during analysis This prevents HTTP 429 rate limiting errors by spreading expense creation over time naturally as files are uploaded, rather than trying to process 60+ expenses rapidly in batches.
This commit is contained in:
dlawler489
parent
c2bdaa3c0d
commit
212dc77df7
1 changed files with 127 additions and 157 deletions
|
|
@ -7,7 +7,6 @@ import { DataManager } from '../utils/dataManager';
|
|||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import { RootState } from '../store';
|
||||
import { addOrder, updateOrder, setOrders } from '../store/slices/orderSlice';
|
||||
import { addExpenses } from '../store/slices/expenseSlice';
|
||||
import { Upload, FileText, Package, Truck, Trash2 } from 'lucide-react';
|
||||
import toast from 'react-hot-toast';
|
||||
import { dateTestResults } from '../utils/testDateParsing';
|
||||
|
|
@ -54,7 +53,6 @@ export default function DataImport() {
|
|||
const [etsyFile, setEtsyFile] = useState<File | null>(null);
|
||||
const [shippingFile, setShippingFile] = useState<File | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isProcessingExpenses, setIsProcessingExpenses] = useState(false);
|
||||
const [results, setResults] = useState<ImportResults | null>(null);
|
||||
|
||||
// PDF Import State
|
||||
|
|
@ -68,11 +66,115 @@ export default function DataImport() {
|
|||
// UI State
|
||||
const [activeTab, setActiveTab] = useState<'csv' | 'pdf'>('csv');
|
||||
const [error, setError] = useState<string>('');
|
||||
const [backgroundProcessing, setBackgroundProcessing] = useState<string>('');
|
||||
|
||||
const etsyFileRef = useRef<HTMLInputElement>(null);
|
||||
const shippingFileRef = useRef<HTMLInputElement>(null);
|
||||
const pdfFileRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
// Background expense processing
|
||||
const processExpensesInBackground = async (etsyFile: File | null, shippingFile: File | null) => {
|
||||
if (!etsyFile && !shippingFile) return;
|
||||
|
||||
try {
|
||||
setBackgroundProcessing('Processing expenses from uploaded files...');
|
||||
|
||||
const existingExpenses = await api.get('/expenses').then(res => res.data);
|
||||
const expensesToCreate: any[] = [];
|
||||
|
||||
if (etsyFile) {
|
||||
const etsyContent = await readFileAsText(etsyFile);
|
||||
const etsyFees = csvImportService.parseEtsyFees(etsyContent);
|
||||
|
||||
// Add Etsy fees (check for duplicates)
|
||||
etsyFees.forEach((fee: EtsyFeeRecord) => {
|
||||
const isDuplicate = existingExpenses.some((expense: any) =>
|
||||
expense.description === fee.description &&
|
||||
Math.abs(expense.amount - fee.amount) < 0.01 &&
|
||||
new Date(expense.date).toDateString() === new Date(fee.date).toDateString() &&
|
||||
expense.vendor === fee.vendor
|
||||
);
|
||||
|
||||
if (!isDuplicate) {
|
||||
expensesToCreate.push({
|
||||
description: fee.description,
|
||||
amount: fee.amount,
|
||||
category: fee.category,
|
||||
date: fee.date,
|
||||
taxDeductible: fee.taxDeductible,
|
||||
vendor: fee.vendor,
|
||||
reference: fee.reference
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (shippingFile) {
|
||||
const shippingContent = await readFileAsText(shippingFile);
|
||||
const shippingRecords = csvImportService.parseAustraliaPostShipping(shippingContent);
|
||||
|
||||
// Add shipping expenses (check for duplicates)
|
||||
shippingRecords.forEach((shipping: ParsedShippingRecord) => {
|
||||
if (shipping.totalCost > 0) {
|
||||
const isDuplicate = existingExpenses.some((expense: any) =>
|
||||
expense.reference === shipping.trackingNumber &&
|
||||
expense.vendor === 'Australia Post' &&
|
||||
Math.abs(expense.amount - shipping.totalCost) < 0.01 &&
|
||||
new Date(expense.date).toDateString() === new Date(shipping.date).toDateString()
|
||||
);
|
||||
|
||||
if (!isDuplicate) {
|
||||
expensesToCreate.push({
|
||||
description: `Australia Post - ${shipping.trackingNumber}`,
|
||||
amount: shipping.totalCost,
|
||||
category: 'Shipping & Postage',
|
||||
date: shipping.date,
|
||||
taxDeductible: true,
|
||||
vendor: 'Australia Post',
|
||||
reference: shipping.trackingNumber
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process expenses with very conservative rate limiting
|
||||
if (expensesToCreate.length > 0) {
|
||||
const DELAY_MS = 3000; // 3 seconds between each expense
|
||||
let created = 0;
|
||||
|
||||
console.log(`Background processing: Creating ${expensesToCreate.length} expenses one at a time...`);
|
||||
|
||||
for (let i = 0; i < expensesToCreate.length; i++) {
|
||||
try {
|
||||
setBackgroundProcessing(`Processing expense ${i + 1}/${expensesToCreate.length}...`);
|
||||
await api.post('/expenses', expensesToCreate[i]);
|
||||
created++;
|
||||
|
||||
// Wait before next expense (except for the last one)
|
||||
if (i < expensesToCreate.length - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, DELAY_MS));
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.response?.status === 11000 || error.response?.data?.message?.includes('duplicate')) {
|
||||
console.log(`Skipping duplicate expense: ${expensesToCreate[i].description}`);
|
||||
} else {
|
||||
console.error(`Failed to create expense: ${expensesToCreate[i].description}`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Background processing completed: ${created} expenses created`);
|
||||
toast.success(`${created} expenses processed in background`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error in background expense processing:', error);
|
||||
toast.error('Error processing expenses in background');
|
||||
} finally {
|
||||
setBackgroundProcessing('');
|
||||
}
|
||||
};
|
||||
|
||||
const handleFileChange = (
|
||||
event: React.ChangeEvent<HTMLInputElement>,
|
||||
type: 'etsy' | 'shipping' | 'pdf'
|
||||
|
|
@ -91,12 +193,22 @@ export default function DataImport() {
|
|||
} else {
|
||||
const file = files[0];
|
||||
if (file && file.type === 'text/csv') {
|
||||
let newEtsyFile = etsyFile;
|
||||
let newShippingFile = shippingFile;
|
||||
|
||||
if (type === 'etsy') {
|
||||
setEtsyFile(file);
|
||||
newEtsyFile = file;
|
||||
} else {
|
||||
setShippingFile(file);
|
||||
newShippingFile = file;
|
||||
}
|
||||
setError('');
|
||||
|
||||
// Start background processing immediately when files are uploaded
|
||||
if (newEtsyFile || newShippingFile) {
|
||||
processExpensesInBackground(newEtsyFile, newShippingFile);
|
||||
}
|
||||
} else {
|
||||
setError('Please select a valid CSV file');
|
||||
}
|
||||
|
|
@ -112,151 +224,6 @@ export default function DataImport() {
|
|||
});
|
||||
};
|
||||
|
||||
// Create expenses from CSV import data
|
||||
const createExpensesFromCsvData = async (
|
||||
shippingRecords: ParsedShippingRecord[],
|
||||
etsyFees: EtsyFeeRecord[]
|
||||
) => {
|
||||
try {
|
||||
setIsProcessingExpenses(true);
|
||||
|
||||
// Get existing expenses to avoid duplicates
|
||||
const existingExpensesRes = await api.get('/expenses?limit=1000');
|
||||
const existingExpenses = existingExpensesRes.data.expenses || [];
|
||||
|
||||
const expensesToCreate: any[] = [];
|
||||
|
||||
// Create granular Etsy fee expenses with proper categorization
|
||||
etsyFees.forEach(fee => {
|
||||
const isDuplicate = existingExpenses.some((expense: any) =>
|
||||
expense.vendor === fee.vendor &&
|
||||
expense.category === fee.category &&
|
||||
expense.description === fee.description &&
|
||||
Math.abs(expense.amount - fee.amount) < 0.01 &&
|
||||
new Date(expense.date).toDateString() === new Date(fee.date).toDateString()
|
||||
);
|
||||
|
||||
if (!isDuplicate) {
|
||||
expensesToCreate.push({
|
||||
description: fee.description,
|
||||
amount: fee.amount,
|
||||
category: fee.category,
|
||||
date: fee.date,
|
||||
taxDeductible: fee.taxDeductible,
|
||||
vendor: fee.vendor,
|
||||
reference: fee.reference
|
||||
});
|
||||
} else {
|
||||
console.log(`Skipping duplicate Etsy fee: ${fee.description} - $${fee.amount} on ${fee.date}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Create shipping expenses (check for duplicates by tracking number, date, and amount)
|
||||
shippingRecords.forEach(shipping => {
|
||||
if (shipping.totalCost > 0) {
|
||||
const isDuplicate = existingExpenses.some((expense: any) =>
|
||||
expense.reference === shipping.trackingNumber &&
|
||||
expense.vendor === 'Australia Post' &&
|
||||
Math.abs(expense.amount - shipping.totalCost) < 0.01 && // Compare amounts (allow for floating point precision)
|
||||
new Date(expense.date).toDateString() === new Date(shipping.date).toDateString() // Compare dates
|
||||
);
|
||||
|
||||
if (!isDuplicate) {
|
||||
expensesToCreate.push({
|
||||
description: `Australia Post - ${shipping.trackingNumber}`,
|
||||
amount: shipping.totalCost,
|
||||
category: 'Shipping & Postage',
|
||||
date: shipping.date,
|
||||
taxDeductible: true,
|
||||
vendor: 'Australia Post',
|
||||
reference: shipping.trackingNumber
|
||||
});
|
||||
} else {
|
||||
console.log(`Skipping duplicate shipping expense: ${shipping.trackingNumber} - $${shipping.totalCost} on ${shipping.date}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Save expenses to database with rate limiting
|
||||
if (expensesToCreate.length > 0) {
|
||||
const savedExpenses: any[] = [];
|
||||
let created = 0;
|
||||
let skippedDuplicates = 0;
|
||||
|
||||
// Process expenses with very conservative rate limiting to prevent 429 errors
|
||||
const BATCH_SIZE = 2; // Further reduced to 2 expenses per batch
|
||||
const DELAY_MS = 2000; // Increased to 2 seconds between batches
|
||||
const totalBatches = Math.ceil(expensesToCreate.length / BATCH_SIZE);
|
||||
|
||||
console.log(`Creating ${expensesToCreate.length} expenses in ${totalBatches} batches to avoid rate limiting...`);
|
||||
|
||||
for (let i = 0; i < expensesToCreate.length; i += BATCH_SIZE) {
|
||||
const batch = expensesToCreate.slice(i, i + BATCH_SIZE);
|
||||
|
||||
// Process batch concurrently
|
||||
const batchPromises = batch.map(async (expense) => {
|
||||
try {
|
||||
const res = await api.post('/expenses', expense);
|
||||
return { success: true, data: res.data };
|
||||
} catch (error: any) {
|
||||
// Check if it's a duplicate error (MongoDB duplicate key error code 11000)
|
||||
if (error.response?.status === 400 && error.response?.data?.message?.includes('duplicate')) {
|
||||
console.log(`Skipping duplicate expense at database level:`, expense.description);
|
||||
return { success: false, isDuplicate: true };
|
||||
} else {
|
||||
console.error('Failed to create expense:', expense, error);
|
||||
return { success: false, isDuplicate: false, error };
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
|
||||
// Process results
|
||||
batchResults.forEach(result => {
|
||||
if (result.success) {
|
||||
savedExpenses.push(result.data);
|
||||
created++;
|
||||
} else if (result.isDuplicate) {
|
||||
skippedDuplicates++;
|
||||
}
|
||||
});
|
||||
|
||||
// Add delay between batches (except for the last batch)
|
||||
if (i + BATCH_SIZE < expensesToCreate.length) {
|
||||
const batchNum = Math.floor(i / BATCH_SIZE) + 1;
|
||||
console.log(`Processed batch ${batchNum}/${totalBatches} (${created + skippedDuplicates}/${expensesToCreate.length} expenses), waiting ${DELAY_MS}ms...`);
|
||||
await new Promise(resolve => setTimeout(resolve, DELAY_MS));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Expense creation completed: ${created} created, ${skippedDuplicates} duplicates skipped`);
|
||||
|
||||
if (savedExpenses.length > 0) {
|
||||
dispatch(addExpenses(savedExpenses));
|
||||
}
|
||||
|
||||
// Provide informative feedback
|
||||
if (created > 0 && skippedDuplicates > 0) {
|
||||
toast.success(`Created ${created} new expenses. Skipped ${skippedDuplicates} duplicates.`);
|
||||
} else if (created > 0) {
|
||||
toast.success(`Created ${created} expenses from CSV data (listing fees, ads, shipping, transaction fees)`);
|
||||
} else if (skippedDuplicates > 0) {
|
||||
toast.success(`All ${skippedDuplicates} expenses already exist - no duplicates created`);
|
||||
} else {
|
||||
toast.error('No expenses were created. Check console for errors.');
|
||||
}
|
||||
} else {
|
||||
toast.success('No new expenses to create from CSV data');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error creating expenses:', error);
|
||||
toast.error('Failed to create expenses from CSV data');
|
||||
} finally {
|
||||
setIsProcessingExpenses(false);
|
||||
}
|
||||
};
|
||||
|
||||
const processCsvFiles = async () => {
|
||||
if (!etsyFile) {
|
||||
setError('Please select an Etsy statement CSV file');
|
||||
|
|
@ -270,9 +237,6 @@ export default function DataImport() {
|
|||
const etsyContent = await readFileAsText(etsyFile);
|
||||
const etsyOrders = csvImportService.parseEtsyStatement(etsyContent);
|
||||
|
||||
// Parse individual Etsy fees for granular expense tracking
|
||||
const etsyFees = csvImportService.parseEtsyFees(etsyContent);
|
||||
|
||||
let shippingRecords: ParsedShippingRecord[] = [];
|
||||
if (shippingFile) {
|
||||
const shippingContent = await readFileAsText(shippingFile);
|
||||
|
|
@ -330,9 +294,6 @@ export default function DataImport() {
|
|||
const ordersRes = await api.get('/orders?limit=1000');
|
||||
dispatch(setOrders(ordersRes.data.orders));
|
||||
|
||||
// Create expenses from CSV data
|
||||
await createExpensesFromCsvData(shippingRecords, etsyFees);
|
||||
|
||||
toast.success(`CSV imported! Created ${res.data.created} new orders and updated ${res.data.updated} existing orders.`);
|
||||
} catch {
|
||||
toast.error('Failed to save orders to database');
|
||||
|
|
@ -812,6 +773,16 @@ export default function DataImport() {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{/* Background Processing Status */}
|
||||
{backgroundProcessing && (
|
||||
<div className="mt-4 bg-blue-50 border border-blue-200 rounded-lg p-4">
|
||||
<div className="flex items-center">
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-600 mr-3"></div>
|
||||
<span className="text-blue-700 text-sm">{backgroundProcessing}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Automatic Expense Creation Notice */}
|
||||
<div className="mt-6 bg-green-50 border border-green-200 rounded-lg p-4">
|
||||
<div className="flex">
|
||||
|
|
@ -843,10 +814,10 @@ export default function DataImport() {
|
|||
<div className="mt-6 flex gap-4">
|
||||
<button
|
||||
onClick={processCsvFiles}
|
||||
disabled={!etsyFile || isLoading || isProcessingExpenses}
|
||||
disabled={!etsyFile || isLoading}
|
||||
className="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 disabled:bg-gray-400"
|
||||
>
|
||||
{isLoading || isProcessingExpenses ? 'Processing...' : 'Analyze Financial Data'}
|
||||
{isLoading ? 'Processing...' : 'Analyze Financial Data'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -978,8 +949,7 @@ export default function DataImport() {
|
|||
</div>
|
||||
<button
|
||||
onClick={createOrdersFromCSV}
|
||||
disabled={isProcessingExpenses}
|
||||
className="flex items-center gap-2 px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 disabled:bg-gray-400 ml-4"
|
||||
className="flex items-center gap-2 px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 ml-4"
|
||||
>
|
||||
<Package className="w-4 h-4" />
|
||||
Re-sync Orders
|
||||
|
|
|
|||
Loading…
Reference in a new issue