diff --git a/src/features/transactions/import/ImportTrxStep1.tsx b/src/features/transactions/import/ImportTrxStep1.tsx index 19990fc6..8125a9d4 100644 --- a/src/features/transactions/import/ImportTrxStep1.tsx +++ b/src/features/transactions/import/ImportTrxStep1.tsx @@ -51,6 +51,7 @@ import { convertStringToFloat, } from '../../../utils/textUtils.ts'; import { IdLabelPair } from '../AddEditTransactionDialog.tsx'; +import { parseClipboardText } from './clipboardParser.ts'; const IMPORT_TRX_FIELD_HEADER_VARIATIONS = { DATE: [ @@ -186,7 +187,7 @@ const ImportTrxStep1 = (props: Props) => { const [selectedAccount, setSelectedAccount] = useState( null, ); - const [rows, setRows] = useState([]); + const [rows, setRows] = useState([]); const [columnMappings, setColumnMappings] = useState< Record >({}); @@ -265,7 +266,7 @@ const ImportTrxStep1 = (props: Props) => { const colIndex = parseInt(dateColumnKey.split('-')[1]); const dateStrings = rows - .map((row) => row.split('\t')[colIndex]?.trim()) + .map((row) => row[colIndex]?.trim()) .filter((s): s is string => !!s && s.length > 0); if (dateStrings.length === 0) { @@ -293,7 +294,7 @@ const ImportTrxStep1 = (props: Props) => { const colIndex = parseInt(dateColumnKey.split('-')[1]); return ( rows - .map((row) => row.split('\t')[colIndex]?.trim()) + .map((row) => row[colIndex]?.trim()) .find((s) => !!s && s.length > 0) || null ); }, [columnMappings, rows]); @@ -320,13 +321,13 @@ const ImportTrxStep1 = (props: Props) => { const column = Object.entries(columnMappings) .find(([_, value]) => value === field)?.[0] ?.split('-')[1]; - if (column) return parseInt(column); + if (column !== undefined) return parseInt(column); return null; }; const trxs: ExportedTransactionItem[] = []; rows.forEach((row) => { - const columns = row.split('\t'); + const columns = row; const date = columns[getCol(FIELD_MAPPING.DATE) ?? -1]; const description = columns[getCol(FIELD_MAPPING.DESCRIPTION) ?? -1]; @@ -337,17 +338,21 @@ const ImportTrxStep1 = (props: Props) => { const typeColumn = getCol(FIELD_MAPPING.TYPE); let amount: number | undefined; let type: TransactionType | undefined; - if (amountColumn && columns[amountColumn] && !typeColumn) { - amount = convertStringToFloat(columns[amountColumn].replace(/ /g, '')); + if ( + amountColumn !== null && + columns[amountColumn] && + typeColumn === null + ) { + amount = convertStringToFloat(columns[amountColumn]); type = amount > 0 ? TransactionType.Income : TransactionType.Expense; - } else if (creditColumn && !typeColumn) { + } else if (creditColumn !== null && typeColumn === null) { amount = convertStringToFloat(columns[creditColumn] ?? ''); type = amount > 0 ? TransactionType.Income : TransactionType.Expense; } - if (!amount && debitColumn && !typeColumn) { + if (!amount && debitColumn !== null && typeColumn === null) { amount = convertStringToFloat(columns[debitColumn] ?? ''); type = amount > 0 ? TransactionType.Expense : TransactionType.Income; - } else if (!amount && amountColumn && typeColumn) { + } else if (!amount && amountColumn !== null && typeColumn !== null) { amount = convertStringToFloat(columns[amountColumn] ?? ''); switch (columns[typeColumn]) { case FIELD_MAPPING.DEBIT: @@ -417,10 +422,8 @@ const ImportTrxStep1 = (props: Props) => { }; const parseClipboardData = (data: string) => { - const allRows = data.split('\n'); - - // Filter out empty/blank rows - const nonEmptyRows = allRows.filter((row) => row.trim().length > 0); + const parsedClipboard = parseClipboardText(data); + const nonEmptyRows = parsedClipboard.rows; if (nonEmptyRows.length === 0) { setRows([]); @@ -428,7 +431,7 @@ const ImportTrxStep1 = (props: Props) => { } // Try to detect headers from first row - const firstRowIsHeader = tryToPrefillHeaders(nonEmptyRows[0].split('\t')); + const firstRowIsHeader = tryToPrefillHeaders(nonEmptyRows[0]); // If first row matched as headers, exclude it from data rows const dataRows = firstRowIsHeader ? nonEmptyRows.slice(1) : nonEmptyRows; @@ -441,10 +444,10 @@ const ImportTrxStep1 = (props: Props) => { [key: string]: string; // This allows for additional string properties } - const buildRowsForTable = (rows: string[]): GridValidRowModel[] => { + const buildRowsForTable = (rows: string[][]): GridValidRowModel[] => { return rows.map( (row, j) => - row.split('\t').reduce((acc: GridValidRowModel, row, i) => { + row.reduce((acc: GridValidRowModel, row, i) => { acc.id = j + '+' + i; acc[`${i}`] = row; // Cast i to string for indexing return acc; @@ -452,10 +455,10 @@ const ImportTrxStep1 = (props: Props) => { ); }; - const buildColumnsForTable = (rows: string[]): GridColDef[] => { - const nColumns = rows[0]?.split('\t').length || 0; + const buildColumnsForTable = (rows: string[][]): GridColDef[] => { + const nColumns = Math.max(...rows.map((row) => row.length), 0); if (nColumns < 1) return []; - return rows[0].split('\t').map((_row, i) => ({ + return Array.from({ length: nColumns }, (_row, i) => ({ field: `column-${i}`, renderHeader: (_params) => { return ( @@ -497,7 +500,7 @@ const ImportTrxStep1 = (props: Props) => { const column = Object.entries(columnMappings) .find(([_, value]) => value === field)?.[0] ?.split('-')[1]; - if (column) return parseInt(column); + if (column !== undefined) return parseInt(column); return null; }; @@ -505,7 +508,7 @@ const ImportTrxStep1 = (props: Props) => { const trxs: ExportedTransactionItem[] = []; const dateFormat = detectedDateFormat || undefined; rows.forEach((row) => { - const columns = row.split('\t'); + const columns = row; const amountAndTypeInferred = inferTrxAmountAndType(columns); const date = columns[getColumnNumberForMapping(FIELD_MAPPING.DATE) ?? -1]; const description = @@ -539,21 +542,21 @@ const ImportTrxStep1 = (props: Props) => { const debitColumn = getColumnNumberForMapping(FIELD_MAPPING.DEBIT); const typeColumn = getColumnNumberForMapping(FIELD_MAPPING.TYPE); - let amount; - let type; + let amount: number | undefined; + let type: TransactionType | undefined; - if (amountColumn && row[amountColumn] && amountColumn && !typeColumn) { - amount = convertStringToFloat(row[amountColumn].replace(/ /g, '')); + if (amountColumn !== null && row[amountColumn] && typeColumn === null) { + amount = convertStringToFloat(row[amountColumn]); type = amount > 0 ? TransactionType.Income : TransactionType.Expense; - } else if (creditColumn && !typeColumn) { + } else if (creditColumn !== null && typeColumn === null) { amount = convertStringToFloat(row[creditColumn] ?? ''); type = amount > 0 ? TransactionType.Income : TransactionType.Expense; } - if (!amount && debitColumn && !typeColumn) { + if (!amount && debitColumn !== null && typeColumn === null) { amount = convertStringToFloat(row[debitColumn] ?? ''); type = amount > 0 ? TransactionType.Expense : TransactionType.Income; - } else if (!amount && amountColumn && typeColumn) { + } else if (!amount && amountColumn !== null && typeColumn !== null) { amount = convertStringToFloat(row[amountColumn] ?? ''); switch (row[typeColumn]) { case FIELD_MAPPING.DEBIT: diff --git a/src/features/transactions/import/clipboardParser.ts b/src/features/transactions/import/clipboardParser.ts new file mode 100644 index 00000000..943a31f6 --- /dev/null +++ b/src/features/transactions/import/clipboardParser.ts @@ -0,0 +1,182 @@ +type Delimiter = '\t' | ',' | ';' | null; + +export type ParsedClipboard = { + rows: string[][]; + delimiter: Delimiter; +}; + +const DATE_RE = /^\d{1,4}[-/.]\d{1,2}[-/.]\d{1,4}$/; +const MONEY_RE = + /^[+-]?\s*(?:\d{1,3}(?:[.,\s]\d{3})+|\d+)(?:[.,]\d{1,4})?\s*(?:[A-Z]{3}|€)?$/i; + +const countCells = (rows: string[][]): number[] => + rows.map((row) => row.length).filter((length) => length > 1); + +const parseDelimitedLine = (line: string, delimiter: Delimiter): string[] => { + if (!delimiter) return [line.trim()]; + + const cells: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i += 1) { + const char = line[i]; + const nextChar = line[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + current += '"'; + i += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + + if (char === delimiter && !inQuotes) { + cells.push(current.trim()); + current = ''; + continue; + } + + current += char; + } + + cells.push(current.trim()); + return cells; +}; + +const parseLines = (lines: string[], delimiter: Delimiter): string[][] => + lines.map((line) => parseDelimitedLine(line, delimiter)); + +const getMode = (values: number[]): number => { + const counts = new Map(); + values.forEach((value) => counts.set(value, (counts.get(value) ?? 0) + 1)); + + let mode = 0; + let modeCount = 0; + counts.forEach((count, value) => { + if (count > modeCount) { + mode = value; + modeCount = count; + } + }); + + return mode; +}; + +const scoreDelimitedRows = (rows: string[][]): number => { + const lengths = countCells(rows); + if (lengths.length === 0) return 0; + + const mode = getMode(lengths); + if (mode < 3) return 0; + + const matchingRows = lengths.filter((length) => length === mode).length; + const consistency = matchingRows / lengths.length; + return mode * consistency * matchingRows; +}; + +const detectDelimiter = (lines: string[]): Delimiter => { + if (lines.some((line) => line.includes('\t'))) return '\t'; + + const commaScore = scoreDelimitedRows(parseLines(lines, ',')); + const semicolonScore = scoreDelimitedRows(parseLines(lines, ';')); + + if (semicolonScore > 0 && semicolonScore >= commaScore) return ';'; + if (commaScore > 0) return ','; + + return null; +}; + +const isEmptyCell = (cell: string): boolean => cell.trim().length === 0; + +const trimRows = (rows: string[][]): string[][] => + rows + .map((row) => row.map((cell) => cell.trim())) + .filter((row) => row.some((cell) => !isEmptyCell(cell))); + +const shouldCompactEmptyCells = (rows: string[][], delimiter: Delimiter) => { + if (delimiter !== '\t') return false; + + const allCells = rows.flat(); + if (allCells.length === 0) return false; + + const emptyRatio = + allCells.filter((cell) => isEmptyCell(cell)).length / allCells.length; + const firstContentIndexes = rows.map((row) => + row.findIndex((cell) => !isEmptyCell(cell)), + ); + const shiftedRows = new Set(firstContentIndexes).size > 1; + + return emptyRatio >= 0.25 || shiftedRows; +}; + +const compactEmptyCells = (rows: string[][]): string[][] => + rows + .map((row) => row.filter((cell) => !isEmptyCell(cell))) + .filter((row) => row.length > 0); + +const fallbackTokenizeRows = (rows: string[][]): string[][] => + rows.map((row) => { + const line = row.join(' ').trim(); + const looseCells = line.split(/\s{2,}/).filter(Boolean); + if (looseCells.length > 1) return looseCells; + + const tokens = line.split(/\s+/).filter(Boolean); + const moneyIndex = tokens.findIndex((token) => MONEY_RE.test(token)); + const dates = tokens.filter((token) => DATE_RE.test(token)); + + if (dates.length === 0 || moneyIndex < 0) return looseCells; + + const descriptionStart = tokens.findIndex( + (token, index) => index > 0 && !DATE_RE.test(token), + ); + + if (descriptionStart < 0) return looseCells; + + return [ + ...dates, + tokens.slice(descriptionStart, moneyIndex).join(' '), + tokens[moneyIndex], + ...tokens.slice(moneyIndex + 1), + ].filter(Boolean); + }); + +const maybeRecoverLooseStatementRows = ( + rows: string[][], + delimiter: Delimiter, +): string[][] => { + if (delimiter) return rows; + + const recoveredRows = fallbackTokenizeRows(rows); + const recoveredCellCounts = recoveredRows.filter((row) => row.length > 1); + + return recoveredCellCounts.length > 0 ? recoveredRows : rows; +}; + +const normalizeRows = (rows: string[][], delimiter: Delimiter): string[][] => { + const trimmedRows = trimRows(rows); + const compactedRows = shouldCompactEmptyCells(trimmedRows, delimiter) + ? compactEmptyCells(trimmedRows) + : trimmedRows; + + return maybeRecoverLooseStatementRows(compactedRows, delimiter); +}; + +export const parseClipboardText = (data: string): ParsedClipboard => { + const lines = data + .replace(/\r\n/g, '\n') + .replace(/\r/g, '\n') + .split('\n') + .filter((line) => line.trim().length > 0); + + if (lines.length === 0) { + return { rows: [], delimiter: null }; + } + + const delimiter = detectDelimiter(lines); + const rows = normalizeRows(parseLines(lines, delimiter), delimiter); + + return { rows, delimiter }; +}; diff --git a/src/utils/textUtils.ts b/src/utils/textUtils.ts index 50241d57..6694ce65 100644 --- a/src/utils/textUtils.ts +++ b/src/utils/textUtils.ts @@ -42,12 +42,26 @@ export const checkIfFieldsAreFilled = ( }; export const convertStringToFloat = (str: string): number => { - if (str.includes(',')) { - // It's a PT-pt currency format - return parseFloat(str.replace('.', '').replace(',', '.')); - } else { - return parseFloat(str); + const sanitized = str + .trim() + .replace(/\s/g, '') + .replace(/[^\d,.\-+]/g, ''); + const lastCommaIndex = sanitized.lastIndexOf(','); + const lastDotIndex = sanitized.lastIndexOf('.'); + + if (lastCommaIndex > -1 && lastDotIndex > -1) { + if (lastCommaIndex > lastDotIndex) { + return parseFloat(sanitized.replace(/\./g, '').replace(',', '.')); + } + + return parseFloat(sanitized.replace(/,/g, '')); } + + if (lastCommaIndex > -1) { + return parseFloat(sanitized.replace(/\./g, '').replace(',', '.')); + } + + return parseFloat(sanitized.replace(/,/g, '')); }; export default {