Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 32 additions & 29 deletions src/features/transactions/import/ImportTrxStep1.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import {
convertStringToFloat,
} from '../../../utils/textUtils.ts';
import { IdLabelPair } from '../AddEditTransactionDialog.tsx';
import { parseClipboardText } from './clipboardParser.ts';

const IMPORT_TRX_FIELD_HEADER_VARIATIONS = {
DATE: [
Expand Down Expand Up @@ -186,7 +187,7 @@ const ImportTrxStep1 = (props: Props) => {
const [selectedAccount, setSelectedAccount] = useState<IdLabelPair | null>(
null,
);
const [rows, setRows] = useState<string[]>([]);
const [rows, setRows] = useState<string[][]>([]);
const [columnMappings, setColumnMappings] = useState<
Record<string, FIELD_MAPPING>
>({});
Expand Down Expand Up @@ -265,7 +266,7 @@ const ImportTrxStep1 = (props: Props) => {

const colIndex = parseInt(dateColumnKey.split('-')[1]);
const dateStrings = rows
.map((row) => row.split('\t')[colIndex]?.trim())
.map((row) => row[colIndex]?.trim())
.filter((s): s is string => !!s && s.length > 0);

if (dateStrings.length === 0) {
Expand Down Expand Up @@ -293,7 +294,7 @@ const ImportTrxStep1 = (props: Props) => {
const colIndex = parseInt(dateColumnKey.split('-')[1]);
return (
rows
.map((row) => row.split('\t')[colIndex]?.trim())
.map((row) => row[colIndex]?.trim())
.find((s) => !!s && s.length > 0) || null
);
}, [columnMappings, rows]);
Expand All @@ -320,13 +321,13 @@ const ImportTrxStep1 = (props: Props) => {
const column = Object.entries(columnMappings)
.find(([_, value]) => value === field)?.[0]
?.split('-')[1];
if (column) return parseInt(column);
if (column !== undefined) return parseInt(column);
return null;
};

const trxs: ExportedTransactionItem[] = [];
rows.forEach((row) => {
const columns = row.split('\t');
const columns = row;
const date = columns[getCol(FIELD_MAPPING.DATE) ?? -1];
const description = columns[getCol(FIELD_MAPPING.DESCRIPTION) ?? -1];

Expand All @@ -337,17 +338,21 @@ const ImportTrxStep1 = (props: Props) => {
const typeColumn = getCol(FIELD_MAPPING.TYPE);
let amount: number | undefined;
let type: TransactionType | undefined;
if (amountColumn && columns[amountColumn] && !typeColumn) {
amount = convertStringToFloat(columns[amountColumn].replace(/ /g, ''));
if (
amountColumn !== null &&
columns[amountColumn] &&
typeColumn === null
) {
amount = convertStringToFloat(columns[amountColumn]);
type = amount > 0 ? TransactionType.Income : TransactionType.Expense;
} else if (creditColumn && !typeColumn) {
} else if (creditColumn !== null && typeColumn === null) {
amount = convertStringToFloat(columns[creditColumn] ?? '');
type = amount > 0 ? TransactionType.Income : TransactionType.Expense;
}
if (!amount && debitColumn && !typeColumn) {
if (!amount && debitColumn !== null && typeColumn === null) {
amount = convertStringToFloat(columns[debitColumn] ?? '');
type = amount > 0 ? TransactionType.Expense : TransactionType.Income;
} else if (!amount && amountColumn && typeColumn) {
} else if (!amount && amountColumn !== null && typeColumn !== null) {
amount = convertStringToFloat(columns[amountColumn] ?? '');
switch (columns[typeColumn]) {
case FIELD_MAPPING.DEBIT:
Expand Down Expand Up @@ -417,18 +422,16 @@ const ImportTrxStep1 = (props: Props) => {
};

const parseClipboardData = (data: string) => {
const allRows = data.split('\n');

// Filter out empty/blank rows
const nonEmptyRows = allRows.filter((row) => row.trim().length > 0);
const parsedClipboard = parseClipboardText(data);
const nonEmptyRows = parsedClipboard.rows;

if (nonEmptyRows.length === 0) {
setRows([]);
return;
}

// Try to detect headers from first row
const firstRowIsHeader = tryToPrefillHeaders(nonEmptyRows[0].split('\t'));
const firstRowIsHeader = tryToPrefillHeaders(nonEmptyRows[0]);

// If first row matched as headers, exclude it from data rows
const dataRows = firstRowIsHeader ? nonEmptyRows.slice(1) : nonEmptyRows;
Expand All @@ -441,21 +444,21 @@ const ImportTrxStep1 = (props: Props) => {
[key: string]: string; // This allows for additional string properties
}

const buildRowsForTable = (rows: string[]): GridValidRowModel[] => {
const buildRowsForTable = (rows: string[][]): GridValidRowModel[] => {
return rows.map(
(row, j) =>
row.split('\t').reduce((acc: GridValidRowModel, row, i) => {
row.reduce((acc: GridValidRowModel, row, i) => {
acc.id = j + '+' + i;
acc[`${i}`] = row; // Cast i to string for indexing
return acc;
}, {} as GridValidRowModel), // Initialize with empty GridValidRowModel
);
};

const buildColumnsForTable = (rows: string[]): GridColDef[] => {
const nColumns = rows[0]?.split('\t').length || 0;
const buildColumnsForTable = (rows: string[][]): GridColDef[] => {
const nColumns = Math.max(...rows.map((row) => row.length), 0);
if (nColumns < 1) return [];
return rows[0].split('\t').map((_row, i) => ({
return Array.from({ length: nColumns }, (_row, i) => ({
field: `column-${i}`,
renderHeader: (_params) => {
return (
Expand Down Expand Up @@ -497,15 +500,15 @@ const ImportTrxStep1 = (props: Props) => {
const column = Object.entries(columnMappings)
.find(([_, value]) => value === field)?.[0]
?.split('-')[1];
if (column) return parseInt(column);
if (column !== undefined) return parseInt(column);
return null;
};

const parseTransactions = () => {
const trxs: ExportedTransactionItem[] = [];
const dateFormat = detectedDateFormat || undefined;
rows.forEach((row) => {
const columns = row.split('\t');
const columns = row;
const amountAndTypeInferred = inferTrxAmountAndType(columns);
const date = columns[getColumnNumberForMapping(FIELD_MAPPING.DATE) ?? -1];
const description =
Expand Down Expand Up @@ -539,21 +542,21 @@ const ImportTrxStep1 = (props: Props) => {
const debitColumn = getColumnNumberForMapping(FIELD_MAPPING.DEBIT);
const typeColumn = getColumnNumberForMapping(FIELD_MAPPING.TYPE);

let amount;
let type;
let amount: number | undefined;
let type: TransactionType | undefined;

if (amountColumn && row[amountColumn] && amountColumn && !typeColumn) {
amount = convertStringToFloat(row[amountColumn].replace(/ /g, ''));
if (amountColumn !== null && row[amountColumn] && typeColumn === null) {
amount = convertStringToFloat(row[amountColumn]);
type = amount > 0 ? TransactionType.Income : TransactionType.Expense;
} else if (creditColumn && !typeColumn) {
} else if (creditColumn !== null && typeColumn === null) {
amount = convertStringToFloat(row[creditColumn] ?? '');
type = amount > 0 ? TransactionType.Income : TransactionType.Expense;
}

if (!amount && debitColumn && !typeColumn) {
if (!amount && debitColumn !== null && typeColumn === null) {
amount = convertStringToFloat(row[debitColumn] ?? '');
type = amount > 0 ? TransactionType.Expense : TransactionType.Income;
} else if (!amount && amountColumn && typeColumn) {
} else if (!amount && amountColumn !== null && typeColumn !== null) {
amount = convertStringToFloat(row[amountColumn] ?? '');
switch (row[typeColumn]) {
case FIELD_MAPPING.DEBIT:
Expand Down
182 changes: 182 additions & 0 deletions src/features/transactions/import/clipboardParser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
type Delimiter = '\t' | ',' | ';' | null;

export type ParsedClipboard = {
rows: string[][];
delimiter: Delimiter;
};

const DATE_RE = /^\d{1,4}[-/.]\d{1,2}[-/.]\d{1,4}$/;
const MONEY_RE =
/^[+-]?\s*(?:\d{1,3}(?:[.,\s]\d{3})+|\d+)(?:[.,]\d{1,4})?\s*(?:[A-Z]{3}|€)?$/i;

const countCells = (rows: string[][]): number[] =>
rows.map((row) => row.length).filter((length) => length > 1);

const parseDelimitedLine = (line: string, delimiter: Delimiter): string[] => {
if (!delimiter) return [line.trim()];

const cells: string[] = [];
let current = '';
let inQuotes = false;

for (let i = 0; i < line.length; i += 1) {
const char = line[i];
const nextChar = line[i + 1];

if (char === '"') {
if (inQuotes && nextChar === '"') {
current += '"';
i += 1;
} else {
inQuotes = !inQuotes;
}
continue;
}

if (char === delimiter && !inQuotes) {
cells.push(current.trim());
current = '';
continue;
}

current += char;
}

cells.push(current.trim());
return cells;
};

const parseLines = (lines: string[], delimiter: Delimiter): string[][] =>
lines.map((line) => parseDelimitedLine(line, delimiter));

const getMode = (values: number[]): number => {
const counts = new Map<number, number>();
values.forEach((value) => counts.set(value, (counts.get(value) ?? 0) + 1));

let mode = 0;
let modeCount = 0;
counts.forEach((count, value) => {
if (count > modeCount) {
mode = value;
modeCount = count;
}
});

return mode;
};

const scoreDelimitedRows = (rows: string[][]): number => {
const lengths = countCells(rows);
if (lengths.length === 0) return 0;

const mode = getMode(lengths);
if (mode < 3) return 0;

const matchingRows = lengths.filter((length) => length === mode).length;
const consistency = matchingRows / lengths.length;
return mode * consistency * matchingRows;
};

const detectDelimiter = (lines: string[]): Delimiter => {
if (lines.some((line) => line.includes('\t'))) return '\t';

const commaScore = scoreDelimitedRows(parseLines(lines, ','));
const semicolonScore = scoreDelimitedRows(parseLines(lines, ';'));

if (semicolonScore > 0 && semicolonScore >= commaScore) return ';';
if (commaScore > 0) return ',';

return null;
};

const isEmptyCell = (cell: string): boolean => cell.trim().length === 0;

const trimRows = (rows: string[][]): string[][] =>
rows
.map((row) => row.map((cell) => cell.trim()))
.filter((row) => row.some((cell) => !isEmptyCell(cell)));

const shouldCompactEmptyCells = (rows: string[][], delimiter: Delimiter) => {
if (delimiter !== '\t') return false;

const allCells = rows.flat();
if (allCells.length === 0) return false;

const emptyRatio =
allCells.filter((cell) => isEmptyCell(cell)).length / allCells.length;
const firstContentIndexes = rows.map((row) =>
row.findIndex((cell) => !isEmptyCell(cell)),
);
const shiftedRows = new Set(firstContentIndexes).size > 1;

return emptyRatio >= 0.25 || shiftedRows;
};

const compactEmptyCells = (rows: string[][]): string[][] =>
rows
.map((row) => row.filter((cell) => !isEmptyCell(cell)))
.filter((row) => row.length > 0);

const fallbackTokenizeRows = (rows: string[][]): string[][] =>
rows.map((row) => {
const line = row.join(' ').trim();
const looseCells = line.split(/\s{2,}/).filter(Boolean);
if (looseCells.length > 1) return looseCells;

const tokens = line.split(/\s+/).filter(Boolean);
const moneyIndex = tokens.findIndex((token) => MONEY_RE.test(token));
const dates = tokens.filter((token) => DATE_RE.test(token));

if (dates.length === 0 || moneyIndex < 0) return looseCells;

const descriptionStart = tokens.findIndex(
(token, index) => index > 0 && !DATE_RE.test(token),
);

if (descriptionStart < 0) return looseCells;

return [
...dates,
tokens.slice(descriptionStart, moneyIndex).join(' '),
tokens[moneyIndex],
...tokens.slice(moneyIndex + 1),
].filter(Boolean);
});

const maybeRecoverLooseStatementRows = (
rows: string[][],
delimiter: Delimiter,
): string[][] => {
if (delimiter) return rows;

const recoveredRows = fallbackTokenizeRows(rows);
const recoveredCellCounts = recoveredRows.filter((row) => row.length > 1);

return recoveredCellCounts.length > 0 ? recoveredRows : rows;
};

const normalizeRows = (rows: string[][], delimiter: Delimiter): string[][] => {
const trimmedRows = trimRows(rows);
const compactedRows = shouldCompactEmptyCells(trimmedRows, delimiter)
? compactEmptyCells(trimmedRows)
: trimmedRows;

return maybeRecoverLooseStatementRows(compactedRows, delimiter);
};

export const parseClipboardText = (data: string): ParsedClipboard => {
const lines = data
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n')
.split('\n')
.filter((line) => line.trim().length > 0);

if (lines.length === 0) {
return { rows: [], delimiter: null };
}

const delimiter = detectDelimiter(lines);
const rows = normalizeRows(parseLines(lines, delimiter), delimiter);

return { rows, delimiter };
};
24 changes: 19 additions & 5 deletions src/utils/textUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,26 @@ export const checkIfFieldsAreFilled = (
};

export const convertStringToFloat = (str: string): number => {
if (str.includes(',')) {
// It's a PT-pt currency format
return parseFloat(str.replace('.', '').replace(',', '.'));
} else {
return parseFloat(str);
const sanitized = str
.trim()
.replace(/\s/g, '')
.replace(/[^\d,.\-+]/g, '');
const lastCommaIndex = sanitized.lastIndexOf(',');
const lastDotIndex = sanitized.lastIndexOf('.');

if (lastCommaIndex > -1 && lastDotIndex > -1) {
if (lastCommaIndex > lastDotIndex) {
return parseFloat(sanitized.replace(/\./g, '').replace(',', '.'));
}

return parseFloat(sanitized.replace(/,/g, ''));
}

if (lastCommaIndex > -1) {
return parseFloat(sanitized.replace(/\./g, '').replace(',', '.'));
}

return parseFloat(sanitized.replace(/,/g, ''));
};

export default {
Expand Down
Loading