From 2e865e333cffb14231c49ccdf782ea2c1e908c45 Mon Sep 17 00:00:00 2001 From: Volodymyr Huzar Date: Sun, 10 Dec 2023 17:35:00 +0100 Subject: [PATCH 1/5] feat(velobank): add support for velobank CSV Velobank CSV is manually created from pdf --- converters/velobankConverter.js | 29 ++++++++++++++++++++++++++ converters/velobankConverter.spec.js | 31 ++++++++++++++++++++++++++++ e2e/index.test.js | 14 +++++++++++++ e2e/testDataVelobank.csv | 2 ++ index.html | 8 +++++-- main.js | 3 +++ 6 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 converters/velobankConverter.js create mode 100644 converters/velobankConverter.spec.js create mode 100644 e2e/testDataVelobank.csv diff --git a/converters/velobankConverter.js b/converters/velobankConverter.js new file mode 100644 index 0000000..e7a3d81 --- /dev/null +++ b/converters/velobankConverter.js @@ -0,0 +1,29 @@ +const { parse } = require('csv-parse/sync') +const { transform } = require('stream-transform') +const categoryResolver = require('../categoryResolver') +const { sanitize } = require('../utils') + +const columns = ['date', 'realDate', 'description', 'payer', 'amount', 'balance'] + +const parseAmount = amount => parseFloat(amount.replace(' ', '').replace(/w/, '').replace(',', '.')) + +const parsePayer = payer => { + const payerWithoutLocation = payer.split(',')[0] + return payerWithoutLocation.replace('w ', '').trim() +} + +const getExpenseManagerRecord = recordCategoryResolver => record => { + const payer = parsePayer(record[columns[3]]); + const amount = parseAmount(record[columns[4]]) + const { category, subCategory } = recordCategoryResolver(payer, amount); + return record[columns[0]] + ',' + + amount + ',' + category + ',' + subCategory + + ',Credit Card,,,' + sanitize(payer) + + ',,,GetIn\n' +} + +exports.convertCvsFileData = (input, categoriesMapping) => { + const getExpenseManagerWithMapping = getExpenseManagerRecord(categoryResolver.resolveCategory(categoriesMapping)) + return transform(parse(input, { delimiter: ',', columns, relax: true, relax_column_count: true, bom: true }), + record => getExpenseManagerWithMapping(record)) +} diff --git a/converters/velobankConverter.spec.js b/converters/velobankConverter.spec.js new file mode 100644 index 0000000..717da57 --- /dev/null +++ b/converters/velobankConverter.spec.js @@ -0,0 +1,31 @@ +const citiConverter = require('./velobankConverter') + +const categoriesMapping = { + 'Play': { category: 'Utilities', subCategory: 'Telephone' }, + 'Employer': { category: 'Income', subCategory: 'Salary' }, + 'Deposit': { category: 'Income', subCategory: 'Deposit' } +}; + +test('should be able to convert Velobank manually created CSV files format', (done) => { + const input = `30.10.2019,24.11.2023,"Przelew z rachunku: xxxxx,",Employer,"9 839,29 PLN","1 574,59 PLN" +28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN","w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN"` + const expected = `30.10.2019,9839.29,Income,Salary,Credit Card,,,Employer,,,GetIn +28.10.2019,-10,Utilities,Telephone,Credit Card,,,Play,,,GetIn +` + let transformedData = ''; + + const converter = citiConverter.convertCvsFileData(input, categoriesMapping) + .on('readable', () => { + let row = converter.read() + while (row) { + transformedData += row + row = converter.read() + } + }) + .on('finish', () => { + setTimeout(() => { + expect(transformedData).toEqual(expected) + done() + }) + }) +}) diff --git a/e2e/index.test.js b/e2e/index.test.js index 3ef61b0..0299650 100644 --- a/e2e/index.test.js +++ b/e2e/index.test.js @@ -80,3 +80,17 @@ test('should download converted csv without mapping for Santander', async t => { await t.expect(file).eql('30.10.2019,9839.29,Income,,Credit Card,,,Employer,,,Santander\n28.10.2019,-10,,,Credit Card,,,Play,,,Santander\n') }) .after(() => unlinkSync(expectedFile)) + +test('should download converted csv without mapping for Velobank', async t => { + expectedFile = join(`${homedir()}`, 'Downloads', 'testDataVelobank-converted.csv') + await t + .setFilesToUpload(Selector('#file'), 'testDataVelobank.csv') + .click(Selector('#velobank')) + .click(Selector('button')) + + await waitForFile(expectedFile) + const file = readFileSync(expectedFile, 'utf-8') + + await t.expect(file).eql('30.10.2019,9839.29,Income,,Credit Card,,,Employer,,,GetIn\n28.10.2019,-10,,,Credit Card,,,Play,,,GetIn\n') +}) + .after(() => unlinkSync(expectedFile)) diff --git a/e2e/testDataVelobank.csv b/e2e/testDataVelobank.csv new file mode 100644 index 0000000..171d918 --- /dev/null +++ b/e2e/testDataVelobank.csv @@ -0,0 +1,2 @@ +30.10.2019,24.11.2023,"Przelew z rachunku: xxxxx,",Employer,"9 839,29 PLN","1 574,59 PLN" +28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN","w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN" diff --git a/index.html b/index.html index 95f9e23..99ba907 100644 --- a/index.html +++ b/index.html @@ -27,13 +27,17 @@
- - + +
+
+ + +
diff --git a/main.js b/main.js index 0fdcc3b..af2c745 100644 --- a/main.js +++ b/main.js @@ -8,6 +8,7 @@ const nestConverter = require("./converters/nestConverter"); const santanderConverter = require("./converters/santanderConverter"); const pekaoConverter = require("./converters/pekaoConverter"); const monobankConverter = require("./converters/monobankConverter"); +const velobankConverter = require("./converters/velobankConverter"); const app = express(); const port = process.env.PORT || 3000; @@ -39,6 +40,8 @@ function selectConverter(bank) { return pekaoConverter; case "monobank": return monobankConverter; + case "velobank": + return velobankConverter; default: return cityConverter; } From d5f5b32011f4c5e2ede23d80fe91f6bf7e150c35 Mon Sep 17 00:00:00 2001 From: Volodymyr Huzar Date: Sun, 10 Dec 2023 18:41:09 +0100 Subject: [PATCH 2/5] fix(velobank): use new csv format The PDF to CSV formatter is changed now so the coverter is udapted --- converters/velobankConverter.js | 18 +++++++++++------- converters/velobankConverter.spec.js | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/converters/velobankConverter.js b/converters/velobankConverter.js index e7a3d81..921e40a 100644 --- a/converters/velobankConverter.js +++ b/converters/velobankConverter.js @@ -3,18 +3,22 @@ const { transform } = require('stream-transform') const categoryResolver = require('../categoryResolver') const { sanitize } = require('../utils') -const columns = ['date', 'realDate', 'description', 'payer', 'amount', 'balance'] +const columns = ['date', 'realDate', 'description', 'amount', 'balance'] -const parseAmount = amount => parseFloat(amount.replace(' ', '').replace(/w/, '').replace(',', '.')) +const parseAmount = amount => parseFloat(amount.replace(' ', '').replace(',', '.')) -const parsePayer = payer => { - const payerWithoutLocation = payer.split(',')[0] - return payerWithoutLocation.replace('w ', '').trim() +const parseDescription = description => { + const descriptionParts = description.split(',') + if (descriptionParts.length === 1) { + return descriptionParts[0] + } + const payerPart = descriptionParts[1] + return payerPart.replace('w ', '').replace(/\d+ PLN/, '').trim() } const getExpenseManagerRecord = recordCategoryResolver => record => { - const payer = parsePayer(record[columns[3]]); - const amount = parseAmount(record[columns[4]]) + const payer = parseDescription(record[columns[2]]); + const amount = parseAmount(record[columns[3]]) const { category, subCategory } = recordCategoryResolver(payer, amount); return record[columns[0]] + ',' + amount + ',' + category + ',' + subCategory diff --git a/converters/velobankConverter.spec.js b/converters/velobankConverter.spec.js index 717da57..913814e 100644 --- a/converters/velobankConverter.spec.js +++ b/converters/velobankConverter.spec.js @@ -7,8 +7,8 @@ const categoriesMapping = { }; test('should be able to convert Velobank manually created CSV files format', (done) => { - const input = `30.10.2019,24.11.2023,"Przelew z rachunku: xxxxx,",Employer,"9 839,29 PLN","1 574,59 PLN" -28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN","w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN"` + const input = `30.10.2019,24.11.2023,Employer,"9 839,29 PLN","1 574,59 PLN" +28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN"` const expected = `30.10.2019,9839.29,Income,Salary,Credit Card,,,Employer,,,GetIn 28.10.2019,-10,Utilities,Telephone,Credit Card,,,Play,,,GetIn ` From 81b6ea3d457eaad5c73a744cd8f7d952fc39e9c9 Mon Sep 17 00:00:00 2001 From: Volodymyr Huzar Date: Sun, 4 Feb 2024 19:06:51 +0100 Subject: [PATCH 3/5] chore: add more descriptive errors --- main.js | 3 ++- mappingNormalizer.js | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/main.js b/main.js index af2c745..fd7918b 100644 --- a/main.js +++ b/main.js @@ -66,7 +66,8 @@ app.post("/", ({ files, body: { bank } }, response) => { .on("error", getHttpErrorHandler(response)) .pipe(response); } catch (err) { - response.status(500).send(err); + console.error('Unexpected error is occured', err) + response.status(500); } } else { response.status(400).send("No file was uploaded."); diff --git a/mappingNormalizer.js b/mappingNormalizer.js index 3c58485..9b8c096 100644 --- a/mappingNormalizer.js +++ b/mappingNormalizer.js @@ -2,6 +2,9 @@ exports.normalizeCategories = (categories) => categories && Object.keys(categori const category = categories[categoryKey]; return Object.keys(category).reduce((subCategoriesWithPayers, subCategoryKey) => { const payers = category[subCategoryKey] || [] + if (!Array.isArray(payers)) { + throw new Error(`Payers for "${subCategoryKey}" is not an array`) + } return payers.reduce((payersWithSubCategory, payer) => ({ ...payersWithSubCategory, [payer]: { category: categoryKey, subCategory: subCategoryKey } }), subCategoriesWithPayers) }, payerByCategory) From a258293ccd4966fd2a341ff61da38ac851ebba59 Mon Sep 17 00:00:00 2001 From: Volodymyr Huzar Date: Sun, 4 Feb 2024 19:54:46 +0100 Subject: [PATCH 4/5] feat(santander): clean up payer from card details --- converters/santanderConverter.js | 6 +++++- converters/santanderConverter.spec.js | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/converters/santanderConverter.js b/converters/santanderConverter.js index bbbecad..3bb0fca 100644 --- a/converters/santanderConverter.js +++ b/converters/santanderConverter.js @@ -16,12 +16,16 @@ const columns = [ ]; const parseAmount = (amount) => parseFloat(amount.replace(",", ".")); +function normalisePayer(payer) { + return payer.replace(/^.*PLN /, '') +} function descriptionToPayer(description) { if (description.startsWith("Suma nagród za płatności")) { return "Santander returns"; } - return description; + return normalisePayer(description); } + function parsePayer(payer, description) { return payer || descriptionToPayer(description); } diff --git a/converters/santanderConverter.spec.js b/converters/santanderConverter.spec.js index 118a913..aabffd6 100644 --- a/converters/santanderConverter.spec.js +++ b/converters/santanderConverter.spec.js @@ -7,7 +7,7 @@ test('should be able to convert Santander CSV files format', (done) => { }; const input = `2022-10-23,25-09-2022,'11 2222 3333 4444,Owner address,PLN,,,17, 21-10-2022,19-04-2019,Description 1,KASOWNIK 12,11 2222 3333,"-3,20",,1, -20-10-2022,19-04-2019,Description 2,H AND M GAL. BALTYCKA,22 3333 44444,"-48,78",,2,` +20-10-2022,19-04-2019,VISA SEL 123456******1234 PŁATNOŚĆ KARTĄ 48.78 PLN H AND M GAL. BALTYCKA,,22 3333 44444,"-48,78",,2,` const expected = `19.04.2019,-3.2,Transport,Public transport,Credit Card,,,KASOWNIK 12,,,Santander 19.04.2019,-48.78,Personal,Clothing,Credit Card,,,H AND M GAL. BALTYCKA,,,Santander ` From 51dbefebd2a14cb0ee11a2bc81542918fa6875d8 Mon Sep 17 00:00:00 2001 From: Volodymyr Huzar Date: Sun, 4 Feb 2024 20:25:58 +0100 Subject: [PATCH 5/5] test(velobank): fix e2e test data --- e2e/testDataVelobank.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/testDataVelobank.csv b/e2e/testDataVelobank.csv index 171d918..f774ccf 100644 --- a/e2e/testDataVelobank.csv +++ b/e2e/testDataVelobank.csv @@ -1,2 +1,2 @@ -30.10.2019,24.11.2023,"Przelew z rachunku: xxxxx,",Employer,"9 839,29 PLN","1 574,59 PLN" -28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN","w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN" +30.10.2019,24.11.2023,Employer,"9 839,29 PLN","1 574,59 PLN" +28.10.2019,24.11.2023,"Operacja kartą na kwotę 17,99 PLN w Play, GDANSK, PL","-10,00 PLN","2 374,59 PLN"