From e7190b50bd33d8682c34478728b77af389e75d56 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Tue, 7 Mar 2023 18:36:48 +0100 Subject: [PATCH 1/2] fix: Use correct import path for FILES_DOCTYPE The `doctypes` module which exports the `FILES_DOCTYPE` const used in the settings helpers can be imported directly as `doctypes` but not `src/doctypes`. --- src/ducks/settings/helpers.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ducks/settings/helpers.js b/src/ducks/settings/helpers.js index 1aeec1f387..3027aca9c5 100644 --- a/src/ducks/settings/helpers.js +++ b/src/ducks/settings/helpers.js @@ -12,9 +12,13 @@ import { Q } from 'cozy-client' import { getDocumentFromState } from 'cozy-client/dist/store' import { translate } from 'cozy-ui/transpiled/react/I18n' -import { FILES_DOCTYPE } from 'src/doctypes' import { DOCTYPE, DEFAULTS_SETTINGS } from 'ducks/settings/constants' -import { ACCOUNT_DOCTYPE, GROUP_DOCTYPE, SETTINGS_DOCTYPE } from 'doctypes' +import { + ACCOUNT_DOCTYPE, + FILES_DOCTYPE, + GROUP_DOCTYPE, + SETTINGS_DOCTYPE +} from 'doctypes' import { getAccountLabel } from 'ducks/account/helpers' import { getGroupLabel, getGroupAccountIds } from 'ducks/groups/helpers' import { From 6e8f94da9a0e0b5073817a2ff34b007b25413685 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Thu, 23 Feb 2023 14:24:41 +0100 Subject: [PATCH 2/2] feat: Allow exporting transactions/accounts as CSV The `export` service fetches all transactions with their associated relationships and transforms them into CSV lines following our own schema. It also transforms accounts without transactions into CSV lines following the same schema. The result is uploaded as a CSV file into the user's root directory. To do so Banks now requires a permission to create and update `io.cozy.files` documents. The CSV is generated by the `@fast-csv` library which offers an easy to use interface with a streamable transformation (although we don't stream the content to cozy-stack yet as we end up with `EPIPE` errors when cozy-stack responds with a 409 status response). The exported accounts without transactions can be found at the top of the file so they can easily be removed by the user if they want to process their transactions in a spreadsheet editor. The service can be launched programmatically with a 10m debounce period. --- src/ducks/export/services.js | 121 +++++++++++++++++++++++++++-- src/ducks/export/services.spec.js | 123 +++++++++++++++++++++++++----- src/targets/services/export.js | 39 ++++++---- 3 files changed, 246 insertions(+), 37 deletions(-) diff --git a/src/ducks/export/services.js b/src/ducks/export/services.js index 32d817f0ce..2ecde2bd96 100644 --- a/src/ducks/export/services.js +++ b/src/ducks/export/services.js @@ -2,8 +2,9 @@ import { Q } from 'cozy-client' import { format } from '@fast-csv/format' import formatDate from 'date-fns/format' -import { TRANSACTION_DOCTYPE } from 'doctypes' -import categories from 'ducks/categories/tree' +import { ACCOUNT_DOCTYPE, TRANSACTION_DOCTYPE } from 'doctypes' +import { getCategoryId, getApplicationDate } from 'ducks/transactions/helpers' +import { getCategoryName } from 'ducks/categories/categoriesMap' import { DATE_FORMAT } from './constants' const dateStr = date => @@ -25,6 +26,27 @@ export const fetchTransactionsToExport = async client => { return client.hydrateDocuments(TRANSACTION_DOCTYPE, transactions) } +/** + * Fetch all accounts and keep only those not associated with the given + * transactions. + * + * @param {CozyClient} client A CozyClient instance + * @param {object} options + * @param {Array} options.transactions Transactions used to filter accounts + * + * @return {Promise>} Promise that resolves with an array of io.cozy.bank.operations documents + */ +export const fetchAccountsToExport = async (client, { transactions }) => { + const accounts = await client.queryAll(Q(ACCOUNT_DOCTYPE).limitBy(1000)) + + return accounts.filter( + account => + !transactions.some( + transaction => transaction.account?.data?._id === account._id + ) + ) +} + /** * Create a Transform Stream to transform input data into CSV lines. * @@ -44,17 +66,29 @@ export const createFormatStream = () => { 'Amount', 'Currency', 'Type', + 'Expected?', + 'Expected debit date', 'Reimbursement status', 'Bank name', 'Account name', + 'Custom account name', 'Account number', + 'Account originalNumber', 'Account type', + 'Account balance', + 'Account coming balance', + 'Account IBAN', + 'Account vendorDeleted', + 'Recurrent?', 'Recurrence name', + 'Recurrence status', + 'Recurrence frequency', 'Tag 1', 'Tag 2', 'Tag 3', 'Tag 4', - 'Tag 5' + 'Tag 5', + 'Unique ID' ] }) } @@ -71,17 +105,20 @@ export const transactionsToCSV = function* (transactions) { const account = transaction.account?.data const recurrence = transaction.recurrence?.data const tags = transaction.tags?.data + const categoryId = getCategoryId(transaction) const data = [ dateStr(transaction.date), dateStr(transaction.realisationDate), - dateStr(transaction.applicationDate), + dateStr(getApplicationDate(transaction)), transaction.label, transaction.originalBankLabel, - categories[transaction.cozyCategoryId], + getCategoryName(categoryId), transaction.amount, transaction.currency, transaction.type, + transaction.isComing ? 'yes' : 'no', + transaction.valueDate, transaction.reimbursementStatus ] @@ -89,12 +126,27 @@ export const transactionsToCSV = function* (transactions) { data.push( account?.institutionLabel, account?.label, + account?.shortLabel, account?.number, - account?.type + account?.originalNumber, + account?.type, + account?.balance, + account?.comingBalance, + account?.iban, + account?.vendorDeleted ) // Transaction's recurrence information - data.push(recurrence?.manualLabel || recurrence?.automaticLabel) + data.push( + recurrence != null + ? 'yes' + : transaction.relationships?.recurrence?.data?._id === 'not-recurrent' + ? 'no' + : undefined, + recurrence?.manualLabel || recurrence?.automaticLabel, + recurrence?.status, + recurrence?.stats?.deltas?.median + ) // Transaction's tags information for (let i = 0; i < 5; i++) { @@ -105,6 +157,61 @@ export const transactionsToCSV = function* (transactions) { } } + // Unique identifier + data.push(transaction.vendorId || transaction.linxoId) + + yield data + } +} + +/** + * Generator that transforms the given accounts into our own CSV format and yields the result line by line. + * + * @param {Array} accounts The list of accounts to transform + * + * @return {IterableIterator>} + */ +export const accountsWitoutTransactionsToCSV = function* (accounts) { + for (const account of accounts) { + // Transaction information + const data = [ + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined + ] + + // Account information + data.push( + account.institutionLabel, + account.label, + account.shortLabel, + account.number, + account.originalNumber, + account.type, + account.balance, + account.comingBalance, + account.iban, + account.vendorDeleted + ) + + // Recurrence information + data.push(undefined, undefined, undefined, undefined) + + // Tags information + data.push(undefined, undefined, undefined, undefined, undefined) + + // Unique identifier + data.push(account.vendorId || account.linxoId) + yield data } } diff --git a/src/ducks/export/services.spec.js b/src/ducks/export/services.spec.js index a32af84844..8fecc08c48 100644 --- a/src/ducks/export/services.spec.js +++ b/src/ducks/export/services.spec.js @@ -8,9 +8,28 @@ const setup = () => { id: '19e2519131deafeb36dad340765635ac', _id: '19e2519131deafeb36dad340765635ac', institutionLabel: 'Société Générale', - label: 'Compte Courant', + label: 'Isabelle Durand Compte Courant', + shortLabel: 'Compte Courant', number: '00031738274', - type: 'Checkings' + originalNumber: '0974200031738274', + type: 'Checkings', + balance: 123.4, + comingBalance: 123.4, + iban: 'FR65023382980003173827423', + vendorId: '12345' + }, + loan: { + id: '29e2519131deafeb36dad340765635ac', + _id: '29e2519131deafeb36dad340765635ac', + institutionLabel: 'Société Générale', + label: 'Isabelle Durand PRET IMMO', + shortLabel: 'Pret Immobilier', + number: 'T00031733728', + originalNumber: 'T00031733728', + type: 'Loan', + balance: -128037.32, + comingBalance: -128037.32, + vendorId: '12346' } } @@ -42,7 +61,8 @@ const setup = () => { categoryIds: ['401080'], latestAmount: -63, latestDate: '2021-09-10T12:00:00.000Z', - manualLabel: 'Abonnement Gaz' + manualLabel: 'Abonnement Gaz', + stats: { deltas: { median: 58.5 } } } } @@ -55,6 +75,7 @@ const setup = () => { }, amount: -63, cozyCategoryId: '401080', + cozyCategoryProba: 1, currency: 'EUR', date: '2021-09-10T12:00:00.000Z', label: 'GAZ', @@ -64,7 +85,8 @@ const setup = () => { recurrence: { data: recurrences.gaz }, - tags: {} + tags: {}, + vendorId: '23456' }, { _id: '0008d7b9134d67cb079d10acc530902f', @@ -74,6 +96,7 @@ const setup = () => { }, amount: 78.3, cozyCategoryId: '400840', + cozyCategoryProba: 1, currency: 'EUR', date: '2021-11-12T12:00:00.000Z', label: 'REMBOURSEMENT FACTURE 0001', @@ -84,7 +107,8 @@ const setup = () => { recurrence: {}, tags: { data: [tags.melun, tags.vacances, tags.remboursements] - } + }, + vendorId: '23457' }, { _id: '0008d7b9134d67cb079d10acc530902f', @@ -94,6 +118,7 @@ const setup = () => { }, amount: -78.3, cozyCategoryId: '400840', + cozyCategoryProba: 1, currency: 'EUR', date: '2021-10-23T12:00:00.000Z', label: 'TRAVEL AGENCY FACTURE 0001', @@ -104,19 +129,23 @@ const setup = () => { recurrence: {}, tags: { data: [tags.melun, tags.vacances] - } + }, + vendorId: '23458' }, { _id: '000dcebc4d23ebd5bd68a16c38d5fe63', id: '000dcebc4d23ebd5bd68a16c38d5fe63', amount: 1.5, - cozyCategoryId: '400110', + automaticCategoryId: '400110', currency: 'EUR', date: '2021-12-24T12:00:00.000Z', label: 'BOULANGERIE AU BON PAIN', originalBankLabel: 'BOULANGERIE AU BON PAIN Card.***4', realisationDate: '2021-12-24T12:00:00.000Z', - type: 'credit card' + isComing: true, + valueDate: '2021-12-31T12:00:00.000Z', + type: 'credit card', + vendorId: '23459' } ] @@ -141,17 +170,29 @@ describe('createFormatStream', () => { -63, 'EUR', undefined, + 'no', + undefined, undefined, 'Société Générale', + 'Isabelle Durand Compte Courant', 'Compte Courant', '00031738274', + '0974200031738274', 'Checkings', + 123.4, + 123.4, + 'FR65023382980003173827423', + undefined, + 'yes', 'Abonnement Gaz', undefined, + 58.5, + undefined, undefined, undefined, undefined, - undefined + undefined, + '23456' ] const stream = createFormatStream() @@ -167,8 +208,8 @@ describe('createFormatStream', () => { expect(String(output)).toEqual( [ // XXX: The header line is included in the output - '"Date";"Realisation date";"Assigned date";"Label";"Original bank label";"Category name";"Amount";"Currency";"Type";"Reimbursement status";"Bank name";"Account name";"Account number";"Account type";"Recurrence name";"Tag 1";"Tag 2";"Tag 3";"Tag 4";"Tag 5"', - '"2021-09-10";"2021-09-10";"";"GAZ";"PRLV SEPA GAZ";"power";"-63";"EUR";"";"";"Société Générale";"Compte Courant";"00031738274";"Checkings";"Abonnement Gaz";"";"";"";"";""' + '"Date";"Realisation date";"Assigned date";"Label";"Original bank label";"Category name";"Amount";"Currency";"Type";"Expected?";"Expected debit date";"Reimbursement status";"Bank name";"Account name";"Custom account name";"Account number";"Account originalNumber";"Account type";"Account balance";"Account coming balance";"Account IBAN";"Account vendorDeleted";"Recurrent?";"Recurrence name";"Recurrence status";"Recurrence frequency";"Tag 1";"Tag 2";"Tag 3";"Tag 4";"Tag 5";"Unique ID"', + '"2021-09-10";"2021-09-10";"";"GAZ";"PRLV SEPA GAZ";"power";"-63";"EUR";"";"no";"";"";"Société Générale";"Isabelle Durand Compte Courant";"Compte Courant";"00031738274";"0974200031738274";"Checkings";"123.4";"123.4";"FR65023382980003173827423";"";"yes";"Abonnement Gaz";"";"58.5";"";"";"";"";"";"23456"' ].join('\n') ) }) @@ -189,24 +230,36 @@ describe('transactionsToCSV', () => { expect(generator.next().value).toEqual([ '2021-09-10', '2021-09-10', - '2021-09-10', + undefined, 'GAZ', 'PRLV SEPA GAZ', 'power', -63, 'EUR', undefined, + 'no', + undefined, undefined, 'Société Générale', + 'Isabelle Durand Compte Courant', 'Compte Courant', '00031738274', + '0974200031738274', 'Checkings', + 123.4, + 123.4, + 'FR65023382980003173827423', + undefined, + 'yes', 'Abonnement Gaz', undefined, + 58.5, + undefined, + undefined, undefined, undefined, undefined, - undefined + '23456' ]) expect(generator.next().value).toEqual([ @@ -219,17 +272,29 @@ describe('transactionsToCSV', () => { 78.3, 'EUR', 'transfer', + 'no', + undefined, undefined, 'Société Générale', + 'Isabelle Durand Compte Courant', 'Compte Courant', '00031738274', + '0974200031738274', 'Checkings', + 123.4, + 123.4, + 'FR65023382980003173827423', + undefined, + undefined, + undefined, + undefined, undefined, 'Vacances à Melun', 'Vacances', 'Remboursements', undefined, - undefined + undefined, + '23457' ]) expect(generator.next().value).toEqual([ @@ -242,17 +307,29 @@ describe('transactionsToCSV', () => { -78.3, 'EUR', 'credit card', + 'no', + undefined, 'reimbursed', 'Société Générale', + 'Isabelle Durand Compte Courant', 'Compte Courant', '00031738274', + '0974200031738274', 'Checkings', + 123.4, + 123.4, + 'FR65023382980003173827423', + undefined, + undefined, + undefined, + undefined, undefined, 'Vacances à Melun', 'Vacances', undefined, undefined, - undefined + undefined, + '23458' ]) expect(generator.next().value).toEqual([ @@ -261,10 +338,22 @@ describe('transactionsToCSV', () => { undefined, 'BOULANGERIE AU BON PAIN', 'BOULANGERIE AU BON PAIN Card.***4', - 'supermarket', + 'awaiting', 1.5, 'EUR', 'credit card', + 'yes', + '2021-12-31T12:00:00.000Z', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, undefined, undefined, undefined, @@ -275,7 +364,7 @@ describe('transactionsToCSV', () => { undefined, undefined, undefined, - undefined + '23459' ]) }) }) diff --git a/src/targets/services/export.js b/src/targets/services/export.js index 6c2f86320d..619700fbc1 100644 --- a/src/targets/services/export.js +++ b/src/targets/services/export.js @@ -1,10 +1,11 @@ import stream from 'stream' -import util from 'util' import flag from 'cozy-flags' import { uploadFileWithConflictStrategy } from 'cozy-client/dist/models/file' import { + accountsWitoutTransactionsToCSV, createFormatStream, + fetchAccountsToExport, fetchTransactionsToExport, transactionsToCSV } from 'ducks/export/services' @@ -12,8 +13,6 @@ import logger from 'ducks/export/logger' import { DATA_EXPORT_DIR_ID, DATA_EXPORT_NAME } from 'ducks/export/constants' import { runService } from './service' -const pipeline = util.promisify(stream.pipeline) - const main = async ({ client }) => { if (require.main !== module && process.env.NODE_ENV !== 'production') { client.registerPlugin(flag.plugin) @@ -31,20 +30,34 @@ const main = async ({ client }) => { const transactions = await fetchTransactionsToExport(client) logger('info', `Fetched ${transactions.length} transactions`) - if (transactions.length === 0) { - logger('info', 'No transactions to export') + logger('info', 'Fetching accounts without transactions...') + const accounts = await fetchAccountsToExport(client, { transactions }) + logger('info', `Fetched ${accounts.length} accounts`) + + if (transactions.length === 0 && accounts.length === 0) { + logger('info', 'No data to export') return } - // Transform the transactions into a CSV file - logger('info', `Creating transformation stream...`) - const data = createFormatStream() - logger('info', `Transforming data to CSV...`) - pipeline(transactionsToCSV(transactions), data) + logger('info', `Creating streams...`) + const csv = createFormatStream() + const accountsStream = stream.Readable.from( + accountsWitoutTransactionsToCSV(accounts) + ) + const transactionsStream = stream.Readable.from( + transactionsToCSV(transactions) + ) + + accountsStream.on('end', () => { + logger('info', `Transforming transactions to CSV...`) + transactionsStream.pipe(csv) + }) + + logger('info', `Transforming accounts to CSV...`) + accountsStream.pipe(csv, { end: false }) - // Upload the CSV file to the Cozy - logger('info', `Uploading CSV file to Cozy...`) - await uploadFileWithConflictStrategy(client, data, { + logger('info', `Starting CSV file upload to Cozy...`) + await uploadFileWithConflictStrategy(client, csv, { name: DATA_EXPORT_NAME, dirId: DATA_EXPORT_DIR_ID, conflictStrategy: 'erase'