From 20da794f4e90fb2a4c501f7b13755a9595994b6a Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sun, 7 Jan 2024 16:10:09 +0100 Subject: [PATCH 1/3] docs: move Nordigen docs to reader/nordigen --- README.md | 34 ++++-------------------- reader/nordigen/README.md | 18 +++++++++++++ reader/nordigen/hooks/logsnag-example.sh | 4 +++ 3 files changed, 27 insertions(+), 29 deletions(-) create mode 100644 reader/nordigen/README.md diff --git a/README.md b/README.md index df3b574..4348ec5 100644 --- a/README.md +++ b/README.md @@ -59,28 +59,6 @@ docker run \ ghcr.io/martinohansen/ynabber:latest ``` - -### Requisition URL Hooks - -In order to allow bank account data to flow to YNAB, this application requires an authentication with Nordigen. That URL is called "requistion URL" and is available in the docker logs. For some banks, this access is only valid for 90 days. This application requires a relogin after. In order to make that process easier (i.e. by sending the requistion URL to the phone) ynabber supports hooks when creating a requisition URL. In order to set it up, one first creates a shell-script, for example named `hook.sh`: - -```bash -#! /bin/sh - -echo "Hi from hook 👋 -status: $1 -link: $2 -at: $(date)" -fi -``` - -And then configures a hook in the configuration file: -```bash -NORDIGEN_REQUISITION_HOOK=/data/hook.sh -``` - -When using ynabber throuch docker, keep in mind that the docker container does not support a vast array of command line tools (i.e. no bash, wget instead of cURL). - ## Readers Currently tested readers and verified banks, but any bank supported by Nordigen @@ -88,17 +66,15 @@ should work. | Reader | Bank | | |----------|-----------------|---| -| Nordigen | ALANDSBANKEN_AABAFI22 | ✅ -| | NORDEA_NDEADKKK | ✅[^1] +| [Nordigen](/reader/nordigen/)[^1] | ALANDSBANKEN_AABAFI22 | ✅ +| | NORDEA_NDEADKKK | ✅ | | NORDEA_NDEAFIHH | ✅ | | NORWEGIAN_FI_NORWNOK1 | ✅ | | S_PANKKI_SBANFIHH | ✅ -Please open an [issue](https://github.com/martinohansen/ynabber/issues/new) if +[^1]: Please open an [issue](https://github.com/martinohansen/ynabber/issues/new) if you have problems with a specific bank. -[^1]: Requires setting NORDIGEN_TRANSACTION_ID to "InternalTransactionId" - ## Writers The default writer is YNAB (that's really what this tool is set out to handle) @@ -106,8 +82,8 @@ but we also have a JSON writer that can be used for testing purposes. | Writer | Description | |---------|---------------| -| YNAB | Pushes transactions to YNAB | -| JSON | Writes transactions to stdout in JSON format | +| [YNAB](/writer/ynab/) | Pushes transactions to YNAB | +| [JSON](/writer/json/) | Writes transactions to stdout in JSON format | ## Contributing diff --git a/reader/nordigen/README.md b/reader/nordigen/README.md new file mode 100644 index 0000000..e063f20 --- /dev/null +++ b/reader/nordigen/README.md @@ -0,0 +1,18 @@ +# Nordigen + +This reader reads transactions from [Nordigen](https://nordigen.com/en/), now +acquired by GoCardless. + +## Requisition Hook + +In order to allow bank account data to flow, you must be authenticated to your +bank. To authenticate a requisition URL is available in the logs. For some banks +this access is only valid for 90 days, whereafter a reauthorization is required. +To ease that process a requisition hook is made available. + +See [config.go](../../config.go) for information on how to configure it. + +### Examples + +A few shell scripts that can be used as targets for the hook are available in +the [hooks](./hooks/) directory. diff --git a/reader/nordigen/hooks/logsnag-example.sh b/reader/nordigen/hooks/logsnag-example.sh index f3536c4..527f2af 100755 --- a/reader/nordigen/hooks/logsnag-example.sh +++ b/reader/nordigen/hooks/logsnag-example.sh @@ -1,4 +1,8 @@ #!/bin/sh + +# This is an example of how to use Logsnag (https://logsnag.com/) to send a +# notifications. + reqURL=$2 logsnagToken="" From e5bc6e9a775166f23aa47890d47b01fadec0132e Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sun, 7 Jan 2024 17:06:39 +0100 Subject: [PATCH 2/3] feat(nordigen): set transaction id based on bankid Remove the TransactionID config option and instead set the transaction id based on the BankID. Less config options makes for a simpler program. --- config.go | 9 ------- reader/nordigen/mapper.go | 13 +++++---- reader/nordigen/nordigen.go | 45 +++++++++++++++++++++----------- reader/nordigen/nordigen_test.go | 13 ++++----- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/config.go b/config.go index e13dc58..87b134f 100644 --- a/config.go +++ b/config.go @@ -77,15 +77,6 @@ type Nordigen struct { // "foo,bar" PayeeStrip []string `envconfig:"NORDIGEN_PAYEE_STRIP"` - // TransactionID picks the field to use as transaction ID. This is relevant - // for some banks where the ID provided by the bank is not consistent. For - // example with NORDEA_NDEADKKK the TransactionId changes with time, which - // might cause hard to debug duplicate entries in YNAB. Only change this if - // you have a good reason to do so. - // - // Valid options are: TransactionId, InternalTransactionId - TransactionID string `envconfig:"NORDIGEN_TRANSACTION_ID" default:"TransactionId"` - // RequisitionHook is a exec hook thats executed at various stages of the // requisition process. The hook is executed with the following arguments: // diff --git a/reader/nordigen/mapper.go b/reader/nordigen/mapper.go index 943b045..7c26786 100644 --- a/reader/nordigen/mapper.go +++ b/reader/nordigen/mapper.go @@ -10,14 +10,17 @@ import ( ) type Mapper interface { - Map(ynabber.Config, ynabber.Account, nordigen.Transaction) ynabber.Transaction + Map(ynabber.Account, nordigen.Transaction) (ynabber.Transaction, error) } // Default mapping for all banks unless a more specific mapping exists -type Default struct{} +type Default struct { + PayeeSource []string + TransactionID string +} // Map Nordigen transactions using the default mapper -func (Default) Map(cfg ynabber.Config, a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { +func (mapper Default) Map(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { amount, err := strconv.ParseFloat(t.TransactionAmount.Amount, 64) if err != nil { return ynabber.Transaction{}, fmt.Errorf("failed to convert string to float: %w", err) @@ -31,7 +34,7 @@ func (Default) Map(cfg ynabber.Config, a ynabber.Account, t nordigen.Transaction // Get the Payee from the first data source that returns data in the order // defined by config payee := "" - for _, source := range cfg.Nordigen.PayeeSource { + for _, source := range mapper.PayeeSource { if payee == "" { switch source { // Unstructured should properly have been called "remittance" but @@ -64,7 +67,7 @@ func (Default) Map(cfg ynabber.Config, a ynabber.Account, t nordigen.Transaction // Get the ID from the first data source that returns data as defined in the // config var id string - switch cfg.Nordigen.TransactionID { + switch mapper.TransactionID { case "InternalTransactionId": id = t.InternalTransactionId default: diff --git a/reader/nordigen/nordigen.go b/reader/nordigen/nordigen.go index 2a8cbda..355afbd 100644 --- a/reader/nordigen/nordigen.go +++ b/reader/nordigen/nordigen.go @@ -36,37 +36,52 @@ func payeeStripNonAlphanumeric(payee string) (x string) { return strings.TrimSpace(x) } -func transactionToYnabber(cfg ynabber.Config, account ynabber.Account, t nordigen.Transaction) (y ynabber.Transaction, err error) { - // Pick an appropriate mapper based on the BankID provided or fallback to - // our default best effort mapper. - switch cfg.Nordigen.BankID { +// Mapper returns a mapper to transform the banks transaction to Ynabber +func (r Reader) Mapper() Mapper { + switch r.Config.Nordigen.BankID { + case "NORDEA_NDEADKKK": + return Default{ + PayeeSource: r.Config.Nordigen.PayeeSource, + // Nordea seems to think it makes sense to change the ID with time, + // I think its changing once a statement is booked. This causes + // duplicate entries in YNAB because the ID is used in the dedup + // hash. + TransactionID: "InternalTransactionId", + } + default: - y, err = Default{}.Map(cfg, account, t) + return Default{ + PayeeSource: r.Config.Nordigen.PayeeSource, + } } +} - // Return now if any of the mappings resulted in error +func (r Reader) toYnabber(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { + transaction, err := r.Mapper().Map(a, t) if err != nil { - return y, err + return ynabber.Transaction{}, err } // Execute strip method on payee if defined in config - if cfg.Nordigen.PayeeStrip != nil { - y.Payee = y.Payee.Strip(cfg.Nordigen.PayeeStrip) + if r.Config.Nordigen.PayeeStrip != nil { + transaction.Payee = transaction.Payee.Strip(r.Config.Nordigen.PayeeStrip) } - return y, err + return transaction, nil } -func transactionsToYnabber(cfg ynabber.Config, account ynabber.Account, t nordigen.AccountTransactions) (x []ynabber.Transaction, err error) { +func (r Reader) toYnabbers(a ynabber.Account, t nordigen.AccountTransactions) ([]ynabber.Transaction, error) { + y := []ynabber.Transaction{} for _, v := range t.Transactions.Booked { - transaction, err := transactionToYnabber(cfg, account, v) + transaction, err := r.toYnabber(a, v) if err != nil { return nil, err } + // Append transaction - x = append(x, transaction) + y = append(y, transaction) } - return x, nil + return y, nil } func (r Reader) Bulk() (t []ynabber.Transaction, err error) { @@ -111,7 +126,7 @@ func (r Reader) Bulk() (t []ynabber.Transaction, err error) { log.Printf("Transactions received from Nordigen: %+v", transactions) } - x, err := transactionsToYnabber(*r.Config, account, transactions) + x, err := r.toYnabbers(account, transactions) if err != nil { return nil, fmt.Errorf("failed to convert transaction: %w", err) } diff --git a/reader/nordigen/nordigen_test.go b/reader/nordigen/nordigen_test.go index 5cc3d09..845bb77 100644 --- a/reader/nordigen/nordigen_test.go +++ b/reader/nordigen/nordigen_test.go @@ -14,8 +14,11 @@ func TestTransactionToYnabber(t *testing.T) { var defaultConfig ynabber.Config _ = envconfig.Process("", &defaultConfig) + reader := Reader{ + Config: &defaultConfig, + } + type args struct { - cfg ynabber.Config account ynabber.Account t nordigen.Transaction } @@ -26,7 +29,7 @@ func TestTransactionToYnabber(t *testing.T) { wantErr bool }{ {name: "milliunits a", - args: args{cfg: ynabber.Config{}, account: ynabber.Account{}, t: nordigen.Transaction{ + args: args{account: ynabber.Account{}, t: nordigen.Transaction{ BookingDate: "0001-01-01", TransactionAmount: struct { Amount string "json:\"amount,omitempty\"" @@ -41,7 +44,7 @@ func TestTransactionToYnabber(t *testing.T) { wantErr: false, }, {name: "milliunits b", - args: args{cfg: ynabber.Config{}, account: ynabber.Account{}, t: nordigen.Transaction{ + args: args{account: ynabber.Account{}, t: nordigen.Transaction{ BookingDate: "0001-01-01", TransactionAmount: struct { Amount string "json:\"amount,omitempty\"" @@ -60,7 +63,6 @@ func TestTransactionToYnabber(t *testing.T) { // default config to highlight any breaking changes. name: "NORDEA_NDEADKKK", args: args{ - cfg: defaultConfig, account: ynabber.Account{Name: "foo", IBAN: "bar"}, t: nordigen.Transaction{ TransactionId: "H00000000000000000000", @@ -100,7 +102,6 @@ func TestTransactionToYnabber(t *testing.T) { // Test transaction from SEB_KORT_AB_NO_SKHSFI21 name: "SEB_KORT_AB_NO_SKHSFI21", args: args{ - cfg: defaultConfig, account: ynabber.Account{Name: "foo", IBAN: "bar"}, t: nordigen.Transaction{ TransactionId: "foobar", @@ -140,7 +141,7 @@ func TestTransactionToYnabber(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := transactionToYnabber(tt.args.cfg, tt.args.account, tt.args.t) + got, err := reader.toYnabber(tt.args.account, tt.args.t) if (err != nil) != tt.wantErr { t.Errorf("error = %+v, wantErr %+v", err, tt.wantErr) return From bd340d1a14f55f507e6f63ffd80b60baa4be0ffc Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sun, 7 Jan 2024 17:40:18 +0100 Subject: [PATCH 3/3] feat(nordigen): add Nordea specific mapper Add mapper thats specific to Nordea which has some quirks that needs to be handled differently than the default mapper. I would like to get rid of the logic inside the default mapper eventually and have all the logic in bank specific mappers. The goal of this is to have less config values and less stuff the user needs to reason about. Its getting hard to remember how setting X affects each bank and what it needs to be for each bank, so lets just put it directly in code. --- reader/nordigen/mapper.go | 77 ++++++++++++++++++++++++-------- reader/nordigen/mapper_test.go | 50 +++++++++++++++++++++ reader/nordigen/nordigen.go | 20 --------- reader/nordigen/nordigen_test.go | 62 ++++++++----------------- ynabber_test.go | 12 +++++ 5 files changed, 139 insertions(+), 82 deletions(-) create mode 100644 reader/nordigen/mapper_test.go diff --git a/reader/nordigen/mapper.go b/reader/nordigen/mapper.go index 7c26786..dcd04ab 100644 --- a/reader/nordigen/mapper.go +++ b/reader/nordigen/mapper.go @@ -13,22 +13,49 @@ type Mapper interface { Map(ynabber.Account, nordigen.Transaction) (ynabber.Transaction, error) } -// Default mapping for all banks unless a more specific mapping exists -type Default struct { - PayeeSource []string - TransactionID string +// Mapper returns a mapper to transform the banks transaction to Ynabber +func (r Reader) Mapper() Mapper { + switch r.Config.Nordigen.BankID { + case "NORDEA_NDEADKKK": + return Nordea{} + + default: + return Default{ + PayeeSource: r.Config.Nordigen.PayeeSource, + } + } } -// Map Nordigen transactions using the default mapper -func (mapper Default) Map(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { +func parseAmount(t nordigen.Transaction) (float64, error) { amount, err := strconv.ParseFloat(t.TransactionAmount.Amount, 64) if err != nil { - return ynabber.Transaction{}, fmt.Errorf("failed to convert string to float: %w", err) + return 0, fmt.Errorf("failed to convert string to float: %w", err) } + return amount, nil +} +func parseDate(t nordigen.Transaction) (time.Time, error) { date, err := time.Parse("2006-01-02", t.BookingDate) if err != nil { - return ynabber.Transaction{}, fmt.Errorf("failed to parse string to time: %w", err) + return time.Time{}, fmt.Errorf("failed to parse string to time: %w", err) + } + return date, nil +} + +// Default mapping for all banks unless a more specific mapping exists +type Default struct { + PayeeSource []string +} + +// Map t using the default mapper +func (mapper Default) Map(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { + amount, err := parseAmount(t) + if err != nil { + return ynabber.Transaction{}, err + } + date, err := parseDate(t) + if err != nil { + return ynabber.Transaction{}, err } // Get the Payee from the first data source that returns data in the order @@ -64,21 +91,35 @@ func (mapper Default) Map(a ynabber.Account, t nordigen.Transaction) (ynabber.Tr } } - // Get the ID from the first data source that returns data as defined in the - // config - var id string - switch mapper.TransactionID { - case "InternalTransactionId": - id = t.InternalTransactionId - default: - id = t.TransactionId + return ynabber.Transaction{ + Account: a, + ID: ynabber.ID(t.TransactionId), + Date: date, + Payee: ynabber.Payee(payee), + Memo: t.RemittanceInformationUnstructured, + Amount: ynabber.MilliunitsFromAmount(amount), + }, nil +} + +// Nordea implements a specific mapper for Nordea +type Nordea struct{} + +// Map t using the Nordea mapper +func (mapper Nordea) Map(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { + amount, err := parseAmount(t) + if err != nil { + return ynabber.Transaction{}, err + } + date, err := parseDate(t) + if err != nil { + return ynabber.Transaction{}, err } return ynabber.Transaction{ Account: a, - ID: ynabber.ID(id), + ID: ynabber.ID(t.InternalTransactionId), Date: date, - Payee: ynabber.Payee(payee), + Payee: ynabber.Payee(payeeStripNonAlphanumeric(t.RemittanceInformationUnstructured)), Memo: t.RemittanceInformationUnstructured, Amount: ynabber.MilliunitsFromAmount(amount), }, nil diff --git a/reader/nordigen/mapper_test.go b/reader/nordigen/mapper_test.go new file mode 100644 index 0000000..f60de10 --- /dev/null +++ b/reader/nordigen/mapper_test.go @@ -0,0 +1,50 @@ +package nordigen + +import ( + "fmt" + "testing" + + "github.com/frieser/nordigen-go-lib/v2" +) + +func TestParseAmount(t *testing.T) { + tests := []struct { + transaction nordigen.Transaction + want float64 + wantErr bool + }{ + { + transaction: nordigen.Transaction{ + TransactionAmount: struct { + Amount string "json:\"amount,omitempty\"" + Currency string "json:\"currency,omitempty\"" + }{Amount: "328.18"}, + }, + want: 328.18, + wantErr: false, + }, + { + transaction: nordigen.Transaction{ + TransactionAmount: struct { + Amount string "json:\"amount,omitempty\"" + Currency string "json:\"currency,omitempty\"" + }{Amount: "32818"}, + }, + want: 32818, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("Amount: %s", tt.transaction.TransactionAmount.Amount), func(t *testing.T) { + got, err := parseAmount(tt.transaction) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/reader/nordigen/nordigen.go b/reader/nordigen/nordigen.go index 355afbd..7984bc6 100644 --- a/reader/nordigen/nordigen.go +++ b/reader/nordigen/nordigen.go @@ -36,26 +36,6 @@ func payeeStripNonAlphanumeric(payee string) (x string) { return strings.TrimSpace(x) } -// Mapper returns a mapper to transform the banks transaction to Ynabber -func (r Reader) Mapper() Mapper { - switch r.Config.Nordigen.BankID { - case "NORDEA_NDEADKKK": - return Default{ - PayeeSource: r.Config.Nordigen.PayeeSource, - // Nordea seems to think it makes sense to change the ID with time, - // I think its changing once a statement is booked. This causes - // duplicate entries in YNAB because the ID is used in the dedup - // hash. - TransactionID: "InternalTransactionId", - } - - default: - return Default{ - PayeeSource: r.Config.Nordigen.PayeeSource, - } - } -} - func (r Reader) toYnabber(a ynabber.Account, t nordigen.Transaction) (ynabber.Transaction, error) { transaction, err := r.Mapper().Map(a, t) if err != nil { diff --git a/reader/nordigen/nordigen_test.go b/reader/nordigen/nordigen_test.go index 845bb77..15c6fc1 100644 --- a/reader/nordigen/nordigen_test.go +++ b/reader/nordigen/nordigen_test.go @@ -9,66 +9,34 @@ import ( "github.com/martinohansen/ynabber" ) -func TestTransactionToYnabber(t *testing.T) { +func TestToYnabber(t *testing.T) { var defaultConfig ynabber.Config _ = envconfig.Process("", &defaultConfig) - reader := Reader{ - Config: &defaultConfig, - } - type args struct { account ynabber.Account t nordigen.Transaction } tests := []struct { - name string + bankID string + reader Reader args args want ynabber.Transaction wantErr bool }{ - {name: "milliunits a", - args: args{account: ynabber.Account{}, t: nordigen.Transaction{ - BookingDate: "0001-01-01", - TransactionAmount: struct { - Amount string "json:\"amount,omitempty\"" - Currency string "json:\"currency,omitempty\"" - }{ - Amount: "328.18", - }, - }}, - want: ynabber.Transaction{ - Amount: ynabber.Milliunits(328180), - }, - wantErr: false, - }, - {name: "milliunits b", - args: args{account: ynabber.Account{}, t: nordigen.Transaction{ - BookingDate: "0001-01-01", - TransactionAmount: struct { - Amount string "json:\"amount,omitempty\"" - Currency string "json:\"currency,omitempty\"" - }{ - Amount: "32818", - }, - }}, - want: ynabber.Transaction{ - Amount: ynabber.Milliunits(32818000), - }, - wantErr: false, - }, { // Tests a common Nordigen transaction from NORDEA_NDEADKKK with the // default config to highlight any breaking changes. - name: "NORDEA_NDEADKKK", + bankID: "NORDEA_NDEADKKK", + reader: Reader{Config: &defaultConfig}, args: args{ account: ynabber.Account{Name: "foo", IBAN: "bar"}, t: nordigen.Transaction{ - TransactionId: "H00000000000000000000", - EntryReference: "", - BookingDate: "2023-02-24", - ValueDate: "2023-02-24", + InternalTransactionId: "H00000000000000000000", + EntryReference: "", + BookingDate: "2023-02-24", + ValueDate: "2023-02-24", TransactionAmount: struct { Amount string "json:\"amount,omitempty\"" Currency string "json:\"currency,omitempty\"" @@ -100,7 +68,8 @@ func TestTransactionToYnabber(t *testing.T) { }, { // Test transaction from SEB_KORT_AB_NO_SKHSFI21 - name: "SEB_KORT_AB_NO_SKHSFI21", + bankID: "SEB_KORT_AB_NO_SKHSFI21", + reader: Reader{Config: &defaultConfig}, args: args{ account: ynabber.Account{Name: "foo", IBAN: "bar"}, t: nordigen.Transaction{ @@ -140,8 +109,13 @@ func TestTransactionToYnabber(t *testing.T) { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := reader.toYnabber(tt.args.account, tt.args.t) + t.Run(tt.bankID, func(t *testing.T) { + + // Set the BankID to the test case but keep the rest of the config + // as is + tt.reader.Config.Nordigen.BankID = tt.bankID + + got, err := tt.reader.toYnabber(tt.args.account, tt.args.t) if (err != nil) != tt.wantErr { t.Errorf("error = %+v, wantErr %+v", err, tt.wantErr) return diff --git a/ynabber_test.go b/ynabber_test.go index 9140b3d..d817803 100644 --- a/ynabber_test.go +++ b/ynabber_test.go @@ -28,6 +28,18 @@ func TestMilliunitsFromAmount(t *testing.T) { if want != got { t.Fatalf("amount with no separator: %s != %s", want, got) } + + want = Milliunits(328180) + got = MilliunitsFromAmount(328.18) + if want != got { + t.Fatalf("amount with no separator: %s != %s", want, got) + } + + want = Milliunits(32818000) + got = MilliunitsFromAmount(32818) + if want != got { + t.Fatalf("amount with no separator: %s != %s", want, got) + } } func TestPayee_Strip(t *testing.T) {