From 1baa9621c66d29dcf7368d152cad335c5fe003c5 Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Fri, 11 Nov 2022 13:35:38 +0100 Subject: [PATCH 1/4] refactor(YNAB): separate func BulkWriter into two Create a separate function for parsing a Ynabber transactions to YNAB. To prepare for a new ImportID generator. --- cmd/ynabber/main.go | 4 +- config.go | 12 ++++- writer/ynab/ynab.go | 118 +++++++++++++++++++++----------------------- 3 files changed, 69 insertions(+), 65 deletions(-) diff --git a/cmd/ynabber/main.go b/cmd/ynabber/main.go index 5f4e2f2..08cb794 100644 --- a/cmd/ynabber/main.go +++ b/cmd/ynabber/main.go @@ -31,9 +31,9 @@ func main() { // Handle movement of PayeeStrip from YNAB to Nordigen config strut if cfg.Nordigen.PayeeStrip == nil { - if cfg.YNAB.PayeeStrip != nil { + if cfg.PayeeStrip != nil { log.Printf("Config YNABBER_PAYEE_STRIP is depreciated, please use NORDIGEN_PAYEE_STRIP instead") - cfg.Nordigen.PayeeStrip = cfg.YNAB.PayeeStrip + cfg.Nordigen.PayeeStrip = cfg.PayeeStrip } } diff --git a/config.go b/config.go index 0a0a59d..520b891 100644 --- a/config.go +++ b/config.go @@ -23,6 +23,9 @@ type Config struct { // only YNAB is supported. Writers []string `envconfig:"YNABBER_WRITERS" default:"ynab"` + // PayeeStrip is depreciated please use Nordigen.PayeeStrip instead + PayeeStrip []string `envconfig:"YNABBER_PAYEE_STRIP"` + // Reader and/or writer specific settings Nordigen Nordigen YNAB YNAB @@ -62,8 +65,13 @@ type Nordigen struct { // YNAB related settings type YNAB struct { - // PayeeStrip is depreciated please use Nordigen.PayeeStrip instead - PayeeStrip []string `envconfig:"YNABBER_PAYEE_STRIP"` + // BudgetID for the budget you want to import transactions into. You can + // find the ID in the URL of YNAB: https://app.youneedabudget.com//budget + BudgetID string `envconfig:"YNAB_BUDGETID"` + + // Token is your personal access token as obtained from the YNAB developer + // settings section + Token string `envconfig:"YNAB_TOKEN"` // Set cleared status, possible values: cleared, uncleared, reconciled . // Default is uncleared for historical reasons but recommend setting this diff --git a/writer/ynab/ynab.go b/writer/ynab/ynab.go index e68de06..c4f5db4 100644 --- a/writer/ynab/ynab.go +++ b/writer/ynab/ynab.go @@ -8,7 +8,6 @@ import ( "log" "net/http" "net/http/httputil" - "os" "regexp" "strings" @@ -19,78 +18,75 @@ import ( const maxMemoSize int = 200 // Max size of memo field in YNAB API const maxPayeeSize int = 100 // Max size of payee field in YNAB API -func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error { +var space = regexp.MustCompile(`\s+`) // Matches all whitespace characters + +// Ytransaction is a single YNAB transaction +type Ytransaction struct { + AccountID string `json:"account_id"` + Date string `json:"date"` + Amount string `json:"amount"` + PayeeName string `json:"payee_name"` + Memo string `json:"memo"` + ImportID string `json:"import_id"` + Cleared string `json:"cleared"` + Approved bool `json:"approved"` +} + +// Ytransactions is multiple YNAB transactions +type Ytransactions struct { + Transactions []Ytransaction `json:"transactions"` +} - budgetID, found := os.LookupEnv("YNAB_BUDGETID") - if !found { - return fmt.Errorf("env variable YNAB_BUDGETID: %w", ynabber.ErrNotFound) +func ynabberToYNAB(cfg ynabber.Config, t ynabber.Transaction) Ytransaction { + date := t.Date.Format("2006-01-02") + amount := t.Amount.String() + + // Trim consecutive spaces from memo and truncate if too long + memo := strings.TrimSpace(space.ReplaceAllString(t.Memo, " ")) + if len(memo) > maxMemoSize { + log.Printf("Memo on account %s on date %s is too long - truncated to %d characters", + t.Account.Name, date, maxMemoSize) + memo = memo[0:(maxMemoSize - 1)] } - token, found := os.LookupEnv("YNAB_TOKEN") - if !found { - return fmt.Errorf("env variable YNAB_TOKEN: %w", ynabber.ErrNotFound) + + // Trim consecutive spaces from payee and truncate if too long + payee := strings.TrimSpace(space.ReplaceAllString(string(t.Payee), " ")) + if len(payee) > maxPayeeSize { + log.Printf("Payee on account %s on date %s is too long - truncated to %d characters", + t.Account.Name, date, maxPayeeSize) + payee = payee[0:(maxPayeeSize - 1)] } + // Generating YNAB compliant import ID, output example: + // YBBR:-741000:2021-02-18:92f2beb1 + hash := sha256.Sum256([]byte(t.Memo)) + id := fmt.Sprintf("YBBR:%s:%s:%x", amount, date, hash[:2]) + + return Ytransaction{ + AccountID: uuid.UUID(t.Account.ID).String(), + Date: date, + Amount: amount, + PayeeName: payee, + Memo: memo, + ImportID: id, + Cleared: cfg.YNAB.Cleared, + Approved: false, + } +} + +func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error { if len(t) == 0 { log.Println("No transactions to write") return nil } - type Ytransaction struct { - AccountID string `json:"account_id"` - Date string `json:"date"` - Amount string `json:"amount"` - PayeeName string `json:"payee_name"` - Memo string `json:"memo"` - ImportID string `json:"import_id"` - Cleared string `json:"cleared"` - Approved bool `json:"approved"` - } - type Ytransactions struct { - Transactions []Ytransaction `json:"transactions"` - } - + // Build array of transactions to send to YNAB y := new(Ytransactions) - space := regexp.MustCompile(`\s+`) // Reused on every iteration for _, v := range t { - date := v.Date.Format("2006-01-02") - amount := v.Amount.String() - - // Trim consecutive spaces from memo and truncate if too long - memo := strings.TrimSpace(space.ReplaceAllString(v.Memo, " ")) - if len(memo) > maxMemoSize { - log.Printf("Memo on account %s on date %s is too long - truncated to %d characters", - v.Account.Name, date, maxMemoSize) - memo = memo[0:(maxMemoSize - 1)] - } - - // Trim consecutive spaces from payee and truncate if too long - payee := strings.TrimSpace(space.ReplaceAllString(string(v.Payee), " ")) - if len(payee) > maxPayeeSize { - log.Printf("Payee on account %s on date %s is too long - truncated to %d characters", - v.Account.Name, date, maxPayeeSize) - payee = payee[0:(maxPayeeSize - 1)] - } - - // Generating YNAB compliant import ID, output example: - // YBBR:-741000:2021-02-18:92f2beb1 - hash := sha256.Sum256([]byte(v.Memo)) - id := fmt.Sprintf("YBBR:%s:%s:%x", amount, date, hash[:2]) - - x := Ytransaction{ - AccountID: uuid.UUID(v.Account.ID).String(), - Date: date, - Amount: amount, - PayeeName: payee, - Memo: memo, - ImportID: id, - Cleared: cfg.YNAB.Cleared, - Approved: false, - } - - y.Transactions = append(y.Transactions, x) + y.Transactions = append(y.Transactions, ynabberToYNAB(cfg, v)) } - url := fmt.Sprintf("https://api.youneedabudget.com/v1/budgets/%s/transactions", budgetID) + url := fmt.Sprintf("https://api.youneedabudget.com/v1/budgets/%s/transactions", cfg.YNAB.BudgetID) payload, err := json.Marshal(y) if err != nil { @@ -108,7 +104,7 @@ func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error { return err } req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cfg.YNAB.Token)) res, err := client.Do(req) if err != nil { From bd5a8fb4f483771af547dc0fcf71611a5f805aaa Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sat, 12 Nov 2022 13:09:44 +0100 Subject: [PATCH 2/4] fix: creating uuids we do not intent to store We should really not create UUIDs if we don't intent to store them for anywhere. So in favor of keeping this a stateless program we default to the transaction ID from Nordigen and keep it a string in Ynabber. If the ID from Nordigen is empty we warn the user in the logs. --- go.mod | 5 +---- go.sum | 2 -- reader/nordigen/nordigen.go | 12 ++++++++++-- reader/nordigen/nordigen_test.go | 6 ++---- writer/ynab/ynab.go | 3 +-- ynabber.go | 12 +----------- 6 files changed, 15 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index baf38c1..1b92712 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,6 @@ module github.com/martinohansen/ynabber go 1.17 -require ( - github.com/frieser/nordigen-go-lib/v2 v2.1.1 - github.com/google/uuid v1.3.0 -) +require github.com/frieser/nordigen-go-lib/v2 v2.1.1 require github.com/kelseyhightower/envconfig v1.4.0 diff --git a/go.sum b/go.sum index a65d54e..257d3e1 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/frieser/nordigen-go-lib/v2 v2.1.1 h1:nApiepW2oh+gPL2my9dvkHEVN6w30cmpAf0glNJkHCY= github.com/frieser/nordigen-go-lib/v2 v2.1.1/go.mod h1:NejYisqD8GvynCN0vDGw7J66slnj7jB25c8tS1tr8bw= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= diff --git a/reader/nordigen/nordigen.go b/reader/nordigen/nordigen.go index 19148b6..82d7a12 100644 --- a/reader/nordigen/nordigen.go +++ b/reader/nordigen/nordigen.go @@ -17,11 +17,14 @@ import ( const timeLayout = "2006-01-02" +// TODO(Martin): Move accountParser from Nordigen to YNAB package. We want to +// map Ynabber transaction to YNAB and not so much Nordigen to Ynabber like this +// is during currently. func accountParser(account string, accountMap map[string]string) (ynabber.Account, error) { for from, to := range accountMap { if account == from { return ynabber.Account{ - ID: ynabber.ID(ynabber.IDFromString(to)), + ID: ynabber.ID(to), Name: from, }, nil } @@ -46,6 +49,11 @@ func payeeStripNonAlphanumeric(payee string) (x string) { } func transactionToYnabber(cfg ynabber.Config, account ynabber.Account, t nordigen.Transaction) (x ynabber.Transaction, err error) { + id := t.TransactionId + if id == "" { + log.Printf("Transaction ID is empty, this might cause duplicate entires in YNAB") + } + memo := t.RemittanceInformationUnstructured amount, err := strconv.ParseFloat(t.TransactionAmount.Amount, 64) @@ -89,7 +97,7 @@ func transactionToYnabber(cfg ynabber.Config, account ynabber.Account, t nordige return ynabber.Transaction{ Account: account, - ID: ynabber.ID(ynabber.IDFromString(t.TransactionId)), + ID: ynabber.ID(id), Date: date, Payee: ynabber.Payee(payee), Memo: memo, diff --git a/reader/nordigen/nordigen_test.go b/reader/nordigen/nordigen_test.go index 9146281..a8cf53a 100644 --- a/reader/nordigen/nordigen_test.go +++ b/reader/nordigen/nordigen_test.go @@ -21,8 +21,7 @@ func TestTransactionToYnabber(t *testing.T) { }{ {name: "milliunits a", args: args{cfg: ynabber.Config{}, account: ynabber.Account{}, t: nordigen.Transaction{ - TransactionId: "00000000-0000-0000-0000-000000000000", - BookingDate: "0001-01-01", + BookingDate: "0001-01-01", TransactionAmount: struct { Amount string "json:\"amount,omitempty\"" Currency string "json:\"currency,omitempty\"" @@ -37,8 +36,7 @@ func TestTransactionToYnabber(t *testing.T) { }, {name: "milliunits b", args: args{cfg: ynabber.Config{}, account: ynabber.Account{}, t: nordigen.Transaction{ - TransactionId: "00000000-0000-0000-0000-000000000000", - BookingDate: "0001-01-01", + BookingDate: "0001-01-01", TransactionAmount: struct { Amount string "json:\"amount,omitempty\"" Currency string "json:\"currency,omitempty\"" diff --git a/writer/ynab/ynab.go b/writer/ynab/ynab.go index c4f5db4..d0c2125 100644 --- a/writer/ynab/ynab.go +++ b/writer/ynab/ynab.go @@ -11,7 +11,6 @@ import ( "regexp" "strings" - "github.com/google/uuid" "github.com/martinohansen/ynabber" ) @@ -63,7 +62,7 @@ func ynabberToYNAB(cfg ynabber.Config, t ynabber.Transaction) Ytransaction { id := fmt.Sprintf("YBBR:%s:%s:%x", amount, date, hash[:2]) return Ytransaction{ - AccountID: uuid.UUID(t.Account.ID).String(), + AccountID: string(t.Account.ID), Date: date, Amount: amount, PayeeName: payee, diff --git a/ynabber.go b/ynabber.go index 8ce7aa1..44a50fb 100644 --- a/ynabber.go +++ b/ynabber.go @@ -4,8 +4,6 @@ import ( "encoding/json" "strconv" "time" - - "github.com/google/uuid" ) type AccountMap map[string]string @@ -24,7 +22,7 @@ type Account struct { Name string } -type ID uuid.UUID +type ID string type Payee string @@ -44,14 +42,6 @@ type Ynabber interface { bulkWriter([]Transaction) error } -func IDFromString(id string) uuid.UUID { - x, err := uuid.Parse(id) - if err != nil { - return uuid.New() - } - return x -} - func (m Milliunits) String() string { return strconv.FormatInt(int64(m), 10) } From 6178b12993849d80dd5ee0573d2980e018ff67ed Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sat, 12 Nov 2022 16:24:17 +0100 Subject: [PATCH 3/4] refactor(nordigen): move dataFile to its own func --- reader/nordigen/nordigen.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/reader/nordigen/nordigen.go b/reader/nordigen/nordigen.go index 82d7a12..a566124 100644 --- a/reader/nordigen/nordigen.go +++ b/reader/nordigen/nordigen.go @@ -117,13 +117,8 @@ func transactionsToYnabber(cfg ynabber.Config, account ynabber.Account, t nordig return x, nil } -func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) { - c, err := nordigen.NewClient(cfg.Nordigen.SecretID, cfg.Nordigen.SecretKey) - if err != nil { - return nil, fmt.Errorf("failed to create client: %w", err) - } - - // Select persistent dataFile +// dataFile returns a persistent path +func dataFile(cfg ynabber.Config) string { dataFile := "" if cfg.Nordigen.Datafile != "" { if path.IsAbs(cfg.Nordigen.Datafile) { @@ -135,7 +130,7 @@ func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) { dataFileBankSpecific := fmt.Sprintf("%s/%s-%s.json", cfg.DataDir, "ynabber", cfg.Nordigen.BankID) dataFileGeneric := fmt.Sprintf("%s/%s.json", cfg.DataDir, "ynabber") dataFile = dataFileBankSpecific - _, err = os.Stat(dataFileBankSpecific) + _, err := os.Stat(dataFileBankSpecific) if errors.Is(err, os.ErrNotExist) { _, err := os.Stat(dataFileGeneric) if errors.Is(err, os.ErrNotExist) { @@ -147,11 +142,19 @@ func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) { } } } + return dataFile +} + +func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) { + c, err := nordigen.NewClient(cfg.Nordigen.SecretID, cfg.Nordigen.SecretKey) + if err != nil { + return nil, fmt.Errorf("failed to create client: %w", err) + } Authorization := Authorization{ Client: *c, BankID: cfg.Nordigen.BankID, - File: dataFile, + File: dataFile(cfg), } r, err := Authorization.Wrapper() if err != nil { From 7242dab9d04b72e839e10ad2b9619601d43fe494 Mon Sep 17 00:00:00 2001 From: Martin Hansen Date: Sat, 12 Nov 2022 16:25:53 +0100 Subject: [PATCH 4/4] refactor(config): move decode to config.go --- config.go | 26 ++++++++++++++++++++++++++ ynabber.go | 12 ------------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/config.go b/config.go index 520b891..eded6b8 100644 --- a/config.go +++ b/config.go @@ -1,9 +1,35 @@ package ynabber import ( + "encoding/json" "time" ) +const DateFormat = "2006-01-02" + +type Date time.Time + +// Decode implements `envconfig.Decoder` for Date to parse string to time.Time +func (date *Date) Decode(value string) error { + time, err := time.Parse(DateFormat, value) + if err != nil { + return err + } + *date = Date(time) + return nil +} + +type AccountMap map[string]string + +// Decode implements `envconfig.Decoder` for AccountMap to decode JSON properly +func (accountMap *AccountMap) Decode(value string) error { + err := json.Unmarshal([]byte(value), &accountMap) + if err != nil { + return err + } + return nil +} + // Config is loaded from the environment during execution with cmd/ynabber type Config struct { // DataDir is the path for storing files e.g. Nordigen authorization diff --git a/ynabber.go b/ynabber.go index 44a50fb..13fc520 100644 --- a/ynabber.go +++ b/ynabber.go @@ -1,22 +1,10 @@ package ynabber import ( - "encoding/json" "strconv" "time" ) -type AccountMap map[string]string - -// Decode implements `envconfig.Decoder` for AccountMap to decode JSON properly -func (input *AccountMap) Decode(value string) error { - err := json.Unmarshal([]byte(value), &input) - if err != nil { - return err - } - return nil -} - type Account struct { ID ID Name string