Merge pull request #30 from martinohansen/martin/move-payee-parsing

Martin/move payee parsing

cmd/ynabber/main.go

@@ -20,6 +20,14 @@ func main() {
 		log.Fatal(err.Error())
 	}
 
+	// Handle movement of PayeeStrip from YNAB to Nordigen config strut
+	if cfg.Nordigen.PayeeStrip == nil {
+		if cfg.YNAB.PayeeStrip != nil {
+			log.Printf("Config YNABBER_PAYEE_STRIP is depreciated, please use NORDIGEN_PAYEE_STRIP instead")
+			cfg.Nordigen.PayeeStrip = cfg.YNAB.PayeeStrip
+		}
+	}
+
 	if cfg.Debug {
 		log.Printf("Config: %+v\n", cfg)
 	}

config.go

@@ -38,12 +38,23 @@ type Config struct {
 		// SecretKey is used to create requisition
 		SecretKey string `envconfig:"NORDIGEN_SECRET_KEY"`
 
-		// List of source for payee candidates
-		// Currently only name(=debtorName or creditorName) or unstructured are possible
-		PayeeSource []string `envconfig:"NORDIGEN_PAYEE_SOURCE" default:unstructured,name"`
+		// PayeeSource is a list of sources for Payee candidates, the first
+		// method that yields a result will be used. Valid options are:
+		// unstructured and name.
+		//
+		// Option unstructured equals to the `RemittanceInformationUnstructured`
+		// filed from Nordigen while name equals either `debtorName` or
+		// `creditorName`.
+		PayeeSource []string `envconfig:"NORDIGEN_PAYEE_SOURCE" default:"unstructured,name"`
+
+		// PayeeStrip is a list of words to remove from Payee. For example:
+		// "foo,bar"
+		PayeeStrip []string `envconfig:"NORDIGEN_PAYEE_STRIP"`
+	}
 
-		// PayeeStrip is a list of words to remove from the Payee before sending
-		// to YNAB when using unstructured data as source. For example: "foo,bar"
+	// YNAB related settings
+	YNAB struct {
+		// PayeeStrip is depreciated please use Nordigen.PayeeStrip instead
 		PayeeStrip []string `envconfig:"YNABBER_PAYEE_STRIP"`
 	}
 }

reader/nordigen/nordigen.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/frieser/nordigen-go-lib/v2"
@@ -26,7 +28,22 @@ func accountParser(account string, accountMap map[string]string) (ynabber.Accoun
 	return ynabber.Account{}, fmt.Errorf("account not found in map: %w", ynabber.ErrNotFound)
 }
 
-func transactionsToYnabber(config ynabber.Config, account ynabber.Account, t nordigen.AccountTransactions) (x []ynabber.Transaction, err error) {
+// payeeStrip returns payee with elements of strips removed
+func payeeStrip(payee string, strips []string) (x string) {
+	for _, strip := range strips {
+		x = strings.ReplaceAll(payee, strip, "")
+	}
+	return strings.TrimSpace(x)
+}
+
+// payeeStripNonAlphanumeric removes all non-alphanumeric characters from payee
+func payeeStripNonAlphanumeric(payee string) (x string) {
+	reg := regexp.MustCompile(`[^\p{L}]+`)
+	x = reg.ReplaceAllString(payee, " ")
+	return strings.TrimSpace(x)
+}
+
+func transactionsToYnabber(cfg ynabber.Config, account ynabber.Account, t nordigen.AccountTransactions) (x []ynabber.Transaction, err error) {
 	for _, v := range t.Transactions.Booked {
 		memo := v.RemittanceInformationUnstructured
 
@@ -41,27 +58,30 @@ func transactionsToYnabber(config ynabber.Config, account ynabber.Account, t nor
 			return nil, fmt.Errorf("failed to parse string to time: %w", err)
 		}
 
-		// Find payee
-		payee := ynabber.Payee("")
-		for _, source := range config.Nordigen.PayeeSource {
+		// Get the Payee data source
+		payee := ""
+		for _, source := range cfg.Nordigen.PayeeSource {
 			if payee == "" {
-				if source == "name" {
+				switch source {
+				case "name":
 					// Creditor/debtor name can be used as is
 					if v.CreditorName != "" {
-						payee = ynabber.Payee(v.CreditorName)
+						payee = v.CreditorName
 					} else if v.DebtorName != "" {
-						payee = ynabber.Payee(v.DebtorName)
+						payee = v.DebtorName
 					}
-				} else if source == "unstructured" {
+				case "unstructured":
 					// Unstructured data may need some formatting
-					payee = ynabber.Payee(v.RemittanceInformationUnstructured)
-					s, err := payee.Parsed(config.Nordigen.PayeeStrip)
-					if err != nil {
-						log.Printf("Failed to parse payee: %s: %s", string(payee), err)
+					payee = v.RemittanceInformationUnstructured
+
+					// Parse Payee according the user specified strips and
+					// remove non-alphanumeric
+					if cfg.Nordigen.PayeeStrip != nil {
+						payee = payeeStrip(payee, cfg.Nordigen.PayeeStrip)
 					}
-					payee = ynabber.Payee(s)
-				} else {
-					return nil, fmt.Errorf("Unrecognized payee data source %s - fix configuration", source)
+					payee = payeeStripNonAlphanumeric(payee)
+				default:
+					return nil, fmt.Errorf("unrecognized PayeeSource: %s", source)
 				}
 			}
 		}
@@ -71,7 +91,7 @@ func transactionsToYnabber(config ynabber.Config, account ynabber.Account, t nor
 			Account: account,
 			ID:      ynabber.ID(ynabber.IDFromString(v.TransactionId)),
 			Date:    date,
-			Payee:   payee,
+			Payee:   ynabber.Payee(payee),
 			Memo:    memo,
 			Amount:  milliunits,
 		})
@@ -85,28 +105,27 @@ func BulkReader(cfg ynabber.Config) (t []ynabber.Transaction, err error) {
 		return nil, fmt.Errorf("failed to create client: %w", err)
 	}
 
-	// Select persistent datafile
-	datafile_bankspecific := fmt.Sprintf("%s/%s-%s.json", cfg.DataDir, "ynabber", cfg.Nordigen.BankID)
-	datafile_generic := fmt.Sprintf("%s/%s.json", cfg.DataDir, "ynabber")
-	datafile := datafile_bankspecific
+	// Select persistent dataFile
+	dataFileBankSpecific := fmt.Sprintf("%s/%s-%s.json", cfg.DataDir, "ynabber", cfg.Nordigen.BankID)
+	dataFileGeneric := fmt.Sprintf("%s/%s.json", cfg.DataDir, "ynabber")
+	dataFile := dataFileBankSpecific
 
-	_, err = os.Stat(datafile_bankspecific)
+	_, err = os.Stat(dataFileBankSpecific)
 	if errors.Is(err, os.ErrNotExist) {
-		_, err := os.Stat(datafile_generic)
+		_, err := os.Stat(dataFileGeneric)
 		if errors.Is(err, os.ErrNotExist) {
-			// If bank specific does not exists and neither does generic, use bankspecific
-			datafile = datafile_bankspecific
+			// If bank specific does not exists and neither does generic, use dataFileBankSpecific
+			dataFile = dataFileBankSpecific
 		} else {
-			// Generic datafile exists(old naming) but not the bank speific, use old file
-			datafile = datafile_generic
-			log.Printf("Using non-bank specific persistent datafile %s, consider moving to %s\n", datafile_generic, datafile_bankspecific)
+			// Generic dataFile exists(old naming) but not the bank specific, use dataFileGeneric
+			dataFile = dataFileGeneric
 		}
 	}
 
 	Authorization := Authorization{
 		Client: *c,
 		BankID: cfg.Nordigen.BankID,
-		File:   datafile,
+		File:   dataFile,
 	}
 	r, err := Authorization.Wrapper()
 	if err != nil {

reader/nordigen/nordigen_test.go

@@ -69,3 +69,32 @@ func TestAccountParser(t *testing.T) {
 		})
 	}
 }
+
+func TestPayeeStripNonAlphanumeric(t *testing.T) {
+	want := "Im just alphanumeric"
+	got := payeeStripNonAlphanumeric("Im just alphanumeric")
+	if want != got {
+		t.Fatalf("alphanumeric: %s != %s", want, got)
+	}
+
+	want = "你好世界"
+	got = payeeStripNonAlphanumeric("你好世界")
+	if want != got {
+		t.Fatalf("non-english: %s != %s", want, got)
+	}
+
+	want = "Im not j ust alphanumeric"
+	got = payeeStripNonAlphanumeric("Im! not j.ust alphanumeric42 69")
+	if want != got {
+		t.Fatalf("non-alphanumeric: %s != %s", want, got)
+	}
+}
+
+func TestPayeeStrip(t *testing.T) {
+	want := "Im here"
+	got := payeeStrip("Im not here", []string{"not "})
+	if want != got {
+		t.Fatalf("strip words: %s != %s", want, got)
+	}
+
+}

writer/ynab/ynab.go

@@ -16,9 +16,10 @@ import (
 	"github.com/martinohansen/ynabber"
 )
 
+const maxMemoSize int = 200  // Max size of memo field in YNAB API
+const maxPayeeSize int = 100 // Max size of payee field in YNAB API
+
 func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
-	const ynab_maxmemo int = 200  // Max size of memo field in YNAB API
-	const ynab_maxpayee int = 100 // Max size of payee field in YNAB API
 
 	budgetID, found := os.LookupEnv("YNAB_BUDGETID")
 	if !found {
@@ -54,18 +55,18 @@ func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
 
 		// Trim consecutive spaces from memo and truncate if too long
 		memo := strings.TrimSpace(space.ReplaceAllString(v.Memo, " "))
-		if len(memo) > ynab_maxmemo {
+		if len(memo) > maxMemoSize {
 			log.Printf("Memo on account %s on date %s is too long - truncated to %d characters",
-				v.Account.Name, date, ynab_maxmemo)
-			memo = memo[0:(ynab_maxmemo - 1)]
+				v.Account.Name, date, maxMemoSize)
+			memo = memo[0:(maxMemoSize - 1)]
 		}
 
 		// Trim consecutive spaces from payee and truncate if too long
 		payee := strings.TrimSpace(space.ReplaceAllString(string(v.Payee), " "))
-		if len(payee) > ynab_maxpayee {
+		if len(payee) > maxPayeeSize {
 			log.Printf("Payee on account %s on date %s is too long - truncated to %d characters",
-				v.Account.Name, date, ynab_maxpayee)
-			payee = payee[0:(ynab_maxpayee - 1)]
+				v.Account.Name, date, maxPayeeSize)
+			payee = payee[0:(maxPayeeSize - 1)]
 		}
 
 		// Generating YNAB compliant import ID, output example:
@@ -95,7 +96,7 @@ func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
 	client := &http.Client{}
 
 	if cfg.Debug {
-		log.Printf("Request: %s\n", payload)
+		log.Printf("Request to YNAB: %s\n", payload)
 	}
 
 	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
@@ -113,7 +114,7 @@ func BulkWriter(cfg ynabber.Config, t []ynabber.Transaction) error {
 
 	if cfg.Debug {
 		b, _ := httputil.DumpResponse(res, true)
-		log.Printf("Response: %s\n", b)
+		log.Printf("Response from YNAB: %s\n", b)
 	}
 
 	if res.StatusCode != http.StatusCreated {

ynabber.go

@@ -2,9 +2,7 @@ package ynabber
 
 import (
 	"encoding/json"
-	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -54,18 +52,6 @@ func IDFromString(id string) uuid.UUID {
 	return x
 }
 
-// Parsed removes all non-alphanumeric characters and elements of strips from p
-func (p Payee) Parsed(strips []string) (string, error) {
-	reg := regexp.MustCompile(`[^\p{L}]+`)
-	x := reg.ReplaceAllString(string(p), " ")
-
-	for _, strip := range strips {
-		x = strings.ReplaceAll(x, strip, "")
-	}
-
-	return strings.TrimSpace(x), nil
-}
-
 func (m Milliunits) String() string {
 	return strconv.FormatInt(int64(m), 10)
 }

ynabber_test.go

@@ -29,29 +29,3 @@ func TestMilliunitsFromAmount(t *testing.T) {
 		t.Fatalf("amount with no separator: %s != %s", want, got)
 	}
 }
-
-func TestPayeeParsed(t *testing.T) {
-	want := "Im just alphanumeric"
-	got, _ := Payee("Im just alphanumeric").Parsed([]string{})
-	if want != got {
-		t.Fatalf("alphanumeric: %s != %s", want, got)
-	}
-
-	want = "你好世界"
-	got, _ = Payee("你好世界").Parsed([]string{})
-	if want != got {
-		t.Fatalf("non-english: %s != %s", want, got)
-	}
-
-	want = "Im here"
-	got, _ = Payee("Im not here").Parsed([]string{"not "})
-	if want != got {
-		t.Fatalf("strip words: %s != %s", want, got)
-	}
-
-	want = "Im not j ust alphanumeric"
-	got, _ = Payee("Im! not j.ust alphanumeric42 69").Parsed([]string{})
-	if want != got {
-		t.Fatalf("non-alphanumeric: %s != %s", want, got)
-	}
-}