From ea8f4963b7884b487d65dfaada276d15ed79d7cb Mon Sep 17 00:00:00 2001 From: Benjamin Cane Date: Sun, 22 Oct 2023 17:44:28 -0700 Subject: [PATCH] Fixing minor item on csv parser --- functions/src/init/go.mod | 6 +++++- functions/src/init/go.sum | 4 ++++ functions/src/init/main.go | 16 +++++++++++++++- go.mod | 10 +++++++++- go.sum | 8 ++++++++ pkg/airport/parsers/csv/csv.go | 5 +---- 6 files changed, 42 insertions(+), 7 deletions(-) diff --git a/functions/src/init/go.mod b/functions/src/init/go.mod index 06633fd..a6931e2 100644 --- a/functions/src/init/go.mod +++ b/functions/src/init/go.mod @@ -2,9 +2,13 @@ module github.com/tarmac-project/example-airport-lookup-go/functions/src/init go 1.21 -require github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 +require ( + github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e + github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 +) require ( + github.com/enescakir/emoji v1.0.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect ) diff --git a/functions/src/init/go.sum b/functions/src/init/go.sum index 7a604da..650f2c6 100644 --- a/functions/src/init/go.sum +++ b/functions/src/init/go.sum @@ -1,5 +1,9 @@ +github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= +github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e h1:YK8n/1grlaZra2bEDFg3xfatS5lB7Jx3JyqRSYb+TFI= +github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e/go.mod h1:NM0cKjAxO2xObefEpFZGgnu5g8iVBDk/xnjysv3uHeU= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= diff --git a/functions/src/init/main.go b/functions/src/init/main.go index 2fdd5e2..d558dba 100644 --- a/functions/src/init/main.go +++ b/functions/src/init/main.go @@ -1,8 +1,10 @@ package main import ( + "bytes" "fmt" "github.com/tarmac-project/tarmac/pkg/sdk" + "github.com/tarmac-project/example-airport-lookup-go/pkg/airport/parsers/csv" ) type Function struct { @@ -13,13 +15,25 @@ func (f *Function) Handler(_ []byte) ([]byte, error) { f.tarmac.Logger.Info("Airport raw data download starting") // Fetch the airport data - _, err := f.tarmac.Function.Call("fetch", []byte("")) + data, err := f.tarmac.Function.Call("fetch", []byte("")) if err != nil { f.tarmac.Logger.Error(fmt.Sprintf("Failed to fetch airport data", err)) return []byte(""), fmt.Errorf("Failed to fetch airport data: %s", err) } // Parse the data + parser, err := csv.New(bytes.NewReader(data)) + if err != nil { + f.tarmac.Logger.Error(fmt.Sprintf("Failed to create CSV parser", err)) + return []byte(""), fmt.Errorf("Failed to create CSV parser: %s", err) + } + + airports, err := parser.Parse() + if err != nil { + f.tarmac.Logger.Error(fmt.Sprintf("Failed to parse airport data", err)) + return []byte(""), fmt.Errorf("Failed to parse airport data: %s", err) + } + f.tarmac.Logger.Info(fmt.Sprintf("Fetched %d airports", len(airports))) // Update the database diff --git a/go.mod b/go.mod index aec1ef3..601869f 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,12 @@ module github.com/tarmac-project/example-airport-lookup-go go 1.21 -require github.com/enescakir/emoji v1.0.0 +require ( + github.com/enescakir/emoji v1.0.0 + github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 +) + +require ( + github.com/valyala/fastjson v1.6.4 // indirect + github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect +) diff --git a/go.sum b/go.sum index 96e3ba7..f081014 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,10 @@ github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ= +github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/wapc/wapc-guest-tinygo v0.3.3 h1:jLebiwjVSHLGnS+BRabQ6+XOV7oihVWAc05Hf1SbeR0= +github.com/wapc/wapc-guest-tinygo v0.3.3/go.mod h1:mzM3CnsdSYktfPkaBdZ8v88ZlfUDEy5Jh5XBOV3fYcw= diff --git a/pkg/airport/parsers/csv/csv.go b/pkg/airport/parsers/csv/csv.go index 39a9ab0..0470b36 100644 --- a/pkg/airport/parsers/csv/csv.go +++ b/pkg/airport/parsers/csv/csv.go @@ -48,11 +48,8 @@ func (p *Parser) Parse() ([]airport.Airport, error) { // Convert the record to an Airport a, err := RecordToAirport(rec) if err != nil { - // Skip headers and records with not enough fields - if err == ErrIsHeader || err == ErrNotEnoughFields { + // Skip any record level errors continue - } - return nil, fmt.Errorf("unable to convert record to airport - %w", err) } // Append the Airport to the slice airports = append(airports, a)