diff --git a/Makefile b/Makefile index 0b96297..0aa6296 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ build: ## Run TinyGo build via Docker because its easier docker run --rm -v `pwd`:/build -w /build/functions/build/init tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/init.wasm -target wasi /build/functions/src/init/main.go docker run --rm -v `pwd`:/build -w /build/functions/build/data/fetch tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/fetch.wasm -target wasi /build/functions/src/data/fetch/main.go + docker run --rm -v `pwd`:/build -w /build/functions/build/data/load tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/load.wasm -target wasi /build/functions/src/data/load/main.go .PHONY: tests tests: @@ -15,6 +16,12 @@ tests: go tool cover -html=coverage/coverage.out -o coverage/coverage.html docker-compose: - docker compose up + docker compose up -d mysql + docker compose up airport-lookup-example run: build docker-compose +run-nobuild: docker-compose + +clean: + rm -rf functions/build + docker compose down --remove-orphans diff --git a/config/tarmac.json b/config/tarmac.json index 02da565..4d1d45c 100644 --- a/config/tarmac.json +++ b/config/tarmac.json @@ -6,6 +6,9 @@ "init": { "filepath": "/functions/init.wasm" }, + "load": { + "filepath": "/functions/load.wasm" + }, "fetch": { "filepath": "/functions/fetch.wasm" } @@ -15,6 +18,15 @@ "type": "init", "function": "init" }, + { + "type": "scheduled_task", + "function": "load", + "frequency": 900 + }, + { + "type": "function", + "function": "load" + }, { "type": "function", "function": "fetch" diff --git a/docker-compose.yml b/docker-compose.yml index 4e01c82..a484a0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,28 @@ version: '3.8' services: airport-lookup-example: - image: tarmac:local + image: madflojo/tarmac:unstable ports: - 80:8080 environment: - "APP_ENABLE_TLS=false" - "APP_LISTEN_ADDR=0.0.0.0:8080" - - "APP_DEBUG=true" - - "APP_TRACE=true" + - "APP_DEBUG=false" + - "APP_TRACE=false" - "APP_WASM_FUNCTION_CONFIG=/config/tarmac.json" + - "APP_ENABLE_SQL=true" + - "APP_SQL_TYPE=mysql" + - "APP_SQL_DSN=root:example@tcp(mysql:3306)/example" volumes: - "./config:/config" - "./functions/build:/functions" + depends_on: + - mysql + mysql: + image: bitnami/mysql:latest + restart: always + environment: + MYSQL_ROOT_PASSWORD: example + MYSQL_DATABASE: example + ports: + - 3306:3306 diff --git a/functions/src/data/fetch/main.go b/functions/src/data/fetch/main.go index ddaf22d..cb7d401 100644 --- a/functions/src/data/fetch/main.go +++ b/functions/src/data/fetch/main.go @@ -10,12 +10,12 @@ type Function struct { } func (f *Function) Handler(_ []byte) ([]byte, error) { - f.tarmac.Logger.Debug("Fetch function initiated, Downloading airports.csv") + f.tarmac.Logger.Info("Downloading airports.csv") rsp, err := f.tarmac.HTTP.Get("https://raw.githubusercontent.com/davidmegginson/ourairports-data/main/airports.csv") if err != nil { return []byte(""), fmt.Errorf("failed to get airports.csv: %w", err) } - f.tarmac.Logger.Trace(fmt.Sprintf("airports.csv downloaded with return code: %d", rsp.StatusCode)) + f.tarmac.Logger.Info(fmt.Sprintf("airports.csv downloaded with return code: %d", rsp.StatusCode)) if rsp.StatusCode >= 299 { f.tarmac.Logger.Error(fmt.Sprintf("airports.csv download failed with return code: %d", rsp.StatusCode)) diff --git a/functions/src/data/load/go.mod b/functions/src/data/load/go.mod new file mode 100644 index 0000000..6d7c2ac --- /dev/null +++ b/functions/src/data/load/go.mod @@ -0,0 +1,14 @@ +module github.com/tarmac-project/example-airport-lookup-go/functions/src/data/load + +go 1.21.1 + +require ( + github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f + github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 +) + +require ( + github.com/enescakir/emoji v1.0.0 // indirect + github.com/valyala/fastjson v1.6.4 // indirect + github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect +) diff --git a/functions/src/data/load/go.sum b/functions/src/data/load/go.sum new file mode 100644 index 0000000..dcfd35b --- /dev/null +++ b/functions/src/data/load/go.sum @@ -0,0 +1,12 @@ +github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= +github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f h1:XhPXDWSza1ae0WBqq3iwRGR5N2/x5W0gFamNULathXM= +github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f/go.mod h1:mpM+9904v8UU6HzL/giS7yLzuGvezbDQvdKw6byxO80= +github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ= +github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/wapc/wapc-guest-tinygo v0.3.3 h1:jLebiwjVSHLGnS+BRabQ6+XOV7oihVWAc05Hf1SbeR0= +github.com/wapc/wapc-guest-tinygo v0.3.3/go.mod h1:mzM3CnsdSYktfPkaBdZ8v88ZlfUDEy5Jh5XBOV3fYcw= diff --git a/functions/src/data/load/main.go b/functions/src/data/load/main.go new file mode 100644 index 0000000..e391176 --- /dev/null +++ b/functions/src/data/load/main.go @@ -0,0 +1,126 @@ +package main + +import ( + "bytes" + "fmt" + "github.com/tarmac-project/example-airport-lookup-go/pkg/airport/parsers/csv" + "github.com/tarmac-project/tarmac/pkg/sdk" + "html" +) + +type Function struct { + tarmac *sdk.Tarmac +} + +func (f *Function) Handler(_ []byte) ([]byte, error) { + f.tarmac.Logger.Info("Airport raw data download starting") + + // Fetch the airport data + data, err := f.tarmac.Function.Call("fetch", []byte("")) + if err != nil { + f.tarmac.Logger.Error(fmt.Sprintf("Failed to fetch airport data - %s", err)) + return []byte(""), fmt.Errorf("Failed to fetch airport data: %s", err) + } + + f.tarmac.Logger.Info("Airport raw data download complete, parsing data") + + // Parse the data + parser, err := csv.New(bytes.NewReader(data)) + if err != nil { + f.tarmac.Logger.Error(fmt.Sprintf("Failed to create CSV parser - %s", err)) + return []byte(""), fmt.Errorf("Failed to create CSV parser: %s", err) + } + + airports, err := parser.Parse() + if err != nil { + f.tarmac.Logger.Error(fmt.Sprintf("Failed to parse airport data - %s", err)) + return []byte(""), fmt.Errorf("Failed to parse airport data: %s", err) + } + f.tarmac.Logger.Info(fmt.Sprintf("Fetched %d airports", len(airports))) + + // Update the database + success := 0 + failure := 0 + for _, airport := range airports { + query := fmt.Sprintf(`INSERT INTO airports ( + local_code, + name, + type, + type_emoji, + continent, + iso_country, + iso_region, + municipality, + emoji, + status + ) VALUES ( + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s') + ON DUPLICATE KEY UPDATE + name = '%s', + type = '%s', + type_emoji = '%s', + continent = '%s', + iso_country = '%s', + iso_region = '%s', + municipality = '%s', + emoji = '%s', + status = '%s';`, + html.EscapeString(airport.LocalCode), + html.EscapeString(airport.Name), + html.EscapeString(airport.Type), + html.EscapeString(airport.TypeEmoji), + html.EscapeString(airport.Continent), + html.EscapeString(airport.ISOCountry), + html.EscapeString(airport.ISORegion), + html.EscapeString(airport.Municipality), + html.EscapeString(airport.Emoji), + html.EscapeString(airport.Status), + html.EscapeString(airport.Name), + html.EscapeString(airport.Type), + html.EscapeString(airport.TypeEmoji), + html.EscapeString(airport.Continent), + html.EscapeString(airport.ISOCountry), + html.EscapeString(airport.ISORegion), + html.EscapeString(airport.Municipality), + html.EscapeString(airport.Emoji), + html.EscapeString(airport.Status), + ) + f.tarmac.Logger.Trace(fmt.Sprintf("Executing query: %s", query)) + + _, err := f.tarmac.SQL.Query(query) + if err != nil { + f.tarmac.Logger.Debug(fmt.Sprintf("Failed to execute query - %s", err)) + failure++ + continue + } + success++ + } + f.tarmac.Logger.Info(fmt.Sprintf("Executed %d queries successfully, %d failures", success, failure)) + + return []byte(""), nil +} + +func main() { + var err error + + // Initialize Function + f := &Function{} + + // Initialize the Tarmac SDK + f.tarmac, err = sdk.New(sdk.Config{ + Namespace: "airport-lookup", + Handler: f.Handler, + }) + if err != nil { + return + } +} diff --git a/functions/src/init/go.mod b/functions/src/init/go.mod index a6931e2..745e685 100644 --- a/functions/src/init/go.mod +++ b/functions/src/init/go.mod @@ -1,14 +1,10 @@ module github.com/tarmac-project/example-airport-lookup-go/functions/src/init -go 1.21 +go 1.21.1 -require ( - github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e - github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 -) +require github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 require ( - github.com/enescakir/emoji v1.0.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect ) diff --git a/functions/src/init/go.sum b/functions/src/init/go.sum index 650f2c6..7a604da 100644 --- a/functions/src/init/go.sum +++ b/functions/src/init/go.sum @@ -1,9 +1,5 @@ -github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= -github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= -github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e h1:YK8n/1grlaZra2bEDFg3xfatS5lB7Jx3JyqRSYb+TFI= -github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e/go.mod h1:NM0cKjAxO2xObefEpFZGgnu5g8iVBDk/xnjysv3uHeU= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= diff --git a/functions/src/init/main.go b/functions/src/init/main.go index 5cf185e..cb101a7 100644 --- a/functions/src/init/main.go +++ b/functions/src/init/main.go @@ -1,9 +1,7 @@ package main import ( - "bytes" "fmt" - "github.com/tarmac-project/example-airport-lookup-go/pkg/airport/parsers/csv" "github.com/tarmac-project/tarmac/pkg/sdk" ) @@ -12,30 +10,36 @@ type Function struct { } func (f *Function) Handler(_ []byte) ([]byte, error) { - f.tarmac.Logger.Info("Airport raw data download starting") - - // Fetch the airport data - data, err := f.tarmac.Function.Call("fetch", []byte("")) + f.tarmac.Logger.Info("Initializing Airport Lookup Service") + + // Create MySQL Database structure + query := `CREATE TABLE IF NOT EXISTS airports ( + local_code VARCHAR(4) NOT NULL UNIQUE, + name VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + type_emoji VARCHAR(255), + continent VARCHAR(255), + iso_country VARCHAR(255) NOT NULL, + iso_region VARCHAR(255), + municipality VARCHAR(255), + emoji VARCHAR(255), + status VARCHAR(255), + PRIMARY KEY (local_code) + );` + _, err := f.tarmac.SQL.Query(query) if err != nil { - f.tarmac.Logger.Error(fmt.Sprintf("Failed to fetch airport data - %s", err)) - return []byte(""), fmt.Errorf("Failed to fetch airport data: %s", err) + f.tarmac.Logger.Error(fmt.Sprintf("Failed to create table - %s", err)) + return []byte(""), fmt.Errorf("Failed to create table: %s", err) } + f.tarmac.Logger.Info("Created database table") - // Parse the data - parser, err := csv.New(bytes.NewReader(data)) + // Load Airport Data + _, err = f.tarmac.Function.Call("load", []byte("")) if err != nil { - f.tarmac.Logger.Error(fmt.Sprintf("Failed to create CSV parser - %s", err)) - return []byte(""), fmt.Errorf("Failed to create CSV parser: %s", err) + f.tarmac.Logger.Error(fmt.Sprintf("Failed to load airport data - %s", err)) + return []byte(""), fmt.Errorf("Failed to load airport data: %s", err) } - - airports, err := parser.Parse() - if err != nil { - f.tarmac.Logger.Error(fmt.Sprintf("Failed to parse airport data - %s", err)) - return []byte(""), fmt.Errorf("Failed to parse airport data: %s", err) - } - f.tarmac.Logger.Info(fmt.Sprintf("Fetched %d airports", len(airports))) - - // Update the database + f.tarmac.Logger.Info("Loaded airport data") return []byte(""), nil } diff --git a/go.mod b/go.mod index 601869f..7d77cd1 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,10 @@ module github.com/tarmac-project/example-airport-lookup-go go 1.21 -require ( - github.com/enescakir/emoji v1.0.0 - github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 -) +require github.com/enescakir/emoji v1.0.0 require ( + github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect ) diff --git a/go.sum b/go.sum index f081014..f8a6176 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog= github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0= -github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= -github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ= github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=