Merge pull request #3 from madflojo/parsecsv

Adds Database SQL load

Makefile

@@ -6,6 +6,7 @@ build:
 	## Run TinyGo build via Docker because its easier
 	docker run --rm -v `pwd`:/build -w /build/functions/build/init tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/init.wasm -target wasi /build/functions/src/init/main.go
 	docker run --rm -v `pwd`:/build -w /build/functions/build/data/fetch tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/fetch.wasm -target wasi /build/functions/src/data/fetch/main.go
+	docker run --rm -v `pwd`:/build -w /build/functions/build/data/load tinygo/tinygo:0.25.0 tinygo build -o /build/functions/build/load.wasm -target wasi /build/functions/src/data/load/main.go
 
 .PHONY: tests
 tests:
@@ -15,6 +16,12 @@ tests:
 	go tool cover -html=coverage/coverage.out -o coverage/coverage.html
 
 docker-compose:
-	docker compose up
+	docker compose up -d mysql
+	docker compose up airport-lookup-example
 
 run: build docker-compose
+run-nobuild: docker-compose
+
+clean:
+	rm -rf functions/build
+	docker compose down --remove-orphans

config/tarmac.json

@@ -6,6 +6,9 @@
                 "init": {
                     "filepath": "/functions/init.wasm"
                 },
+                "load": {
+                    "filepath": "/functions/load.wasm"
+                },
                 "fetch": {
                     "filepath": "/functions/fetch.wasm"
                 }
@@ -15,6 +18,15 @@
                     "type": "init",
                     "function": "init"
                 },
+                {
+                    "type": "scheduled_task",
+                    "function": "load",
+                    "frequency": 900
+                },
+                {
+                    "type": "function",
+                    "function": "load"
+                },
                 {
                     "type": "function",
                     "function": "fetch"

docker-compose.yml

@@ -1,15 +1,28 @@
 version: '3.8'
 services:
   airport-lookup-example:
-    image: tarmac:local
+    image: madflojo/tarmac:unstable
     ports:
       - 80:8080
     environment:
       - "APP_ENABLE_TLS=false"
       - "APP_LISTEN_ADDR=0.0.0.0:8080"
-      - "APP_DEBUG=true"
-      - "APP_TRACE=true"
+      - "APP_DEBUG=false"
+      - "APP_TRACE=false"
       - "APP_WASM_FUNCTION_CONFIG=/config/tarmac.json"
+      - "APP_ENABLE_SQL=true"
+      - "APP_SQL_TYPE=mysql"
+      - "APP_SQL_DSN=root:example@tcp(mysql:3306)/example"
     volumes:
       - "./config:/config"
       - "./functions/build:/functions"
+    depends_on:
+      - mysql
+  mysql:
+    image: bitnami/mysql:latest
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: example
+      MYSQL_DATABASE: example
+    ports:
+      - 3306:3306

functions/src/data/fetch/main.go

@@ -10,12 +10,12 @@ type Function struct {
 }
 
 func (f *Function) Handler(_ []byte) ([]byte, error) {
-	f.tarmac.Logger.Debug("Fetch function initiated, Downloading airports.csv")
+	f.tarmac.Logger.Info("Downloading airports.csv")
 	rsp, err := f.tarmac.HTTP.Get("https://raw.githubusercontent.com/davidmegginson/ourairports-data/main/airports.csv")
 	if err != nil {
 		return []byte(""), fmt.Errorf("failed to get airports.csv: %w", err)
 	}
-	f.tarmac.Logger.Trace(fmt.Sprintf("airports.csv downloaded with return code: %d", rsp.StatusCode))
+	f.tarmac.Logger.Info(fmt.Sprintf("airports.csv downloaded with return code: %d", rsp.StatusCode))
 
 	if rsp.StatusCode >= 299 {
 		f.tarmac.Logger.Error(fmt.Sprintf("airports.csv download failed with return code: %d", rsp.StatusCode))

functions/src/data/load/go.mod

@@ -0,0 +1,14 @@
+module github.com/tarmac-project/example-airport-lookup-go/functions/src/data/load
+
+go 1.21.1
+
+require (
+	github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f
+	github.com/tarmac-project/tarmac/pkg/sdk v0.5.0
+)
+
+require (
+	github.com/enescakir/emoji v1.0.0 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
+	github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect
+)

functions/src/data/load/go.sum

@@ -0,0 +1,12 @@
+github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog=
+github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0=
+github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20=
+github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f h1:XhPXDWSza1ae0WBqq3iwRGR5N2/x5W0gFamNULathXM=
+github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023010400-e4b3efdea82f/go.mod h1:mpM+9904v8UU6HzL/giS7yLzuGvezbDQvdKw6byxO80=
+github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ=
+github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
+github.com/wapc/wapc-guest-tinygo v0.3.3 h1:jLebiwjVSHLGnS+BRabQ6+XOV7oihVWAc05Hf1SbeR0=
+github.com/wapc/wapc-guest-tinygo v0.3.3/go.mod h1:mzM3CnsdSYktfPkaBdZ8v88ZlfUDEy5Jh5XBOV3fYcw=

functions/src/data/load/main.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/tarmac-project/example-airport-lookup-go/pkg/airport/parsers/csv"
+	"github.com/tarmac-project/tarmac/pkg/sdk"
+	"html"
+)
+
+type Function struct {
+	tarmac *sdk.Tarmac
+}
+
+func (f *Function) Handler(_ []byte) ([]byte, error) {
+	f.tarmac.Logger.Info("Airport raw data download starting")
+
+	// Fetch the airport data
+	data, err := f.tarmac.Function.Call("fetch", []byte(""))
+	if err != nil {
+		f.tarmac.Logger.Error(fmt.Sprintf("Failed to fetch airport data - %s", err))
+		return []byte(""), fmt.Errorf("Failed to fetch airport data: %s", err)
+	}
+
+	f.tarmac.Logger.Info("Airport raw data download complete, parsing data")
+
+	// Parse the data
+	parser, err := csv.New(bytes.NewReader(data))
+	if err != nil {
+		f.tarmac.Logger.Error(fmt.Sprintf("Failed to create CSV parser - %s", err))
+		return []byte(""), fmt.Errorf("Failed to create CSV parser: %s", err)
+	}
+
+	airports, err := parser.Parse()
+	if err != nil {
+		f.tarmac.Logger.Error(fmt.Sprintf("Failed to parse airport data - %s", err))
+		return []byte(""), fmt.Errorf("Failed to parse airport data: %s", err)
+	}
+	f.tarmac.Logger.Info(fmt.Sprintf("Fetched %d airports", len(airports)))
+
+	// Update the database
+	success := 0
+	failure := 0
+	for _, airport := range airports {
+		query := fmt.Sprintf(`INSERT INTO airports (
+      local_code,
+      name,
+      type,
+      type_emoji,
+      continent,
+      iso_country,
+      iso_region,
+      municipality,
+      emoji,
+      status
+    ) VALUES (
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s',
+      '%s')
+    ON DUPLICATE KEY UPDATE
+      name = '%s',
+      type = '%s',
+      type_emoji = '%s',
+      continent = '%s',
+      iso_country = '%s',
+      iso_region = '%s',
+      municipality = '%s',
+      emoji = '%s',
+      status = '%s';`,
+			html.EscapeString(airport.LocalCode),
+			html.EscapeString(airport.Name),
+			html.EscapeString(airport.Type),
+			html.EscapeString(airport.TypeEmoji),
+			html.EscapeString(airport.Continent),
+			html.EscapeString(airport.ISOCountry),
+			html.EscapeString(airport.ISORegion),
+			html.EscapeString(airport.Municipality),
+			html.EscapeString(airport.Emoji),
+			html.EscapeString(airport.Status),
+			html.EscapeString(airport.Name),
+			html.EscapeString(airport.Type),
+			html.EscapeString(airport.TypeEmoji),
+			html.EscapeString(airport.Continent),
+			html.EscapeString(airport.ISOCountry),
+			html.EscapeString(airport.ISORegion),
+			html.EscapeString(airport.Municipality),
+			html.EscapeString(airport.Emoji),
+			html.EscapeString(airport.Status),
+		)
+		f.tarmac.Logger.Trace(fmt.Sprintf("Executing query: %s", query))
+
+		_, err := f.tarmac.SQL.Query(query)
+		if err != nil {
+			f.tarmac.Logger.Debug(fmt.Sprintf("Failed to execute query - %s", err))
+			failure++
+			continue
+		}
+		success++
+	}
+	f.tarmac.Logger.Info(fmt.Sprintf("Executed %d queries successfully, %d failures", success, failure))
+
+	return []byte(""), nil
+}
+
+func main() {
+	var err error
+
+	// Initialize Function
+	f := &Function{}
+
+	// Initialize the Tarmac SDK
+	f.tarmac, err = sdk.New(sdk.Config{
+		Namespace: "airport-lookup",
+		Handler:   f.Handler,
+	})
+	if err != nil {
+		return
+	}
+}

functions/src/init/go.mod

@@ -1,14 +1,10 @@
 module github.com/tarmac-project/example-airport-lookup-go/functions/src/init
 
-go 1.21
+go 1.21.1
 
-require (
-	github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e
-	github.com/tarmac-project/tarmac/pkg/sdk v0.5.0
-)
+require github.com/tarmac-project/tarmac/pkg/sdk v0.5.0
 
 require (
-	github.com/enescakir/emoji v1.0.0 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect
 )

functions/src/init/go.sum

@@ -1,9 +1,5 @@
-github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog=
-github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0=
 github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20=
 github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
-github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e h1:YK8n/1grlaZra2bEDFg3xfatS5lB7Jx3JyqRSYb+TFI=
-github.com/tarmac-project/example-airport-lookup-go v0.0.0-20231023001804-82de5425cb1e/go.mod h1:NM0cKjAxO2xObefEpFZGgnu5g8iVBDk/xnjysv3uHeU=
 github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ=
 github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E=
 github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=

functions/src/init/main.go

@@ -1,9 +1,7 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"github.com/tarmac-project/example-airport-lookup-go/pkg/airport/parsers/csv"
 	"github.com/tarmac-project/tarmac/pkg/sdk"
 )
 
@@ -12,30 +10,36 @@ type Function struct {
 }
 
 func (f *Function) Handler(_ []byte) ([]byte, error) {
-	f.tarmac.Logger.Info("Airport raw data download starting")
-
-	// Fetch the airport data
-	data, err := f.tarmac.Function.Call("fetch", []byte(""))
+	f.tarmac.Logger.Info("Initializing Airport Lookup Service")
+
+	// Create MySQL Database structure
+	query := `CREATE TABLE IF NOT EXISTS airports (
+    local_code VARCHAR(4) NOT NULL UNIQUE,
+    name VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    type_emoji VARCHAR(255),
+    continent VARCHAR(255),
+    iso_country VARCHAR(255) NOT NULL,
+    iso_region VARCHAR(255),
+    municipality VARCHAR(255),
+    emoji VARCHAR(255),
+    status VARCHAR(255),
+    PRIMARY KEY (local_code)
+  );`
+	_, err := f.tarmac.SQL.Query(query)
 	if err != nil {
-		f.tarmac.Logger.Error(fmt.Sprintf("Failed to fetch airport data - %s", err))
-		return []byte(""), fmt.Errorf("Failed to fetch airport data: %s", err)
+		f.tarmac.Logger.Error(fmt.Sprintf("Failed to create table - %s", err))
+		return []byte(""), fmt.Errorf("Failed to create table: %s", err)
 	}
+	f.tarmac.Logger.Info("Created database table")
 
-	// Parse the data
-	parser, err := csv.New(bytes.NewReader(data))
+	// Load Airport Data
+	_, err = f.tarmac.Function.Call("load", []byte(""))
 	if err != nil {
-		f.tarmac.Logger.Error(fmt.Sprintf("Failed to create CSV parser - %s", err))
-		return []byte(""), fmt.Errorf("Failed to create CSV parser: %s", err)
+		f.tarmac.Logger.Error(fmt.Sprintf("Failed to load airport data - %s", err))
+		return []byte(""), fmt.Errorf("Failed to load airport data: %s", err)
 	}
-
-	airports, err := parser.Parse()
-	if err != nil {
-		f.tarmac.Logger.Error(fmt.Sprintf("Failed to parse airport data - %s", err))
-		return []byte(""), fmt.Errorf("Failed to parse airport data: %s", err)
-	}
-	f.tarmac.Logger.Info(fmt.Sprintf("Fetched %d airports", len(airports)))
-
-	// Update the database
+	f.tarmac.Logger.Info("Loaded airport data")
 
 	return []byte(""), nil
 }

go.mod

@@ -2,12 +2,10 @@ module github.com/tarmac-project/example-airport-lookup-go
 
 go 1.21
 
-require (
-	github.com/enescakir/emoji v1.0.0
-	github.com/tarmac-project/tarmac/pkg/sdk v0.5.0
-)
+require github.com/enescakir/emoji v1.0.0
 
 require (
+	github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/wapc/wapc-guest-tinygo v0.3.3 // indirect
 )

go.sum

@@ -1,7 +1,5 @@
 github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog=
 github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0=
-github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20=
-github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
 github.com/tarmac-project/tarmac/pkg/sdk v0.5.0 h1:QKsEf6SXTYrJM9/B4cNoM4RS3/rzuViJaiutEcdSRZQ=
 github.com/tarmac-project/tarmac/pkg/sdk v0.5.0/go.mod h1:UTKYV0QFdkJDgV2sJcnuCujVy49MCd8bgi2JmwviJ6E=
 github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=