Create ApiSpecDoc event listener.

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     hooks:
       - id: golangci-lint
   - repo: https://github.com/zricethezav/gitleaks
-    rev: v8.10.3
+    rev: v8.11.0
     hooks:
       - id: gitleaks
 

cmd/main.go

@@ -1,8 +1,11 @@
 package main
 
-import "fmt"
+import (
+	"os"
+
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal"
+)
 
 func main() {
-	// Feel free to delete this file.
-	fmt.Println("Hello Gophers")
+	os.Exit(internal.Start())
 }

docker/docker-compose-dev.yml

@@ -0,0 +1,40 @@
+version: '3.9'
+
+volumes:
+  rabbit-data:
+    driver: local
+
+services:
+  rabbit:
+    image: rabbitmq:3-management #you may open management UI via http://localhost:15672/#/ login&password == guest
+    container_name: rabbit
+    #hostname required here to work with the volume on persistent queues.
+    #Rabbit saves data by folders whose names are generated from the host. To have data restored on container restart we need to commit the host.
+    hostname: rabbit
+    volumes:
+      - rabbit-data:/var/lib/rabbitmq
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+
+  data-scraping-service:
+    container_name: dss
+    build:
+      context: ../../api-hub_data-scraping-service
+      dockerfile: Dockerfile
+    environment:
+      QUEUE_URL: amqp://guest:guest@rabbit:5672/
+    restart: unless-stopped
+    depends_on:
+      - rabbit
+
+  storage-and-update-service:
+    container_name: sus
+    build:
+      context: ../.
+      dockerfile: Dockerfile
+    environment:
+      QUEUE_URL: amqp://guest:guest@rabbit:5672/
+    restart: unless-stopped
+    depends_on:
+      - rabbit

go.mod

@@ -1,3 +1,21 @@
-module github.com/rog-golang-buddies
+module github.com/rog-golang-buddies/api-hub_storage-and-update-service
 
 go 1.18
+
+require (
+	github.com/golang/mock v1.6.0
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/rog-golang-buddies/api_hub_common v0.0.3
+	github.com/stretchr/testify v1.8.0
+	github.com/wagslane/go-rabbitmq v0.10.0
+	go.uber.org/zap v1.23.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rabbitmq/amqp091-go v1.4.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,70 @@
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rabbitmq/amqp091-go v1.4.0 h1:T2G+J9W9OY4p64Di23J6yH7tOkMocgnESvYeBjuG9cY=
+github.com/rabbitmq/amqp091-go v1.4.0/go.mod h1:JsV0ofX5f1nwOGafb8L5rBItt9GyhfQfcJj+oyz0dGg=
+github.com/rog-golang-buddies/api_hub_common v0.0.3 h1:ABg+GOuVfhjL5x3yjqBZRVvWnJ9izs/yS1iARqVi7s0=
+github.com/rog-golang-buddies/api_hub_common v0.0.3/go.mod h1:9qNVWp7RgAcS7xVtPbALrWIYXh0DWKZakw7tWo2sE6k=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/wagslane/go-rabbitmq v0.10.0 h1:y9Bw8Q/9gOvsHfjMOGQjCW3033aYTKabxDm8eyjUGjs=
+github.com/wagslane/go-rabbitmq v0.10.0/go.mod h1:u6xM1V7OO4D0szUy/F6Bya/9r0lLae/2FXBijkAQmn0=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app.go

@@ -0,0 +1,67 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal/config"
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal/logger"
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal/queue"
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal/queue/handler"
+	"github.com/rog-golang-buddies/api-hub_storage-and-update-service/internal/queue/publisher"
+)
+
+func Start() int {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	conf, err := config.ReadConfig() //read configuration from env
+	if err != nil {
+		fmt.Println("error while reading configuration")
+		return 1
+	}
+	log, err := logger.NewLogger(conf)
+	if err != nil {
+		fmt.Println("error creating logger: ", err)
+		return 1
+	}
+
+	//initialize publisher connection to the queue
+	//this library assumes using one publisher and one consumer per application
+	//https://github.com/wagslane/go-rabbitmq/issues/79
+	pub, err := publisher.NewPublisher(conf.Queue, log)
+	if err != nil {
+		log.Error("error while starting publisher: ", err)
+		return 1
+	}
+	defer publisher.ClosePublisher(pub, log)
+	//initialize consumer connection to the queue
+	consumer, err := queue.NewConsumer(conf.Queue, log)
+	if err != nil {
+		log.Error("error while connecting to the queue: ", err)
+		return 1
+	}
+	defer queue.CloseConsumer(consumer, log)
+
+	handl := handler.NewApiSpecDocHandler(pub, conf.Queue, log)
+	listener := queue.NewListener()
+	err = listener.Start(ctx, consumer, &conf.Queue, handl)
+	if err != nil {
+		log.Error("error while listening queue ", err)
+		return 1
+	}
+
+	<-ctx.Done()
+
+	log.Info("application stopped gracefully (not)")
+	return 0
+}
+
+// func createDefaultProcessor(log logger.Logger, config *config.ApplicationConfig) (process.UrlProcessor, error) {
+// 	recognizer := recognize.NewRecognizer(log)
+// 	parsers := []parse.Parser{openapi.NewOpenApi(log)}
+// 	converter := parse.NewConverter(log, parsers)
+// 	loader := load.NewContentLoader(log, &config.Web)
+
+// return process.NewProcessor(recognizer, converter, loader)
+// }

internal/config/application.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/kelseyhightower/envconfig"
+)
+
+type ApplicationConfig struct {
+	Env    Environment `default:"dev"`
+	Logger LoggerConfig
+	Queue  QueueConfig
+}
+
+// ReadConfig reads configuration from the environment and populates the structure with it
+func ReadConfig() (*ApplicationConfig, error) {
+	var conf ApplicationConfig
+	if err := envconfig.Process("", &conf); err != nil {
+		return nil, err
+	}
+	fmt.Printf("conf: %+v\n", conf)
+	return &conf, nil
+}

internal/config/application_test.go

@@ -0,0 +1,38 @@
+package config
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadConfig(t *testing.T) {
+	expUrlRequestQueue := "url_request_queue"
+	err := os.Setenv("QUEUE_URL_REQUEST_QUEUE", expUrlRequestQueue)
+	assert.Nil(t, err)
+
+	expScrResQueue := "scraping_res_queue"
+	err = os.Setenv("QUEUE_SCRAPING_RESULT_QUEUE", expScrResQueue)
+	assert.Nil(t, err)
+
+	expNotificationQueue := "test_notifications"
+	err = os.Setenv("QUEUE_NOTIFICATION_QUEUE", expNotificationQueue)
+	assert.Nil(t, err)
+
+	expQueueUrl := "test_url"
+	err = os.Setenv("QUEUE_URL", expQueueUrl)
+	assert.Nil(t, err)
+
+	expQueueConcurrency := 50
+	err = os.Setenv("QUEUE_CONCURRENCY", strconv.Itoa(expQueueConcurrency))
+	assert.Nil(t, err)
+
+	conf, err := ReadConfig()
+	assert.Nil(t, err)
+	assert.Equal(t, expScrResQueue, conf.Queue.SaveASDRequestQueue)
+	assert.Equal(t, expNotificationQueue, conf.Queue.NotificationQueue)
+	assert.Equal(t, expQueueUrl, conf.Queue.Url)
+	assert.Equal(t, expQueueConcurrency, conf.Queue.Concurrency)
+}

internal/config/logger.go

@@ -0,0 +1,6 @@
+package config
+
+// LoggerConfig represents configuration of the logger
+type LoggerConfig struct {
+	Level LoggerLevel `default:"info"` //Level of minimum logging
+}

internal/config/queue.go

@@ -0,0 +1,13 @@
+package config
+
+// QueueConfig queue configuration
+type QueueConfig struct {
+	//SaveASDRequestQueue represents a queue name to send processed ApiSpecDoc
+	SaveASDRequestQueue string `default:"storage-update-asd" envconfig:"SCRAPING_RESULT_QUEUE"`
+	//NotificationQueue represents a queue name to notify a user about an error or success (if required)
+	NotificationQueue string `default:"gateway-scrape-notifications" envconfig:"NOTIFICATION_QUEUE"`
+	//Url is a RabbitMQ url
+	Url string `default:"amqp://guest:guest@localhost:5672/"`
+	//Concurrency represents number of parallel handlers
+	Concurrency int `default:"30"`
+}

internal/config/types.go

@@ -0,0 +1,40 @@
+package config
+
+import "go.uber.org/zap/zapcore"
+
+// Environment defines the application environment to adjust settings to it.
+type Environment string
+
+const (
+	Dev  Environment = "dev"
+	Prod Environment = "prod"
+)
+
+// LoggerLevel defines the minimum logging level to process
+type LoggerLevel string
+
+const (
+	TraceLevel LoggerLevel = "trace"
+	DebugLevel LoggerLevel = "debug"
+	InfoLevel  LoggerLevel = "info"
+	WarnLevel  LoggerLevel = "warn"
+	ErrorLevel LoggerLevel = "error"
+	PanicLevel LoggerLevel = "panic"
+)
+
+func (ll LoggerLevel) ToZapLevel() zapcore.Level {
+	switch ll {
+	case TraceLevel, DebugLevel:
+		return zapcore.DebugLevel
+	case InfoLevel:
+		return zapcore.InfoLevel
+	case WarnLevel:
+		return zapcore.WarnLevel
+	case ErrorLevel:
+		return zapcore.ErrorLevel
+	case PanicLevel:
+		return zapcore.PanicLevel
+	default:
+		return zapcore.InfoLevel
+	}
+}

internal/dto/scrapingResult.go

@@ -0,0 +1,9 @@
+package dto
+
+import "github.com/rog-golang-buddies/api_hub_common/apispecdoc"
+
+type ScrapingResult struct {
+	IsNotifyUser bool
+
+	ApiSpecDoc *apispecdoc.ApiSpecDoc
+}