feat: remove kafka package from utils

opengovern · Jan 9, 2024 · 9bbc473 · 9bbc473
1 parent 12f6072
commit 9bbc473
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 323 deletions.
diff --git a/pkg/es/doc.go b/pkg/es/doc.go
@@ -0,0 +1,22 @@
+package es
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+const (
+	EsIndexHeader = "elasticsearch_index"
+)
+
+type Doc interface {
+	KeysAndIndex() ([]string, string)
+}
+
+func HashOf(strings ...string) string {
+	h := sha256.New()
+	for _, s := range strings {
+		h.Write([]byte(s))
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
diff --git a/pkg/kafka/consume.go b/pkg/kafka/consume.go
diff --git a/pkg/kafka/producer.go b/pkg/kafka/producer.go
diff --git a/pkg/kafka/sender.go b/pkg/kafka/sender.go
diff --git a/pkg/pipeline/sender.go b/pkg/pipeline/sender.go
@@ -7,14 +7,13 @@ import (
 	"fmt"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/kaytu-io/kaytu-util/pkg/kafka"
 	"io"
 	"net/http"
 	"strings"
 	"time"
 )
 
-func sendToPipelineIndividually(ingestionPipelineEndpoint string, resourcesToSend []kafka.Doc) error {
+func sendToPipelineIndividually(ingestionPipelineEndpoint string, resourcesToSend []doc.Doc) error {
 	httpClient := &http.Client{Timeout: 10 * time.Second}
 	if len(resourcesToSend) == 0 {
 		return nil
@@ -72,7 +71,7 @@ func sendToPipelineIndividually(ingestionPipelineEndpoint string, resourcesToSen
 	return nil
 }
 
-func SendToPipeline(ingestionPipelineEndpoint string, resourcesToSend []kafka.Doc) error {
+func SendToPipeline(ingestionPipelineEndpoint string, resourcesToSend []doc.Doc) error {
 	httpClient := &http.Client{Timeout: 10 * time.Second}
 	if len(resourcesToSend) == 0 {
 		return nil