feat: add github plugin (#2410)

go.mod

@@ -34,6 +34,8 @@ require (
 	github.com/google/go-github/v66 v66.0.0
 	github.com/google/uuid v1.6.0
 	github.com/haoel/downsampling v0.0.0-20221012062717-1132fe8afe24
+	github.com/hashicorp/go-hclog v1.6.3
+	github.com/hashicorp/go-plugin v1.6.0
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/jackc/pgtype v1.14.0
 	github.com/jackc/pgx/v5 v5.7.1
@@ -47,6 +49,7 @@ require (
 	github.com/opensearch-project/opensearch-go/v2 v2.3.0
 	github.com/opensearch-project/opensearch-go/v4 v4.2.0
 	github.com/ory/dockertest/v3 v3.10.0
+	github.com/pganalyze/pg_query_go/v4 v4.2.3
 	github.com/prometheus/client_golang v1.20.5
 	github.com/sashabaranov/go-openai v1.20.3
 	github.com/sony/sonyflake v1.2.0
@@ -55,6 +58,7 @@ require (
 	github.com/swaggo/echo-swagger v1.3.0
 	github.com/swaggo/swag v1.16.1
 	github.com/turbot/steampipe-plugin-sdk/v5 v5.10.1
+	github.com/zaffka/zap-to-hclog v0.10.6
 	go.opentelemetry.io/otel v1.31.0
 	go.opentelemetry.io/otel/trace v1.31.0
 	go.uber.org/zap v1.27.0
@@ -182,9 +186,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-getter v1.7.5 // indirect
-	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-plugin v1.6.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
@@ -245,7 +247,6 @@ require (
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pganalyze/pg_query_go/v4 v4.2.3 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -997,10 +997,6 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/opencontainers/runc v1.2.0 h1:qke7ZVCmJcKrJVY2iHJVC+0kql9uYdkusOPsQOOeBw4=
 github.com/opencontainers/runc v1.2.0/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc=
-github.com/opengovern/og-util v1.7.0 h1:2L3jXkasNLM+JO+vUmbmTCRf4ptjLvjDYqe/760CkFY=
-github.com/opengovern/og-util v1.7.0/go.mod h1:ob8RSaNZG3RWnLH3lDXQaVpIf+B8Ry7ds3twYlDJPp0=
-github.com/opengovern/og-util v1.7.1 h1:N2JgJWeMHGkfkGPgGd0Rr6w+Cjpbjc6+qfoidzjomFE=
-github.com/opengovern/og-util v1.7.1/go.mod h1:ob8RSaNZG3RWnLH3lDXQaVpIf+B8Ry7ds3twYlDJPp0=
 github.com/opengovern/og-util v1.7.2 h1:eLGgqSHQmYuSehRBXJcn61pD7uslCzZe9wbwJaEKbc0=
 github.com/opengovern/og-util v1.7.2/go.mod h1:ob8RSaNZG3RWnLH3lDXQaVpIf+B8Ry7ds3twYlDJPp0=
 github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ=
@@ -1209,6 +1205,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zaffka/zap-to-hclog v0.10.6 h1:dNxbL5drL6sVUDHtCMbokJLWrYn5wSKAWTXjgWFadx0=
+github.com/zaffka/zap-to-hclog v0.10.6/go.mod h1:wLqRe/Fa1MkfUY9EtnCDiz2CqhTPNZPh0/pRE9lLi04=
 github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
 github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI=

services/integration/cmd.go

@@ -99,7 +99,7 @@ func Command() *cobra.Command {
 				}
 			}
 
-			typeManager := integration_type.NewIntegrationTypeManager()
+			typeManager := integration_type.NewIntegrationTypeManager(logger)
 
 			err = IntegrationTypesMigration(logger, db, typeManager, IntegrationsJsonFilePath)
 			if err != nil {

services/integration/integration-type/integrations.go

@@ -1,6 +1,10 @@
 package integration_type
 
 import (
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-plugin"
+	"go.uber.org/zap"
+	"os/exec"
 	"strings"
 
 	"github.com/opengovern/og-util/pkg/integration"
@@ -18,8 +22,6 @@ import (
 	dopplerConfigs "github.com/opengovern/opencomply/services/integration/integration-type/doppler-account/configs"
 	"github.com/opengovern/opencomply/services/integration/integration-type/entra-id-directory"
 	entraidConfigs "github.com/opengovern/opencomply/services/integration/integration-type/entra-id-directory/configs"
-	githubaccount "github.com/opengovern/opencomply/services/integration/integration-type/github-account"
-	githubConfigs "github.com/opengovern/opencomply/services/integration/integration-type/github-account/configs"
 	google_workspace_account "github.com/opengovern/opencomply/services/integration/integration-type/google-workspace-account"
 	googleConfig "github.com/opengovern/opencomply/services/integration/integration-type/google-workspace-account/configs"
 	"github.com/opengovern/opencomply/services/integration/integration-type/interfaces"
@@ -31,13 +33,14 @@ import (
 	openaiConfigs "github.com/opengovern/opencomply/services/integration/integration-type/openai-integration/configs"
 	render "github.com/opengovern/opencomply/services/integration/integration-type/render-account"
 	renderConfigs "github.com/opengovern/opencomply/services/integration/integration-type/render-account/configs"
+	hczap "github.com/zaffka/zap-to-hclog"
 )
 
 var integrationTypes = map[integration.Type]interfaces.IntegrationType{
-	awsConfigs.IntegrationTypeAwsCloudAccount:           &aws_account.Integration{},
-	azureConfigs.IntegrationTypeAzureSubscription:       &azure_subscription.Integration{},
-	entraidConfigs.IntegrationTypeEntraidDirectory:      &entra_id_directory.Integration{},
-	githubConfigs.IntegrationTypeGithubAccount:          &githubaccount.Integration{},
+	awsConfigs.IntegrationTypeAwsCloudAccount:      &aws_account.Integration{},
+	azureConfigs.IntegrationTypeAzureSubscription:  &azure_subscription.Integration{},
+	entraidConfigs.IntegrationTypeEntraidDirectory: &entra_id_directory.Integration{},
+	//githubConfigs.IntegrationTypeGithubAccount:          &githubaccount.Integration{},
 	digitalOceanConfigs.IntegrationTypeDigitalOceanTeam: &digitalocean_team.Integration{},
 	cloudflareConfigs.IntegrationNameCloudflareAccount:  &cloudflareaccount.Integration{},
 	openaiConfigs.IntegrationTypeOpenaiIntegration:      &openaiproject.Integration{},
@@ -50,11 +53,52 @@ var integrationTypes = map[integration.Type]interfaces.IntegrationType{
 }
 
 type IntegrationTypeManager struct {
+	logger           *zap.Logger
+	hcLogger         hclog.Logger
 	IntegrationTypes map[integration.Type]interfaces.IntegrationType
 }
 
-func NewIntegrationTypeManager() *IntegrationTypeManager {
+func NewIntegrationTypeManager(logger *zap.Logger) *IntegrationTypeManager {
+	hcLogger := hczap.Wrap(logger)
+
+	// TODO Load plugins from scanning plugins folder for zip files
+	plugins := make(map[string]string)
+	plugins["github_account"] = "/plugins/github_account/integration-plugin"
+
+	for pluginName, pluginPath := range plugins {
+		client := plugin.NewClient(&plugin.ClientConfig{
+			HandshakeConfig: interfaces.HandshakeConfig,
+			Plugins:         map[string]plugin.Plugin{pluginName: &interfaces.IntegrationTypePlugin{}},
+			Cmd:             exec.Command(pluginPath),
+			Logger:          hcLogger,
+		})
+
+		rpcClient, err := client.Client()
+		if err != nil {
+			logger.Error("failed to create plugin client", zap.Error(err), zap.String("plugin", pluginName), zap.String("path", pluginPath))
+			continue
+		}
+
+		// Request the plugin
+		raw, err := rpcClient.Dispense("greeter")
+		if err != nil {
+			logger.Error("failed to dispense plugin", zap.Error(err), zap.String("plugin", pluginName), zap.String("path", pluginPath))
+			continue
+		}
+
+		// Cast the raw interface to the appropriate interface
+		itInterface, ok := raw.(interfaces.IntegrationType)
+		if !ok {
+			logger.Error("failed to cast plugin to integration type", zap.String("plugin", pluginName), zap.String("path", pluginPath))
+			continue
+		}
+
+		integrationTypes[itInterface.GetIntegrationType()] = itInterface
+	}
+
 	return &IntegrationTypeManager{
+		logger:           logger,
+		hcLogger:         hcLogger,
 		IntegrationTypes: integrationTypes,
 	}
 }

services/integration/integration-type/interfaces/integration.go

@@ -1,10 +1,18 @@
 package interfaces
 
 import (
+	"github.com/hashicorp/go-plugin"
 	"github.com/opengovern/og-util/pkg/integration"
 	"github.com/opengovern/opencomply/services/integration/models"
+	"net/rpc"
 )
 
+var HandshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion:  1,
+	MagicCookieKey:   "platform-integration-plugin",
+	MagicCookieValue: "integration",
+}
+
 type IntegrationConfiguration struct {
 	NatsScheduledJobsTopic   string
 	NatsManualJobsTopic      string
@@ -31,3 +39,113 @@ type IntegrationType interface {
 
 // IntegrationCreator IntegrationType interface, credentials, error
 type IntegrationCreator func() IntegrationType
+
+type IntegrationTypeRPC struct {
+	client *rpc.Client
+}
+
+func (i *IntegrationTypeRPC) GetIntegrationType() integration.Type {
+	var integrationType integration.Type
+	err := i.client.Call("PluginGetIntegrationType", struct{}{}, &integrationType)
+	if err != nil {
+		panic(err)
+	}
+	return integrationType
+}
+
+func (i *IntegrationTypeRPC) GetConfiguration() IntegrationConfiguration {
+	var configuration IntegrationConfiguration
+	err := i.client.Call("Plugin.GetConfiguration", struct{}{}, &configuration)
+	if err != nil {
+		panic(err)
+	}
+	return configuration
+}
+
+func (i *IntegrationTypeRPC) GetResourceTypesByLabels(labels map[string]string) (map[string]*ResourceTypeConfiguration, error) {
+	var resourceTypes map[string]*ResourceTypeConfiguration
+	err := i.client.Call("Plugin.GetResourceTypesByLabels", labels, &resourceTypes)
+	return resourceTypes, err
+}
+
+type HealthCheckRequest struct {
+	JsonData    []byte
+	ProviderId  string
+	Labels      map[string]string
+	Annotations map[string]string
+}
+
+func (i *IntegrationTypeRPC) HealthCheck(jsonData []byte, providerId string, labels map[string]string, annotations map[string]string) (bool, error) {
+	var result bool
+	err := i.client.Call("Plugin.HealthCheck", HealthCheckRequest{
+		JsonData:    jsonData,
+		ProviderId:  providerId,
+		Labels:      labels,
+		Annotations: annotations,
+	}, &result)
+	return result, err
+}
+
+func (i *IntegrationTypeRPC) DiscoverIntegrations(jsonData []byte) ([]models.Integration, error) {
+	var integrations []models.Integration
+	err := i.client.Call("Plugin.DiscoverIntegrations", jsonData, &integrations)
+	return integrations, err
+}
+
+func (i *IntegrationTypeRPC) GetResourceTypeFromTableName(tableName string) string {
+	var resourceType string
+	err := i.client.Call("Plugin.GetResourceTypeFromTableName", tableName, &resourceType)
+	if err != nil {
+		panic(err)
+	}
+	return resourceType
+}
+
+type IntegrationTypeRPCServer struct {
+	Impl IntegrationType
+}
+
+func (i *IntegrationTypeRPCServer) GetIntegrationType(_ struct{}, integrationType *integration.Type) error {
+	*integrationType = i.Impl.GetIntegrationType()
+	return nil
+}
+
+func (i *IntegrationTypeRPCServer) GetConfiguration(_ struct{}, configuration *IntegrationConfiguration) error {
+	*configuration = i.Impl.GetConfiguration()
+	return nil
+}
+
+func (i *IntegrationTypeRPCServer) GetResourceTypesByLabels(labels map[string]string, resourceTypes *map[string]*ResourceTypeConfiguration) error {
+	var err error
+	*resourceTypes, err = i.Impl.GetResourceTypesByLabels(labels)
+	return err
+}
+
+func (i *IntegrationTypeRPCServer) HealthCheck(request HealthCheckRequest, result *bool) error {
+	var err error
+	*result, err = i.Impl.HealthCheck(request.JsonData, request.ProviderId, request.Labels, request.Annotations)
+	return err
+}
+
+func (i *IntegrationTypeRPCServer) DiscoverIntegrations(jsonData []byte, integrations *[]models.Integration) error {
+	var err error
+	*integrations, err = i.Impl.DiscoverIntegrations(jsonData)
+	return err
+}
+
+func (i *IntegrationTypeRPCServer) GetResourceTypeFromTableName(tableName string, resourceType *string) error {
+	*resourceType = i.Impl.GetResourceTypeFromTableName(tableName)
+	return nil
+}
+
+type IntegrationTypePlugin struct {
+	Impl IntegrationType
+}
+
+func (p *IntegrationTypePlugin) Server(*plugin.MuxBroker) (any, error) {
+	return &IntegrationTypeRPCServer{Impl: p.Impl}, nil
+}
+
+func (IntegrationTypePlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (any, error) {
+	return &IntegrationTypeRPC{client: c}, nil
+}