Merge remote-tracking branch 'origin/main' into fix-steampipe-upgrade

pkg/auth/auth0/service.go

@@ -92,9 +92,23 @@ func (a *Service) GetOrCreateUser(userID, email string) (*User, error) {
 	}
 
 	if user == nil || user.UserId == "" {
+		appMetadataJsonb := pgtype.JSONB{}
+		err = appMetadataJsonb.Set([]byte(""))
+		if err != nil {
+			return nil, err
+		}
+
+		userMetadataJsonb := pgtype.JSONB{}
+		err = userMetadataJsonb.Set([]byte(""))
+		if err != nil {
+			return nil, err
+		}
+
 		user = &db.User{
-			Email:  email,
-			UserId: userID,
+			Email:        email,
+			UserId:       userID,
+			AppMetadata:  appMetadataJsonb,
+			UserMetadata: userMetadataJsonb,
 		}
 		err = a.database.CreateUser(user)
 		if err != nil {
@@ -106,6 +120,10 @@ func (a *Service) GetOrCreateUser(userID, email string) (*User, error) {
 	if err != nil {
 		return nil, err
 	}
+	if resp.AppMetadata.WorkspaceAccess == nil {
+		resp.AppMetadata.WorkspaceAccess = map[string]api.Role{}
+	}
+	resp.AppMetadata.WorkspaceAccess["main"] = api.AdminRole
 
 	return resp, nil
 }
@@ -121,6 +139,11 @@ func (a *Service) GetUser(userID string) (*User, error) {
 		return nil, err
 	}
 
+	if resp.AppMetadata.WorkspaceAccess == nil {
+		resp.AppMetadata.WorkspaceAccess = map[string]api.Role{}
+	}
+	resp.AppMetadata.WorkspaceAccess["main"] = api.AdminRole
+
 	return resp, nil
 }
 
@@ -136,6 +159,12 @@ func (a *Service) SearchByEmail(email string) ([]User, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if u.AppMetadata.WorkspaceAccess == nil {
+			u.AppMetadata.WorkspaceAccess = map[string]api.Role{}
+		}
+		u.AppMetadata.WorkspaceAccess["main"] = api.AdminRole
+
 		resp = append(resp, *u)
 	}
 

pkg/auth/server.go

@@ -209,6 +209,11 @@ func (s *Server) Check(ctx context.Context, req *envoyauth.CheckRequest) (*envoy
 			zap.String("email", user.Email),
 			zap.Error(err))
 	}
+	if user.WorkspaceAccess == nil {
+		user.WorkspaceAccess = map[string]api3.Role{}
+	}
+
+	user.WorkspaceAccess["main"] = "admin"
 
 	rb, err := s.GetWorkspaceByName(workspaceName, user)
 	if err != nil {

pkg/workspace/server.go

@@ -231,8 +231,8 @@ func (s *Server) CreateWorkspace(c echo.Context) error {
 	if request.Name == "" {
 		return echo.NewHTTPError(http.StatusBadRequest, "name is empty")
 	}
-	if request.Name == "kaytu" || request.Name == "workspaces" {
-		return echo.NewHTTPError(http.StatusBadRequest, "name cannot be kaytu or workspaces")
+	if request.Name == "kaytu" || request.Name == "main" || request.Name == "workspaces" {
+		return echo.NewHTTPError(http.StatusBadRequest, "name cannot be kaytu, main or workspaces")
 	}
 	if !regexp.MustCompile(`^[a-zA-Z0-9\-]*$`).MatchString(request.Name) {
 		return echo.NewHTTPError(http.StatusBadRequest, "name is invalid. only characters, numbers and - is allowed")
@@ -903,6 +903,10 @@ func (s *Server) ListWorkspaces(c echo.Context) error {
 			hasRoleInWorkspace = true
 		}
 
+		if workspace.OwnerId != nil && *workspace.OwnerId == "kaytu|owner|all" {
+			hasRoleInWorkspace = true
+		}
+
 		if workspace.OwnerId == nil || (*workspace.OwnerId != userId && !hasRoleInWorkspace) {
 			continue
 		}

pkg/workspace/statemanager/service.go

@@ -11,12 +11,16 @@ import (
 	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
 	aws2 "github.com/kaytu-io/kaytu-aws-describer/aws"
 	authclient "github.com/kaytu-io/kaytu-engine/pkg/auth/client"
+	"github.com/kaytu-io/kaytu-engine/pkg/workspace/api"
 	workspaceConfig "github.com/kaytu-io/kaytu-engine/pkg/workspace/config"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/db"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/transactions"
+	api2 "github.com/kaytu-io/kaytu-util/pkg/api"
 	"github.com/kaytu-io/kaytu-util/pkg/config"
+	"github.com/kaytu-io/kaytu-util/pkg/httpclient"
 	"github.com/kaytu-io/kaytu-util/pkg/vault"
 	contourv1 "github.com/projectcontour/contour/apis/projectcontour/v1"
+	"github.com/sony/sonyflake"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -118,6 +122,65 @@ func New(ctx context.Context, cfg workspaceConfig.Config,
 	}, nil
 }
 
+func (s *Service) CreateWorkspace(ctx context.Context) error {
+	sf := sonyflake.NewSonyflake(sonyflake.Settings{})
+	id, err := sf.NextID()
+	if err != nil {
+		return err
+	}
+
+	awsUID, err := sf.NextID()
+	if err != nil {
+		return err
+	}
+
+	ownerAll := "kaytu|owner|all"
+	awsUniqueID := fmt.Sprintf("aws-uid-%d", awsUID)
+	workspace := &db.Workspace{
+		ID:                       fmt.Sprintf("ws-%d", id),
+		Name:                     "main",
+		AWSUniqueId:              &awsUniqueID,
+		OwnerId:                  &ownerAll,
+		Status:                   api.StateID_Provisioning,
+		Size:                     api.SizeXS,
+		Tier:                     api.Tier_Free,
+		OrganizationID:           nil,
+		IsCreated:                false,
+		IsBootstrapInputFinished: false,
+		AnalyticsJobID:           0,
+		InsightJobsID:            "",
+		ComplianceTriggered:      false,
+	}
+
+	if err := s.db.CreateWorkspace(workspace); err != nil {
+		return err
+	}
+
+	for _, tr := range []api.TransactionID{api.Transaction_CreateMasterCredential,
+		api.Transaction_EnsureCredentialExists, api.Transaction_CreateServiceAccountRoles,
+		api.Transaction_EnsureCredentialOnboarded, api.Transaction_EnsureDiscoveryFinished,
+		api.Transaction_EnsureJobsRunning, api.Transaction_EnsureJobsFinished,
+		api.Transaction_CreateRoleBinding} {
+		err := s.db.CreateWorkspaceTransaction(&db.WorkspaceTransaction{
+			WorkspaceID:   workspace.ID,
+			TransactionID: tr,
+			CreatedAt:     time.Now(),
+			UpdatedAt:     time.Now(),
+			Done:          true,
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	err = s.authClient.UpdateWorkspaceMap(&httpclient.Context{UserRole: api2.InternalRole})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s *Service) StartReconciler(ctx context.Context) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -152,6 +215,12 @@ func (s *Service) StartReconciler(ctx context.Context) {
 			if err := s.syncHelmValues(ctx, workspaces); err != nil {
 				s.logger.Error(fmt.Sprintf("syncing helm values: %v", err))
 			}
+
+			if len(workspaces) == 0 {
+				if err := s.CreateWorkspace(ctx); err != nil {
+					s.logger.Error(fmt.Sprintf("creating workspace if empty: %v", err))
+				}
+			}
 		}
 		if s.cfg.EnvType == config.EnvTypeProd && s.cfg.DoReserve {
 			err = s.handleReservation(ctx)

pkg/workspace/statemanager/workspace.go

@@ -21,7 +21,7 @@ func (s *Service) getTransactionByTransactionID(currentState state.State, tid ap
 	case api.Transaction_EnsureCredentialExists:
 		transaction = transactions.NewEnsureCredentialExists(s.db)
 	case api.Transaction_CreateHelmRelease:
-		transaction = transactions.NewCreateHelmRelease(s.kubeClient, s.vault, s.cfg, s.db)
+		transaction = transactions.NewCreateHelmRelease(s.kubeClient, s.vault, s.vaultSecretHandler, s.cfg, s.db, s.logger)
 	//case api.Transaction_CreateInsightBucket:
 	//	transaction = transactions.NewCreateInsightBucket(s.s3Client)
 	case api.Transaction_CreateMasterCredential:

pkg/workspace/transactions/create_helm_release.go

@@ -2,44 +2,47 @@ package transactions
 
 import (
 	"context"
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	types2 "github.com/aws/aws-sdk-go-v2/service/iam/types"
 	"github.com/fluxcd/helm-controller/api/v2beta1"
 	helmv2 "github.com/fluxcd/helm-controller/api/v2beta1"
 	apimeta "github.com/fluxcd/pkg/apis/meta"
+	api6 "github.com/hashicorp/vault/api"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/api"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/config"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/db"
 	"github.com/kaytu-io/kaytu-engine/pkg/workspace/internal/helm"
 	types3 "github.com/kaytu-io/kaytu-engine/pkg/workspace/types"
 	"github.com/kaytu-io/kaytu-util/pkg/vault"
+	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"strings"
 )
 
 type CreateHelmRelease struct {
-	kubeClient k8sclient.Client // the kubernetes client
-	vault      vault.VaultSourceConfig
-	cfg        config.Config
-	db         *db.Database
+	kubeClient         k8sclient.Client // the kubernetes client
+	vault              vault.VaultSourceConfig
+	cfg                config.Config
+	db                 *db.Database
+	logger             *zap.Logger
+	vaultSecretHandler vault.VaultSecretHandler
 }
 
-func NewCreateHelmRelease(
-	kubeClient k8sclient.Client,
-	vault vault.VaultSourceConfig,
-	cfg config.Config,
-	db *db.Database,
-) *CreateHelmRelease {
+func NewCreateHelmRelease(kubeClient k8sclient.Client, vault vault.VaultSourceConfig, handler vault.VaultSecretHandler, cfg config.Config, db *db.Database, logger *zap.Logger) *CreateHelmRelease {
 	return &CreateHelmRelease{
-		kubeClient: kubeClient,
-		vault:      vault,
-		cfg:        cfg,
-		db:         db,
+		kubeClient:         kubeClient,
+		vaultSecretHandler: handler,
+		vault:              vault,
+		cfg:                cfg,
+		db:                 db,
+		logger:             logger,
 	}
 }
 
@@ -214,6 +217,24 @@ func (t *CreateHelmRelease) createHelmRelease(ctx context.Context, workspace db.
 		return fmt.Errorf("create helm release: %w", err)
 	}
 
+	if t.cfg.Vault.Provider == vault.HashiCorpVault {
+		_, err := vault.NewHashiCorpVaultClient(ctx, t.logger, t.cfg.Vault.HashiCorp, settings.Vault.KeyID)
+		if err != nil {
+			if strings.Contains(err.Error(), api6.ErrSecretNotFound.Error()) || strings.Contains(err.Error(), "secret value is nil") {
+				b := make([]byte, 32)
+				_, err := rand.Read(b)
+				if err != nil {
+					return err
+				}
+
+				_, err = t.vaultSecretHandler.SetSecret(ctx, settings.Vault.KeyID, b)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
 	return nil
 }
 

services/migrator/job/git.go

@@ -3,7 +3,6 @@ package job
 import (
 	"encoding/json"
 	"github.com/go-git/go-git/v5"
-	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/kaytu-io/kaytu-engine/pkg/metadata/client"
 	"github.com/kaytu-io/kaytu-engine/pkg/metadata/models"
 	"github.com/kaytu-io/kaytu-engine/services/migrator/config"
@@ -35,13 +34,13 @@ func GitClone(conf config.MigratorConfig, logger *zap.Logger) (string, error) {
 
 	refs := make([]string, 0, 2)
 
-	gitAuth := githttp.BasicAuth{
-		Username: "abc123",
-		Password: gitConfig.githubToken,
-	}
+	//gitAuth := githttp.BasicAuth{
+	//	Username: "abc123",
+	//	Password: gitConfig.githubToken,
+	//}
 	os.RemoveAll(config.ConfigzGitPath)
 	res, err := git.PlainClone(config.ConfigzGitPath, false, &git.CloneOptions{
-		Auth:     &gitAuth,
+		//Auth:     &gitAuth,
 		URL:      gitConfig.AnalyticsGitURL,
 		Progress: os.Stdout,
 	})
@@ -58,7 +57,7 @@ func GitClone(conf config.MigratorConfig, logger *zap.Logger) (string, error) {
 
 	os.RemoveAll(config.ControlEnrichmentGitPath)
 	res, err = git.PlainClone(config.ControlEnrichmentGitPath, false, &git.CloneOptions{
-		Auth:     &gitAuth,
+		//Auth:     &gitAuth,
 		URL:      gitConfig.ControlEnrichmentGitURL,
 		Progress: os.Stdout,
 	})