From e84e399f1d3d4b5ee469cbd4ac3ca72d433f9d10 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 27 Aug 2024 09:56:28 +0000 Subject: [PATCH] Auto-create ECR registries before docker push --- go.mod | 9 ++-- go.sum | 10 +++++ pkg/leeway/build.go | 5 +++ pkg/leeway/images.go | 95 +++++++++++++++++++++++++++++++++++++++++++ pkg/leeway/package.go | 30 ++++++++++++++ 5 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 pkg/leeway/images.go diff --git a/go.mod b/go.mod index d107eb4..4c921af 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/gitpod-io/leeway go 1.21 require ( - github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.18.45 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 @@ -37,17 +37,18 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.32.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect - github.com/aws/smithy-go v1.15.0 // indirect + github.com/aws/smithy-go v1.20.4 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/go.sum b/go.sum index 06ec2cc..2b9eafa 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= @@ -12,12 +14,18 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 h1:mtJRt80k1oGw7QQPluAx github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90/go.mod h1:lYwZTkeMQWPvNU+u7oYArdNhQ8EKiSGU76jVv0w2GH4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.2 h1:2RjzMZp/8PXJUMqiKkDSp7RVj6inF5DpVel35THjV+I= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.2/go.mod h1:kdk+WJbHcGVbIlRQfSrKyuKkbWDdD8I9NScyS5vZ8eQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE= @@ -36,6 +44,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwF github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= diff --git a/pkg/leeway/build.go b/pkg/leeway/build.go index 102455c..f442148 100644 --- a/pkg/leeway/build.go +++ b/pkg/leeway/build.go @@ -1426,6 +1426,11 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p buildcmd = append(buildcmd, ".") commands[PackageBuildPhaseBuild] = append(commands[PackageBuildPhaseBuild], buildcmd) + err = cfg.EnsureRegistryExists() + if err != nil { + log.Warnf("failed to create the registries: %v", err) + } + if len(cfg.Image) == 0 { // we don't push the image, let's export it ef := strings.TrimSuffix(result, ".gz") diff --git a/pkg/leeway/images.go b/pkg/leeway/images.go new file mode 100644 index 0000000..be9a3ca --- /dev/null +++ b/pkg/leeway/images.go @@ -0,0 +1,95 @@ +package leeway + +import ( + "context" + "errors" + "fmt" + "strings" + + "os/exec" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecr" + "github.com/aws/aws-sdk-go-v2/service/ecr/types" +) + +type ImageAdapter interface { + Create(imageName string) error + Sign(imageName, profileARN string) error +} + +// ECRAdapter implements the ImageAdapter interface for AWS ECR +type ECRAdapter struct { + ecrClient *ecr.Client +} + +// NewECRAdapter initializes an ECRAdapter with an AWS ECR client +func NewECRAdapter() (*ECRAdapter, error) { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return nil, fmt.Errorf("unable to load SDK config, %v", err) + } + + client := ecr.NewFromConfig(cfg) + return &ECRAdapter{ + ecrClient: client, + }, nil +} + +// Create checks if the ECR image exists and creates it if it doesn't +func (e *ECRAdapter) Create(image string) error { + imageName := getRepoNameFromImage(image) + + _, err := e.ecrClient.DescribeImages(context.TODO(), &ecr.DescribeImagesInput{ + RepositoryName: aws.String(imageName), + }) + if err == nil { + fmt.Printf("Image %s already exists\n", imageName) + return nil + } + + if !isRepositoryNotFoundErr(err) { + return fmt.Errorf("failed to check if ECR image %s exists: %w", imageName, err) + } + + _, err = e.ecrClient.CreateRepository(context.TODO(), &ecr.CreateRepositoryInput{ + RepositoryName: aws.String(imageName), + }) + if err != nil { + return fmt.Errorf("failed to create ECR image: %w", err) + } + + fmt.Printf("Image %s created successfully\n", imageName) + return nil +} + +// Sign uses the notation tool to sign the ECR image +func (e *ECRAdapter) Sign(imageName, profileARN string) error { + cmd := exec.Command("notation", "sign", "--profile", profileARN, imageName) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to sign the image: %v, output: %s", err, string(output)) + } + + fmt.Printf("Image %s signed successfully\n", imageName) + return nil +} + +// isImageNotFoundErr checks if the error is an ImageNotFoundException +func isRepositoryNotFoundErr(err error) bool { + var notFoundErr *types.RepositoryNotFoundException + return errors.As(err, ¬FoundErr) +} + +// getRepoNameFromImage extracts and returns the full repository name from the image string. +func getRepoNameFromImage(image string) string { + // Split the image string by slashes + parts := strings.Split(image, "/") + // Combine the parts after the domain name to get the full repository name + repoName := strings.Join(parts[1:], "/") + // Split the repo name by colon to remove the tag if present + repoParts := strings.Split(repoName, ":") + // Return the first part, which is the full repo name without the tag + return repoParts[0] +} diff --git a/pkg/leeway/package.go b/pkg/leeway/package.go index 39c2c7c..8fc68ed 100644 --- a/pkg/leeway/package.go +++ b/pkg/leeway/package.go @@ -551,6 +551,36 @@ func (cfg DockerPkgConfig) AdditionalSources(workspaceOrigin string) []string { return []string{cfg.Dockerfile} } +// CreateAndSign processes the Docker image: creates and signs it using the appropriate adapter +func (cfg *DockerPkgConfig) EnsureRegistryExists() error { + for _, image := range cfg.Image { + var adapter ImageAdapter + var err error + + if isECRImage(image) { + adapter, err = NewECRAdapter() + if err != nil { + return fmt.Errorf("failed to create ECR adapter: %w", err) + } + } else { + log.Debugf("no supported adapter for image registry for image: %s", image) + return nil + } + + if err := adapter.Create(image); err != nil { + return fmt.Errorf("failed to create image %s: %w", image, err) + } + } + + return nil +} + +// isECRImage checks if the image is hosted on AWS ECR +func isECRImage(image string) bool { + // AWS ECR images typically start with account-id.dkr.ecr.region.amazonaws.com + return strings.Contains(image, ".dkr.ecr.") +} + // GenericPkgConfig configures a generic package type GenericPkgConfig struct { Commands [][]string `yaml:"commands"`