From 825aff2187682656d9c998417b81f8d39bb856bc Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Wed, 20 Sep 2023 13:46:58 +0000 Subject: [PATCH] 1 --- components/image-builder-bob/cmd/proxy.go | 3 +- .../image-builder-bob/leeway.Dockerfile | 2 +- .../image-builder-bob/pkg/proxy/auth.go | 18 ++- .../image-builder-bob/pkg/proxy/proxy.go | 117 +++++++++++++++++- components/image-builder-mk3/pkg/auth/auth.go | 2 + .../pkg/orchestrator/orchestrator.go | 2 +- 6 files changed, 137 insertions(+), 7 deletions(-) diff --git a/components/image-builder-bob/cmd/proxy.go b/components/image-builder-bob/cmd/proxy.go index 443d0f7ed36ba5..3b21b84faba6cc 100644 --- a/components/image-builder-bob/cmd/proxy.go +++ b/components/image-builder-bob/cmd/proxy.go @@ -59,6 +59,7 @@ var proxyCmd = &cobra.Command{ } auth := func() docker.Authorizer { return docker.NewDockerAuthorizer(docker.WithAuthCreds(authP.Authorize)) } + mirrorAuth := func() docker.Authorizer { return docker.NewDockerAuthorizer(docker.WithAuthCreds(authA.Authorize)) } prx, err := proxy.NewProxy(&url.URL{Host: "localhost:8080", Scheme: "http"}, map[string]proxy.Repo{ "base": { Host: reference.Domain(baseref), @@ -72,7 +73,7 @@ var proxyCmd = &cobra.Command{ Tag: targettag, Auth: auth, }, - }) + }, mirrorAuth) if err != nil { log.Fatal(err) } diff --git a/components/image-builder-bob/leeway.Dockerfile b/components/image-builder-bob/leeway.Dockerfile index 13ac28c830ed86..5923817adc9872 100644 --- a/components/image-builder-bob/leeway.Dockerfile +++ b/components/image-builder-bob/leeway.Dockerfile @@ -2,7 +2,7 @@ # Licensed under the GNU Affero General Public License (AGPL). # See License.AGPL.txt in the project root for license information. -FROM moby/buildkit:v0.12.2 +FROM eu.gcr.io/gitpod-core-dev/build/buildkit:v0.12.2-gitpod USER root RUN apk --no-cache add sudo bash \ diff --git a/components/image-builder-bob/pkg/proxy/auth.go b/components/image-builder-bob/pkg/proxy/auth.go index b06253b1d67391..825b4286fc4b4e 100644 --- a/components/image-builder-bob/pkg/proxy/auth.go +++ b/components/image-builder-bob/pkg/proxy/auth.go @@ -7,12 +7,22 @@ package proxy import ( "encoding/base64" "encoding/json" + "regexp" "strings" "github.com/gitpod-io/gitpod/common-go/log" "github.com/sirupsen/logrus" ) +var ecrRegistryRegexp = regexp.MustCompile(`\d{12}.dkr.ecr.\w+-\w+-\w+.amazonaws.com`) + +const DummyECRRegistryDomain = "000000000000.dkr.ecr.dummy-host-zone.amazonaws.com" + +// isECRRegistry returns true if the registry domain is an ECR registry +func isECRRegistry(domain string) bool { + return ecrRegistryRegexp.MatchString(domain) +} + // authConfig configures authentication for a single host type authConfig struct { Username string `json:"username"` @@ -32,7 +42,13 @@ func (a MapAuthorizer) Authorize(host string) (user, pass string, err error) { res, ok := a[host] if !ok { - return + if !isECRRegistry(host) { + return + } + res, ok = a[DummyECRRegistryDomain] + if !ok { + return + } } user, pass = res.Username, res.Password diff --git a/components/image-builder-bob/pkg/proxy/proxy.go b/components/image-builder-bob/pkg/proxy/proxy.go index 2c431dab262bc3..b7c4ca8bf7fa9c 100644 --- a/components/image-builder-bob/pkg/proxy/proxy.go +++ b/components/image-builder-bob/pkg/proxy/proxy.go @@ -20,7 +20,7 @@ import ( const authKey = "authKey" -func NewProxy(host *url.URL, aliases map[string]Repo) (*Proxy, error) { +func NewProxy(host *url.URL, aliases map[string]Repo, mirrorAuth func() docker.Authorizer) (*Proxy, error) { if host.Host == "" || host.Scheme == "" { return nil, fmt.Errorf("host Host or Scheme are missing") } @@ -41,8 +41,9 @@ type Proxy struct { Host url.URL Aliases map[string]Repo - mu sync.Mutex - proxies map[string]*httputil.ReverseProxy + mu sync.Mutex + proxies map[string]*httputil.ReverseProxy + mirrorAuth func() docker.Authorizer } type Repo struct { @@ -126,6 +127,23 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { break } } + + // get mirror host + if host := r.URL.Query().Get("ns"); host != "" && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + if host == "docker.io" { + host = "registry-1.docker.io" + } + r.URL.Host = host + r.Host = host + + auth := proxy.mirrorAuth + r = r.WithContext(context.WithValue(ctx, authKey, auth)) + + r.RequestURI = "" + proxy.mirror(host).ServeHTTP(w, r) + return + } + if repo == nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return @@ -273,3 +291,96 @@ func (proxy *Proxy) reverse(alias string) *httputil.ReverseProxy { proxy.proxies[alias] = rp return rp } + +// mirror produces an authentication-adding reverse proxy for given host +func (proxy *Proxy) mirror(host string) *httputil.ReverseProxy { + proxy.mu.Lock() + defer proxy.mu.Unlock() + + if rp, ok := proxy.proxies[host]; ok { + return rp + } + + rp := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: host}) + + client := retryablehttp.NewClient() + client.RetryMax = 3 + client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + if err != nil { + log.WithError(err).Warn("saw error during CheckRetry") + return false, err + } + auth, ok := ctx.Value(authKey).(docker.Authorizer) + if !ok || auth == nil { + return false, nil + } + if resp.StatusCode == http.StatusUnauthorized { + // the docker authorizer only refreshes OAuth tokens after two + // successive 401 errors for the same URL. Rather than issue the same + // request multiple times to tickle the token-refreshing logic, just + // provide the same response twice to trick it into refreshing the + // cached OAuth token. Call AddResponses() twice, first to invalidate + // the existing token (with two responses), second to fetch a new one + // (with one response). + // TODO: fix after one of these two PRs are merged and available: + // https://github.com/containerd/containerd/pull/8735 + // https://github.com/containerd/containerd/pull/8388 + err := auth.AddResponses(ctx, []*http.Response{resp, resp}) + if err != nil { + log.WithError(err).WithField("URL", resp.Request.URL.String()).Warn("cannot add responses although response was Unauthorized") + return false, nil + } + + err = auth.AddResponses(ctx, []*http.Response{resp}) + if err != nil { + log.WithError(err).WithField("URL", resp.Request.URL.String()).Warn("cannot add responses although response was Unauthorized") + return false, nil + } + return true, nil + } + if resp.StatusCode == http.StatusBadRequest { + log.WithField("URL", resp.Request.URL.String()).Warn("bad request") + return true, nil + } + + return false, nil + } + client.RequestLogHook = func(l retryablehttp.Logger, r *http.Request, i int) { + // Total hack: we need a place to modify the request before retrying, and this log + // hook seems to be the only place. We need to modify the request, because + // maybe we just added the host authorizer in the previous CheckRetry call. + // + // The ReverseProxy sets the X-Forwarded-For header with the host machine + // address. If on a cluster with IPV6 enabled, this will be "::1" (IPV6 equivalent + // of "127.0.0.1"). This can have the knock-on effect of receiving an IPV6 + // URL, e.g. auth.ipv6.docker.com instead of auth.docker.com which may not + // exist. By forcing the value to be "127.0.0.1", we ensure consistency + // across clusters. + // + // @link https://golang.org/src/net/http/httputil/reverseproxy.go + r.Header.Set("X-Forwarded-For", "127.0.0.1") + + auth, ok := r.Context().Value(authKey).(docker.Authorizer) + if !ok || auth == nil { + return + } + _ = auth.Authorize(r.Context(), r) + } + client.ResponseLogHook = func(l retryablehttp.Logger, r *http.Response) {} + + rp.Transport = &retryablehttp.RoundTripper{ + Client: client, + } + rp.ModifyResponse = func(r *http.Response) error { + if r.StatusCode == http.StatusBadGateway { + // BadGateway makes containerd retry - we don't want that because we retry the upstream + // requests internally. + r.StatusCode = http.StatusInternalServerError + r.Status = http.StatusText(http.StatusInternalServerError) + } + + return nil + } + proxy.proxies[host] = rp + return rp +} diff --git a/components/image-builder-mk3/pkg/auth/auth.go b/components/image-builder-mk3/pkg/auth/auth.go index 6e4afe6047a5f2..59cd534d7ca3a3 100644 --- a/components/image-builder-mk3/pkg/auth/auth.go +++ b/components/image-builder-mk3/pkg/auth/auth.go @@ -220,6 +220,8 @@ func (a *Authentication) Empty() bool { var ecrRegistryRegexp = regexp.MustCompile(`\d{12}.dkr.ecr.\w+-\w+-\w+.amazonaws.com`) +const DummyECRRegistryDomain = "000000000000.dkr.ecr.dummy-host-zone.amazonaws.com" + // isECRRegistry returns true if the registry domain is an ECR registry func isECRRegistry(domain string) bool { return ecrRegistryRegexp.MatchString(domain) diff --git a/components/image-builder-mk3/pkg/orchestrator/orchestrator.go b/components/image-builder-mk3/pkg/orchestrator/orchestrator.go index cf49dbec328bd6..9e96f514e017f8 100644 --- a/components/image-builder-mk3/pkg/orchestrator/orchestrator.go +++ b/components/image-builder-mk3/pkg/orchestrator/orchestrator.go @@ -336,7 +336,7 @@ func (o *Orchestrator) Build(req *protocol.BuildRequest, resp protocol.ImageBuil wsref, err := reference.ParseNamed(wsrefstr) var additionalAuth []byte if err == nil { - ath := reqauth.GetImageBuildAuthFor(ctx, o.Auth, []string{reference.Domain(pbaseref)}, []string{ + ath := reqauth.GetImageBuildAuthFor(ctx, o.Auth, []string{reference.Domain(pbaseref), auth.DummyECRRegistryDomain}, []string{ reference.Domain(wsref), }) additionalAuth, err = json.Marshal(ath)