From 825aff2187682656d9c998417b81f8d39bb856bc Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Wed, 20 Sep 2023 13:46:58 +0000 Subject: [PATCH 1/3] 1 --- components/image-builder-bob/cmd/proxy.go | 3 +- .../image-builder-bob/leeway.Dockerfile | 2 +- .../image-builder-bob/pkg/proxy/auth.go | 18 ++- .../image-builder-bob/pkg/proxy/proxy.go | 117 +++++++++++++++++- components/image-builder-mk3/pkg/auth/auth.go | 2 + .../pkg/orchestrator/orchestrator.go | 2 +- 6 files changed, 137 insertions(+), 7 deletions(-) diff --git a/components/image-builder-bob/cmd/proxy.go b/components/image-builder-bob/cmd/proxy.go index 443d0f7ed36ba5..3b21b84faba6cc 100644 --- a/components/image-builder-bob/cmd/proxy.go +++ b/components/image-builder-bob/cmd/proxy.go @@ -59,6 +59,7 @@ var proxyCmd = &cobra.Command{ } auth := func() docker.Authorizer { return docker.NewDockerAuthorizer(docker.WithAuthCreds(authP.Authorize)) } + mirrorAuth := func() docker.Authorizer { return docker.NewDockerAuthorizer(docker.WithAuthCreds(authA.Authorize)) } prx, err := proxy.NewProxy(&url.URL{Host: "localhost:8080", Scheme: "http"}, map[string]proxy.Repo{ "base": { Host: reference.Domain(baseref), @@ -72,7 +73,7 @@ var proxyCmd = &cobra.Command{ Tag: targettag, Auth: auth, }, - }) + }, mirrorAuth) if err != nil { log.Fatal(err) } diff --git a/components/image-builder-bob/leeway.Dockerfile b/components/image-builder-bob/leeway.Dockerfile index 13ac28c830ed86..5923817adc9872 100644 --- a/components/image-builder-bob/leeway.Dockerfile +++ b/components/image-builder-bob/leeway.Dockerfile @@ -2,7 +2,7 @@ # Licensed under the GNU Affero General Public License (AGPL). # See License.AGPL.txt in the project root for license information. -FROM moby/buildkit:v0.12.2 +FROM eu.gcr.io/gitpod-core-dev/build/buildkit:v0.12.2-gitpod USER root RUN apk --no-cache add sudo bash \ diff --git a/components/image-builder-bob/pkg/proxy/auth.go b/components/image-builder-bob/pkg/proxy/auth.go index b06253b1d67391..825b4286fc4b4e 100644 --- a/components/image-builder-bob/pkg/proxy/auth.go +++ b/components/image-builder-bob/pkg/proxy/auth.go @@ -7,12 +7,22 @@ package proxy import ( "encoding/base64" "encoding/json" + "regexp" "strings" "github.com/gitpod-io/gitpod/common-go/log" "github.com/sirupsen/logrus" ) +var ecrRegistryRegexp = regexp.MustCompile(`\d{12}.dkr.ecr.\w+-\w+-\w+.amazonaws.com`) + +const DummyECRRegistryDomain = "000000000000.dkr.ecr.dummy-host-zone.amazonaws.com" + +// isECRRegistry returns true if the registry domain is an ECR registry +func isECRRegistry(domain string) bool { + return ecrRegistryRegexp.MatchString(domain) +} + // authConfig configures authentication for a single host type authConfig struct { Username string `json:"username"` @@ -32,7 +42,13 @@ func (a MapAuthorizer) Authorize(host string) (user, pass string, err error) { res, ok := a[host] if !ok { - return + if !isECRRegistry(host) { + return + } + res, ok = a[DummyECRRegistryDomain] + if !ok { + return + } } user, pass = res.Username, res.Password diff --git a/components/image-builder-bob/pkg/proxy/proxy.go b/components/image-builder-bob/pkg/proxy/proxy.go index 2c431dab262bc3..b7c4ca8bf7fa9c 100644 --- a/components/image-builder-bob/pkg/proxy/proxy.go +++ b/components/image-builder-bob/pkg/proxy/proxy.go @@ -20,7 +20,7 @@ import ( const authKey = "authKey" -func NewProxy(host *url.URL, aliases map[string]Repo) (*Proxy, error) { +func NewProxy(host *url.URL, aliases map[string]Repo, mirrorAuth func() docker.Authorizer) (*Proxy, error) { if host.Host == "" || host.Scheme == "" { return nil, fmt.Errorf("host Host or Scheme are missing") } @@ -41,8 +41,9 @@ type Proxy struct { Host url.URL Aliases map[string]Repo - mu sync.Mutex - proxies map[string]*httputil.ReverseProxy + mu sync.Mutex + proxies map[string]*httputil.ReverseProxy + mirrorAuth func() docker.Authorizer } type Repo struct { @@ -126,6 +127,23 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { break } } + + // get mirror host + if host := r.URL.Query().Get("ns"); host != "" && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + if host == "docker.io" { + host = "registry-1.docker.io" + } + r.URL.Host = host + r.Host = host + + auth := proxy.mirrorAuth + r = r.WithContext(context.WithValue(ctx, authKey, auth)) + + r.RequestURI = "" + proxy.mirror(host).ServeHTTP(w, r) + return + } + if repo == nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return @@ -273,3 +291,96 @@ func (proxy *Proxy) reverse(alias string) *httputil.ReverseProxy { proxy.proxies[alias] = rp return rp } + +// mirror produces an authentication-adding reverse proxy for given host +func (proxy *Proxy) mirror(host string) *httputil.ReverseProxy { + proxy.mu.Lock() + defer proxy.mu.Unlock() + + if rp, ok := proxy.proxies[host]; ok { + return rp + } + + rp := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: host}) + + client := retryablehttp.NewClient() + client.RetryMax = 3 + client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + if err != nil { + log.WithError(err).Warn("saw error during CheckRetry") + return false, err + } + auth, ok := ctx.Value(authKey).(docker.Authorizer) + if !ok || auth == nil { + return false, nil + } + if resp.StatusCode == http.StatusUnauthorized { + // the docker authorizer only refreshes OAuth tokens after two + // successive 401 errors for the same URL. Rather than issue the same + // request multiple times to tickle the token-refreshing logic, just + // provide the same response twice to trick it into refreshing the + // cached OAuth token. Call AddResponses() twice, first to invalidate + // the existing token (with two responses), second to fetch a new one + // (with one response). + // TODO: fix after one of these two PRs are merged and available: + // https://github.com/containerd/containerd/pull/8735 + // https://github.com/containerd/containerd/pull/8388 + err := auth.AddResponses(ctx, []*http.Response{resp, resp}) + if err != nil { + log.WithError(err).WithField("URL", resp.Request.URL.String()).Warn("cannot add responses although response was Unauthorized") + return false, nil + } + + err = auth.AddResponses(ctx, []*http.Response{resp}) + if err != nil { + log.WithError(err).WithField("URL", resp.Request.URL.String()).Warn("cannot add responses although response was Unauthorized") + return false, nil + } + return true, nil + } + if resp.StatusCode == http.StatusBadRequest { + log.WithField("URL", resp.Request.URL.String()).Warn("bad request") + return true, nil + } + + return false, nil + } + client.RequestLogHook = func(l retryablehttp.Logger, r *http.Request, i int) { + // Total hack: we need a place to modify the request before retrying, and this log + // hook seems to be the only place. We need to modify the request, because + // maybe we just added the host authorizer in the previous CheckRetry call. + // + // The ReverseProxy sets the X-Forwarded-For header with the host machine + // address. If on a cluster with IPV6 enabled, this will be "::1" (IPV6 equivalent + // of "127.0.0.1"). This can have the knock-on effect of receiving an IPV6 + // URL, e.g. auth.ipv6.docker.com instead of auth.docker.com which may not + // exist. By forcing the value to be "127.0.0.1", we ensure consistency + // across clusters. + // + // @link https://golang.org/src/net/http/httputil/reverseproxy.go + r.Header.Set("X-Forwarded-For", "127.0.0.1") + + auth, ok := r.Context().Value(authKey).(docker.Authorizer) + if !ok || auth == nil { + return + } + _ = auth.Authorize(r.Context(), r) + } + client.ResponseLogHook = func(l retryablehttp.Logger, r *http.Response) {} + + rp.Transport = &retryablehttp.RoundTripper{ + Client: client, + } + rp.ModifyResponse = func(r *http.Response) error { + if r.StatusCode == http.StatusBadGateway { + // BadGateway makes containerd retry - we don't want that because we retry the upstream + // requests internally. + r.StatusCode = http.StatusInternalServerError + r.Status = http.StatusText(http.StatusInternalServerError) + } + + return nil + } + proxy.proxies[host] = rp + return rp +} diff --git a/components/image-builder-mk3/pkg/auth/auth.go b/components/image-builder-mk3/pkg/auth/auth.go index 6e4afe6047a5f2..59cd534d7ca3a3 100644 --- a/components/image-builder-mk3/pkg/auth/auth.go +++ b/components/image-builder-mk3/pkg/auth/auth.go @@ -220,6 +220,8 @@ func (a *Authentication) Empty() bool { var ecrRegistryRegexp = regexp.MustCompile(`\d{12}.dkr.ecr.\w+-\w+-\w+.amazonaws.com`) +const DummyECRRegistryDomain = "000000000000.dkr.ecr.dummy-host-zone.amazonaws.com" + // isECRRegistry returns true if the registry domain is an ECR registry func isECRRegistry(domain string) bool { return ecrRegistryRegexp.MatchString(domain) diff --git a/components/image-builder-mk3/pkg/orchestrator/orchestrator.go b/components/image-builder-mk3/pkg/orchestrator/orchestrator.go index cf49dbec328bd6..9e96f514e017f8 100644 --- a/components/image-builder-mk3/pkg/orchestrator/orchestrator.go +++ b/components/image-builder-mk3/pkg/orchestrator/orchestrator.go @@ -336,7 +336,7 @@ func (o *Orchestrator) Build(req *protocol.BuildRequest, resp protocol.ImageBuil wsref, err := reference.ParseNamed(wsrefstr) var additionalAuth []byte if err == nil { - ath := reqauth.GetImageBuildAuthFor(ctx, o.Auth, []string{reference.Domain(pbaseref)}, []string{ + ath := reqauth.GetImageBuildAuthFor(ctx, o.Auth, []string{reference.Domain(pbaseref), auth.DummyECRRegistryDomain}, []string{ reference.Domain(wsref), }) additionalAuth, err = json.Marshal(ath) From a1bfd462f46a49a6f90cd503868888a37ad5db3e Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Wed, 20 Sep 2023 15:46:16 +0000 Subject: [PATCH 2/3] fix --- .../image-builder-bob/pkg/proxy/auth.go | 47 ++++++++++++++++--- .../image-builder-bob/pkg/proxy/proxy.go | 14 +++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/components/image-builder-bob/pkg/proxy/auth.go b/components/image-builder-bob/pkg/proxy/auth.go index 825b4286fc4b4e..fbd76e5269ee71 100644 --- a/components/image-builder-bob/pkg/proxy/auth.go +++ b/components/image-builder-bob/pkg/proxy/auth.go @@ -23,6 +23,18 @@ func isECRRegistry(domain string) bool { return ecrRegistryRegexp.MatchString(domain) } +// isDockerHubRegistry returns true if the registry domain is an docker hub +func isDockerHubRegistry(domain string) bool { + switch domain { + case "registry-1.docker.io": + fallthrough + case "docker.io": + return true + default: + return false + } +} + // authConfig configures authentication for a single host type authConfig struct { Username string `json:"username"` @@ -40,15 +52,36 @@ func (a MapAuthorizer) Authorize(host string) (user, pass string, err error) { }).Info("authorizing registry access") }() - res, ok := a[host] - if !ok { - if !isECRRegistry(host) { - return + explicitHostMatcher := func() (authConfig, bool) { + res, ok := a[host] + return res, ok + } + ecrHostMatcher := func() (authConfig, bool) { + if isECRRegistry(host) { + res, ok := a[DummyECRRegistryDomain] + return res, ok } - res, ok = a[DummyECRRegistryDomain] - if !ok { - return + return authConfig{}, false + } + dockerHubHostMatcher := func() (authConfig, bool) { + if isDockerHubRegistry(host) { + res, ok := a["docker.io"] + return res, ok } + return authConfig{}, false + } + + matchers := []func() (authConfig, bool){explicitHostMatcher, ecrHostMatcher, dockerHubHostMatcher} + res, ok := authConfig{}, false + for _, matcher := range matchers { + res, ok = matcher() + if ok { + break + } + } + + if !ok { + return } user, pass = res.Username, res.Password diff --git a/components/image-builder-bob/pkg/proxy/proxy.go b/components/image-builder-bob/pkg/proxy/proxy.go index b7c4ca8bf7fa9c..4406f207dd31cb 100644 --- a/components/image-builder-bob/pkg/proxy/proxy.go +++ b/components/image-builder-bob/pkg/proxy/proxy.go @@ -31,9 +31,10 @@ func NewProxy(host *url.URL, aliases map[string]Repo, mirrorAuth func() docker.A aliases[k] = v } return &Proxy{ - Host: *host, - Aliases: aliases, - proxies: make(map[string]*httputil.ReverseProxy), + Host: *host, + Aliases: aliases, + proxies: make(map[string]*httputil.ReverseProxy), + mirrorAuth: mirrorAuth, }, nil } @@ -130,13 +131,12 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // get mirror host if host := r.URL.Query().Get("ns"); host != "" && (r.Method == http.MethodGet || r.Method == http.MethodHead) { - if host == "docker.io" { - host = "registry-1.docker.io" - } + host, _ = docker.DefaultHost(host) + r.URL.Host = host r.Host = host - auth := proxy.mirrorAuth + auth := proxy.mirrorAuth() r = r.WithContext(context.WithValue(ctx, authKey, auth)) r.RequestURI = "" From da85c631ece0f3762c78f0642ae97d1ecc6d3e4b Mon Sep 17 00:00:00 2001 From: Pudong Date: Fri, 22 Sep 2023 16:22:36 +0800 Subject: [PATCH 3/3] update buildkitd --- components/image-builder-bob/leeway.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/image-builder-bob/leeway.Dockerfile b/components/image-builder-bob/leeway.Dockerfile index 5923817adc9872..777be782fb0e1b 100644 --- a/components/image-builder-bob/leeway.Dockerfile +++ b/components/image-builder-bob/leeway.Dockerfile @@ -2,7 +2,7 @@ # Licensed under the GNU Affero General Public License (AGPL). # See License.AGPL.txt in the project root for license information. -FROM eu.gcr.io/gitpod-core-dev/build/buildkit:v0.12.2-gitpod +FROM eu.gcr.io/gitpod-core-dev/build/buildkit:v0.12.2-gitpod.1 USER root RUN apk --no-cache add sudo bash \