Merge pull request #2018 from crazy-max/driver-client-meta

driver(docker): opt to set additional dial meta to the client
docker · Oct 16, 2023 · 05af608 · 05af608
2 parents 05b8821 + 511e413
commit 05af608
Show file tree

Hide file tree

Showing 22 changed files with 137 additions and 93 deletions.
diff --git a/builder/builder.go b/builder/builder.go
@@ -208,7 +208,7 @@ type driverFactory struct {
 }
 
 // Factory returns the driver factory.
-func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
+func (b *Builder) Factory(ctx context.Context, dialMeta map[string][]string) (_ driver.Factory, err error) {
 	b.driverFactory.once.Do(func() {
 		if b.Driver != "" {
 			b.driverFactory.Factory, err = driver.GetFactory(b.Driver, true)
@@ -231,7 +231,7 @@ func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
 			if _, err = dockerapi.Ping(ctx); err != nil {
 				return
 			}
-			b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false)
+			b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false, dialMeta)
 			if err != nil {
 				return
 			}

diff --git a/builder/node.go b/builder/node.go
@@ -42,9 +42,35 @@ func (b *Builder) Nodes() []Node {
 	return b.nodes
 }
 
+type LoadNodesOption func(*loadNodesOptions)
+
+type loadNodesOptions struct {
+	data     bool
+	dialMeta map[string][]string
+}
+
+func WithData() LoadNodesOption {
+	return func(o *loadNodesOptions) {
+		o.data = true
+	}
+}
+
+func WithDialMeta(dialMeta map[string][]string) LoadNodesOption {
+	return func(o *loadNodesOptions) {
+		o.dialMeta = dialMeta
+	}
+}
+
 // LoadNodes loads and returns nodes for this builder.
 // TODO: this should be a method on a Node object and lazy load data for each driver.
-func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err error) {
+func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []Node, err error) {
+	lno := loadNodesOptions{
+		data: false,
+	}
+	for _, opt := range opts {
+		opt(&lno)
+	}
+
 	eg, _ := errgroup.WithContext(ctx)
 	b.nodes = make([]Node, len(b.NodeGroup.Nodes))
 
@@ -54,7 +80,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 		}
 	}()
 
-	factory, err := b.Factory(ctx)
+	factory, err := b.Factory(ctx, lno.dialMeta)
 	if err != nil {
 		return nil, err
 	}
@@ -113,15 +139,15 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 					}
 				}
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash)
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash, lno.dialMeta)
 				if err != nil {
 					node.Err = err
 					return nil
 				}
 				node.Driver = d
 				node.ImageOpt = imageopt
 
-				if withData {
+				if lno.data {
 					if err := node.loadData(ctx); err != nil {
 						node.Err = err
 					}
@@ -136,7 +162,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 	}
 
 	// TODO: This should be done in the routine loading driver data
-	if withData {
+	if lno.data {
 		kubernetesDriverCount := 0
 		for _, d := range b.nodes {
 			if d.DriverInfo != nil && len(d.DriverInfo.DynamicNodes) > 0 {

diff --git a/commands/bake.go b/commands/bake.go
@@ -114,7 +114,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 		if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
 			return errors.Wrapf(err, "failed to update builder last activity time")
 		}
-		nodes, err = b.LoadNodes(ctx, false)
+		nodes, err = b.LoadNodes(ctx)
 		if err != nil {
 			return err
 		}

diff --git a/commands/build.go b/commands/build.go
@@ -252,7 +252,7 @@ func runBuild(dockerCli command.Cli, options buildOptions) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = b.LoadNodes(ctx, false)
+	_, err = b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

diff --git a/commands/create.go b/commands/create.go
@@ -123,7 +123,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 			if len(args) > 0 {
 				arg = args[0]
 			}
-			f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true)
+			f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true, nil)
 			if err != nil {
 				return err
 			}
@@ -270,7 +270,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	nodes, err := b.LoadNodes(timeoutCtx, true)
+	nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 	if err != nil {
 		return err
 	}

diff --git a/commands/diskusage.go b/commands/diskusage.go
@@ -39,7 +39,7 @@ func runDiskUsage(dockerCli command.Cli, opts duOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

diff --git a/commands/inspect.go b/commands/inspect.go
@@ -40,15 +40,15 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	nodes, err := b.LoadNodes(timeoutCtx, true)
+	nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 	if in.bootstrap {
 		var ok bool
 		ok, err = b.Boot(ctx)
 		if err != nil {
 			return err
 		}
 		if ok {
-			nodes, err = b.LoadNodes(timeoutCtx, true)
+			nodes, err = b.LoadNodes(timeoutCtx, builder.WithData())
 		}
 	}
 

diff --git a/commands/ls.go b/commands/ls.go
@@ -49,7 +49,7 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
 	for _, b := range builders {
 		func(b *builder.Builder) {
 			eg.Go(func() error {
-				_, _ = b.LoadNodes(timeoutCtx, true)
+				_, _ = b.LoadNodes(timeoutCtx, builder.WithData())
 				return nil
 			})
 		}(b)

diff --git a/commands/prune.go b/commands/prune.go
@@ -60,7 +60,7 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

diff --git a/commands/rm.go b/commands/rm.go
@@ -55,7 +55,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}
@@ -139,7 +139,7 @@ func rmAllInactive(ctx context.Context, txn *store.Txn, dockerCli command.Cli, i
 	for _, b := range builders {
 		func(b *builder.Builder) {
 			eg.Go(func() error {
-				nodes, err := b.LoadNodes(timeoutCtx, true)
+				nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 				if err != nil {
 					return errors.Wrapf(err, "cannot load %s", b.Name)
 				}

diff --git a/commands/stop.go b/commands/stop.go
@@ -25,7 +25,7 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 	if err != nil {
 		return err
 	}
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

diff --git a/controller/build/build.go b/controller/build/build.go
@@ -171,7 +171,7 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 	if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
 	}
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return nil, nil, err
 	}

diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go
@@ -428,6 +428,10 @@ func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
 	}
 }
 
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	return nil, errors.New("host-gateway is not supported by the docker-container driver")
+}
+
 func demuxConn(c net.Conn) net.Conn {
 	pr, pw := io.Pipe()
 	// TODO: rewrite parser with Reader() to avoid goroutine switch

diff --git a/driver/docker-container/factory.go b/driver/docker-container/factory.go
@@ -29,7 +29,7 @@ func (*factory) Usage() string {
 	return "docker-container"
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}

diff --git a/driver/docker/driver.go b/driver/docker/driver.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"strings"
+	"sync"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/util/progress"
@@ -15,6 +16,9 @@ import (
 type Driver struct {
 	factory driver.Factory
 	driver.InitConfig
+
+	features    features
+	hostGateway hostGateway
 }
 
 func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
@@ -54,7 +58,7 @@ func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error {
 func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 	opts := []client.ClientOpt{
 		client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
-			return d.DockerAPI.DialHijack(ctx, "/grpc", "h2c", nil)
+			return d.DockerAPI.DialHijack(ctx, "/grpc", "h2c", d.DialMeta)
 		}), client.WithSessionDialer(func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) {
 			return d.DockerAPI.DialHijack(ctx, "/session", proto, meta)
 		}),
@@ -70,24 +74,67 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 	return client.New(ctx, "", opts...)
 }
 
+type features struct {
+	once sync.Once
+	list map[driver.Feature]bool
+}
+
 func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
-	var useContainerdSnapshotter bool
-	c, err := d.Client(ctx)
-	if err == nil {
-		workers, _ := c.ListWorkers(ctx)
+	d.features.once.Do(func() {
+		var useContainerdSnapshotter bool
+		if c, err := d.Client(ctx); err == nil {
+			workers, _ := c.ListWorkers(ctx)
+			for _, w := range workers {
+				if _, ok := w.Labels["org.mobyproject.buildkit.worker.snapshotter"]; ok {
+					useContainerdSnapshotter = true
+				}
+			}
+			c.Close()
+		}
+		d.features.list = map[driver.Feature]bool{
+			driver.OCIExporter:    useContainerdSnapshotter,
+			driver.DockerExporter: useContainerdSnapshotter,
+			driver.CacheExport:    useContainerdSnapshotter,
+			driver.MultiPlatform:  useContainerdSnapshotter,
+		}
+	})
+	return d.features.list
+}
+
+type hostGateway struct {
+	once sync.Once
+	ip   net.IP
+	err  error
+}
+
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	d.hostGateway.once.Do(func() {
+		c, err := d.Client(ctx)
+		if err != nil {
+			d.hostGateway.err = err
+			return
+		}
+		defer c.Close()
+		workers, err := c.ListWorkers(ctx)
+		if err != nil {
+			d.hostGateway.err = errors.Wrap(err, "listing workers")
+			return
+		}
 		for _, w := range workers {
-			if _, ok := w.Labels["org.mobyproject.buildkit.worker.snapshotter"]; ok {
-				useContainerdSnapshotter = true
+			// should match github.com/docker/docker/builder/builder-next/worker/label.HostGatewayIP const
+			if v, ok := w.Labels["org.mobyproject.buildkit.worker.moby.host-gateway-ip"]; ok && v != "" {
+				ip := net.ParseIP(v)
+				if ip == nil {
+					d.hostGateway.err = errors.Errorf("failed to parse host-gateway IP: %s", v)
+					return
+				}
+				d.hostGateway.ip = ip
+				return
 			}
 		}
-		c.Close()
-	}
-	return map[driver.Feature]bool{
-		driver.OCIExporter:    useContainerdSnapshotter,
-		driver.DockerExporter: useContainerdSnapshotter,
-		driver.CacheExport:    useContainerdSnapshotter,
-		driver.MultiPlatform:  useContainerdSnapshotter,
-	}
+		d.hostGateway.err = errors.New("host-gateway IP not found")
+	})
+	return d.hostGateway.ip, d.hostGateway.err
 }
 
 func (d *Driver) Factory() driver.Factory {

diff --git a/driver/docker/factory.go b/driver/docker/factory.go
@@ -26,12 +26,12 @@ func (*factory) Usage() string {
 	return "docker"
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}
 
-	c, err := api.DialHijack(ctx, "/grpc", "h2c", nil)
+	c, err := api.DialHijack(ctx, "/grpc", "h2c", dialMeta)
 	if err != nil {
 		return priorityUnsupported
 	}

diff --git a/driver/driver.go b/driver/driver.go
@@ -3,6 +3,7 @@ package driver
 import (
 	"context"
 	"io"
+	"net"
 
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/util/progress"
@@ -60,6 +61,7 @@ type Driver interface {
 	Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error
 	Client(ctx context.Context) (*client.Client, error)
 	Features(ctx context.Context) map[Feature]bool
+	HostGatewayIP(ctx context.Context) (net.IP, error)
 	IsMobyDriver() bool
 	Config() InitConfig
 }

diff --git a/driver/kubernetes/driver.go b/driver/kubernetes/driver.go
@@ -236,3 +236,7 @@ func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
 		driver.MultiPlatform:  true, // Untested (needs multiple Driver instances)
 	}
 }
+
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	return nil, errors.New("host-gateway is not supported by the kubernetes driver")
+}
diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go
@@ -34,7 +34,7 @@ func (*factory) Usage() string {
 	return DriverName
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}