bake: display read definition files in build output

Signed-off-by: CrazyMax <[email protected]>

bake/bake.go

@@ -11,15 +11,18 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	composecli "github.com/compose-spec/compose-go/cli"
 	"github.com/docker/buildx/bake/hclparser"
 	"github.com/docker/buildx/build"
 	controllerapi "github.com/docker/buildx/controller/pb"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
+	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/config"
 	hcl "github.com/hashicorp/hcl/v2"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/pkg/errors"
@@ -54,28 +57,34 @@ func defaultFilenames() []string {
 	return names
 }
 
-func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) {
+func ReadLocalFiles(names []string, stdin io.Reader, l progress.SubLogger) ([]File, error) {
 	isDefault := false
 	if len(names) == 0 {
 		isDefault = true
 		names = defaultFilenames()
 	}
 	out := make([]File, 0, len(names))
 
+	setStatus := func(st *client.VertexStatus) {
+		if l != nil {
+			l.SetStatus(st)
+		}
+	}
+
 	for _, n := range names {
 		var dt []byte
 		var err error
 		if n == "-" {
-			dt, err = io.ReadAll(stdin)
+			dt, err = readWithStatus(stdin, setStatus)
 			if err != nil {
 				return nil, err
 			}
 		} else {
-			dt, err = os.ReadFile(n)
+			dt, err = readFileWithStatus(n, isDefault, setStatus)
+			if dt == nil && err == nil {
+				continue
+			}
 			if err != nil {
-				if isDefault && errors.Is(err, os.ErrNotExist) {
-					continue
-				}
 				return nil, err
 			}
 		}
@@ -84,6 +93,88 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) {
 	return out, nil
 }
 
+func readFileWithStatus(fname string, isDefault bool, setStatus func(st *client.VertexStatus)) (dt []byte, err error) {
+	st := &client.VertexStatus{
+		ID: "reading " + fname,
+	}
+
+	defer func() {
+		now := time.Now()
+		st.Completed = &now
+		if dt != nil || err != nil {
+			setStatus(st)
+		}
+	}()
+
+	now := time.Now()
+	st.Started = &now
+
+	f, err := os.Open(fname)
+	if err != nil {
+		if isDefault && errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	setStatus(st)
+
+	info, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	st.Total = info.Size()
+	setStatus(st)
+
+	buf := make([]byte, 1024)
+	for {
+		n, err := f.Read(buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		dt = append(dt, buf[:n]...)
+		st.Current += int64(n)
+		setStatus(st)
+	}
+
+	return dt, nil
+}
+
+func readWithStatus(r io.Reader, setStatus func(st *client.VertexStatus)) (dt []byte, err error) {
+	st := &client.VertexStatus{
+		ID: "reading from stdin",
+	}
+
+	defer func() {
+		now := time.Now()
+		st.Completed = &now
+		setStatus(st)
+	}()
+
+	now := time.Now()
+	st.Started = &now
+	setStatus(st)
+
+	buf := make([]byte, 1024)
+	for {
+		n, err := r.Read(buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		dt = append(dt, buf[:n]...)
+		st.Current += int64(n)
+		setStatus(st)
+	}
+
+	return dt, nil
+}
+
 func ListTargets(files []File) ([]string, error) {
 	c, err := ParseFiles(files, nil)
 	if err != nil {

bake/bake_test.go

@@ -1408,7 +1408,7 @@ func TestReadLocalFilesDefault(t *testing.T) {
 			for _, tf := range tt.filenames {
 				require.NoError(t, os.WriteFile(tf, []byte(tf), 0644))
 			}
-			files, err := ReadLocalFiles(nil, nil)
+			files, err := ReadLocalFiles(nil, nil, nil)
 			require.NoError(t, err)
 			if len(files) == 0 {
 				require.Equal(t, len(tt.expected), len(files))

bake/remote.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
+	"time"
 
 	"github.com/docker/buildx/builder"
 	controllerapi "github.com/docker/buildx/controller/pb"
@@ -76,11 +77,14 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name
 			return nil, err
 		}
 
-		if filename != "" {
-			files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
-		} else {
-			files, err = filesFromRef(ctx, ref, names)
-		}
+		progress.Wrap("[internal] load remote bake definitions", pw.Write, func(sub progress.SubLogger) error {
+			if filename != "" {
+				files, err = filesFromURLRef(ctx, c, ref, inp, filename, names, sub)
+			} else {
+				files, err = filesFromRef(ctx, ref, names, sub)
+			}
+			return nil
+		})
 		return nil, err
 	}, ch)
 
@@ -110,7 +114,7 @@ func isArchive(header []byte) bool {
 	return err == nil
 }
 
-func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
+func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string, l progress.SubLogger) ([]File, error) {
 	stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
 	if err != nil {
 		return nil, err
@@ -148,13 +152,20 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere
 			return nil, err
 		}
 
-		return filesFromRef(ctx, ref, names)
+		return filesFromRef(ctx, ref, names, l)
 	}
 
 	inp.State = nil
 	name := inp.URL
 	inp.URL = ""
 
+	now := time.Now()
+	l.SetStatus(&client.VertexStatus{
+		ID:        "reading " + name,
+		Started:   &now,
+		Completed: &now,
+	})
+
 	if len(dt) > stat.Size() {
 		if stat.Size() > 1024*512 {
 			return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
@@ -171,7 +182,7 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere
 	return []File{{Name: name, Data: dt}}, nil
 }
 
-func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
+func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string, l progress.SubLogger) ([]File, error) {
 	// TODO: auto-remove parent dir in needed
 	var files []File
 
@@ -189,6 +200,12 @@ func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) (
 			}
 			return nil, err
 		}
+		now := time.Now()
+		l.SetStatus(&client.VertexStatus{
+			ID:        "reading " + name,
+			Started:   &now,
+			Completed: &now,
+		})
 		dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
 		if err != nil {
 			return nil, err

commands/bake.go

@@ -150,7 +150,10 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 	if url != "" {
 		files, inp, err = bake.ReadRemoteFiles(ctx, nodes, url, in.files, printer)
 	} else {
-		files, err = bake.ReadLocalFiles(in.files, dockerCli.In())
+		progress.Wrap("[internal] load local bake definitions", printer.Write, func(sub progress.SubLogger) error {
+			files, err = bake.ReadLocalFiles(in.files, dockerCli.In(), sub)
+			return nil
+		})
 	}
 	if err != nil {
 		return err

tests/bake.go

@@ -19,6 +19,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) {
 
 var bakeTests = []func(t *testing.T, sb integration.Sandbox){
 	testBakeLocal,
+	testBakeLocalMulti,
 	testBakeRemote,
 	testBakeRemoteCmdContext,
 	testBakeRemoteCmdContextOverride,
@@ -44,8 +45,45 @@ target "default" {
 	)
 	dirDest := t.TempDir()
 
-	out, err := bakeCmd(sb, withDir(dir), withArgs("--set", "*.output=type=local,dest="+dirDest))
+	cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest))
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, out)
+	require.Contains(t, string(out), `#1 [internal] load local bake definitions`)
+	require.Contains(t, string(out), `#1 reading docker-bake.hcl`)
+
+	require.FileExists(t, filepath.Join(dirDest, "foo"))
+}
+
+func testBakeLocalMulti(t *testing.T, sb integration.Sandbox) {
+	dockerfile := []byte(`
+FROM scratch
+COPY foo /foo
+	`)
+	bakefile := []byte(`
+target "default" {
+}
+`)
+	composefile := []byte(`
+services:
+  app:
+    build: {}
+`)
+
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+		fstest.CreateFile("compose.yaml", composefile, 0600),
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+	dirDest := t.TempDir()
+
+	cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest))
+	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, out)
+	require.Contains(t, string(out), `#1 [internal] load local bake definitions`)
+	require.Contains(t, string(out), `#1 reading compose.yaml`)
+	require.Contains(t, string(out), `#1 reading docker-bake.hcl`)
 
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }
@@ -74,8 +112,11 @@ EOT
 	gitutil.GitCommit(git, t, "initial commit")
 	addr := gitutil.GitServeHTTP(git, t)
 
-	out, err := bakeCmd(sb, withDir(dir), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest))
+	cmd := buildxCmd(sb, withDir(dir), withArgs("bake", addr, "--progress=plain", "--set", "*.output=type=local,dest="+dirDest))
+	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, out)
+	require.Contains(t, string(out), `#1 [internal] load remote bake definitions`)
+	require.Contains(t, string(out), `#1 reading docker-bake.hcl`)
 
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }

util/cobrautil/completion/completion.go

@@ -19,7 +19,7 @@ func Disable(cmd *cobra.Command, args []string, toComplete string) ([]string, co
 
 func BakeTargets(files []string) ValidArgsFn {
 	return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-		f, err := bake.ReadLocalFiles(files, nil)
+		f, err := bake.ReadLocalFiles(files, nil, nil)
 		if err != nil {
 			return nil, cobra.ShellCompDirectiveError
 		}