diff --git a/bake/bake.go b/bake/bake.go index a2a5499c6844..7031086fd826 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "time" composecli "github.com/compose-spec/compose-go/cli" "github.com/docker/buildx/bake/hclparser" @@ -18,8 +19,10 @@ import ( controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/platformutil" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/config" hcl "github.com/hashicorp/hcl/v2" + "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" @@ -54,7 +57,7 @@ func defaultFilenames() []string { return names } -func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { +func ReadLocalFiles(names []string, stdin io.Reader, l progress.SubLogger) ([]File, error) { isDefault := false if len(names) == 0 { isDefault = true @@ -62,20 +65,26 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { } out := make([]File, 0, len(names)) + setStatus := func(st *client.VertexStatus) { + if l != nil { + l.SetStatus(st) + } + } + for _, n := range names { var dt []byte var err error if n == "-" { - dt, err = io.ReadAll(stdin) + dt, err = readWithProgress(stdin, setStatus) if err != nil { return nil, err } } else { - dt, err = os.ReadFile(n) + dt, err = readFileWithProgress(n, isDefault, setStatus) + if dt == nil && err == nil { + continue + } if err != nil { - if isDefault && errors.Is(err, os.ErrNotExist) { - continue - } return nil, err } } @@ -84,6 +93,88 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { return out, nil } +func readFileWithProgress(fname string, isDefault bool, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading " + fname, + } + + defer func() { + now := time.Now() + st.Completed = &now + if dt != nil || err != nil { + setStatus(st) + } + }() + + now := time.Now() + st.Started = &now + + f, err := os.Open(fname) + if err != nil { + if isDefault && errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + defer f.Close() + setStatus(st) + + info, err := f.Stat() + if err != nil { + return nil, err + } + st.Total = info.Size() + setStatus(st) + + buf := make([]byte, 1024) + for { + n, err := f.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + st.Current += int64(n) + setStatus(st) + } + + return dt, nil +} + +func readWithProgress(r io.Reader, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading from stdin", + } + + defer func() { + now := time.Now() + st.Completed = &now + setStatus(st) + }() + + now := time.Now() + st.Started = &now + setStatus(st) + + buf := make([]byte, 1024) + for { + n, err := r.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + st.Current += int64(n) + setStatus(st) + } + + return dt, nil +} + func ListTargets(files []File) ([]string, error) { c, err := ParseFiles(files, nil) if err != nil { diff --git a/bake/bake_test.go b/bake/bake_test.go index 37789c9467d8..1d8e86c2c9c4 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -1440,7 +1440,7 @@ func TestReadLocalFilesDefault(t *testing.T) { for _, tf := range tt.filenames { require.NoError(t, os.WriteFile(tf, []byte(tf), 0644)) } - files, err := ReadLocalFiles(nil, nil) + files, err := ReadLocalFiles(nil, nil, nil) require.NoError(t, err) if len(files) == 0 { require.Equal(t, len(tt.expected), len(files)) diff --git a/bake/remote.go b/bake/remote.go index 3dd1e0d16798..a4752e86f3b1 100644 --- a/bake/remote.go +++ b/bake/remote.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "context" + "time" "github.com/docker/buildx/builder" controllerapi "github.com/docker/buildx/controller/pb" @@ -76,11 +77,14 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name return nil, err } - if filename != "" { - files, err = filesFromURLRef(ctx, c, ref, inp, filename, names) - } else { - files, err = filesFromRef(ctx, ref, names) - } + progress.Wrap("[internal] load remote bake definitions", pw.Write, func(sub progress.SubLogger) error { + if filename != "" { + files, err = filesFromURLRef(ctx, c, ref, inp, filename, names, sub) + } else { + files, err = filesFromRef(ctx, ref, names, sub) + } + return nil + }) return nil, err }, ch) @@ -110,7 +114,8 @@ func isArchive(header []byte) bool { return err == nil } -func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) { +func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string, l progress.SubLogger) ([]File, error) { + now := time.Now() stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename}) if err != nil { return nil, err @@ -148,13 +153,19 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere return nil, err } - return filesFromRef(ctx, ref, names) + return filesFromRef(ctx, ref, names, l) } inp.State = nil name := inp.URL inp.URL = "" + l.SetStatus(&client.VertexStatus{ + ID: "reading " + name, + Started: &now, + Completed: &now, + }) + if len(dt) > stat.Size() { if stat.Size() > 1024*512 { return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size") @@ -171,7 +182,7 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere return []File{{Name: name, Data: dt}}, nil } -func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) { +func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string, l progress.SubLogger) ([]File, error) { // TODO: auto-remove parent dir in needed var files []File @@ -189,6 +200,12 @@ func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ( } return nil, err } + now := time.Now() + l.SetStatus(&client.VertexStatus{ + ID: "reading " + name, + Started: &now, + Completed: &now, + }) dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name}) if err != nil { return nil, err diff --git a/commands/bake.go b/commands/bake.go index 958b8299a404..f81d64fe88c9 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -150,7 +150,10 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com if url != "" { files, inp, err = bake.ReadRemoteFiles(ctx, nodes, url, in.files, printer) } else { - files, err = bake.ReadLocalFiles(in.files, dockerCli.In()) + progress.Wrap("[internal] load local bake definitions", printer.Write, func(sub progress.SubLogger) error { + files, err = bake.ReadLocalFiles(in.files, dockerCli.In(), sub) + return nil + }) } if err != nil { return err diff --git a/tests/bake.go b/tests/bake.go index 33d534618be5..40cb7afe3071 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -20,6 +20,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) { var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakeLocal, + testBakeLocalMulti, testBakeRemote, testBakeRemoteCmdContext, testBakeRemoteCmdContextOverride, @@ -47,8 +48,45 @@ target "default" { ) dirDest := t.TempDir() - out, err := bakeCmd(sb, withDir(dir), withArgs("--set", "*.output=type=local,dest="+dirDest)) + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest)) + out, err := cmd.CombinedOutput() + require.NoError(t, err, out) + require.Contains(t, string(out), `#1 [internal] load local bake definitions`) + require.Contains(t, string(out), `#1 reading docker-bake.hcl`) + + require.FileExists(t, filepath.Join(dirDest, "foo")) +} + +func testBakeLocalMulti(t *testing.T, sb integration.Sandbox) { + dockerfile := []byte(` +FROM scratch +COPY foo /foo + `) + bakefile := []byte(` +target "default" { +} +`) + composefile := []byte(` +services: + app: + build: {} +`) + + dir := tmpdir( + t, + fstest.CreateFile("docker-bake.hcl", bakefile, 0600), + fstest.CreateFile("compose.yaml", composefile, 0600), + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte("foo"), 0600), + ) + dirDest := t.TempDir() + + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest)) + out, err := cmd.CombinedOutput() require.NoError(t, err, out) + require.Contains(t, string(out), `#1 [internal] load local bake definitions`) + require.Contains(t, string(out), `#1 reading compose.yaml`) + require.Contains(t, string(out), `#1 reading docker-bake.hcl`) require.FileExists(t, filepath.Join(dirDest, "foo")) } @@ -77,8 +115,11 @@ EOT gitutil.GitCommit(git, t, "initial commit") addr := gitutil.GitServeHTTP(git, t) - out, err := bakeCmd(sb, withDir(dir), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest)) + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", addr, "--progress=plain", "--set", "*.output=type=local,dest="+dirDest)) + out, err := cmd.CombinedOutput() require.NoError(t, err, out) + require.Contains(t, string(out), `#1 [internal] load remote bake definitions`) + require.Contains(t, string(out), `#1 reading docker-bake.hcl`) require.FileExists(t, filepath.Join(dirDest, "foo")) } diff --git a/util/cobrautil/completion/completion.go b/util/cobrautil/completion/completion.go index 607545378026..095cac521090 100644 --- a/util/cobrautil/completion/completion.go +++ b/util/cobrautil/completion/completion.go @@ -19,7 +19,7 @@ func Disable(cmd *cobra.Command, args []string, toComplete string) ([]string, co func BakeTargets(files []string) ValidArgsFn { return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - f, err := bake.ReadLocalFiles(files, nil) + f, err := bake.ReadLocalFiles(files, nil, nil) if err != nil { return nil, cobra.ShellCompDirectiveError }