From 23c2498dee0977d8972c8ff025f0c8cbcb6c268f Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 23 Oct 2023 15:26:11 +0200 Subject: [PATCH] bake: display read definition files in build output Signed-off-by: CrazyMax --- bake/bake.go | 103 ++++++++++++++++++++++-- bake/bake_test.go | 2 +- commands/bake.go | 5 +- tests/bake.go | 40 ++++++++- util/cobrautil/completion/completion.go | 2 +- 5 files changed, 142 insertions(+), 10 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index a2a5499c684..7031086fd82 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "time" composecli "github.com/compose-spec/compose-go/cli" "github.com/docker/buildx/bake/hclparser" @@ -18,8 +19,10 @@ import ( controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/platformutil" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/config" hcl "github.com/hashicorp/hcl/v2" + "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" @@ -54,7 +57,7 @@ func defaultFilenames() []string { return names } -func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { +func ReadLocalFiles(names []string, stdin io.Reader, l progress.SubLogger) ([]File, error) { isDefault := false if len(names) == 0 { isDefault = true @@ -62,20 +65,26 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { } out := make([]File, 0, len(names)) + setStatus := func(st *client.VertexStatus) { + if l != nil { + l.SetStatus(st) + } + } + for _, n := range names { var dt []byte var err error if n == "-" { - dt, err = io.ReadAll(stdin) + dt, err = readWithProgress(stdin, setStatus) if err != nil { return nil, err } } else { - dt, err = os.ReadFile(n) + dt, err = readFileWithProgress(n, isDefault, setStatus) + if dt == nil && err == nil { + continue + } if err != nil { - if isDefault && errors.Is(err, os.ErrNotExist) { - continue - } return nil, err } } @@ -84,6 +93,88 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { return out, nil } +func readFileWithProgress(fname string, isDefault bool, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading " + fname, + } + + defer func() { + now := time.Now() + st.Completed = &now + if dt != nil || err != nil { + setStatus(st) + } + }() + + now := time.Now() + st.Started = &now + + f, err := os.Open(fname) + if err != nil { + if isDefault && errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + defer f.Close() + setStatus(st) + + info, err := f.Stat() + if err != nil { + return nil, err + } + st.Total = info.Size() + setStatus(st) + + buf := make([]byte, 1024) + for { + n, err := f.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + st.Current += int64(n) + setStatus(st) + } + + return dt, nil +} + +func readWithProgress(r io.Reader, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading from stdin", + } + + defer func() { + now := time.Now() + st.Completed = &now + setStatus(st) + }() + + now := time.Now() + st.Started = &now + setStatus(st) + + buf := make([]byte, 1024) + for { + n, err := r.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + st.Current += int64(n) + setStatus(st) + } + + return dt, nil +} + func ListTargets(files []File) ([]string, error) { c, err := ParseFiles(files, nil) if err != nil { diff --git a/bake/bake_test.go b/bake/bake_test.go index 37789c9467d..1d8e86c2c9c 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -1440,7 +1440,7 @@ func TestReadLocalFilesDefault(t *testing.T) { for _, tf := range tt.filenames { require.NoError(t, os.WriteFile(tf, []byte(tf), 0644)) } - files, err := ReadLocalFiles(nil, nil) + files, err := ReadLocalFiles(nil, nil, nil) require.NoError(t, err) if len(files) == 0 { require.Equal(t, len(tt.expected), len(files)) diff --git a/commands/bake.go b/commands/bake.go index 958b8299a40..f81d64fe88c 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -150,7 +150,10 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com if url != "" { files, inp, err = bake.ReadRemoteFiles(ctx, nodes, url, in.files, printer) } else { - files, err = bake.ReadLocalFiles(in.files, dockerCli.In()) + progress.Wrap("[internal] load local bake definitions", printer.Write, func(sub progress.SubLogger) error { + files, err = bake.ReadLocalFiles(in.files, dockerCli.In(), sub) + return nil + }) } if err != nil { return err diff --git a/tests/bake.go b/tests/bake.go index 33d534618be..83a171ea573 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -20,6 +20,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) { var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakeLocal, + testBakeLocalMulti, testBakeRemote, testBakeRemoteCmdContext, testBakeRemoteCmdContextOverride, @@ -47,8 +48,45 @@ target "default" { ) dirDest := t.TempDir() - out, err := bakeCmd(sb, withDir(dir), withArgs("--set", "*.output=type=local,dest="+dirDest)) + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest)) + out, err := cmd.CombinedOutput() + require.NoError(t, err, out) + require.Contains(t, string(out), `#1 [internal] load local bake definitions`) + require.Contains(t, string(out), `#1 reading docker-bake.hcl`) + + require.FileExists(t, filepath.Join(dirDest, "foo")) +} + +func testBakeLocalMulti(t *testing.T, sb integration.Sandbox) { + dockerfile := []byte(` +FROM scratch +COPY foo /foo + `) + bakefile := []byte(` +target "default" { +} +`) + composefile := []byte(` +services: + app: + build: {} +`) + + dir := tmpdir( + t, + fstest.CreateFile("docker-bake.hcl", bakefile, 0600), + fstest.CreateFile("compose.yaml", composefile, 0600), + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte("foo"), 0600), + ) + dirDest := t.TempDir() + + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--progress=plain", "--set", "*.output=type=local,dest="+dirDest)) + out, err := cmd.CombinedOutput() require.NoError(t, err, out) + require.Contains(t, string(out), `#1 [internal] load local bake definitions`) + require.Contains(t, string(out), `#1 reading compose.yaml`) + require.Contains(t, string(out), `#1 reading docker-bake.hcl`) require.FileExists(t, filepath.Join(dirDest, "foo")) } diff --git a/util/cobrautil/completion/completion.go b/util/cobrautil/completion/completion.go index 60754537802..095cac52109 100644 --- a/util/cobrautil/completion/completion.go +++ b/util/cobrautil/completion/completion.go @@ -19,7 +19,7 @@ func Disable(cmd *cobra.Command, args []string, toComplete string) ([]string, co func BakeTargets(files []string) ValidArgsFn { return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - f, err := bake.ReadLocalFiles(files, nil) + f, err := bake.ReadLocalFiles(files, nil, nil) if err != nil { return nil, cobra.ShellCompDirectiveError }