diff --git a/README.md b/README.md index adf6f99d5..fb5b40358 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,15 @@ Flags: -h, --help help for gdu -i, --ignore-dirs strings Absolute paths to ignore (separated by comma) (default [/proc,/dev,/sys,/run]) -I, --ignore-dirs-pattern strings Absolute path patterns to ignore (separated by comma) + -f, --input-file string Import analysis from JSON file -l, --log-file string Path to a logfile (default "/dev/null") -m, --max-cores int Set max cores that GDU will use. 8 cores available (default 8) -c, --no-color Do not use colorized output -x, --no-cross Do not cross filesystem boundaries - -H, --no-hidden Ignore hidden directories (beggining with dot) + -H, --no-hidden Ignore hidden directories (beginning with dot) -p, --no-progress Do not show progress in non-interactive mode -n, --non-interactive Do not run in interactive mode + -o, --output-file string Export all info into file as JSON -a, --show-apparent-size Show apparent size -d, --show-disks Show all mounted disks -v, --version Print version @@ -103,10 +105,15 @@ Flags: gdu -np / # do not show progress, useful when using its output in a script gdu / > file # write stats to file, do not start interactive mode -Gdu has two modes: interactive (default) and non-interactive. + gdu -o- / | gzip >report.json.gz # write all info to JSON file for later analysis + zcat report.json.gz | gdu -f- # read analysis from file + +Gdu has three modes: interactive (default), non-interactive and export. Non-interactive mode is started automtically when TTY is not detected (using [go-isatty](https://github.com/mattn/go-isatty)), for example if the output is being piped to a file, or it can be started explicitly by using a flag. +Export mode (flag `-o`) outputs all usage data as JSON, which can then be later opened using the `-f` flag. + Hard links are counted only once. ## File flags diff --git a/cmd/gdu/app/app.go b/cmd/gdu/app/app.go index f98a14ed8..4517ee3c4 100644 --- a/cmd/gdu/app/app.go +++ b/cmd/gdu/app/app.go @@ -12,6 +12,7 @@ import ( "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/dundee/gdu/v5/stdout" "github.com/dundee/gdu/v5/tui" "github.com/gdamore/tcell/v2" @@ -22,6 +23,7 @@ import ( type UI interface { ListDevices(getter device.DevicesInfoGetter) error AnalyzePath(path string, parentDir *analyze.Dir) error + ReadAnalysis(input io.Reader) error SetIgnoreDirPaths(paths []string) SetIgnoreDirPatterns(paths []string) error SetIgnoreHidden(value bool) @@ -31,6 +33,8 @@ type UI interface { // Flags define flags accepted by Run type Flags struct { LogFile string + InputFile string + OutputFile string IgnoreDirs []string IgnoreDirPatterns []string MaxCores int @@ -71,7 +75,10 @@ func (a *App) Run() error { log.SetOutput(f) path := a.getPath() - ui := a.createUI() + ui, err := a.createUI() + if err != nil { + return err + } if err := a.setNoCross(path); err != nil { return err @@ -116,9 +123,28 @@ func (a *App) setMaxProcs() { log.Printf("Max cores set to %d", runtime.GOMAXPROCS(0)) } -func (a *App) createUI() UI { +func (a *App) createUI() (UI, error) { var ui UI + if a.Flags.OutputFile != "" { + var output io.Writer + var err error + if a.Flags.OutputFile == "-" { + output = os.Stdout + } else { + output, err = os.OpenFile(a.Flags.OutputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return nil, fmt.Errorf("opening output file: %w", err) + } + } + ui = report.CreateExportUI( + a.Writer, + output, + !a.Flags.NoProgress && a.Istty, + ) + return ui, nil + } + if a.Flags.NonInteractive || !a.Istty { ui = stdout.CreateStdoutUI( a.Writer, @@ -134,7 +160,7 @@ func (a *App) createUI() UI { } tview.Styles.BorderColor = tcell.ColorDefault } - return ui + return ui, nil } func (a *App) setNoCross(path string) error { @@ -154,6 +180,21 @@ func (a *App) runAction(ui UI, path string) error { if err := ui.ListDevices(a.Getter); err != nil { return fmt.Errorf("loading mount points: %w", err) } + } else if a.Flags.InputFile != "" { + var input io.Reader + var err error + if a.Flags.InputFile == "-" { + input = os.Stdin + } else { + input, err = os.OpenFile(a.Flags.InputFile, os.O_RDONLY, 0644) + if err != nil { + return fmt.Errorf("opening input file: %w", err) + } + } + + if err := ui.ReadAnalysis(input); err != nil { + return fmt.Errorf("reading analysis: %w", err) + } } else { if err := ui.AnalyzePath(path, nil); err != nil { return fmt.Errorf("scanning dir: %w", err) diff --git a/cmd/gdu/app/app_linux_test.go b/cmd/gdu/app/app_linux_test.go index eb705acea..739f32ae0 100644 --- a/cmd/gdu/app/app_linux_test.go +++ b/cmd/gdu/app/app_linux_test.go @@ -51,3 +51,15 @@ func TestLogError(t *testing.T) { assert.Empty(t, out) assert.Contains(t, err.Error(), "permission denied") } + +func TestOutputFileError(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", OutputFile: "/xyzxyz"}, + []string{}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Empty(t, out) + assert.Contains(t, err.Error(), "permission denied") +} diff --git a/cmd/gdu/app/app_test.go b/cmd/gdu/app/app_test.go index 7bcdc1a0a..7c344d830 100644 --- a/cmd/gdu/app/app_test.go +++ b/cmd/gdu/app/app_test.go @@ -2,6 +2,7 @@ package app import ( "bytes" + "os" "runtime" "strings" "testing" @@ -99,6 +100,61 @@ func TestAnalyzePathWithGui(t *testing.T) { assert.Nil(t, err) } +func TestAnalyzePathWithExport(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + defer func() { + os.Remove("output.json") + }() + + out, err := runApp( + &Flags{LogFile: "/dev/null", OutputFile: "output.json"}, + []string{"test_dir"}, + true, + testdev.DevicesInfoGetterMock{}, + ) + + assert.NotEmpty(t, out) + assert.Nil(t, err) +} + +func TestReadAnalysisFromFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "../../../internal/testdata/test.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.NotEmpty(t, out) + assert.Contains(t, out, "main.go") + assert.Nil(t, err) +} + +func TestReadWrongAnalysisFromFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "../../../internal/testdata/wrong.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Empty(t, out) + assert.Contains(t, err.Error(), "Array of maps not found") +} + +func TestReadWrongAnalysisFromNotExistingFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "xxx.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Empty(t, out) + assert.Contains(t, err.Error(), "no such file or directory") +} + func TestAnalyzePathWithErr(t *testing.T) { fin := testdir.CreateTestDir() defer fin() @@ -144,6 +200,24 @@ func TestListDevices(t *testing.T) { assert.Nil(t, err) } +func TestListDevicesToFile(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + defer func() { + os.Remove("output.json") + }() + + out, err := runApp( + &Flags{LogFile: "/dev/null", ShowDisks: true, OutputFile: "output.json"}, + []string{}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Equal(t, "", out) + assert.Contains(t, err.Error(), "not supported") +} + func TestListDevicesWithGui(t *testing.T) { fin := testdir.CreateTestDir() defer fin() diff --git a/cmd/gdu/main.go b/cmd/gdu/main.go index 799c97f7a..3e0838c3a 100644 --- a/cmd/gdu/main.go +++ b/cmd/gdu/main.go @@ -32,6 +32,8 @@ func init() { af = &app.Flags{} flags := rootCmd.Flags() flags.StringVarP(&af.LogFile, "log-file", "l", "/dev/null", "Path to a logfile") + flags.StringVarP(&af.OutputFile, "output-file", "o", "", "Export all info into file as JSON") + flags.StringVarP(&af.InputFile, "input-file", "f", "", "Import analysis from JSON file") flags.IntVarP(&af.MaxCores, "max-cores", "m", runtime.NumCPU(), fmt.Sprintf("Set max cores that GDU will use. %d cores available", runtime.NumCPU())) flags.BoolVarP(&af.ShowVersion, "version", "v", false, "Print version") @@ -61,7 +63,7 @@ func runE(command *cobra.Command, args []string) error { var termApp *tview.Application - if !af.ShowVersion && !af.NonInteractive && istty { + if !af.ShowVersion && !af.NonInteractive && istty && af.OutputFile == "" { screen, err := tcell.NewScreen() if err != nil { return fmt.Errorf("Error creating screen: %w", err) diff --git a/gdu.1 b/gdu.1 index 2a3b32b48..be3456a17 100644 --- a/gdu.1 +++ b/gdu.1 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pandoc 2.13 +.\" Automatically generated by Pandoc 2.14.0.2 .\" .TH "gdu" "1" "Jan 2021" "" "" .hy @@ -47,6 +47,13 @@ interactive mode .PP \f[B]-a\f[R], \f[B]--show-apparent-size\f[R][=false] Show apparent size .PP +\f[B]-f\f[R], \f[B]-\[em]input-file\f[R] Import analysis from JSON file. +If the file is \[dq]-\[dq], read from standard input. +.PP +\f[B]-o\f[R], \f[B]-\[em]output-file\f[R] Export all info into file as +JSON. +If the file is \[dq]-\[dq], write to standard output. +.PP \f[B]-v\f[R], \f[B]--version\f[R][=false] Print version .SH FILE FLAGS .PP diff --git a/gdu.1.md b/gdu.1.md index d9706ee08..dad9fd3bf 100644 --- a/gdu.1.md +++ b/gdu.1.md @@ -49,6 +49,10 @@ non-interactive mode **-a**, **\--show-apparent-size**\[=false\] Show apparent size +**-f**, **\----input-file** Import analysis from JSON file. If the file is \"-\", read from standard input. + +**-o**, **\----output-file** Export all info into file as JSON. If the file is \"-\", write to standard output. + **-v**, **\--version**\[=false\] Print version # FILE FLAGS diff --git a/internal/common/ui.go b/internal/common/ui.go index 3d8c0bd66..6039fb1bc 100644 --- a/internal/common/ui.go +++ b/internal/common/ui.go @@ -18,3 +18,21 @@ type UI struct { ShowApparentSize bool PathChecker func(string) (fs.FileInfo, error) } + +// file size constants +const ( + _ = iota + KB float64 = 1 << (10 * iota) + MB + GB + TB + PB + EB +) + +// file count constants +const ( + K int = 1e3 + M int = 1e6 + G int = 1e9 +) diff --git a/internal/testdata/test.json b/internal/testdata/test.json new file mode 100644 index 000000000..192e1d2ba --- /dev/null +++ b/internal/testdata/test.json @@ -0,0 +1,7 @@ +[1,2,{"progname":"gdu","progver":"development","timestamp":1626807263}, +[{"name":"/home/gdu"}, +[{"name":"app"}, +{"name":"app.go","asize":4638,"dsize":8192}, +{"name":"app_linux_test.go","asize":1410,"dsize":4096}, +{"name":"app_test.go","asize":4974,"dsize":8192}], +{"name":"main.go","asize":3205,"dsize":4096}]] diff --git a/internal/testdata/wrong.json b/internal/testdata/wrong.json new file mode 100644 index 000000000..8adb9bb60 --- /dev/null +++ b/internal/testdata/wrong.json @@ -0,0 +1 @@ +[1,2,3,4] diff --git a/pkg/analyze/encode.go b/pkg/analyze/encode.go new file mode 100644 index 000000000..1b9038fce --- /dev/null +++ b/pkg/analyze/encode.go @@ -0,0 +1,80 @@ +package analyze + +import ( + "encoding/json" + "fmt" + "io" +) + +// EncodeJSON writes JSON representation of dir +func (f *Dir) EncodeJSON(writer io.Writer, topLevel bool) error { + buff := make([]byte, 0, 20) + + buff = append(buff, []byte(`[{"name":`)...) + + if topLevel { + if err := addString(&buff, f.GetPath()); err != nil { + return err + } + } else { + if err := addString(&buff, f.GetName()); err != nil { + return err + } + } + + buff = append(buff, '}') + if f.Files.Len() > 0 { + buff = append(buff, ',') + } + buff = append(buff, '\n') + + if _, err := writer.Write(buff); err != nil { + return err + } + + for i, item := range f.Files { + if i > 0 { + if _, err := writer.Write([]byte(",\n")); err != nil { + return err + } + } + err := item.EncodeJSON(writer, false) + if err != nil { + return err + } + } + + if _, err := writer.Write([]byte("]")); err != nil { + return err + } + return nil +} + +// EncodeJSON writes JSON representation of file +func (f *File) EncodeJSON(writer io.Writer, topLevel bool) error { + buff := make([]byte, 0, 20) + + buff = append(buff, []byte(`{"name":`)...) + if err := addString(&buff, f.GetName()); err != nil { + return err + } + buff = append(buff, []byte(`,"asize":`)...) + buff = append(buff, []byte(fmt.Sprint(f.GetSize()))...) + buff = append(buff, []byte(`,"dsize":`)...) + buff = append(buff, []byte(fmt.Sprint(f.GetUsage()))...) + buff = append(buff, '}') + + if _, err := writer.Write(buff); err != nil { + return err + } + return nil +} + +func addString(buff *[]byte, val string) error { + b, err := json.Marshal(val) + if err != nil { + return err + } + *buff = append(*buff, b...) + return err +} diff --git a/pkg/analyze/encode_test.go b/pkg/analyze/encode_test.go new file mode 100644 index 000000000..efa1ad984 --- /dev/null +++ b/pkg/analyze/encode_test.go @@ -0,0 +1,55 @@ +package analyze + +import ( + "bytes" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func init() { + log.SetLevel(log.WarnLevel) +} + +func TestEncode(t *testing.T) { + dir := &Dir{ + File: &File{ + Name: "test_dir", + Size: 10, + Usage: 18, + }, + ItemCount: 4, + BasePath: ".", + } + + subdir := &Dir{ + File: &File{ + Name: "nested", + Size: 9, + Usage: 14, + Parent: dir, + }, + ItemCount: 3, + } + file := &File{ + Name: "file2", + Size: 3, + Usage: 4, + Parent: subdir, + } + file2 := &File{ + Name: "file", + Size: 5, + Usage: 6, + Parent: subdir, + } + dir.Files = Files{subdir} + subdir.Files = Files{file, file2} + + var buff bytes.Buffer + err := dir.EncodeJSON(&buff, true) + + assert.Nil(t, err) + assert.Contains(t, buff.String(), `"name":"nested"`) +} diff --git a/pkg/analyze/file.go b/pkg/analyze/file.go index 7a4687454..9c90bb95a 100644 --- a/pkg/analyze/file.go +++ b/pkg/analyze/file.go @@ -1,6 +1,7 @@ package analyze import ( + "io" "os" "path/filepath" ) @@ -19,6 +20,7 @@ type Item interface { GetUsage() int64 GetItemCount() int GetParent() *Dir + EncodeJSON(writer io.Writer, topLevel bool) error getItemStats(links AlreadyCountedHardlinks) (int, int64, int64) } diff --git a/report/export.go b/report/export.go new file mode 100644 index 000000000..48b4b5a25 --- /dev/null +++ b/report/export.go @@ -0,0 +1,193 @@ +package report + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/dundee/gdu/v5/build" + "github.com/dundee/gdu/v5/internal/common" + "github.com/dundee/gdu/v5/pkg/analyze" + "github.com/dundee/gdu/v5/pkg/device" + "github.com/fatih/color" +) + +// UI struct +type UI struct { + *common.UI + output io.Writer + exportOutput io.Writer + red *color.Color + orange *color.Color + writtenChan chan struct{} +} + +// CreateExportUI creates UI for stdout +func CreateExportUI(output io.Writer, exportOutput io.Writer, showProgress bool) *UI { + ui := &UI{ + UI: &common.UI{ + ShowProgress: showProgress, + Analyzer: analyze.CreateAnalyzer(), + PathChecker: os.Stat, + }, + output: output, + exportOutput: exportOutput, + writtenChan: make(chan struct{}), + } + ui.red = color.New(color.FgRed).Add(color.Bold) + ui.orange = color.New(color.FgYellow).Add(color.Bold) + + return ui +} + +// StartUILoop stub +func (ui *UI) StartUILoop() error { + return nil +} + +// ListDevices lists mounted devices and shows their disk usage +func (ui *UI) ListDevices(getter device.DevicesInfoGetter) error { + return errors.New("Exporting devices list is not supported") +} + +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + return errors.New("Reading analysis is not possible while exporting") +} + +// AnalyzePath analyzes recursively disk usage in given path +func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { + var ( + dir *analyze.Dir + wait sync.WaitGroup + waitWritten sync.WaitGroup + ) + abspath, _ := filepath.Abs(path) + + _, err := ui.PathChecker(abspath) + if err != nil { + return err + } + + if ui.ShowProgress { + waitWritten.Add(1) + go func() { + defer waitWritten.Done() + ui.updateProgress() + }() + } + + wait.Add(1) + go func() { + defer wait.Done() + dir = ui.Analyzer.AnalyzeDir(abspath, ui.CreateIgnoreFunc()) + }() + + wait.Wait() + + sort.Sort(dir.Files) + + var buff bytes.Buffer + + buff.Write([]byte(`[1,2,{"progname":"gdu","progver":"`)) + buff.Write([]byte(build.Version)) + buff.Write([]byte(`","timestamp":`)) + buff.Write([]byte(fmt.Sprint(time.Now().Unix()))) + buff.Write([]byte("},\n")) + + if err = dir.EncodeJSON(&buff, true); err != nil { + return err + } + if _, err = buff.Write([]byte("]\n")); err != nil { + return err + } + if _, err = buff.WriteTo(ui.exportOutput); err != nil { + return err + } + + switch f := ui.exportOutput.(type) { + case *os.File: + f.Close() + } + + if ui.ShowProgress { + ui.writtenChan <- struct{}{} + waitWritten.Wait() + } + + return nil +} + +func (ui *UI) updateProgress() { + waitingForWrite := false + + emptyRow := "\r" + for j := 0; j < 100; j++ { + emptyRow += " " + } + + progressRunes := []rune(`⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧`) + + progressChan := ui.Analyzer.GetProgressChan() + doneChan := ui.Analyzer.GetDoneChan() + + var progress analyze.CurrentProgress + + i := 0 + for { + fmt.Fprint(ui.output, emptyRow) + + select { + case progress = <-progressChan: + case <-doneChan: + fmt.Fprint(ui.output, "\r") + waitingForWrite = true + case <-ui.writtenChan: + fmt.Fprint(ui.output, "\r") + return + default: + } + + fmt.Fprintf(ui.output, "\r %s ", string(progressRunes[i])) + + if waitingForWrite { + fmt.Fprint(ui.output, "Writing output file...") + } else { + fmt.Fprint(ui.output, "Scanning... Total items: "+ + ui.red.Sprint(progress.ItemCount)+ + " size: "+ + ui.formatSize(progress.TotalSize)) + } + + time.Sleep(100 * time.Millisecond) + i++ + i %= 10 + } +} + +func (ui *UI) formatSize(size int64) string { + fsize := float64(size) + + switch { + case fsize >= common.EB: + return ui.orange.Sprintf("%.1f", fsize/common.EB) + " EiB" + case fsize >= common.PB: + return ui.orange.Sprintf("%.1f", fsize/common.PB) + " PiB" + case fsize >= common.TB: + return ui.orange.Sprintf("%.1f", fsize/common.TB) + " TiB" + case fsize >= common.GB: + return ui.orange.Sprintf("%.1f", fsize/common.GB) + " GiB" + case fsize >= common.MB: + return ui.orange.Sprintf("%.1f", fsize/common.MB) + " MiB" + case fsize >= common.KB: + return ui.orange.Sprintf("%.1f", fsize/common.KB) + " KiB" + default: + return ui.orange.Sprintf("%d", size) + " B" + } +} diff --git a/report/export_test.go b/report/export_test.go new file mode 100644 index 000000000..7548c161e --- /dev/null +++ b/report/export_test.go @@ -0,0 +1,116 @@ +package report + +import ( + "bytes" + "os" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/dundee/gdu/v5/internal/testdir" + "github.com/dundee/gdu/v5/pkg/device" + "github.com/stretchr/testify/assert" +) + +func init() { + log.SetLevel(log.WarnLevel) +} + +func TestAnalyzePath(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, false) + ui.SetIgnoreDirPaths([]string{"/xxx"}) + err := ui.AnalyzePath("test_dir", nil) + assert.Nil(t, err) + err = ui.StartUILoop() + + assert.Nil(t, err) + assert.Contains(t, reportOutput.String(), `"name":"nested"`) +} + +func TestAnalyzePathWithProgress(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + ui.SetIgnoreDirPaths([]string{"/xxx"}) + err := ui.AnalyzePath("test_dir", nil) + assert.Nil(t, err) + err = ui.StartUILoop() + + assert.Nil(t, err) + assert.Contains(t, reportOutput.String(), `"name":"nested"`) +} + +func TestShowDevices(t *testing.T) { + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + err := ui.ListDevices(device.Getter) + + assert.Contains(t, err.Error(), "not supported") +} + +func TestReadAnalysisWhileExporting(t *testing.T) { + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + err := ui.ReadAnalysis(output) + + assert.Contains(t, err.Error(), "not possible while exporting") +} + +func TestExportToFile(t *testing.T) { + fin := testdir.CreateTestDir() + defer fin() + + reportOutput, err := os.OpenFile("output.json", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + assert.Nil(t, err) + defer func() { + os.Remove("output.json") + }() + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + ui.SetIgnoreDirPaths([]string{"/xxx"}) + err = ui.AnalyzePath("test_dir", nil) + assert.Nil(t, err) + err = ui.StartUILoop() + assert.Nil(t, err) + + reportOutput, err = os.OpenFile("output.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + _, err = reportOutput.Seek(0, 0) + assert.Nil(t, err) + buff := make([]byte, 200) + _, err = reportOutput.Read(buff) + assert.Nil(t, err) + + assert.Contains(t, string(buff), `"name":"nested"`) +} + +func TestFormatSize(t *testing.T) { + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + + assert.Contains(t, ui.formatSize(1), "B") + assert.Contains(t, ui.formatSize(1<<10+1), "KiB") + assert.Contains(t, ui.formatSize(1<<20+1), "MiB") + assert.Contains(t, ui.formatSize(1<<30+1), "GiB") + assert.Contains(t, ui.formatSize(1<<40+1), "TiB") + assert.Contains(t, ui.formatSize(1<<50+1), "PiB") + assert.Contains(t, ui.formatSize(1<<60+1), "EiB") +} diff --git a/report/import.go b/report/import.go new file mode 100644 index 000000000..bab245f81 --- /dev/null +++ b/report/import.go @@ -0,0 +1,82 @@ +package report + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + + "github.com/dundee/gdu/v5/pkg/analyze" +) + +// ReadAnalysis reads analysis report from JSON file and returns directory item +func ReadAnalysis(input io.Reader) (*analyze.Dir, error) { + var data interface{} + + var buff bytes.Buffer + buff.ReadFrom(input) + json.Unmarshal(buff.Bytes(), &data) + + dataArray, ok := data.([]interface{}) + if !ok { + return nil, errors.New("JSON file does not contain top level array") + } + if len(dataArray) < 4 { + return nil, errors.New("Top level array must have at least 4 items") + } + + items, ok := dataArray[3].([]interface{}) + if !ok { + return nil, errors.New("Array of maps not found in the top level array on 4th position") + } + + return processDir(items) +} + +func processDir(items []interface{}) (*analyze.Dir, error) { + dir := &analyze.Dir{ + File: &analyze.File{ + Flag: ' ', + }, + } + dirMap, ok := items[0].(map[string]interface{}) + if !ok { + return nil, errors.New("Directory item is not a map") + } + name, ok := dirMap["name"].(string) + if !ok { + return nil, errors.New("Directory name is not a string") + } + + slashPos := strings.LastIndex(name, "/") + if slashPos > -1 { + dir.Name = name[slashPos+1:] + dir.BasePath = name[:slashPos+1] + } else { + dir.Name = name + } + + for _, v := range items[1:] { + switch item := v.(type) { + case map[string]interface{}: + file := &analyze.File{} + file.Name = item["name"].(string) + file.Size = int64(item["asize"].(float64)) + file.Usage = int64(item["dsize"].(float64)) + file.Parent = dir + file.Flag = ' ' + + dir.Files.Append(file) + case []interface{}: + subdir, err := processDir(item) + if err != nil { + return nil, err + } + subdir.Parent = dir + dir.Files.Append(subdir) + } + } + + return dir, nil +} diff --git a/report/import_test.go b/report/import_test.go new file mode 100644 index 000000000..c76172c11 --- /dev/null +++ b/report/import_test.go @@ -0,0 +1,81 @@ +package report + +import ( + "bytes" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/assert" +) + +func init() { + log.SetLevel(log.WarnLevel) +} + +func TestReadAnalysis(t *testing.T) { + buff := bytes.NewBuffer([]byte(` + [1,2,{"progname":"gdu","progver":"development","timestamp":1626806293}, + [{"name":"/home/xxx"}, + {"name":"gdu.json","asize":33805233,"dsize":33808384}, + [{"name":"app"}, + {"name":"app.go","asize":4638,"dsize":8192}, + {"name":"app_linux_test.go","asize":1410,"dsize":4096}, + {"name":"app_test.go","asize":4974,"dsize":8192}], + {"name":"main.go","asize":3205,"dsize":4096}]] + `)) + + dir, err := ReadAnalysis(buff) + + assert.Nil(t, err) + assert.Equal(t, "xxx", dir.GetName()) + assert.Equal(t, "/home/xxx", dir.GetPath()) +} + +func TestReadAnalysisWithEmptyInput(t *testing.T) { + buff := bytes.NewBuffer([]byte(``)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "JSON file does not contain top level array", err.Error()) +} + +func TestReadAnalysisWithEmptyArray(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Top level array must have at least 4 items", err.Error()) +} + +func TestReadAnalysisWithWrongContent(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,4]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Array of maps not found in the top level array on 4th position", err.Error()) +} + +func TestReadAnalysisWithEmptyDirContent(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[{}]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory name is not a string", err.Error()) +} + +func TestReadAnalysisWithWrongDirItem(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[1, 2, 3]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory item is not a map", err.Error()) +} + +func TestReadAnalysisWithWrongSubdirItem(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[{"name":"xxx"}, [1,2,3]]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory item is not a map", err.Error()) +} diff --git a/stdout/stdout.go b/stdout/stdout.go index 7da9f7b6b..b572e9a58 100644 --- a/stdout/stdout.go +++ b/stdout/stdout.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "runtime" "sort" "sync" "time" @@ -13,6 +14,7 @@ import ( "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/fatih/color" ) @@ -25,6 +27,8 @@ type UI struct { blue *color.Color } +var progressRunes = []rune(`⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧`) + // CreateStdoutUI creates UI for stdout func CreateStdoutUI(output io.Writer, useColors bool, showProgress bool, showApparentSize bool) *UI { ui := &UI{ @@ -141,6 +145,12 @@ func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { wait.Wait() + ui.showDir(dir) + + return nil +} + +func (ui *UI) showDir(dir *analyze.Dir) { sort.Sort(dir.Files) var lineFormat string @@ -173,18 +183,89 @@ func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { file.GetName()) } } +} + +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + var ( + dir *analyze.Dir + wait sync.WaitGroup + err error + doneChan chan struct{} + ) + + if ui.ShowProgress { + wait.Add(1) + doneChan = make(chan struct{}) + go func() { + defer wait.Done() + ui.showReadingProgress(doneChan) + }() + } + + wait.Add(1) + go func() { + defer wait.Done() + dir, err = report.ReadAnalysis(input) + if err != nil { + if ui.ShowProgress { + doneChan <- struct{}{} + } + return + } + runtime.GC() + + links := make(analyze.AlreadyCountedHardlinks, 10) + dir.UpdateStats(links) + + if ui.ShowProgress { + doneChan <- struct{}{} + } + }() + + wait.Wait() + + if err != nil { + return err + } + + ui.showDir(dir) return nil } +func (ui *UI) showReadingProgress(doneChan chan struct{}) { + emptyRow := "\r" + for j := 0; j < 40; j++ { + emptyRow += " " + } + + i := 0 + for { + fmt.Fprint(ui.output, emptyRow) + + select { + case <-doneChan: + fmt.Fprint(ui.output, "\r") + return + default: + } + + fmt.Fprintf(ui.output, "\r %s ", string(progressRunes[i])) + fmt.Fprint(ui.output, "Reading analysis from file...") + + time.Sleep(100 * time.Millisecond) + i++ + i %= 10 + } +} + func (ui *UI) updateProgress() { emptyRow := "\r" for j := 0; j < 100; j++ { emptyRow += " " } - progressRunes := []rune(`⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧`) - progressChan := ui.Analyzer.GetProgressChan() doneChan := ui.Analyzer.GetDoneChan() @@ -215,15 +296,21 @@ func (ui *UI) updateProgress() { } func (ui *UI) formatSize(size int64) string { + fsize := float64(size) + switch { - case size > 1e12: - return ui.orange.Sprintf("%.1f", float64(size)/math.Pow(2, 40)) + " TiB" - case size > 1e9: - return ui.orange.Sprintf("%.1f", float64(size)/math.Pow(2, 30)) + " GiB" - case size > 1e6: - return ui.orange.Sprintf("%.1f", float64(size)/math.Pow(2, 20)) + " MiB" - case size > 1e3: - return ui.orange.Sprintf("%.1f", float64(size)/math.Pow(2, 10)) + " KiB" + case fsize >= common.EB: + return ui.orange.Sprintf("%.1f", fsize/common.EB) + " EiB" + case fsize >= common.PB: + return ui.orange.Sprintf("%.1f", fsize/common.PB) + " PiB" + case fsize >= common.TB: + return ui.orange.Sprintf("%.1f", fsize/common.TB) + " TiB" + case fsize >= common.GB: + return ui.orange.Sprintf("%.1f", fsize/common.GB) + " GiB" + case fsize >= common.MB: + return ui.orange.Sprintf("%.1f", fsize/common.MB) + " MiB" + case fsize >= common.KB: + return ui.orange.Sprintf("%.1f", fsize/common.KB) + " KiB" default: return ui.orange.Sprintf("%d", size) + " B" } diff --git a/stdout/stdout_test.go b/stdout/stdout_test.go index 9ad6c8dac..fe514effb 100644 --- a/stdout/stdout_test.go +++ b/stdout/stdout_test.go @@ -2,6 +2,7 @@ package stdout import ( "bytes" + "os" "testing" log "github.com/sirupsen/logrus" @@ -86,7 +87,6 @@ func TestItemRows(t *testing.T) { err := ui.AnalyzePath("test_dir", nil) assert.Nil(t, err) - assert.Contains(t, output.String(), "TiB") assert.Contains(t, output.String(), "GiB") assert.Contains(t, output.String(), "MiB") assert.Contains(t, output.String(), "KiB") @@ -128,11 +128,63 @@ func TestShowDevicesWithColor(t *testing.T) { assert.Contains(t, output.String(), "xxx") } +func TestReadAnalysisWithColor(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, true, true, true) + err = ui.ReadAnalysis(input) + + assert.Nil(t, err) + assert.Contains(t, output.String(), "main.go") +} + +func TestReadAnalysisBw(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, false, false, false) + err = ui.ReadAnalysis(input) + + assert.Nil(t, err) + assert.Contains(t, output.String(), "main.go") +} + +func TestReadAnalysisWithWrongFile(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/wrong.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, true, true, true) + err = ui.ReadAnalysis(input) + + assert.NotNil(t, err) +} + func TestMaxInt(t *testing.T) { assert.Equal(t, 5, maxInt(2, 5)) assert.Equal(t, 4, maxInt(4, 2)) } +func TestFormatSize(t *testing.T) { + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, true, true, true) + + assert.Contains(t, ui.formatSize(1), "B") + assert.Contains(t, ui.formatSize(1<<10+1), "KiB") + assert.Contains(t, ui.formatSize(1<<20+1), "MiB") + assert.Contains(t, ui.formatSize(1<<30+1), "GiB") + assert.Contains(t, ui.formatSize(1<<40+1), "TiB") + assert.Contains(t, ui.formatSize(1<<50+1), "PiB") + assert.Contains(t, ui.formatSize(1<<60+1), "EiB") +} + // func printBuffer(buff *bytes.Buffer) { // for i, x := range buff.String() { // println(i, string(x)) diff --git a/tui/actions.go b/tui/actions.go index 09800ac09..c07c36af6 100644 --- a/tui/actions.go +++ b/tui/actions.go @@ -3,6 +3,7 @@ package tui import ( "bufio" "fmt" + "io" "os" "path/filepath" "runtime" @@ -10,6 +11,7 @@ import ( "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/gdamore/tcell/v2" "github.com/rivo/tview" ) @@ -113,6 +115,57 @@ func (ui *UI) AnalyzePath(path string, parentDir *analyze.Dir) error { return nil } +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + ui.progress = tview.NewTextView().SetText("Reading analysis from file...") + ui.progress.SetBorder(true).SetBorderPadding(2, 2, 2, 2) + ui.progress.SetTitle(" Reading... ") + ui.progress.SetDynamicColors(true) + + flex := tview.NewFlex(). + AddItem(nil, 0, 1, false). + AddItem(tview.NewFlex().SetDirection(tview.FlexRow). + AddItem(nil, 10, 1, false). + AddItem(ui.progress, 8, 1, false). + AddItem(nil, 10, 1, false), 0, 50, false). + AddItem(nil, 0, 1, false) + + ui.pages.AddPage("progress", flex, true, true) + + go func() { + var err error + ui.currentDir, err = report.ReadAnalysis(input) + if err != nil { + ui.app.QueueUpdateDraw(func() { + ui.pages.RemovePage("progress") + ui.showErr("Error reading file", err) + }) + if ui.done != nil { + ui.done <- struct{}{} + } + return + } + runtime.GC() + + ui.topDirPath = ui.currentDir.GetPath() + ui.topDir = ui.currentDir + + links := make(analyze.AlreadyCountedHardlinks, 10) + ui.topDir.UpdateStats(links) + + ui.app.QueueUpdateDraw(func() { + ui.showDir() + ui.pages.RemovePage("progress") + }) + + if ui.done != nil { + ui.done <- struct{}{} + } + }() + + return nil +} + func (ui *UI) deleteSelected(shouldEmpty bool) { row, column := ui.table.GetSelection() selectedItem := ui.table.GetCell(row, column).GetReference().(analyze.Item) diff --git a/tui/actions_test.go b/tui/actions_test.go index 3f88ff8e0..401d58162 100644 --- a/tui/actions_test.go +++ b/tui/actions_test.go @@ -1,6 +1,7 @@ package tui import ( + "os" "testing" "github.com/dundee/gdu/v5/internal/testanalyze" @@ -160,6 +161,46 @@ func TestAnalyzePathWithParentDir(t *testing.T) { assert.Contains(t, ui.table.GetCell(1, 0).Text, "aaa") } +func TestReadAnalysis(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + app := testapp.CreateMockedApp(true) + ui := CreateUI(app, true, true) + ui.done = make(chan struct{}) + + err = ui.ReadAnalysis(input) + assert.Nil(t, err) + + <-ui.done // wait for reading + + assert.Equal(t, "gdu", ui.currentDir.Name) + + for _, f := range ui.app.(*testapp.MockedApp).UpdateDraws { + f() + } +} + +func TestReadAnalysisWithWrongFile(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/wrong.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + app := testapp.CreateMockedApp(true) + ui := CreateUI(app, true, true) + ui.done = make(chan struct{}) + + err = ui.ReadAnalysis(input) + assert.Nil(t, err) + + <-ui.done // wait for reading + + for _, f := range ui.app.(*testapp.MockedApp).UpdateDraws { + f() + } + + assert.True(t, ui.pages.HasPage("error")) +} + func TestViewDirContents(t *testing.T) { app := testapp.CreateMockedApp(true) ui := CreateUI(app, false, true) diff --git a/tui/format.go b/tui/format.go index ec9e0e58d..a18802da4 100644 --- a/tui/format.go +++ b/tui/format.go @@ -3,27 +3,10 @@ package tui import ( "fmt" + "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/analyze" ) -// file size constants -const ( - _ = iota - KB float64 = 1 << (10 * iota) - MB - GB - TB - PB - EB -) - -// file count constants -const ( - K int = 1e3 - M int = 1e6 - G int = 1e9 -) - func (ui *UI) formatFileRow(item analyze.Item) string { var part int @@ -88,18 +71,18 @@ func (ui *UI) formatSize(size int64, reverseColor bool, transparentBg bool) stri fsize := float64(size) switch { - case fsize >= EB: - return fmt.Sprintf("%.1f%s EiB", fsize/EB, color) - case fsize >= PB: - return fmt.Sprintf("%.1f%s PiB", fsize/PB, color) - case fsize >= TB: - return fmt.Sprintf("%.1f%s TiB", fsize/TB, color) - case fsize >= GB: - return fmt.Sprintf("%.1f%s GiB", fsize/GB, color) - case fsize >= MB: - return fmt.Sprintf("%.1f%s MiB", fsize/MB, color) - case fsize >= KB: - return fmt.Sprintf("%.1f%s KiB", fsize/KB, color) + case fsize >= common.EB: + return fmt.Sprintf("%.1f%s EiB", fsize/common.EB, color) + case fsize >= common.PB: + return fmt.Sprintf("%.1f%s PiB", fsize/common.PB, color) + case fsize >= common.TB: + return fmt.Sprintf("%.1f%s TiB", fsize/common.TB, color) + case fsize >= common.GB: + return fmt.Sprintf("%.1f%s GiB", fsize/common.GB, color) + case fsize >= common.MB: + return fmt.Sprintf("%.1f%s MiB", fsize/common.MB, color) + case fsize >= common.KB: + return fmt.Sprintf("%.1f%s KiB", fsize/common.KB, color) default: return fmt.Sprintf("%d%s B", size, color) } @@ -110,12 +93,12 @@ func (ui *UI) formatCount(count int) string { color := "[-::]" switch { - case count >= G: - row += fmt.Sprintf("%.1f%sG", float64(count)/float64(G), color) - case count >= M: - row += fmt.Sprintf("%.1f%sM", float64(count)/float64(M), color) - case count >= K: - row += fmt.Sprintf("%.1f%sk", float64(count)/float64(K), color) + case count >= common.G: + row += fmt.Sprintf("%.1f%sG", float64(count)/float64(common.G), color) + case count >= common.M: + row += fmt.Sprintf("%.1f%sM", float64(count)/float64(common.M), color) + case count >= common.K: + row += fmt.Sprintf("%.1f%sk", float64(count)/float64(common.K), color) default: row += fmt.Sprintf("%d%s", count, color) }