feat: sequential scanning

README.md

@@ -58,6 +58,7 @@ Flags:
   -n, --non-interactive               Do not run in interactive mode
   -o, --output-file string            Export all info into file as JSON
   -r, --read-from-storage             Read analysis data from persistent key-value storage
+      --sequential                    Use sequential scanning (intended for rotating HDDs)
   -a, --show-apparent-size            Show apparent size
   -d, --show-disks                    Show all mounted disks
   -B, --show-relative-size            Show relative size

cmd/gdu/app/app.go

@@ -43,37 +43,38 @@
 
 // Flags define flags accepted by Run
 type Flags struct {
-	CfgFile           string   `yaml:"-"`
-	LogFile           string   `yaml:"log-file"`
-	InputFile         string   `yaml:"input-file"`
-	OutputFile        string   `yaml:"output-file"`
-	IgnoreDirs        []string `yaml:"ignore-dirs"`
-	IgnoreDirPatterns []string `yaml:"ignore-dir-patterns"`
-	IgnoreFromFile    string   `yaml:"ignore-from-file"`
-	MaxCores          int      `yaml:"max-cores"`
-	ShowDisks         bool     `yaml:"-"`
-	ShowApparentSize  bool     `yaml:"show-apparent-size"`
-	ShowRelativeSize  bool     `yaml:"show-relative-size"`
-	ShowVersion       bool     `yaml:"-"`
-	NoColor           bool     `yaml:"no-color"`
-	NoMouse           bool     `yaml:"no-mouse"`
-	NonInteractive    bool     `yaml:"non-interactive"`
-	NoProgress        bool     `yaml:"no-progress"`
-	NoCross           bool     `yaml:"no-cross"`
-	NoHidden          bool     `yaml:"no-hidden"`
-	FollowSymlinks    bool     `yaml:"follow-symlinks"`
-	Profiling         bool     `yaml:"profiling"`
-	ConstGC           bool     `yaml:"const-gc"`
-	UseStorage        bool     `yaml:"use-storage"`
-	StoragePath       string   `yaml:"storage-path"`
-	ReadFromStorage   bool     `yaml:"read-from-storage"`
-	Summarize         bool     `yaml:"summarize"`
-	UseSIPrefix       bool     `yaml:"use-si-prefix"`
-	NoPrefix          bool     `yaml:"no-prefix"`
-	WriteConfig       bool     `yaml:"-"`
-	ChangeCwd         bool     `yaml:"change-cwd"`
-	Style             Style    `yaml:"style"`
-	Sorting           Sorting  `yaml:"sorting"`
+	CfgFile            string   `yaml:"-"`
+	LogFile            string   `yaml:"log-file"`
+	InputFile          string   `yaml:"input-file"`
+	OutputFile         string   `yaml:"output-file"`
+	IgnoreDirs         []string `yaml:"ignore-dirs"`
+	IgnoreDirPatterns  []string `yaml:"ignore-dir-patterns"`
+	IgnoreFromFile     string   `yaml:"ignore-from-file"`
+	MaxCores           int      `yaml:"max-cores"`
+	SequentialScanning bool     `yaml:"sequential-scanning"`
+	ShowDisks          bool     `yaml:"-"`
+	ShowApparentSize   bool     `yaml:"show-apparent-size"`
+	ShowRelativeSize   bool     `yaml:"show-relative-size"`
+	ShowVersion        bool     `yaml:"-"`
+	NoColor            bool     `yaml:"no-color"`
+	NoMouse            bool     `yaml:"no-mouse"`
+	NonInteractive     bool     `yaml:"non-interactive"`
+	NoProgress         bool     `yaml:"no-progress"`
+	NoCross            bool     `yaml:"no-cross"`
+	NoHidden           bool     `yaml:"no-hidden"`
+	FollowSymlinks     bool     `yaml:"follow-symlinks"`
+	Profiling          bool     `yaml:"profiling"`
+	ConstGC            bool     `yaml:"const-gc"`
+	UseStorage         bool     `yaml:"use-storage"`
+	StoragePath        string   `yaml:"storage-path"`
+	ReadFromStorage    bool     `yaml:"read-from-storage"`
+	Summarize          bool     `yaml:"summarize"`
+	UseSIPrefix        bool     `yaml:"use-si-prefix"`
+	NoPrefix           bool     `yaml:"no-prefix"`
+	WriteConfig        bool     `yaml:"-"`
+	ChangeCwd          bool     `yaml:"change-cwd"`
+	Style              Style    `yaml:"style"`
+	Sorting            Sorting  `yaml:"sorting"`
 }
 
 // Style define style config
@@ -144,6 +145,9 @@
 	if a.Flags.UseStorage {
 		ui.SetAnalyzer(analyze.CreateStoredAnalyzer(a.Flags.StoragePath))
 	}
+	if a.Flags.SequentialScanning {
+		ui.SetAnalyzer(analyze.CreateSeqAnalyzer())
+	}
 	if a.Flags.FollowSymlinks {
 		ui.SetFollowSymlinks(true)
 	}

cmd/gdu/main.go

@@ -43,6 +43,7 @@ func init() {
 	flags.StringVarP(&af.OutputFile, "output-file", "o", "", "Export all info into file as JSON")
 	flags.StringVarP(&af.InputFile, "input-file", "f", "", "Import analysis from JSON file")
 	flags.IntVarP(&af.MaxCores, "max-cores", "m", runtime.NumCPU(), fmt.Sprintf("Set max cores that GDU will use. %d cores available", runtime.NumCPU()))
+	flags.BoolVar(&af.SequentialScanning, "sequential", false, "Use sequential scanning (intended for rotating HDDs)")
 	flags.BoolVarP(&af.ShowVersion, "version", "v", false, "Print version")
 
 	flags.StringSliceVarP(&af.IgnoreDirs, "ignore-dirs", "i", []string{"/proc", "/dev", "/sys", "/run"}, "Absolute paths to ignore (separated by comma)")

pkg/analyze/dir_linux_test.go

@@ -37,3 +37,29 @@ func TestErr(t *testing.T) {
 	assert.Equal(t, "nested", dir.Files[0].GetName())
 	assert.Equal(t, '!', dir.Files[0].GetFlag())
 }
+
+func TestSeqErr(t *testing.T) {
+	fin := testdir.CreateTestDir()
+	defer fin()
+
+	err := os.Chmod("test_dir/nested", 0)
+	assert.Nil(t, err)
+	defer func() {
+		err = os.Chmod("test_dir/nested", 0755)
+		assert.Nil(t, err)
+	}()
+
+	analyzer := CreateSeqAnalyzer()
+	dir := analyzer.AnalyzeDir(
+		"test_dir", func(_, _ string) bool { return false }, false,
+	).(*Dir)
+	analyzer.GetDone().Wait()
+	dir.UpdateStats(make(fs.HardLinkedItems))
+
+	assert.Equal(t, "test_dir", dir.GetName())
+	assert.Equal(t, 2, dir.ItemCount)
+	assert.Equal(t, '.', dir.GetFlag())
+
+	assert.Equal(t, "nested", dir.Files[0].GetName())
+	assert.Equal(t, '!', dir.Files[0].GetFlag())
+}

pkg/analyze/sequential.go

@@ -0,0 +1,176 @@
+package analyze
+
+import (
+	"os"
+	"path/filepath"
+	"runtime/debug"
+
+	"github.com/dundee/gdu/v5/internal/common"
+	"github.com/dundee/gdu/v5/pkg/fs"
+	log "github.com/sirupsen/logrus"
+)
+
+// SequentialAnalyzer implements Analyzer
+type SequentialAnalyzer struct {
+	progress         *common.CurrentProgress
+	progressChan     chan common.CurrentProgress
+	progressOutChan  chan common.CurrentProgress
+	progressDoneChan chan struct{}
+	doneChan         common.SignalGroup
+	wait             *WaitGroup
+	ignoreDir        common.ShouldDirBeIgnored
+	followSymlinks   bool
+}
+
+// CreateSeqAnalyzer returns Analyzer
+func CreateSeqAnalyzer() *SequentialAnalyzer {
+	return &SequentialAnalyzer{
+		progress: &common.CurrentProgress{
+			ItemCount: 0,
+			TotalSize: int64(0),
+		},
+		progressChan:     make(chan common.CurrentProgress, 1),
+		progressOutChan:  make(chan common.CurrentProgress, 1),
+		progressDoneChan: make(chan struct{}),
+		doneChan:         make(common.SignalGroup),
+		wait:             (&WaitGroup{}).Init(),
+	}
+}
+
+// SetFollowSymlinks sets whether symlink to files should be followed
+func (a *SequentialAnalyzer) SetFollowSymlinks(v bool) {
+	a.followSymlinks = v
+}
+
+// GetProgressChan returns channel for getting progress
+func (a *SequentialAnalyzer) GetProgressChan() chan common.CurrentProgress {
+	return a.progressOutChan
+}
+
+// GetDone returns channel for checking when analysis is done
+func (a *SequentialAnalyzer) GetDone() common.SignalGroup {
+	return a.doneChan
+}
+
+// ResetProgress returns progress
+func (a *SequentialAnalyzer) ResetProgress() {
+	a.progress = &common.CurrentProgress{}
+	a.progressChan = make(chan common.CurrentProgress, 1)
+	a.progressOutChan = make(chan common.CurrentProgress, 1)
+	a.progressDoneChan = make(chan struct{})
+	a.doneChan = make(common.SignalGroup)
+}
+
+// AnalyzeDir analyzes given path
+func (a *SequentialAnalyzer) AnalyzeDir(
+	path string, ignore common.ShouldDirBeIgnored, constGC bool,
+) fs.Item {
+	if !constGC {
+		defer debug.SetGCPercent(debug.SetGCPercent(-1))
+		go manageMemoryUsage(a.doneChan)
+	}
+
+	a.ignoreDir = ignore
+
+	go a.updateProgress()
+	dir := a.processDir(path)
+
+	dir.BasePath = filepath.Dir(path)
+
+	a.progressDoneChan <- struct{}{}
+	a.doneChan.Broadcast()
+
+	return dir
+}
+
+func (a *SequentialAnalyzer) processDir(path string) *Dir {
+	var (
+		file      *File
+		err       error
+		totalSize int64
+		info      os.FileInfo
+		dirCount  int
+	)
+
+	files, err := os.ReadDir(path)
+	if err != nil {
+		log.Print(err.Error())
+	}
+
+	dir := &Dir{
+		File: &File{
+			Name: filepath.Base(path),
+			Flag: getDirFlag(err, len(files)),
+		},
+		ItemCount: 1,
+		Files:     make(fs.Files, 0, len(files)),
+	}
+	setDirPlatformSpecificAttrs(dir, path)
+
+	for _, f := range files {
+		name := f.Name()
+		entryPath := filepath.Join(path, name)
+		if f.IsDir() {
+			if a.ignoreDir(name, entryPath) {
+				continue
+			}
+			dirCount++
+
+			subdir := a.processDir(entryPath)
+			subdir.Parent = dir
+			dir.AddFile(subdir)
+		} else {
+			info, err = f.Info()
+			if err != nil {
+				log.Print(err.Error())
+				dir.Flag = '!'
+				continue
+			}
+			if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
+				err = followSymlink(entryPath, &info)
+				if err != nil {
+					log.Print(err.Error())
+					dir.Flag = '!'
+					continue
+				}
+			}
+
+			file = &File{
+				Name:   name,
+				Flag:   getFlag(info),
+				Size:   info.Size(),
+				Parent: dir,
+			}
+			setPlatformSpecificAttrs(file, info)
+
+			totalSize += info.Size()
+
+			dir.AddFile(file)
+		}
+	}
+
+	a.progressChan <- common.CurrentProgress{
+		CurrentItemName: path,
+		ItemCount:       len(files),
+		TotalSize:       totalSize,
+	}
+	return dir
+}
+
+func (a *SequentialAnalyzer) updateProgress() {
+	for {
+		select {
+		case <-a.progressDoneChan:
+			return
+		case progress := <-a.progressChan:
+			a.progress.CurrentItemName = progress.CurrentItemName
+			a.progress.ItemCount += progress.ItemCount
+			a.progress.TotalSize += progress.TotalSize
+		}
+
+		select {
+		case a.progressOutChan <- *a.progress:
+		default:
+		}
+	}
+}