A large number of changes to the release 0.3.1

.gitignore

@@ -0,0 +1,5 @@
+*.log
+gorsync
+go-rsync
+builds/fpm_packages/packages/*
+data/assets_vfsdata.go

.travis.yml

@@ -0,0 +1,36 @@
+language: go
+
+go:
+#  - "1.6"
+#  - "1.7"
+  - "1.10"
+#  - "tip"
+
+env:
+  - GOARCH=amd64
+
+sudo: false
+#dist: trusty
+dist: xenial
+
+#services:
+#  - docker
+
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq -y gtk+3.0 libgtk-3-dev libnotify-dev
+  - sudo apt-get install -qq -y xvfb
+  - "export DISPLAY=:99.0"
+  - sudo /usr/bin/Xvfb $DISPLAY 2>1 > /dev/null &
+  - "export GTK_VERSION=$(pkg-config --modversion gtk+-3.0 | tr . _| cut -d '_' -f 1-2)"
+  - "export GLib_VERSION=$(pkg-config --modversion glib-2.0 | tr . _| cut -d '_' -f 1-2)"
+  - "export Cairo_VERSION=$(pkg-config --modversion cairo)"
+  - "export Pango_VERSION=$(pkg-config --modversion pango)"
+  - echo "GTK ${GTK_VERSION}, GLib ${GLib_VERSION} (Cairo ${Cairo_VERSION}, Pango ${Pango_VERSION})"
+
+install:
+  #- go get -t -tags "gtk_${GTK_VERSION} glib_${GLib_VERSION}" github.com/d2r2/gotk3/...
+  - go get -t -tags "gtk_${GTK_VERSION} glib_${GLib_VERSION}" github.com/d2r2/go-rsync
+
+script:
+  - go test -tags "gtk_${GTK_VERSION} glib_${GLib_VERSION}" github.com/d2r2/go-rsync

README.md

@@ -0,0 +1,159 @@
+Gorsync Backup: GTK+ RSYNC frontend
+===================================
+
+[![Build Status](https://travis-ci.org/d2r2/go-rsync.svg?branch=master)](https://travis-ci.org/d2r2/go-rsync)
+[![Go Report Card](https://goreportcard.com/badge/github.com/d2r2/go-rsync)](https://goreportcard.com/report/github.com/d2r2/go-rsync)
+[![GoDoc](https://godoc.org/github.com/d2r2/go-rsync?status.svg)](https://godoc.org/github.com/d2r2/go-rsync)
+[![GPLv3 License](http://img.shields.io/badge/License-GPLv3-yellow.svg)](./LICENSE)
+
+Gorsync Backup is a best GTK+ frontend for brilliant RSYNC console utility.  Simple, but powerful.
+Written completely in [Go programming language](https://golang.org/), provides responsive GUI design and intuitive interface. Might be used as training material how to write rich multi-threaded GUI application with GTK+ in Golang.
+
+
+
+Features and benefits
+----------------------
+
+* Multiple backup profiles are supported. Moreover, each profile can be configured to get data from multiple RSYNC sources.
+* 2-pass backup session approach to estimated backup volume in 1st pass. Display predicted time of completion in 2nd pass.
+* Demonstrate "deduplication" on modern file systems, once previous backup sessions found (and significant time reduction in repeated backup processes). Works if backup destination is Ext3/Ext4/NTFS (employ file system hardlink feature).
+* [Improved GOTK3+](https://github.com/d2r2/gotk3) library (GTK+ golang bindings) used for GUI.
+
+
+
+
+Screenshots
+-----------
+Main form:
+
+![image](https://raw.github.com/d2r2/go-rsync/master/docs/gorsync_main_form.png)
+
+Preferences:
+
+![image](https://raw.github.com/d2r2/go-rsync/master/docs/gorsync_preference_dialog.png)
+
+
+
+
+Installation approaches
+-----------------------
+
+##### Build and run from sources:
+
+* Verify, that RSYNC console utility is installed.
+* Download Gorsync Backup sources (with all dependent golang libraries):
+```bash
+$ go get -u github.com/d2r2/go-rsync
+```
+* Compile and deploy application GLIB gsettings schema, with console prompt:
+```bash
+$ sudo ./ui/gtkui/gs_schema_install.sh
+```
+* Finally, run app from terminal:
+```bash
+$ ./gorsync_run.sh
+```
+
+##### Precompiled linux packages (deb, rpm and others) from releases:
+
+Alternative approach to install application is to downloads installation packages from latest release, which can be found in [release page](https://github.com/d2r2/go-rsync/releases). You may find there packages for deb (Debian, Ubuntu), rpm (Fedora, Redhat) and pkg.tar.xz (Arch linux) Linux distributives.
+
+
+##### Archlinux AUR repository:
+
+One can be found in AUR repository https://aur.archlinux.org/ by name "gorsync-git" to download, compile and install latest release. On Archlinux you can use any AUR helper to install application, for instance `yaourt -S gorsync-git`.
+
+
+
+Releases information
+--------------------
+
+##### [v0.3.1](https://github.com/d2r2/go-rsync/releases/tag/v0.3.1) (latest release):
+
+* Internationalization implemented. Localization provided for English, Russian languages.
+* Backup result's notifications: desktop notification and shell script for any kind of automation.
+* Out of disk space protection for backup destination.
+* A lot of improvements in algorithms and GUI.
+* Significant code refactoring and rewrites.
+
+
+
+
+Plans for next releases
+-----------------------
+Short list of preliminary anticipated features for next releases:
+
+* More code comments and improved documentation.
+* Installation packages for all major linux distribution and repositories.
+* Application console parameters. Perhaps some CLI modes.
+
+
+
+
+Gorsync Backup backup process explanation
+-----------------------------------------
+
+* As in regular [RSYNC](https://ss64.com/bash/rsync_options.html) session, Gorsync Backup is doing same job: copy files from one location (source) to another (backup destination). For instance, real life scenario would be: backing up your data from home NAS to flash hard drive attached to your notebook.
+
+* Gorsync Backup can copy from multiple RSYNC sources at once. It could be your pictures, movies, document's from home NAS, routers and smart Linux device configuration files (/etc/... folder) and so on... Thus Gorsync Backup profile let you specify multiple separated RSYNC URL sources to get data from in single backup session and combine them all together in one destination place.
+
+* Gorsync Backup never overwrite existing backup session destination, but use same common target root path, to put data near. For instance, your flash drive backup folder content might looks like:
+```
+$ <destination root folder to stores backup content>
+$             ↳ ~rsync_backup_20180801-012237~
+$             ↳ ~rsync_backup_20180802-013113~
+$             ↳ ~rsync_backup_<date>-<time>~
+...
+$             ↳ ~rsync_backup_20180806-014036~
+$             ↳ ~rsync_backup_(incomplete)_20180807-014024~
+```
+, where each specific backup session stored in separate folder with date and time in the name. "(incomplete)" phrase stands for backup, that occures at the moment. Once backup will be completed, "(incomplete)" will be removed from backup folder name. Another scenario is possible, when backup process has been interrupted for some reason: in this case "(incomplete)" phrase will never get out from folder name. But, in any case it's easy to understand where you have consistent backup results, and where not.
+
+* In its turn, each backup folder has next regular structure:
+```
+$ <root folder to store backup contents on flash drive>
+$             ↳ ~rsync_backup_20180801-012237~
+$                       ↳ ~backup_log~.log
+$                       ↳ ~backup_nodes~.signatures
+$                       ↳ <folder with rsync source #1 content>
+...
+$                       ↳ <folder with rsync source #N content>
+```
+, where `~backup_log~.log` file describe all the details about the steps occurred, including info/warning/error messages if any took place. `~backup_nodes~.signatures` file contains hash imprint for all source URLs, to detect in future backup sessions same data source for "deduplication" purpose.
+
+* Gorsync Backup is splitting backup process to the peaces. Application in every backup session is trying to find optimal data block size to backup at once. To reach this application download folders structure in 1st pass to analyze how to divide the whole process into parts. Ordinary single data block size selected to be not less than 300 MB and no more than 5 GB.
+
+
+
+"Deduplication" capabilities
+----------------------------
+Once you start using Gorsync Backup on regular basis (daily/weekly/monthly), you will find soon that your backup storage will be filled with sets of almost same files (of course changes over time will provides some relatively small deviation between data sets). This redundancy might quickly exhaust your free space, but there is a real magic exists in modern file systems - "hard links"! Hard links allow do not spent space for files, which have been found in previous backup sessions unchanged. Additionally it's significantly speed up backup process. The collaboration of Gorsync Backup with RSYNC know how to activate this feature. Still you have possibility to opt out this feature in application preferences, but in general scenarios you don't need to do this.
+
+Remember, that such legacy file systems as FAT, does not support "hard links", but successors, such as NTFS, Ext3, Ext4 and others have "hard links" supported. So, think in advance which file system to choose for your backup destination.
+
+>*Note*: Gorsync Backup and RSYNC has some limitations with "deduplication" - they can't track file renames and relocations inside backup directory tree to save space and time in next backup session. This is not the application problem, it's RSYNC utility limitation. There is some experimental patches exist to get rid of this limitation, but not in the public RSYNC releases: you can read [this](https://lincolnloop.com/blog/detecting-file-moves-renames-rsync/) and [this](http://javier.io/blog/en/2014/08/06/rsync-rename-move.html).
+
+
+
+
+Collaboration and contribution
+------------------------------
+
+If you want to contribute to the project, read next:
+
+* Localization. Any help is appreciated to translate application to local languages. Use file ./data/assets/translate.en.toml as a source for new language translation.
+* Ready to discuss proposals regarding application improvement and further development scenarios.
+
+
+
+Contact
+-------
+
+Please use [Github issue tracker](https://github.com/d2r2/go-rsync/issues) for filing bugs or feature requests.
+
+
+
+License
+-------
+
+Gorsync Backup is licensed under [GNU GENERAL PUBLIC LICENSE version 3](https://raw.github.com/d2r2/go-rsync/master/LICENSE) by Free Software Foundation, Inc.

backup/abstract.go

@@ -0,0 +1,43 @@
+package backup
+
+import (
+	"time"
+
+	"github.com/d2r2/go-rsync/core"
+)
+
+type BackupNode struct {
+	SourceRsync string `toml:"src_rsync"`
+	DestSubPath string `toml:"dst_subpath"`
+}
+
+// BackupNodePlan contain information about single rsync source backup.
+type BackupNodePlan struct {
+	BackupNode BackupNode
+	RootDir    *core.Dir
+}
+
+// BackupPlan keep all necessary information obtained from
+// preferences and 1st backup pass to start backup process.
+type BackupPlan struct {
+	Config     *Config
+	Nodes      []BackupNodePlan
+	BackupSize core.FolderSize
+}
+
+type Notifier interface {
+	NotifyPlanStage_NodeStructureStartInquiry(sourceID int,
+		sourceRsync string) error
+	NotifyPlanStage_NodeStructureDoneInquiry(sourceID int,
+		sourceRsync string, dir *core.Dir) error
+	NotifyBackupStage_FolderStartBackup(rootDest string,
+		paths core.SrcDstPath, backupType core.FolderBackupType,
+		leftToBackup core.FolderSize,
+		timePassed time.Duration, eta *time.Duration,
+	) error
+	NotifyBackupStage_FolderDoneBackup(rootDest string,
+		paths core.SrcDstPath, backupType core.FolderBackupType,
+		leftToBackup core.FolderSize, sizeDone core.SizeProgress,
+		timePassed time.Duration, eta *time.Duration,
+		sessionErr error) error
+}

backup/common.go

@@ -0,0 +1,15 @@
+package backup
+
+import (
+	"fmt"
+
+	"github.com/d2r2/go-logger"
+)
+
+var LocalLog = logger.NewPackageLogger("backup",
+	// logger.DebugLevel,
+	logger.InfoLevel,
+)
+
+var e = fmt.Errorf
+var f = fmt.Sprintf

backup/config.go

@@ -0,0 +1,130 @@
+package backup
+
+import (
+	"github.com/BurntSushi/toml"
+	"github.com/d2r2/go-rsync/rsync"
+)
+
+type Config struct {
+	SigFileIgnoreBackup                string `toml:"sig_file_ignore_backup"`
+	RsyncRetryCount                    *int   `toml:"retry_count"`
+	AutoManageBackupBlockSize          *bool  `auto_manage_backup_block_size`
+	MaxBackupBlockSizeMb               *int   `toml:"max_backup_block_size_mb"`
+	UsePreviousBackup                  *bool  `toml:"use_previous_backup"`
+	NumberOfPreviousBackupToUse        *int   `toml:"number_of_previous_backup_to_use"`
+	EnableLowLevelLogForRsync          *bool  `toml:"enable_low_level_log_rsync"`
+	EnableIntensiveLowLevelLogForRsync *bool  `toml:"enable_intensive_low_level_log_rsync"`
+	// rsync --compress
+	RsyncCompressFileTransfer *bool `toml:"rsync_compress_file_transfer"`
+	// rsync --links
+	RsyncRecreateSymlinks *bool `toml:"rsync_recreate_symlinks"`
+	// rsync --perms
+	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"`
+	// rsync --group
+	RsyncTransferSourceGroup *bool `toml:"rsync_transfer_source_group"`
+	// rsync --owner
+	RsyncTransferSourceOwner *bool `toml:"rsync_transfer_source_owner"`
+	// rsync --devices
+	RsyncTransferDeviceFiles *bool `toml:"rsync_transfer_device_files"`
+	// rsync --specials
+	RsyncTransferSpecialFiles *bool `toml:"rsync_transfer_special_files"`
+
+	BackupNodes []BackupNode `toml:"backup_node"`
+}
+
+func NewConfig(filePath string) (*Config, error) {
+	var config Config
+	if _, err := toml.DecodeFile(filePath, &config); err != nil {
+		return nil, err
+	}
+	LocalLog.Debug(f("%+v", config))
+	return &config, nil
+}
+
+func (conf *Config) getRsyncParams(addExtraParams ...string) []string {
+	var params []string
+	if conf.RsyncCompressFileTransfer != nil && *conf.RsyncCompressFileTransfer {
+		params = append(params, "--compress")
+	}
+	if conf.RsyncTransferSourceOwner != nil && *conf.RsyncTransferSourceOwner {
+		params = append(params, "--owner")
+	}
+	if conf.RsyncTransferSourceGroup != nil && *conf.RsyncTransferSourceGroup {
+		params = append(params, "--group")
+	}
+	if conf.RsyncTransferSourcePermissions != nil && *conf.RsyncTransferSourcePermissions {
+		params = append(params, "--perms")
+	}
+	if conf.RsyncRecreateSymlinks != nil && *conf.RsyncRecreateSymlinks {
+		params = append(params, "--links")
+	}
+	if conf.RsyncTransferDeviceFiles != nil && *conf.RsyncTransferDeviceFiles {
+		params = append(params, "--devices")
+	}
+	if conf.RsyncTransferSpecialFiles != nil && *conf.RsyncTransferSpecialFiles {
+		params = append(params, "--specials")
+	}
+	params = append(params, addExtraParams...)
+	return params
+}
+
+func (conf *Config) usePreviousBackupEnabled() bool {
+	var usePreviousBackup bool = true
+	if conf.UsePreviousBackup != nil {
+		usePreviousBackup = *conf.UsePreviousBackup
+	}
+	return usePreviousBackup
+}
+
+func (conf *Config) numberOfPreviousBackupToUse() int {
+	var numberOfPreviousBackupToUse int = 1
+	if conf.NumberOfPreviousBackupToUse != nil {
+		numberOfPreviousBackupToUse = *conf.NumberOfPreviousBackupToUse
+	}
+	return numberOfPreviousBackupToUse
+}
+
+func (conf *Config) getRsyncSettings() *rsync.Logging {
+	logging := &rsync.Logging{}
+	if conf.EnableLowLevelLogForRsync != nil {
+		logging.EnableLog = *conf.EnableLowLevelLogForRsync
+	}
+	if conf.EnableIntensiveLowLevelLogForRsync != nil {
+		logging.EnableIntensiveLog = *conf.EnableIntensiveLowLevelLogForRsync
+	}
+	return logging
+}
+
+func (conf *Config) getBackupBlockSizeSettings() *backupBlockSizeSettings {
+	blockSize := &backupBlockSizeSettings{AutoManageBackupBlockSize: true, BackupBlockSize: 500}
+	if conf.AutoManageBackupBlockSize != nil {
+		blockSize.AutoManageBackupBlockSize = *conf.AutoManageBackupBlockSize
+	}
+	if conf.MaxBackupBlockSizeMb != nil {
+		blockSize.BackupBlockSize = uint64(*conf.MaxBackupBlockSizeMb * 1024 * 1024)
+	}
+	return blockSize
+}
+
+type sortConfig struct {
+	BackupNodes []BackupNode
+}
+
+func (s sortConfig) Len() int {
+	return len(s.BackupNodes)
+}
+
+func (s sortConfig) Less(i, j int) bool {
+	if s.BackupNodes[i].SourceRsync < s.BackupNodes[j].SourceRsync &&
+		s.BackupNodes[i].DestSubPath < s.BackupNodes[j].DestSubPath {
+		return true
+	} else {
+		return false
+	}
+}
+
+func (s sortConfig) Swap(i, j int) {
+	node := s.BackupNodes[i]
+	s.BackupNodes[i] = s.BackupNodes[j]
+	s.BackupNodes[j] = node
+}