diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be5d2695 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: 'gomod' + directory: '/' + schedule: + interval: 'weekly' + labels: + - 'dependencies' + commit-message: + prefix: 'Bump' + include: 'scope' + + - package-ecosystem: 'github-actions' + directory: '/' + schedule: + interval: 'weekly' + labels: + - 'dependencies' + commit-message: + prefix: '[CI]' + include: 'scope' diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index beb6e833..8a8aab90 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -9,17 +9,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-go@v4 with: - go-version: '1.21' + go-version: 'stable' - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v5 with: version: latest - args: release --rm-dist + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 39eb79bf..23ce5969 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,10 +14,10 @@ jobs: name: Linter code runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: '1.21' + go-version: 'stable' - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d2d66f9..056c44fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,13 @@ on: branches: - 'main' - '*.x' + paths-ignore: + - 'docs/**' + - '*.md' pull_request: + paths-ignore: + - 'docs/**' + - '*.md' jobs: test-matrix: strategy: @@ -13,7 +19,7 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b9f3cd..d867d679 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## 4.0.0 +### Added +- Support for the latest published Go version (1.21). This project will maintain compatibility with the latest **two major versions** published. + +### Changed +- Bump `github.com/sirupsen/logrus` from 1.9.0 to 1.9.3 ([#378][i378]) +- Bump `github.com/spf13/afero` from 1.9.5 to 1.10.0 ([#379][i379]) +- Bump `github.com/gphotosuploader/google-photos-api-client-go/v3` from 3.0.1 to 3.0.2 +- Bump `golang.org/x/oauth2` from 0.12.0 to 0.13.0 +- Bump `golang.org/x/sync` from 0.3.0 to 0.4.0 ([#377][i377]) +- Bump `golang.org/x/term` from 0.10.0 to 0.13.0 ([#376][i376]) +- [CI] Bump `github.com/stretchr/testify` from 1.7.0 to 1.8.4 ([#380][i380]) +- [CI] Bump `actions/checkout` from 3 to 4 ([#375][i375]) +- [CI] Bump `goreleaser/goreleaser-action` from 4 to 5 ([#374][i374]) +- [CI] Bump `golangci` from 1.52.1 to 1.54.2 + +### Removed +- Support for multiple concurrent workers. The bandwidth to upload items is shared, so we are not expecting any performance problem. +- Removed DEPRECATED configuration parameters from previous versions. + +[i374]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/374 +[i375]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/375 +[i376]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/376 +[i377]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/377 +[i378]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/378 +[i379]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/379 +[i380]: https://github.com/gphotosuploader/gphotos-uploader-cli/pulls/380 + ## 3.5.2 ### Added - Support for the latest published Go version (1.21). This project will maintain compatibility with the latest two major versions published. @@ -50,7 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ### Changed - The command `auth` initiates the [Google authentication to get an OAuth 2.0 token](https://gphotosuploader.github.io/gphotos-uploader-cli/#/getting-started?id=authentication). **It should be used the first time that the CLI is configured**. See [documentation](https://gphotosuploader.github.io/gphotos-uploader-cli/#/getting-started?id=authentication). ### Deprecated -- The OAuth 2.0 authentication based in out-of-band tokens is deprecated by Google. ([#326][i326]) +- Google deprecates the OAuth 2.0 authentication based on out-of-band tokens. ([#326][i326]) [i326]: https://github.com/gphotosuploader/gphotos-uploader-cli/issues/326 @@ -58,7 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ### Added - Support for the latest published Go version (1.18). This project will maintain compatibility with the latest two major versions published. ### Changed -- Dependency has been updated, so potential bugs has been fixed. +- Dependency has been updated, so potential bugs have been fixed. ### Deprecated - Once Go 1.18 has been published, previous Go 1.16 support is deprecated. ### Removed @@ -119,10 +147,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this - Configuration validation. The cli validates the configuration data at starting time. - Information messages to bring more context at runtime. ([#260][i260]) ### Changed -- `Jobs.MakeAlbums` configuration setting has changed to `Jobs.CreateAlbums`. Valid values are "Off", "folderName" and "folderPath". -- **Reduced the number of calls to the API when uploading files**. It's using less than 50% of calls than before. +- `Jobs.MakeAlbums` configuration setting has changed to `Jobs.CreateAlbums`. Valid values are `Off`,`folderName` and `folderPath`. +- **Reduce the number of calls to the API when uploading files**. It's using less than 50% of calls than before. - Move to `golang.org/x/term` from `golang.org/x/crypto/ssh/terminal`, due to deprecation. -- Some parts of the code has been refactored to make cleaner code and increase testability. +- Some parts of the code have been refactored to make cleaner code and increase testability. - `Jobs.Account` configuration setting has been changed to `Account`. Multiple Google Photos accounts are not supported. ([#231][i231]) - Bump `google-photos-api-client-go` from `v2.0.0` to `v2.1.3`. It improves performance. ([#259][i259]) - Bump `golangci-lint` from `1.30.0` to `1.34.1`. @@ -150,14 +178,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 2.0.0 > This is a **major upgrade**, and it has some **non-backwards compatible changes**: -> - `includePatterns` & `excludePatterns` configuration options has changed. +> - `includePatterns` & `excludePatterns` configuration options have changed. > - `includePatterns` has a new default (`_IMAGE_EXTENSIONS_`). > - `uploadVideos` configuration option has been removed. ### Added -- Two new tagged patterns has been added: `_IMAGE_EXTENSIONS_`, matching [supported image file types](https://support.google.com/googleone/answer/6193313), and `_RAW_EXTENSIONS_`, matching [supported RAW file types](https://support.google.com/googleone/answer/6193313). ([#249][i249]) -- Retries management. It's implementing exponential back-off with a maximum of 4 retries by default. ([#253][i253]) +- Two new tagged patterns have been added: `_IMAGE_EXTENSIONS_`, matching [supported image file types](https://support.google.com/googleone/answer/6193313), and `_RAW_EXTENSIONS_`, matching [supported RAW file types](https://support.google.com/googleone/answer/6193313). ([#249][i249]) +- Retry management. It's implementing exponential back-off with a maximum of 4 retries by default. ([#253][i253]) ### Changed -- `includePatterns` & `excludePatterns` configuration options has changed. It's using a new format, please review de [configuration documentation][idocumentation]. +- `includePatterns` & `excludePatterns` configuration options have changed. It's using a new format, please review de [configuration documentation][idocumentation]. - By default, if `includePatterns` is empty, `_IMAGE_EXTENSIONS_` will be used. ([#249][i249]) - Bump `google-photos-api-client-go` from `v2.0.0-beta-1` to `v2.0.0`. ### Fixed @@ -292,7 +320,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.8.6 ### Changed -- Remove `build` from version. Now `version` has all the tag+build information. +- Remove `build` from a version. Now `version` has all the tag+build information. ### Fixed - Fix duplicated album creation. ([#135][i135]) @@ -304,7 +332,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.8.4 ### Fixed -- Fix panic when a unexpected error on media item creation was raised. (#110) +- Fix panic when an unexpected error on media item creation was raised. (#110) ### Changed - Update `gphotosuploader/google-photos-api-client-go` to v1.0.7. @@ -328,11 +356,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this - Updated `google-photos-api-client` to version v1.0.4 to help with broken album creation. (#19) ### Fixed -- Fix duplicated album creation due to concurrency problem. (#19) +- Fix duplicated album creation due to a concurrency problem. (#19) ## 0.8.0 ### Added -- Uploads can be resumed. This will help uploading large files or when connection has fails. Thanks to @pdecat. +- Uploads can be resumed. This will help upload large files or when connection has fails. Thanks to @pdecat. ## 0.7.2 ### Fixed @@ -344,15 +372,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this - Google Auth expired token refresh. Once token is expired, `gphotos-uploader-cli` will try to refresh the token without user intervention. **NOTE**: First time you use this version, you should re-authenticate in order to get the token that allows token refresh. (#103) - Add `--config` flag to specify the folder where configuration is kept. (#104) ### Changed -- Moved CI/CD platform from Travis to [Drone.io](https://cloud.drone.io/gphotosuploader/gphotos-uploader-cli). It has reduce the time to CI by a half. +- Moved CI/CD platform from Travis to [Drone.io](https://cloud.drone.io/gphotosuploader/gphotos-uploader-cli). It has reduced the time to CI by a half. ## 0.6.0 ### Added - `deleteAfterUpload` option has been reactivated, it was removed on v0.4.0. If you use this option in [config file][idocumentation] files will be deleted from local repository after being uploaded to Google Photos. (#25) ### Changed -- This repository has transferred to [GPhotos Uploaders organization](https://github.com/gphotosuploader), so all imports has been updated to the new organization's URL. +- This repository has transferred to [GPhotos Uploaders organization](https://github.com/gphotosuploader), so all imports have been updated to the new organization's URL. ### Removed -- Removed some useless log lines. There are still too much. +- Removed some useless log lines. There is still too much. ## 0.5.0 ### Changed @@ -360,14 +388,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.4.2 ### Fixed -- Fix CI release pipeline to fix application version (#94). Last version was still broken on CI. +- Fix CI release pipeline to fix an application version (#94). The Last version was still broken on CI. ## 0.4.1 ### Added - Add Homebrew tap to allow users to install `gphotos-uploader-cli` using Homebrew. See [install](README.md) section. ### Fixed -- Fix CI release pipeline to fix application version (#94) +- Fix CI release pipeline to fix an application version (#94) ## 0.4.0 ### Added @@ -398,7 +426,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.3.1 ### Changed - Move some dependencies to the new [gphotosuploader](https://github.com/gphotosuploader) organization -- `make test` is not as verbose as before. To make easier to see if there is an error +- `make test` is not as verbose as before. To make it easier to see if there is an error ### Removed - Removed some useless and local vendor files @@ -433,7 +461,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.1.16 - 2019-06-16 ### Fixed -- Fix goreleaser configuration (remove deprecated statement) +- Fix goreleaser configuration (remove a deprecated statement) - Update [Getting started](README.md) documentation ### Removed @@ -441,6 +469,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this ## 0.1.11 - 2018-09-20 ### Added -- [goreleaser](https://goreleaser.com/) will be on charge of publishing [binaries](https://github.com/gphotosuploader/gphotos-uploader-cli/releases) after new release is done +- [goreleaser](https://goreleaser.com/) will be in charge of publishing [binaries](https://github.com/gphotosuploader/gphotos-uploader-cli/releases) after the new release is done [idocumentation]: https://gphotosuploader.github.io/gphotos-uploader-cli/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07c6785b..453884f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,22 +10,19 @@ You are more than welcome to open issues in this project to [suggest new feature ## Contributing Code This project is mainly written in Golang. -> This project will maintain compatibility with the last two Go major versions published. Currently Go 1.12 and Go 1.13. +> This project will maintain compatibility with the last two [golang major versions published](https://go.dev/doc/devel/release). To contribute code: -1. Ensure you are running golang version 1.12 or greater -2. Set the following environment variables: - ``` - GO111MODULE=on - ``` -3. Fork the project -4. Clone the project: `git clone https://github.com/[YOUR_USERNAME]/gphotos-uploader-cli && cd gphotos-uploader-cli` -5. Run `go mod download` to install the dependencies -6. Make changes to the code -7. Run `make build` to build the project -8. Make changes -9. Run tests: `make test` -10. Format your code: `go fmt ./...` -11. Commit changes -12. Push commits -13. Open pull request +1. Ensure you are running a supported golang version +1. Fork the project +1. Clone the project: `git clone https://github.com/[YOUR_USERNAME]/gphotos-uploader-cli && cd gphotos-uploader-cli` +1. Run `go mod download` to install the dependencies +1. Make changes to the code +1. Run `make build` to build the project +1. Make changes +1. Run tests: `make test` +1. Run linter: `make lint` +1. Format your code: `go fmt ./...` +1. Commit changes +1. Push commits +1. Open pull request diff --git a/Makefile b/Makefile index 286972be..caa0c122 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,12 @@ +# Use linker flags to provide version/build settings to the target +CONFIGURATION_PACKAGE := github.com/gphotosuploader/gphotos-uploader-cli/version + # This VERSION could be set calling `make VERSION=0.2.0` VERSION ?= $(shell git describe --tags --always --dirty) +LDFLAGS=-ldflags "-X ${CONFIGURATION_PACKAGE}.versionString=$(VERSION)" -# Use linker flags to provide version/build settings to the target -VERSION_IMPORT_PATH := github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd -RELEASE_VERSION_FLAGS=-X=${VERSION_IMPORT_PATH}.version=$(VERSION) -LDFLAGS=-ldflags "$(RELEASE_VERSION_FLAGS)" +TEST_VERSION="0.0.0-test.preview" +TEST_LDFLAGS=-ldflags "-X ${CONFIGURATION_PACKAGE}.versionString=$(TEST_VERSION)" # go source files, ignore vendor directory PKGS = $(shell go list ./... | grep -v /vendor) @@ -16,7 +18,7 @@ TMP_DIR ?= .tmp COVERAGE_FILE := $(TMP_DIR)/coverage.txt COVERAGE_HTML_FILE := $(TMP_DIR)/coverage.html GOLANGCI := $(TMP_DIR)/golangci-lint -GOLANGCI_VERSION := 1.55.5 +GOLANGCI_VERSION := 1.55.0 # set how to open files based on OS and ARCH. UNAME_OS := $(shell uname -s) @@ -35,7 +37,7 @@ endif test: ## Run all the tests @echo "--> Running tests..." @mkdir -p $(dir $(COVERAGE_FILE)) - @go test -covermode=atomic -coverprofile=$(COVERAGE_FILE) -race -failfast -timeout=30s $(PKGS) + @go test -covermode=atomic -coverprofile=$(COVERAGE_FILE) -race -failfast -timeout=30s ${TEST_LDFLAGS} $(PKGS) .PHONY: cover cover: test ## Run all the tests and opens the coverage report diff --git a/README.md b/README.md index 53345a00..5cfe06ba 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -![logo](.docs/_media/gphotos-uploader-cli-logo.png) +![logo](docs/images/gphotos-uploader-cli-logo.png) -Command line tool to mass upload media folders to your Google Photos account. +Tool to mass upload media folders to your Google Photos account and manage it using a CLI. [![Go Report Card](https://goreportcard.com/badge/github.com/gphotosuploader/gphotos-uploader-cli)](https://goreportcard.com/report/github.com/gphotosuploader/gphotos-uploader-cli) [![codebeat badge](https://codebeat.co/badges/9f3561ad-2838-456e-bc92-68988eeb376b)](https://codebeat.co/projects/github-com-gphotosuploader-gphotos-uploader-cli-main) @@ -11,7 +11,7 @@ Command line tool to mass upload media folders to your Google Photos account. # Google Photos uploader CLI -While the official tool only supports Mac OS and Windows, this brings an uploader to Linux too. Lets you upload photos from, in theory, any OS for which you can compile a Go program. +While the official tool only supports macOS and Windows, this brings an uploader to Linux too. Lets you upload photos from, in theory, any OS for which you can compile a Go program. See the [documentation site](https://gphotosuploader.github.io/gphotos-uploader-cli) for more information about this CLI. diff --git a/UPGRADING.md b/UPGRADING.md index f907b1e3..663e83f0 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,12 +1,39 @@ # Upgrading notes -## Upgrading To 2.0.0 from 1.x.x +## Upgrading To 4.x from 3.x + +### Data folders +There are some changes inside the [configuration folder](https://gphotosuploader.github.io/gphotos-uploader-cli/#/getting-started?id=configure) (usually `~/.gphotos-uploader-cli`): + +- The folder `uploads.db` **MUST be renamed** to `uploaded_files`. +- The folder `resumable_uploads.db` **MUST be renamed** to `ongoing_uploads`. +- The token files, named as email address, **MUST be moved** under the `tokens` folder. + +The content of the configuration folder (e.g `~/.gphotos-uploader-cli`) should be: + +``` +-rw------- config.hjson +drwx------ ongoing_uploads +drwx------ tokens +drwx------ uploaded_files +``` + +> **ATTENTION**: If you don't follow the process above, all the information regarding the previous version will not be kept. + +## Upgrading To 3.x from 2.x + +### Configuration settings +- `Jobs.Account` configuration setting has been changed to `Account`. See [configuration documentation](https://gphotosuploader.github.io/gphotos-uploader-cli/#/configuration). +- `Jobs.MakeAlbums` configuration setting has changed to `Jobs.CreateAlbums`. See [configuration documentation](https://gphotosuploader.github.io/gphotos-uploader-cli/#/configuration?id=createalbums). +- **Multiple Google Photos account support has been removed**. You can use multiple configuration files in the same application folder instead. + +## Upgrading To 2.x from 1.x ### Patterns definition -The `includePatterns` and `excludePatterns` configuration options has changed, see [configuration documentation](.docs/configuration.md). You should modify your configuration to honor the **new format**. +The `includePatterns` and `excludePatterns` configuration options has changed, see [configuration documentation](https://gphotosuploader.github.io/gphotos-uploader-cli/#/configuration). You should modify your configuration to honor the **new format**. -If you were using the tagged patterns (*\_ALL_FILES_* and *\_ALL_VIDEO_FILES_*) you don't need to do anything. +If you were using the tagged patterns (`_ALL_FILES_` and `_ALL_VIDEO_FILES_`) you don't need to do anything. ```bash sourceFolder @@ -23,4 +50,4 @@ Description | Current format | Previous format Include all files | `includePatterns: "**"}` | `includePatterns: {"*"}` Include only PNG files | `includePatterns: "**/*.png"}` | `includePatterns: {"*.png"}` Include PNG files in `foo` folder | `includePatterns: "foo/*.png"}` | `includePatterns: {"*.png"}`
`excludePatterns: {"bar"}` - \ No newline at end of file + diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 00000000..e69de29b diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..043bd0b1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,8 @@ +# Google Photos uploader CLI +> Tool to mass upload media folders to your Google Photos account and manage it using a CLI. + +While the official tool only supports macOS and Windows, this brings an uploader to Linux too. Lets you upload photos from, in theory, any OS for which you can compile a Go program. + +## Documentation + +See The [guide](introduction.md) diff --git a/docs/_coverpage.md b/docs/_coverpage.md new file mode 100644 index 00000000..441edd62 --- /dev/null +++ b/docs/_coverpage.md @@ -0,0 +1,8 @@ +![logo](images/gphotos-uploader-cli-logo.png) + +# gphotos-uploader-cli + +> Tool to mass upload media folders to your Google Photos account and manage it using a CLI. + +[GitHub](https://github.com/gphotosuploader/gphotos-uploader-cli) +[Get Started](introduction.md) diff --git a/docs/_navbar.md b/docs/_navbar.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/_sidebar.md b/docs/_sidebar.md new file mode 100644 index 00000000..14f4d1b5 --- /dev/null +++ b/docs/_sidebar.md @@ -0,0 +1,5 @@ +* [Introduction](introduction.md) +* [Getting Started](getting-started.md) +* [Configuration](configuration.md) +* [UPGRADING](upgrading) +* [CHANGELOG](changelog) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..5b8f41f4 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,164 @@ +# Configuration + +## Configuration options + +> The configuration is kept in the file `config.hjson` inside the configuration folder. You can specify your own folder using `--config /my/config/dir` otherwise default configuration folder is `~/.gphotos-uploader-cli`. + +Example configuration file: + +```hjson +{ + APIAppCredentials: + { + ClientID: YOUR_APP_CLIENT_ID + ClientSecret: YOUR_APP_CLIENT_SECRET + } + Account: YOUR_GOOGLE_PHOTOS_ACCOUNT + SecretsBackendType: file + Jobs: + [ + { + SourceFolder: YOUR_FOLDER_PATH + CreateAlbums: Off + DeleteAfterUpload: false + IncludePatterns: [ "**/*.jpg", "**/*.png" ] + ExcludePatterns: [ "**/ScreenShot*" ] + } + ] +} +``` + +## APIAppCredentials + +Given that `gphotos-uploader-cli` uses OAuth 2 to access Google APIs, authentication is a bit tricky and involves a few manual steps. Please follow the guide below carefully, to give `gphotos-uploader-cli` the required access to your Google Photos account. + +Before you can use `gphotos-uploader-cli`, you must enable the Photos Library API and request an OAuth 2.0 Client ID. + +1. Make sure you're logged in into the Google Account where your photos should be uploaded to. +1. Start by [creating a new project](https://console.cloud.google.com/projectcreate) in Google Cloud Platform and give it a name (example: _Google Photos Uploader_). +1. Enable the [Google Photos Library API](https://console.cloud.google.com/apis/library/photoslibrary.googleapis.com) by clicking the ENABLE button. +1. Configure the [OAuth consent screen](https://console.cloud.google.com/apis/credentials/consent) by setting the application name (example: _gphotos-uploader-cli_) and then click the Save button on the bottom. +1. Create [credentials](https://console.cloud.google.com/apis/credentials) by clicking the **Create credentials → OAuth client ID** option, then pick **Desktop app** as the application type and give it a name (example: _gphotos-uploader-cli_). +1. Copy the **Client ID** and the **Client Secret** and keep them ready to use in the next step. +1. Open the *config file* and set both the `ClientID` and `ClientSecret` options to the ones generated on the previous step. + +## Account +It's the Google Account identity (e-mail address) where the files are going to be uploaded. + +### SecretsBackendType +This option allows you to choose which backend will be used for secret storage. You set `auto` to allow the application to decide which one will be used given your environment. + +Available options for secrets backend are: + +``` +"auto" For auto backend selection +"secret-service" For gnome-keyring support +"keychain" For OS X keychain support +"kwallet" For KDE Secrets Manager support +"file" For encrypted file support - needs interaction to supply a symetric encryption key +``` + +Most of the time `auto` is the proper one. The application will try to use the existing backends in the order [defined by the library](https://github.com/99designs/keyring/blob/2c916c935b9f0286ed72c22a3ccddb491c01c620/keyring.go#L28): + +``` +// This order makes sure the OS-specific backends +// are picked over the more generic backends. +var backendOrder = []BackendType{ + // MacOS + KeychainBackend, + // Linux + SecretServiceBackend, + KWalletBackend, + // General + FileBackend, +} +``` + +## Jobs +List of folders to upload and upload options for each folder. + +### SourceFolder +The folder to upload from. Must be an absolute path. Can expand the home folder tilde shorthand `~`. +> The application will follow any symlink it finds, it does not terminate if there are any non-terminating loops in the file structure. + +### CreateAlbums +It controls how uploaded files will be organized into albums in Google Photos. + +There are three options: +* `Off` will not create any album. +* `folderName` will use the name of the folder (within `SourceFolder`), where the item is uploaded from, to set the album name. +* `folderPath` will use the full path of the folder (relative to `SourceFolder`), where the item is uploaded from, to set the album name. + +``` +# Given SouceFolder: /foo +# and file: /foo/bar/xyz/file.jpg + +CreateAlbums: folderName +# album name would be: xyz + +CreateAlbums: folderPath +# album name would be: bar_xyz +``` + +### DeleteAfterUpload +If set to true, media will be deleted from the local disk after completing the upload. + +## Including and Excluding files +You can include and exclude files by specifying the `includePatterns` and `excludePatterns` options. You can add one or more patterns separated by commas `,`. These patterns are always applied to `sourceFolder`. + +For example, to upload all _JPG and PNG files_ that are not named _*ScreenShots*_ you can configure it like this: +``` +includePatterns: [ "**/*.jpg", "**/*.png" ] +excludePatterns: [ "**/ScreenShot*" ] +``` + +Another example excluding a specific directory (and folders inside it): +``` +includePatterns: [ "_ALL_FILES_" ] +excludePatterns: [ "**/Temp/**" ] +``` + +> If `includePatterns` is empty, `_IMAGE_EXTENSIONS_` will be used. + +### Patterns +Supports the following special terms in the patterns: + +Special Terms | Meaning +------------- | ------- +`*` | matches any sequence of non-path-separators +`**` | matches any sequence of characters, including path separators +`?` | matches any single non-path-separator character +`[class]` | matches any single non-path-separator character against a class of characters ([see below](#character-classes)) +`{alt1,...}` | matches a sequence of characters if one of the comma-separated alternatives matches + +Any character with a special meaning can be escaped with a backslash (`\`). + +#### Character Classes + +Character classes support the following: + +Class | Meaning +---------- | ------- +`[abc]` | matches any single character within the set +`[a-z]` | matches any single character in the range +`[^class]` | matches any single character which does *not* match the class + +#### Tagged patterns +There are some common patterns that have been tagged, you can use them to simplify your configuration. + +> Tagged patterns matches file extensions case insensitively. + +* `_ALL_FILES_`: Matches all files, is the same as using `**`. +* `_IMAGE_EXTENSIONS_`: Matches [Google Photos supported image file types](https://support.google.com/googleone/answer/6193313) and it includes: `jpg, jpeg, png, webp, gif` file extensions case in-sensitively. +* `_RAW_EXTENSIONS_`: Matches [Google Photos supported RAW file types](https://support.google.com/googleone/answer/6193313) and it includes `arw, srf, sr2, crw, cr2, cr3, dng, nef, nrw, orf, raf, raw, rw2` file extensions case in-sensitively. +* `_ALL_VIDEO_FILES_`: Matches [Google Photos supported video file types](https://support.google.com/googleone/answer/6193313) and it includes `mpg, mod, mmv, tod, wmv, asf, avi, divx, mov, m4v, 3gp, 3g2, mp4, m2t, m2ts, mts, mkv` file extensions case in-sensitively. + +## Environment variables + +### GPHOTOS_CLI_TOKENSTORE_KEY + +This variable is used to read the token store key for opening the secrets storage. It works when `SecretsBackendType: file` and it is intended to be used by headless runners. + +```bash +GPHOTOS_CLI_TOKENSTORE_KEY=my-super-secret gphotos-uploader-cli push +``` diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 00000000..a8b19c99 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,98 @@ +# Getting started + +## Install +You can install the pre-compiled binary (in several different ways) or compile from source. + +Here are the steps for each of them: + +### Install the pre-compiled binary + +#### homebrew tap (only on macOS for now): +```bash +brew install gphotosuploader/tap/gphotos-uploader-cli +``` + +#### manually + +Download the pre-compiled binaries from the [releases page](https://github.com/gphotosuploader/gphotos-uploader-cli/releases/latest) and copy to the desired location. + +For Linux: +```bash +LOCATION=$(curl -s https://api.github.com/repos/gphotosuploader/gphotos-uploader-cli/releases/latest \ +| grep browser_download_url \ +| awk '{ print $2 }' \ +| tr -d \" \ +| grep linux); wget --quiet -O - $LOCATION | tar -zxf - +``` + +For macOS: +```bash +LOCATION=$(curl -s https://api.github.com/repos/gphotosuploader/gphotos-uploader-cli/releases/latest \ +| grep browser_download_url \ +| awk '{ print $2 }' \ +| tr -d \" \ +| grep darwin); wget --quiet -O - $LOCATION | tar -zxf - +``` + +### Compiling from source + +> This project will maintain compatibility with the last two Go major versions published. It could work with other versions but we can't support it. + +You can compile the source code in your system. + +```bash +git clone https://github.com/gphotosuploader/gphotos-uploader-cli \ +&& cd gphotos-uploader-cli \ +&& make build +``` + +Or you can use `go get` if you prefer it: + +```bash +go get github.com/gphotosuploader/gphotos-uploader-cli +``` + +## Configure +First initialize the config file using this command: +```bash +gphotos-uploader-cli init +``` + +> Default configuration folder is `~/.gphotos-uploader-cli` but you can specify your own folder using `--config /my/config/dir`. Configuration is kept in the `config.hjson` file inside this folder. + +You must review the [documentation](configuration.md) to specify your **Google Photos API credentials**, `APIAppCredentials`. You should tune your `jobs` configuration also. + +## Authentication +Once it's configured, you need to authenticate your CLI against Google Photos: +```bash +gphotos-uploader-cli auth +``` + +Few manual steps are needed: + +1. You should get an output like this one: + +![Run gphotos-uploader-cli auth](images/run_gphotos_uploader_cli_with_auth.jpeg) + +> **Reaching the `localhost` in headless environments:** Even that the CLI is not intended to be used in headless setups, several users have confirmed that it's possible to set it up. You can forward one port from your real host to the remote host, In order to connect to the remote host, create an SSH tunnel to the remote machine: + +```bash +# Only for headless setups +ssh -L 37551:localhost:37551 myremotehost +``` + +2. Open a browser and point to the previous URL. Select the account to authenticate the CLI with (the same you configured in the config file). You will see something like this: + +![Google asking for Google account](images/select_google_account.jpeg) + +3. After that, you should confirm that you trust on `gphotos-uploader-cli` to access to your Google Photos account, click on **Allow**: + +![Google ask you to verify gphotos-upload-cli](images/allow_gphotos_uploader_cli_to_access.jpeg) + +4. A page with a code is shown in your browser, copy this code and go back to the terminal. + +![Browser's steps are complete](images/you_can_close_the_browser.jpeg) + +5. Go back to your terminal window. The authentication process is complete. + +![Authentication is complete](images/authentication_is_complete.jpeg) diff --git a/docs/images/allow_gphotos_uploader_cli_to_access.jpeg b/docs/images/allow_gphotos_uploader_cli_to_access.jpeg new file mode 100644 index 00000000..14a3e614 Binary files /dev/null and b/docs/images/allow_gphotos_uploader_cli_to_access.jpeg differ diff --git a/docs/images/authentication_is_complete.jpeg b/docs/images/authentication_is_complete.jpeg new file mode 100644 index 00000000..d1f18c26 Binary files /dev/null and b/docs/images/authentication_is_complete.jpeg differ diff --git a/.docs/_media/gphotos-uploader-cli-logo.png b/docs/images/gphotos-uploader-cli-logo.png similarity index 100% rename from .docs/_media/gphotos-uploader-cli-logo.png rename to docs/images/gphotos-uploader-cli-logo.png diff --git a/docs/images/run_gphotos_uploader_cli_with_auth.jpeg b/docs/images/run_gphotos_uploader_cli_with_auth.jpeg new file mode 100644 index 00000000..d05b3a77 Binary files /dev/null and b/docs/images/run_gphotos_uploader_cli_with_auth.jpeg differ diff --git a/docs/images/select_google_account.jpeg b/docs/images/select_google_account.jpeg new file mode 100644 index 00000000..1da980bf Binary files /dev/null and b/docs/images/select_google_account.jpeg differ diff --git a/docs/images/you_can_close_the_browser.jpeg b/docs/images/you_can_close_the_browser.jpeg new file mode 100644 index 00000000..ae585b63 Binary files /dev/null and b/docs/images/you_can_close_the_browser.jpeg differ diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 00000000..419d66fd --- /dev/null +++ b/docs/index.html @@ -0,0 +1,41 @@ + + + + + Document + + + + +
+ + + + + + + diff --git a/docs/introduction.md b/docs/introduction.md new file mode 100644 index 00000000..71db6b1f --- /dev/null +++ b/docs/introduction.md @@ -0,0 +1,42 @@ +# Introduction + +Command line tool to mass upload media folders to your Google Photos account. + +While the official tool only supports Mac OS and Windows, this brings an uploader to Linux too. Lets you upload photos from, in theory, any OS for which you can compile a Go program. +> The Google Photos API which `gphotos-uploader-cli` uses has quite a few limitations, so please read the [limitations section](#limitations) carefully to make sure it is suitable for your use. + +## Features + +- **Customizable configuration**: via JSON-like config file. +- **Filter files with patterns**: include/exclude files & folders using patterns (see [documentation](configuration.md)). +- **Resumable uploads**: Uploads can be resumed, saving time and bandwidth. +- **File deletion after uploading**: Clean up local files after being uploaded. +- **Track already uploaded files**: uploads only new files to save bandwidth. +- **Cache request results**: keep a local cache to reduce number of queries to Google Photos. +- **Secure**: logs you into Google using OAuth (so this app doesn't have to know your password), and stores your temporary access code in your OS's secure storage (keyring/keychain). +- **Retryable**: all the requests are retried using exponential back-off as is recommended by [Google Photos best practices](https://developers.google.com/photos/library/guides/best-practices#error-handling). + +## Limitations +Only images and videos can be uploaded. If you attempt to upload non videos or images or formats that Google Photos doesn't understand, `gphotos-uploader-cli` will upload the file, then Google Photos will give an error when it is put turned into a media item. + +### Photo storage and quality +All media items uploaded to Google Photos using the API [are stored in full resolution](https://support.google.com/photos/answer/6220791) at original quality. **They count toward the user’s storage**. The API does not offer a way to upload in "high quality" mode. + +### Duplicates +If you upload the same image (with the same binary data), twice then Google Photos will deduplicate it. However it will retain the filename from the first upload which may be confusing. In practise this shouldn't cause too many problems. + +### Modified time +The date shown of media in Google Photos is the creation date as determined by the EXIF information, or the upload date if that is not known. +This is not changeable by `gphotos-upload-cli` and is not the modification date of the media on local disk. This means that this CLI cannot use the dates from Google Photos for syncing purposes. + +### Size +The Google Photos API does not return the size of media. This means that when syncing to Google Photos, `gphotos-uploader-cli` can only do a file existence check. +It is possible to read the size of the media, but this needs an extra HTTP HEAD request per media item so is **very slow** and uses up a lot of transactions. + +### Albums +`gphotos-uploader-cli` can only upload files to albums it created. This is a limitation of the Google Photos API. + +`gphotos-uploader-cli` can remove files it uploaded from albums it created only. + +### Rate Limiting +Google Photos imposes a rate limit on all API clients. The quota limit for requests to the Library API is 10,000 requests per project per day. The quota limit for requests to access media bytes (by loading a photo or video from a base URL) is 75,000 requests per project per day. diff --git a/go.mod b/go.mod index a66abc1e..6134c8a7 100644 --- a/go.mod +++ b/go.mod @@ -6,24 +6,24 @@ require ( github.com/99designs/keyring v1.2.2 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 - github.com/gphotosuploader/google-photos-api-client-go/v2 v2.4.2 + github.com/gphotosuploader/google-photos-api-client-go/v3 v3.0.2 github.com/hjson/hjson-go/v4 v4.3.0 github.com/int128/oauth2cli v1.14.0 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d github.com/mitchellh/go-homedir v1.1.0 - github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pierrec/xxHash v0.1.5 + github.com/pkg/errors v0.9.1 github.com/schollz/progressbar/v3 v3.13.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.0 golang.org/x/oauth2 v0.13.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 - google.golang.org/api v0.148.0 ) require ( @@ -31,12 +31,12 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/danieljoos/wincred v1.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12 // indirect - github.com/gadelkareem/cachita v0.2.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -47,16 +47,17 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/int128/listener v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mediocregopher/radix/v3 v3.8.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/api v0.148.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4183fb86..b6853d49 100644 --- a/go.sum +++ b/go.sum @@ -81,9 +81,7 @@ github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222/go.mod h1:PgrCj github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12 h1:pKeuUgeuL6jk/FpxSr0ZVL1XEiOmrcWBvB2rKXu0mMI= github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12/go.mod h1:IYed2VYeQcs7JTN6KiVXjaz6Rv/Qz092Wjc6o5bCJ9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gadelkareem/cachita v0.2.3 h1:VLR5rddytxM6BA+VrOcD0GjRhcXv+xwai/U4Jq/Ba5o= -github.com/gadelkareem/cachita v0.2.3/go.mod h1:xxh2RDZCXnVZQM7A/txZFFyEf3DHX+VWZ+Qgg/WOWVI= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -121,7 +119,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -154,9 +151,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gphotosuploader/google-photos-api-client-go/v2 v2.4.2 h1:DzSy3R/pU2y4M0Qf/nUW0W5f/sNbjQXC8RVNySjAp9o= -github.com/gphotosuploader/google-photos-api-client-go/v2 v2.4.2/go.mod h1:n+y6WPFHAkWP3t7nQiArfZBKDYdQMdPm4hdP1u5QaqA= +github.com/gphotosuploader/google-photos-api-client-go/v3 v3.0.2 h1:kWLV6kXJWvpJguNL9LN6Qd5OJayHHq9uG0zgRkp9EuU= +github.com/gphotosuploader/google-photos-api-client-go/v3 v3.0.2/go.mod h1:/QFXDvwMbVxJ94sZGLsBS+7QMxhb2H3h23E+4kul/jA= github.com/gphotosuploader/googlemirror v0.5.0 h1:9a9CCUnAFo3qHp7U/epmdTiOvAzXCkVq5AQLo8PWBns= github.com/gphotosuploader/googlemirror v0.5.0/go.mod h1:L6A+2KW6d/OwjZ5QH2fGXJXsOtR115tj9w+YxdyjfUI= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= @@ -181,8 +177,6 @@ github.com/int128/listener v1.1.0 h1:2Jb41DWLpkQ3I9bIdBzO8H/tNwMvyl/OBZWtCV5Pjuw github.com/int128/listener v1.1.0/go.mod h1:68WkmTN8PQtLzc9DucIaagAKeGVyMnyyKIkW4Xn47UA= github.com/int128/oauth2cli v1.14.0 h1:r63NoO10ybUXIXUQxih8WOmt5HQpJubdTmhWh22B9VE= github.com/int128/oauth2cli v1.14.0/go.mod h1:LIoVAzgAsS2tDDBc8yopkcgY5oZR0+MJAeECkCwtxhA= -github.com/joomcode/errorx v0.1.0/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= -github.com/joomcode/redispipe v0.9.0/go.mod h1:4S/gpBCZ62pB/3+XLNWDH7jQnB0vxmpddAMBva2adpM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= @@ -193,22 +187,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix.v2 v0.0.0-20181115013041-b67df6e626f9/go.mod h1:fLRUbhbSd5Px2yKUaGYYPltlyxi1guJz1vCmo1RQL50= -github.com/mediocregopher/radix/v3 v3.2.0/go.mod h1:baVzIVpQ8FpvCE6s+XbkoLkBRRI6k/e/HcSNhJDdFjk= -github.com/mediocregopher/radix/v3 v3.8.1 h1:rOkHflVuulFKlwsLY01/M2cM2tWCjDoETcMqKbAWu1M= -github.com/mediocregopher/radix/v3 v3.8.1/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= @@ -224,11 +211,10 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pierrec/xxHash v0.1.5 h1:n/jBpwTHiER4xYvK3/CdPVnLDPchj8eTJFFLUb4QHBo= github.com/pierrec/xxHash v0.1.5/go.mod h1:w2waW5Zoa/Wc4Yqe0wgrIYAGKqRMf7czn2HNKXmuL+I= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -256,12 +242,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= -github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -319,7 +303,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -492,8 +475,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -600,8 +581,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go index 558cb803..c470ed70 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" "net/http" "path/filepath" @@ -10,9 +11,8 @@ import ( "golang.org/x/oauth2" "github.com/gphotosuploader/gphotos-uploader-cli/internal/config" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/leveldbstore" "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/tokenmanager" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/upload_tracker" "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" ) @@ -103,7 +103,7 @@ func StartWithoutConfig(fs afero.Fs, path string) (*App, error) { } // Stop stops the application releasing all service resources. -func (app App) Stop() error { +func (app *App) Stop() error { // Close already uploaded file tracker app.Logger.Debug("Shutting down File Tracker service...") if err := app.FileTracker.Close(); err != nil { @@ -112,9 +112,7 @@ func (app App) Stop() error { // Close upload session tracker app.Logger.Debug("Shutting down Upload Tracker service...") - if err := app.UploadSessionTracker.Close(); err != nil { - return err - } + app.UploadSessionTracker.Close() // Close token manager app.Logger.Debug("Shutting down Token Manager service...") @@ -128,7 +126,7 @@ func (app App) Stop() error { // CreateAppDataDir return the filename after creating the application directory and the configuration file with defaults. // CreateAppDataDir destroys previous application directory. -func (app App) CreateAppDataDir() (string, error) { +func (app *App) CreateAppDataDir() (string, error) { if err := app.emptyDir(app.appDir); err != nil { return "", err } @@ -141,7 +139,7 @@ func (app App) CreateAppDataDir() (string, error) { } // AppDataDirExists return true if the application data dir exists. -func (app App) AppDataDirExists() bool { +func (app *App) AppDataDirExists() bool { exist, err := afero.Exists(app.fs, app.configFilename()) if err != nil { return false @@ -149,7 +147,7 @@ func (app App) AppDataDirExists() bool { return exist } -func (app App) configFilename() string { +func (app *App) configFilename() string { return filepath.Join(app.appDir, DefaultConfigFilename) } @@ -173,27 +171,30 @@ func (app *App) startServices() error { return nil } -func (app App) defaultFileTracker() (*filetracker.FileTracker, error) { - repo, err := filetracker.NewLevelDBRepository(filepath.Join(app.appDir, "uploads.db")) +func (app *App) defaultFileTracker() (*filetracker.FileTracker, error) { + fileTrackerFolder := filepath.Join(app.appDir, "uploaded_files") + repo, err := filetracker.NewLevelDBRepository(fileTrackerFolder) if err != nil { return nil, err } return filetracker.New(repo), nil } -func (app App) defaultTokenManager(backendType string) (*tokenmanager.TokenManager, error) { - kr, err := tokenmanager.NewKeyringRepository(backendType, nil, app.appDir) +func (app *App) defaultTokenManager(backendType string) (*tokenmanager.TokenManager, error) { + tokensFolder := filepath.Join(app.appDir, "tokens") + kr, err := tokenmanager.NewKeyringRepository(backendType, nil, tokensFolder) if err != nil { return nil, err } return tokenmanager.New(kr), nil } -func (app App) defaultUploadsSessionTracker() (*leveldbstore.LevelDBStore, error) { - return leveldbstore.NewStore(filepath.Join(app.appDir, "resumable_uploads.db")) +func (app *App) defaultUploadsSessionTracker() (*upload_tracker.LevelDBStore, error) { + ongoingUploadsTrackerFolder := filepath.Join(app.appDir, "ongoing_uploads") + return upload_tracker.NewStore(ongoingUploadsTrackerFolder) } -func (app App) emptyDir(path string) error { +func (app *App) emptyDir(path string) error { if err := app.fs.RemoveAll(path); err != nil { return err } @@ -202,9 +203,9 @@ func (app App) emptyDir(path string) error { // FileTracker represents a service to track file already uploaded. type FileTracker interface { - Put(file string) error - Exist(file string) bool - Delete(file string) error + MarkAsUploaded(file string) error + IsUploaded(file string) bool + UnmarkAsUploaded(file string) error Close() error } @@ -216,9 +217,11 @@ type TokenManager interface { } // UploadSessionTracker represents a service to keep resumable upload sessions. +// +// See [gphotosuploader/google-photos-api-client-go] Store interface. type UploadSessionTracker interface { - Get(fingerprint string) []byte - Set(fingerprint string, url []byte) + Get(fingerprint string) (string, bool) + Set(fingerprint string, url string) Delete(fingerprint string) - Close() error + Close() } diff --git a/internal/app/oauth.go b/internal/app/oauth.go index 4d041550..9779b4d6 100644 --- a/internal/app/oauth.go +++ b/internal/app/oauth.go @@ -10,7 +10,7 @@ import ( // AuthenticateFromToken returns an HTTP client authenticated in Google Photos. // AuthenticateFromToken will use the token from the Token Manage. -func (app App) AuthenticateFromToken(ctx context.Context) (*http.Client, error) { +func (app *App) AuthenticateFromToken(ctx context.Context) (*http.Client, error) { account := app.Config.Account app.Logger.Infof("Authenticating using token for '%s'", account) diff --git a/internal/cmd/auth.go b/internal/cli/auth/auth.go similarity index 86% rename from internal/cmd/auth.go rename to internal/cli/auth/auth.go index 3b376a03..8f27d80e 100644 --- a/internal/cmd/auth.go +++ b/internal/cli/auth/auth.go @@ -1,4 +1,4 @@ -package cmd +package auth import ( "context" @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" ) // AuthCmd holds the required data for the init cmd @@ -14,7 +14,7 @@ type AuthCmd struct { *flags.GlobalFlags } -func NewAuthCmd(globalFlags *flags.GlobalFlags) *cobra.Command { +func NewCommand(globalFlags *flags.GlobalFlags) *cobra.Command { cmd := &AuthCmd{GlobalFlags: globalFlags} authCmd := &cobra.Command{ diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 00000000..68eec297 --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,98 @@ +package cli + +import ( + "fmt" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/auth" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/list" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/push" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/version" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" + "github.com/mgutz/ansi" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +var ( + longCommandDescription = ` +Google Photos Command Line Interface (CLI) + +This CLI application allows you to upload pictures and videos to Google Photos. You can upload folders to your Google Photos account and organize them in albums automatically. Additionally, you can list albums and media items already uploaded to Google Photos. + +To get started, initialize your settings by running the following command: +$ gphotos-uploader-cli init + +Once configured, you can uploading your files with this command: +$ gphotos-uploder-cli push + +Or you can list your albums in Google Photos by running: +$ gphotos-uploader-cli list albums + +For more information, visit: https://gphotosuploader.github.io/gphotos-uploader-cli. +` + globalFlags *flags.GlobalFlags + + // Os points to the (real) file system. + // Useful for testing. + Os = afero.NewOsFs() +) + +// NewCommand creates a new gphotosCLI command root +func NewCommand() *cobra.Command { + // ArduinoCli is the root command + gphotosCLI := &cobra.Command{ + Use: "gphotos-uploader-cli", + Short: "Google Photos CLI.", + Long: longCommandDescription, + PersistentPreRunE: preRun, + } + + createCliCommandTree(gphotosCLI) + + return gphotosCLI +} + +// this is here only for testing +func createCliCommandTree(cmd *cobra.Command) { + persistentFlags := cmd.PersistentFlags() + globalFlags = flags.SetGlobalFlags(persistentFlags) + + // Add main commands + cmd.AddCommand(version.NewCommand()) + cmd.AddCommand(NewInitCmd(globalFlags)) + cmd.AddCommand(push.NewCommand(globalFlags)) + cmd.AddCommand(auth.NewCommand(globalFlags)) + cmd.AddCommand(list.NewCommand(globalFlags)) + + // TODO: Set flags here instead of passing globalFlags to all commands. + // See: https://github.com/arduino/arduino-cli/blob/master/internal/cli/cli.go + // + //cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Print the logs on the standard output.") + //validLogLevels := []string{"trace", "debug", "info", "warn", "error", "fatal", "panic"} + //cmd.PersistentFlags().String("log-level", "", fmt.Sprintf("Messages with this level and above will be logged. Valid levels are: %s", strings.Join(validLogLevels, ", "))) + //cmd.RegisterFlagCompletionFunc("log-level", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // return validLogLevels, cobra.ShellCompDirectiveDefault + //}) + //cmd.PersistentFlags().String("log-file", "", "Path to the file where logs will be written.") + //validLogFormats := []string{"text", "json"} + //cmd.PersistentFlags().String("log-format", "", fmt.Sprintf("The output format for the logs, can be: %s", strings.Join(validLogFormats, ", "))) + //cmd.RegisterFlagCompletionFunc("log-format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // return validLogFormats, cobra.ShellCompDirectiveDefault + //}) + //cmd.PersistentFlags().StringVar(&configFile, "config-file", "", "The custom config file (if not specified the default will be used).") + //cmd.PersistentFlags().Bool("no-color", false, "Disable colored output.") +} + +func preRun(cobraCmd *cobra.Command, args []string) error { + if globalFlags.Silent && globalFlags.Debug { + return fmt.Errorf("%s and %s cannot be specified at the same time", ansi.Color("--silent", "white+b"), ansi.Color("--debug", "white+b")) + } + if globalFlags.Silent { + log.GetInstance().SetLevel(logrus.FatalLevel) + } + if globalFlags.Debug { + log.GetInstance().SetLevel(logrus.DebugLevel) + } + return nil +} diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go new file mode 100644 index 00000000..564bd2ca --- /dev/null +++ b/internal/cli/cli_test.go @@ -0,0 +1,39 @@ +package cli_test + +import ( + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli" + "github.com/stretchr/testify/assert" + "io" + "testing" +) + +func TestNewCommand(t *testing.T) { + t.Run("Should return error when using --silent and --debug at the same time", func(t *testing.T) { + cmd := cli.NewCommand() + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"version", "--silent", "--debug"}) + + assert.Error(t, cmd.Execute()) + }) + + t.Run("Should return success when using --silent", func(t *testing.T) { + // TODO: Assert that nothing is written to the output when using --silent. + cmd := cli.NewCommand() + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"version", "--silent"}) + + assert.NoError(t, cmd.Execute()) + }) + + t.Run("Should return success when using --debug", func(t *testing.T) { + // TODO: Assert that log message is written when using --debug. + cmd := cli.NewCommand() + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"version", "--debug"}) + + assert.NoError(t, cmd.Execute()) + }) +} diff --git a/internal/cmd/flags/flags.go b/internal/cli/flags/flags.go similarity index 73% rename from internal/cmd/flags/flags.go rename to internal/cli/flags/flags.go index b07524a8..20f26aa0 100644 --- a/internal/cmd/flags/flags.go +++ b/internal/cli/flags/flags.go @@ -10,7 +10,7 @@ const ( defaultApplicationDataFolder = "~/.gphotos-uploader-cli" ) -// GlobalFlags is the flags that contains the global flags +// GlobalFlags is the flags that contain the global flags type GlobalFlags struct { Silent bool Debug bool @@ -21,10 +21,10 @@ type GlobalFlags struct { func SetGlobalFlags(flags *flag.FlagSet) *GlobalFlags { globalFlags := &GlobalFlags{} - flags.BoolVar(&globalFlags.Debug, "debug", false, "Logs very verbose information. Useful for troubleshooting.") - flags.BoolVar(&globalFlags.Silent, "silent", false, "Run in silent mode and prevents any log output except panics & fatals.") + flags.BoolVar(&globalFlags.Debug, "debug", false, "Log very verbose information. Useful for troubleshooting.") + flags.BoolVar(&globalFlags.Silent, "silent", false, "Run in silent mode and prevent any log output except panics.") - flags.StringVar(&globalFlags.CfgDir, "config", defaultApplicationDataPath(), "Sets config folder path. All configuration will be keep in this folder.") + flags.StringVar(&globalFlags.CfgDir, "config", defaultApplicationDataPath(), "Sets the config folder path. All configuration will be kept in this folder.") return globalFlags } diff --git a/internal/cmd/helpers_test.go b/internal/cli/helpers_test.go similarity index 93% rename from internal/cmd/helpers_test.go rename to internal/cli/helpers_test.go index 0499aac4..f83e7938 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cli/helpers_test.go @@ -1,4 +1,4 @@ -package cmd_test +package cli_test import "testing" diff --git a/internal/cmd/init.go b/internal/cli/init.go similarity index 89% rename from internal/cmd/init.go rename to internal/cli/init.go index 667b88a2..89d0d3ac 100644 --- a/internal/cmd/init.go +++ b/internal/cli/init.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "fmt" @@ -7,7 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" ) @@ -24,8 +24,8 @@ func NewInitCmd(globalFlags *flags.GlobalFlags) *cobra.Command { initCmd := &cobra.Command{ Use: "init", - Short: "Initializes the configuration", - Long: `Initializes a new configuration with defaults.`, + Short: "Initialize the configuration", + Long: `Initialize a new configuration with defaults.`, Args: cobra.NoArgs, RunE: cmd.Run, } diff --git a/internal/cmd/init_test.go b/internal/cli/init_test.go similarity index 77% rename from internal/cmd/init_test.go rename to internal/cli/init_test.go index 435039df..eca4df99 100644 --- a/internal/cmd/init_test.go +++ b/internal/cli/init_test.go @@ -1,4 +1,4 @@ -package cmd_test +package cli_test import ( "path/filepath" @@ -7,8 +7,8 @@ import ( "github.com/spf13/afero" "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" ) func TestNewInitCmd(t *testing.T) { @@ -24,15 +24,15 @@ func TestNewInitCmd(t *testing.T) { } t.Cleanup(func() { - cmd.Os = afero.NewOsFs() + cli.Os = afero.NewOsFs() }) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cmd.Os = afero.NewMemMapFs() - createTestConfigurationFile(t, cmd.Os, tc.input) + cli.Os = afero.NewMemMapFs() + createTestConfigurationFile(t, cli.Os, tc.input) - c := cmd.NewInitCmd(&flags.GlobalFlags{CfgDir: tc.input}) + c := cli.NewInitCmd(&flags.GlobalFlags{CfgDir: tc.input}) c.SetArgs(tc.args) err := c.Execute() diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go new file mode 100644 index 00000000..075d81d8 --- /dev/null +++ b/internal/cli/list/list.go @@ -0,0 +1,18 @@ +package list + +import ( + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" + "github.com/spf13/cobra" +) + +func NewCommand(globalFlags *flags.GlobalFlags) *cobra.Command { + listCommand := &cobra.Command{ + Use: "list", + Short: "List albums or media items in Google Photos", + } + + listCommand.AddCommand(initAlbumsCommand(globalFlags)) + listCommand.AddCommand(initMediaItemsCommand(globalFlags)) + + return listCommand +} diff --git a/internal/cli/list/list_albums.go b/internal/cli/list/list_albums.go new file mode 100644 index 00000000..92ddf3a1 --- /dev/null +++ b/internal/cli/list/list_albums.go @@ -0,0 +1,133 @@ +package list + +import ( + "context" + "fmt" + gphotos "github.com/gphotosuploader/google-photos-api-client-go/v3" + "github.com/gphotosuploader/google-photos-api-client-go/v3/albums" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/feedback" + "github.com/spf13/cobra" + "io" + "text/tabwriter" +) + +// ListAlbumsCommandOptions contains the input to the 'list albums' command. +type ListAlbumsCommandOptions struct { + *flags.GlobalFlags + + NoHeaders bool + NoProgress bool +} + +func initAlbumsCommand(globalFlags *flags.GlobalFlags) *cobra.Command { + o := &ListAlbumsCommandOptions{ + GlobalFlags: globalFlags, + + NoHeaders: false, + NoProgress: false, + } + + command := &cobra.Command{ + Use: "albums", + Short: "List albums", + Long: `List all the albums in Google Photos where this CLI has access to.`, + Args: cobra.NoArgs, + RunE: o.Run, + } + + command.Flags().BoolVar(&o.NoHeaders, "no-headers", false, "Don't print the header and footer.") + command.Flags().BoolVar(&o.NoProgress, "no-progress", false, "Don't show the progress bar.") + + return command +} + +func (o *ListAlbumsCommandOptions) Run(cobraCmd *cobra.Command, args []string) error { + ctx := context.Background() + cli, err := app.Start(ctx, o.CfgDir) + if err != nil { + return err + } + defer func() { + _ = cli.Stop() + }() + + photos, err := gphotos.NewClient(cli.Client) + if err != nil { + return err + } + + if o.NoProgress { + cobraCmd.Println("Getting albums from Google Photos...") + } + + cli.Logger.Debug("Calling albums API...") + + albumsList, nextPageToken, err := photos.Albums.PaginatedList(ctx, nil) + if err != nil { + return err + } + + // The progress bar is not shown when using '--no-progress' flag or in '--debug' mode. + showProgressBar := !o.Debug && !o.NoProgress + + bar := feedback.NewTaskProgressBar("Getting albums from Google Photos...", -1, showProgressBar) + + bar.Add(len(albumsList)) + + // Iterate until all pages are got + for nextPageToken != "" { + var response []albums.Album + + cli.Logger.Debugf("Calling albums API for page: %s", nextPageToken) + + options := &albums.PaginatedListOptions{ + PageToken: nextPageToken, + } + response, nextPageToken, err = photos.Albums.PaginatedList(ctx, options) + if err != nil { + return err + } + + // Append current page albums to the final albums list + albumsList = append(albumsList, response...) + + bar.Add(len(response)) + } + + bar.Finish() + + cli.Logger.Debugf("Printing album list... (%d items)", len(albumsList)) + + o.printAlbumsList(albumsList, cobraCmd.OutOrStdout()) + + return nil +} + +func (o *ListAlbumsCommandOptions) printAlbumsList(a []albums.Album, writer io.Writer) { + if len(a) == 0 { + fmt.Fprintln(writer, "No albums were found!") + return + } + + o.printAsTable(a, writer) +} + +func (o *ListAlbumsCommandOptions) printAsTable(a []albums.Album, writer io.Writer) { + w := tabwriter.NewWriter(writer, 0, 0, 1, ' ', 0) + + if !o.NoHeaders { + fmt.Fprintln(w, "TITLE\t ITEMS\t ID\t") + } + + for _, album := range a { + fmt.Fprintf(w, "%s\t %d\t %s\t\n", album.Title, album.TotalMediaItems, album.ID) + } + + if !o.NoHeaders { + fmt.Fprintf(w, "Total: %d albums.\n", len(a)) + } + + w.Flush() +} diff --git a/internal/cli/list/list_media.go b/internal/cli/list/list_media.go new file mode 100644 index 00000000..6c2d462c --- /dev/null +++ b/internal/cli/list/list_media.go @@ -0,0 +1,148 @@ +package list + +import ( + "context" + "fmt" + gphotos "github.com/gphotosuploader/google-photos-api-client-go/v3" + "github.com/gphotosuploader/google-photos-api-client-go/v3/media_items" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/feedback" + "github.com/spf13/cobra" + "io" + "text/tabwriter" +) + +// ListMediaItemsCommandOptions contains the input to the 'list media' command. +type ListMediaItemsCommandOptions struct { + *flags.GlobalFlags + + NoHeaders bool + NoProgress bool + + AlbumID string +} + +func initMediaItemsCommand(globalFlags *flags.GlobalFlags) *cobra.Command { + o := &ListMediaItemsCommandOptions{ + GlobalFlags: globalFlags, + + NoHeaders: false, + NoProgress: false, + + AlbumID: "", + } + + command := &cobra.Command{ + Use: "media-items", + Short: "List media items", + Long: `List all the media items in Google Photos where this CLI has access to.`, + Args: cobra.NoArgs, + RunE: o.Run, + } + + command.Flags().BoolVar(&o.NoHeaders, "no-headers", false, "Don't print the header and footer.") + command.Flags().BoolVar(&o.NoProgress, "no-progress", false, "Don't show the progress bar.") + command.Flags().StringVar(&o.AlbumID, "album-id", "", "Filter results by album ID.") + + return command +} + +func (o *ListMediaItemsCommandOptions) Run(cobraCmd *cobra.Command, args []string) error { + ctx := context.Background() + cli, err := app.Start(ctx, o.CfgDir) + if err != nil { + return err + } + defer func() { + _ = cli.Stop() + }() + + photos, err := gphotos.NewClient(cli.Client) + if err != nil { + return err + } + + if o.AlbumID != "" { + cli.Logger.Debugf("Listing media items for album ID: %s", o.AlbumID) + } + + if o.NoProgress { + cobraCmd.Println("Getting media items from Google Photos...") + } + + cli.Logger.Debug("Calling media items API...") + + options := &media_items.PaginatedListOptions{ + AlbumID: o.AlbumID, + } + + mediaItemsList, nextPageToken, err := photos.MediaItems.PaginatedList(ctx, options) + if err != nil { + return err + } + + // The progress bar is not shown when using '--no-progress' flag or in '--debug' mode. + showProgressBar := !o.Debug && !o.NoProgress + + bar := feedback.NewTaskProgressBar("Getting media items from Google Photos...", -1, showProgressBar) + + bar.Add(len(mediaItemsList)) + + // Iterate until all pages are got + for nextPageToken != "" { + var response []media_items.MediaItem + + cli.Logger.Debugf("Calling media items API for page: %s", nextPageToken) + + options.PageToken = nextPageToken + response, nextPageToken, err = photos.MediaItems.PaginatedList(ctx, options) + if err != nil { + return err + } + + // Append current page media items to the final media items list + mediaItemsList = append(mediaItemsList, response...) + + bar.Add(len(response)) + } + + bar.Finish() + + cli.Logger.Debugf("Printing media items list... (%d items)", len(mediaItemsList)) + + o.printMediaItemsList(mediaItemsList, cobraCmd.OutOrStdout()) + + return nil +} + +func (o *ListMediaItemsCommandOptions) printMediaItemsList(mi []media_items.MediaItem, writer io.Writer) { + if o.AlbumID != "" { + fmt.Fprintf(writer, "Listing media items for album ID: %s\n", o.AlbumID) + } + + if len(mi) == 0 { + fmt.Fprintln(writer, "No media items were found!") + return + } + + o.printAsTable(mi, writer) +} + +func (o *ListMediaItemsCommandOptions) printAsTable(mi []media_items.MediaItem, writer io.Writer) { + w := tabwriter.NewWriter(writer, 0, 0, 1, ' ', 0) + + if !o.NoHeaders { + fmt.Fprintln(w, "FILENAME\t MIME-TYPE\t ID\t") + } + + for _, mediaItem := range mi { + fmt.Fprintf(w, "%s\t %s\t %s\t\n", mediaItem.Filename, mediaItem.MimeType, mediaItem.ID) + } + + if !o.NoHeaders { + fmt.Fprintf(w, "Total: %d media items.\n", len(mi)) + } + + w.Flush() +} diff --git a/internal/cli/push/push.go b/internal/cli/push/push.go new file mode 100644 index 00000000..b0d506fa --- /dev/null +++ b/internal/cli/push/push.go @@ -0,0 +1,179 @@ +package push + +import ( + "context" + gphotos "github.com/gphotosuploader/google-photos-api-client-go/v3" + "github.com/gphotosuploader/google-photos-api-client-go/v3/uploader" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/flags" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/feedback" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/filter" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/upload" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "net/http" +) + +// PushCmd holds the required data for the push cmd +type PushCmd struct { + *flags.GlobalFlags + + // command flags + DryRunMode bool +} + +func NewCommand(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &PushCmd{GlobalFlags: globalFlags} + + pushCmd := &cobra.Command{ + Use: "push", + Short: "Upload local folders to Google Photos", + Long: `Scan configured folders in the configuration and upload all new object to Google Photos.`, + Args: cobra.NoArgs, + RunE: cmd.Run, + } + + pushCmd.Flags().BoolVar(&cmd.DryRunMode, "dry-run", false, "Dry run mode") + + return pushCmd +} + +func (cmd *PushCmd) Run(cobraCmd *cobra.Command, args []string) error { + ctx := context.Background() + cli, err := app.Start(ctx, cmd.CfgDir) + if err != nil { + return err + } + defer func() { + _ = cli.Stop() + }() + + photosService, err := newPhotosService(cli.Client, cli.UploadSessionTracker, cli.Logger) + if err != nil { + return err + } + + if cmd.DryRunMode { + cli.Logger.Info("[DRY-RUN] Running in dry run mode. No file will be uploaded.") + } + + // launch all folder upload jobs + for _, config := range cli.Config.Jobs { + sourceFolder := config.SourceFolder + + filterFiles, err := filter.Compile(config.IncludePatterns, config.ExcludePatterns) + if err != nil { + return err + } + + folder := upload.UploadFolderJob{ + FileTracker: cli.FileTracker, + + SourceFolder: sourceFolder, + CreateAlbums: config.CreateAlbums, + Filter: filterFiles, + } + + // get UploadItem{} to be uploaded to Google Photos. + itemsToUpload, err := folder.ScanFolder(cli.Logger) + if err != nil { + cli.Logger.Fatalf("Failed to process location '%s': %s", config.SourceFolder, err) + continue + } + + totalItems := len(itemsToUpload) + var uploadedItems int + + cli.Logger.Infof("Found %d items to be uploaded processing location '%s'.", totalItems, config.SourceFolder) + + bar := feedback.NewTaskProgressBar("Uploading files...", totalItems, !cmd.Debug) + + itemsGroupedByAlbum := upload.GroupByAlbum(itemsToUpload) + for albumName, files := range itemsGroupedByAlbum { + albumId, err := getOrCreateAlbum(ctx, photosService.Albums, albumName) + if err != nil { + cli.Logger.Failf("Unable to create album '%s': %s", albumName, err) + continue + } + + for _, file := range files { + cli.Logger.Debugf("Processing (%d/%d): %s", uploadedItems+1, totalItems, file) + + if !cmd.DryRunMode { + // Upload the file and add it to PhotosService. + _, err := photosService.UploadToAlbum(ctx, albumId, file.Path) + + // Check if the Google Photos daily quota has been exceeded. + var e *gphotos.ErrDailyQuotaExceeded + if errors.As(err, &e) { + cli.Logger.Failf("returning 'quota exceeded' error") + return err + } + + if err != nil { + cli.Logger.Failf("Error processing %s: %s", file, err) + continue + } + + // Mark the file as uploaded in the FileTracker. + if err := cli.FileTracker.MarkAsUploaded(file.Path); err != nil { + cli.Logger.Warnf("Tracking file as uploaded failed: file=%s, error=%v", file, err) + } + + if config.DeleteAfterUpload { + if err := file.Remove(); err != nil { + cli.Logger.Errorf("Deletion request failed: file=%s, err=%v", file, err) + } + } + } + + bar.Add(1) + uploadedItems++ + } + } + + bar.Finish() + + cli.Logger.Donef("%d processed files: %d successfully, %d with errors", totalItems, uploadedItems, totalItems-uploadedItems) + } + return nil +} + +func newPhotosService(client *http.Client, sessionTracker app.UploadSessionTracker, logger log.Logger) (*gphotos.Client, error) { + u, err := uploader.NewResumableUploader(client) + if err != nil { + return nil, err + } + u.Store = sessionTracker + u.Logger = logger + + photos, err := gphotos.NewClient(client) + if err != nil { + return nil, err + } + + // Use the resumable uploader to allow large file uploading. + photos.Uploader = u + + return photos, nil +} + +// getOrCreateAlbum returns the created (or existent) album in PhotosService. +func getOrCreateAlbum(ctx context.Context, service gphotos.AlbumsService, title string) (string, error) { + // Returns if empty to avoid a PhotosService call. + if title == "" { + return "", nil + } + + if album, err := service.GetByTitle(ctx, title); err == nil { + return album.ID, nil + } + + album, err := service.Create(ctx, title) + if err != nil { + return "", err + } + + return album.ID, nil +} diff --git a/internal/cli/version/version.go b/internal/cli/version/version.go new file mode 100644 index 00000000..f7ff1424 --- /dev/null +++ b/internal/cli/version/version.go @@ -0,0 +1,24 @@ +package version + +import ( + "github.com/gphotosuploader/gphotos-uploader-cli/version" + "github.com/spf13/cobra" +) + +func NewCommand() *cobra.Command { + versionCommand := &cobra.Command{ + Use: "version", + Short: "Shows version number.", + Long: "Shows the version number of Google Photos CLI which is installed on your system.", + Args: cobra.NoArgs, + Run: runVersionCommand, + } + + return versionCommand +} + +func runVersionCommand(cmd *cobra.Command, args []string) { + info := version.VersionInfo + + cmd.Println(info) +} diff --git a/internal/cli/version/version_test.go b/internal/cli/version/version_test.go new file mode 100644 index 00000000..587cf607 --- /dev/null +++ b/internal/cli/version/version_test.go @@ -0,0 +1,26 @@ +package version_test + +import ( + "bytes" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli/version" + versioninfo "github.com/gphotosuploader/gphotos-uploader-cli/version" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewCommand(t *testing.T) { + // Prepare a know version without depending on the build info. + versioninfo.VersionInfo = &versioninfo.Info{ + Application: "fooBarCommand", + VersionString: "fooBarVersion", + } + + actual := new(bytes.Buffer) + versionCommand := version.NewCommand() + versionCommand.SetOut(actual) + _ = versionCommand.Execute() + + expected := "fooBarCommand Version: fooBarVersion\n" + + assert.Equal(t, expected, actual.String()) +} diff --git a/internal/cmd/push.go b/internal/cmd/push.go deleted file mode 100644 index eda3692c..00000000 --- a/internal/cmd/push.go +++ /dev/null @@ -1,195 +0,0 @@ -package cmd - -import ( - "context" - gphotos "github.com/gphotosuploader/google-photos-api-client-go/v2" - "github.com/gphotosuploader/google-photos-api-client-go/v2/albums" - "github.com/gphotosuploader/google-photos-api-client-go/v2/uploader/resumable" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/app" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/filter" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/upload" - "github.com/patrickmn/go-cache" - "github.com/schollz/progressbar/v3" - "github.com/spf13/cobra" - "google.golang.org/api/googleapi" - "net/http" - "regexp" - "time" -) - -var ( - requestQuotaErrorRe = regexp.MustCompile(`Quota exceeded for quota metric 'All requests' and limit 'All requests per day'`) -) - -// PushCmd holds the required data for the push cmd -type PushCmd struct { - *flags.GlobalFlags - - // command flags - NumberOfWorkers int - DryRunMode bool -} - -func NewPushCmd(globalFlags *flags.GlobalFlags) *cobra.Command { - cmd := &PushCmd{GlobalFlags: globalFlags} - - pushCmd := &cobra.Command{ - Use: "push", - Short: "Push local files to Google Photos service", - Long: `Scan configured folders in the configuration and push all new object to Google Photos service.`, - Args: cobra.NoArgs, - RunE: cmd.Run, - } - - pushCmd.Flags().IntVar(&cmd.NumberOfWorkers, "workers", 1, "Number of workers") - pushCmd.Flags().BoolVar(&cmd.DryRunMode, "dry-run", false, "Dry run mode") - - return pushCmd -} - -func (cmd *PushCmd) Run(cobraCmd *cobra.Command, args []string) error { - ctx := context.Background() - cli, err := app.Start(ctx, cmd.CfgDir) - if err != nil { - return err - } - defer func() { - _ = cli.Stop() - }() - - cli.Logger.Info("[DEV] This is a development version. Please be warned that it's not ready for production") - - photosService, err := newPhotosService(cli.Client, cli.UploadSessionTracker, cli.Logger) - if err != nil { - return err - } - - // Get all the albums from Google Photos - cli.Logger.Debug("Getting all albums from Google Photos...") - allAlbums, err := photosService.Albums.List(ctx) - if err != nil { - return err - } - - // Transform an array into map using Album.Title as key - albumMap := make(map[string]cache.Item) - for _, album := range allAlbums { - albumMap[album.Title] = cache.Item{Object: album} - } - - albumCache := cache.NewFrom(cache.NoExpiration, cache.NoExpiration, albumMap) - - cli.Logger.Infof("Found & cached %d albums.", albumCache.ItemCount()) - - // launch all folder upload jobs - for _, config := range cli.Config.Jobs { - sourceFolder := config.SourceFolder - - filterFiles, err := filter.Compile(config.IncludePatterns, config.ExcludePatterns) - if err != nil { - return err - } - - folder := upload.UploadFolderJob{ - FileTracker: cli.FileTracker, - - SourceFolder: sourceFolder, - CreateAlbums: config.CreateAlbums, - Filter: filterFiles, - } - - // get UploadItem{} to be uploaded to Google Photos. - itemsToUpload, err := folder.ScanFolder(cli.Logger) - if err != nil { - cli.Logger.Fatalf("Failed to process location '%s': %s", config.SourceFolder, err) - continue - } - - totalItems := len(itemsToUpload) - var uploadedItems int - - cli.Logger.Infof("Found %d items to be uploaded processing location '%s'.", totalItems, config.SourceFolder) - - bar := progressbar.NewOptions(totalItems, - progressbar.OptionFullWidth(), - progressbar.OptionSetDescription("Uploading files..."), - progressbar.OptionSetPredictTime(false), - progressbar.OptionShowCount(), - progressbar.OptionSetVisibility(!cmd.Debug), - ) - - for _, item := range itemsToUpload { - albumId, err := getOrCreateAlbum(ctx, photosService.Albums, albumCache, item.AlbumName, cli.Logger) - if err != nil { - cli.Logger.Failf("Unable to create album '%s': %s", item.AlbumName, err) - continue - } - - cli.Logger.Debugf("Processing (%d/%d): %s...", uploadedItems+1, totalItems, item) - - if !cmd.DryRunMode { - // Upload the file and add it to PhotosService. - _, err := photosService.UploadFileToAlbum(ctx, albumId, item.Path) - if err != nil { - if googleApiErr, ok := err.(*googleapi.Error); ok { - if requestQuotaErrorRe.MatchString(googleApiErr.Message) { - cli.Logger.Failf("Daily quota exceeded: waiting 12h until quota is recovered") - time.Sleep(12 * time.Hour) - continue - } - } else { - cli.Logger.Failf("Error processing %s", item) - continue - } - } - - // Mark the file as uploaded in the FileTracker. - if err := cli.FileTracker.Put(item.Path); err != nil { - cli.Logger.Warnf("Tracking file as uploaded failed: file=%s, error=%v", item, err) - } - - if config.DeleteAfterUpload { - if err := item.Remove(); err != nil { - cli.Logger.Errorf("Deletion request failed: file=%s, err=%v", item, err) - } - } - } - - _ = bar.Add(1) - uploadedItems++ - } - - _ = bar.Finish() - - cli.Logger.Donef("%d processed files: %d successfully, %d with errors", totalItems, uploadedItems, totalItems-uploadedItems) - } - return nil -} - -func newPhotosService(client *http.Client, sessionTracker app.UploadSessionTracker, logger log.Logger) (*gphotos.Client, error) { - u, err := resumable.NewResumableUploader(client, sessionTracker, resumable.WithLogger(logger)) - if err != nil { - return nil, err - } - return gphotos.NewClient(client, gphotos.WithUploader(u)) -} - -func getOrCreateAlbum(ctx context.Context, service gphotos.AlbumsService, albumsCache *cache.Cache, title string, logger log.Logger) (string, error) { - if album, found := albumsCache.Get(title); found { - log.Debugf("Getting album from cache: %s", title) - return album.(albums.Album).ID, nil - } - - log.Debugf("Creating new album: %s", title) - - album, err := service.Create(ctx, title) - if err != nil { - return "", err - } - - albumsCache.SetDefault(album.Title, *album) - - return album.ID, nil -} diff --git a/internal/cmd/root.go b/internal/cmd/root.go deleted file mode 100644 index 0ce8becd..00000000 --- a/internal/cmd/root.go +++ /dev/null @@ -1,78 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/mgutz/ansi" - "github.com/sirupsen/logrus" - "github.com/spf13/afero" - "github.com/spf13/cobra" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "gphotos-uploader-cli", - SilenceUsage: true, - SilenceErrors: true, - Short: "Welcome to `gphotos-uploader-cli` a Google Photos uploader!", - PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error { - if globalFlags.Silent && globalFlags.Debug { - return fmt.Errorf("%s and %s cannot be specified at the same time", ansi.Color("--silent", "white+b"), ansi.Color("--debug", "white+b")) - } - if globalFlags.Silent { - log.GetInstance().SetLevel(logrus.FatalLevel) - } - if globalFlags.Debug { - log.GetInstance().SetLevel(logrus.DebugLevel) - } - return nil - }, - Long: ` - This application allows you to upload pictures and videos to Google Photos. - You can upload folders to your Google Photos account and organize them in albums automatically. - - Get started by running the init command to configure your settings: - $ gphotos-uploader-cli init - - once it's configured, start uploading your files: - $ gphotos-uploader-cli push - - You can visit https://gphotosuploader.github.io/gphotos-uploader-cli for more information.`, -} - -var globalFlags *flags.GlobalFlags - -// Os points to the (real) file system. -// Useful for testing. -var Os = afero.NewOsFs() - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - // Execute command - err := rootCmd.Execute() - if err != nil { - log.Fatal(err) - os.Exit(1) - } -} - -func init() { - persistentFlags := rootCmd.PersistentFlags() - globalFlags = flags.SetGlobalFlags(persistentFlags) - - // Add main commands - rootCmd.AddCommand(NewVersionCmd()) - rootCmd.AddCommand(NewInitCmd(globalFlags)) - rootCmd.AddCommand(NewPushCmd(globalFlags)) - rootCmd.AddCommand(NewAuthCmd(globalFlags)) -} - -// GetRoot returns the root command -func GetRoot() *cobra.Command { - return rootCmd -} diff --git a/internal/cmd/version.go b/internal/cmd/version.go deleted file mode 100644 index 32b3c6b3..00000000 --- a/internal/cmd/version.go +++ /dev/null @@ -1,40 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -// Base version information. -// -// This is the fallback data used when version information from git is not -// provided via go ldflags. It provides an approximation of the application -// version for ad-hoc builds (e.g. `go build`) that cannot get the version -// information from git -// -// If you are looking at these fields in the git tree, they look strange. They -// are modified on the fly by the build process. -// -// We use semantic version (see https://semver.org/ for more information). When -// releasing a new version, this file is updated by Makefile to reflect the new -// version, a git annotated tag is used to set this version -var ( - version = "v0.0.0" // git tag, output of $(git describe --tags --always --dirty) -) - -type VersionCmd struct{} - -func NewVersionCmd() *cobra.Command { - cmd := &VersionCmd{} - - versionCmd := &cobra.Command{ - Use: "version", - Short: "Prints current version", - Run: cmd.Run, - } - - return versionCmd -} - -func (cmd *VersionCmd) Run(command *cobra.Command, args []string) { - command.Printf("gphotos-uploader-cli %s\n", version) -} diff --git a/internal/cmd/version_test.go b/internal/cmd/version_test.go deleted file mode 100644 index 67a948a5..00000000 --- a/internal/cmd/version_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package cmd_test - -import ( - "bytes" - "io" - "testing" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd" -) - -func TestNewVersionCmd(t *testing.T) { - c := cmd.NewVersionCmd() - b := bytes.NewBufferString("") - c.SetOut(b) - if err := c.Execute(); err != nil { - t.Fatal(err) - } - got, err := io.ReadAll(b) - if err != nil { - t.Fatal(err) - } - want := "gphotos-uploader-cli v0.0.0\n" - if want != string(got) { - t.Fatalf("want: %s, got: %s", want, string(got)) - } -} diff --git a/internal/config/schema.go b/internal/config/schema.go index 273cc8e6..87bd5c2d 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -26,9 +26,6 @@ type APIAppCredentials struct { // FolderUploadJob represents configuration for a folder to be uploaded type FolderUploadJob struct { - // DEPRECATED: Account is deprecated, use Config.Account instead. - Account string `json:"-"` - // SourceFolder is the folder containing the objects to be uploaded. SourceFolder string `json:"SourceFolder"` @@ -39,10 +36,6 @@ type FolderUploadJob struct { // folderName: Creates album with the name based on the folder name. CreateAlbums string `json:"CreateAlbums,omitempty"` - // MakeAlbums is deprecated, use Config.Jobs.CreateAlbums instead. - // DEPRECATED - MakeAlbums MakeAlbums `json:"-"` - // DeleteAfterUpload if it is true, the app will remove files after upload them. DeleteAfterUpload bool `json:"DeleteAfterUpload"` @@ -52,15 +45,3 @@ type FolderUploadJob struct { // ExcludePatterns are the patterns to exclude files. ExcludePatterns []string `json:"ExcludePatterns"` } - -// MakeAlbums is deprecated, use Config.Jobs.CreateAlbums instead -// DEPRECATED -type MakeAlbums struct { - // Enabled is deprecated, use Config.Jobs.CreateAlbums instead. - // DEPRECATED - Enabled bool `json:"-"` - - // Use is deprecated, use Config.Jobs.CreateAlbums instead. - // DEPRECATED - Use string `json:"-"` -} diff --git a/internal/datastore/filetracker/entity_test.go b/internal/datastore/filetracker/entity_test.go index 2c1d4b70..3f0a8b62 100644 --- a/internal/datastore/filetracker/entity_test.go +++ b/internal/datastore/filetracker/entity_test.go @@ -1,10 +1,9 @@ package filetracker_test import ( + "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" "testing" "time" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" ) func TestTrackedFile_Hash(t *testing.T) { diff --git a/internal/datastore/filetracker/filetracker.go b/internal/datastore/filetracker/filetracker.go index 19617105..96cdde34 100644 --- a/internal/datastore/filetracker/filetracker.go +++ b/internal/datastore/filetracker/filetracker.go @@ -1,26 +1,21 @@ package filetracker import ( - "fmt" "os" "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" ) -var ( - // ErrItemNotFound is the expected error if the item is not found. - ErrItemNotFound = fmt.Errorf("item was not found") -) - -// FileTracker allows to track already uploaded files in a repository. +// FileTracker allows tracking already uploaded files in a repository. type FileTracker struct { - repo Repository + repo FileRepository - // Hasher allows to change the way that hashes are calculated. Uses xxHash32Hasher{} by default. + // Hasher allows changing the way that hashes are calculated. + // Uses xxHash32Hasher{} by default. // Useful for testing. Hasher Hasher - logger log.Logger + Logger log.Logger } // Hasher is a Hasher to get the value of the file. @@ -28,26 +23,25 @@ type Hasher interface { Hash(file string) (string, error) } -// Repository is the repository where to track already uploaded files. -type Repository interface { - // Get It returns ErrItemNotFound if the repo does not contains the key. - Get(key string) (TrackedFile, error) +// FileRepository is the repository where to track already uploaded files. +type FileRepository interface { + Get(key string) (TrackedFile, bool) Put(key string, item TrackedFile) error Delete(key string) error Close() error } // New returns a FileTracker using specified repo. -func New(r Repository) *FileTracker { +func New(r FileRepository) *FileTracker { return &FileTracker{ repo: r, - Hasher: xxHash32Hasher{}, - logger: log.GetInstance(), + Hasher: XXHash32Hasher{}, + Logger: log.Discard, } } -// Put marks a file as already uploaded to prevent re-uploads. -func (ft FileTracker) Put(file string) error { +// MarkAsUploaded marks a file as already uploaded. +func (ft FileTracker) MarkAsUploaded(file string) error { fileInfo, err := os.Stat(file) if err != nil { return err @@ -65,27 +59,25 @@ func (ft FileTracker) Put(file string) error { return ft.repo.Put(file, item) } -// Exist checks if the file was already uploaded. -// Exist compares the last modification time of the file against the one in the repository. -// Last time modification comparison tries to reduce the number of times where the hash comparison +// IsUploaded checks if the file was already uploaded. +// First compares the last modification time of the file against the one in the repository. +// Last time modification comparison tries to reduce the number of times when the hash comparison // is needed. // In case that last modification time has changed (or it doesn't exist - retro compatibility), // it compares a hash of the content of the file against the one in the repository. -func (ft FileTracker) Exist(file string) bool { - // Get returns ErrItemNotFound if the repo does not contains the key. - item, err := ft.repo.Get(file) - if err != nil { +func (ft FileTracker) IsUploaded(file string) bool { + item, found := ft.repo.Get(file) + if !found { return false } fileInfo, err := os.Stat(file) if err != nil { - ft.logger.Debugf("Error retrieving file info for '%s' (%s).", file, err) + ft.Logger.Debugf("Error retrieving file info for '%s' (%s).", file, err) return false } if item.ModTime.Equal(fileInfo.ModTime()) { - ft.logger.Debugf("File modification time has not changed for '%s'.", file) return true } @@ -96,12 +88,10 @@ func (ft FileTracker) Exist(file string) bool { // checks if the file is the same (equal value) if item.Hash == hash { - ft.logger.Debugf("File hash has not changed for '%s'.", file) - - // updates file marker with mtime to speed up comparison on next run + // updates file marker with mtime to speed up comparison on the next run item.ModTime = fileInfo.ModTime() if err = ft.repo.Put(file, item); err != nil { - ft.logger.Debugf("Error updating marker for '%s' with modification time (%s).", file, err) + ft.Logger.Debugf("Error updating marker for '%s' with modification time (%s).", file, err) } return true @@ -110,8 +100,8 @@ func (ft FileTracker) Exist(file string) bool { return false } -// Delete un-marks a file as already uploaded. -func (ft FileTracker) Delete(file string) error { +// UnmarkAsUploaded un-marks a file as already uploaded. +func (ft FileTracker) UnmarkAsUploaded(file string) error { return ft.repo.Delete(file) } diff --git a/internal/datastore/filetracker/filetracker_test.go b/internal/datastore/filetracker/filetracker_test.go index 3f33913e..0b603a4c 100644 --- a/internal/datastore/filetracker/filetracker_test.go +++ b/internal/datastore/filetracker/filetracker_test.go @@ -2,9 +2,8 @@ package filetracker_test import ( "errors" - "testing" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" + "testing" ) const ( @@ -18,7 +17,7 @@ var ( ErrTestError = errors.New("error") ) -func TestFileTracker_Put(t *testing.T) { +func TestFileTracker_MarkAsUploaded(t *testing.T) { testCases := []struct { name string input string @@ -34,13 +33,13 @@ func TestFileTracker_Put(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := ft.Put(tc.input) + err := ft.MarkAsUploaded(tc.input) assertExpectedError(t, tc.isErrExpected, err) }) } } -func TestFileTracker_Exist(t *testing.T) { +func TestFileTracker_IsUploaded(t *testing.T) { testCases := []struct { name string input string @@ -58,7 +57,7 @@ func TestFileTracker_Exist(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := ft.Exist(tc.input) + got := ft.IsUploaded(tc.input) if tc.want != got { t.Errorf("want: %t, got: %t", tc.want, got) } @@ -66,7 +65,7 @@ func TestFileTracker_Exist(t *testing.T) { } } -func TestFileTracker_Delete(t *testing.T) { +func TestFileTracker_UnmarkAsUploaded(t *testing.T) { testCases := []struct { name string input string @@ -80,7 +79,7 @@ func TestFileTracker_Delete(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := ft.Delete(tc.input) + err := ft.UnmarkAsUploaded(tc.input) assertExpectedError(t, tc.isErrExpected, err) }) } @@ -109,12 +108,12 @@ type mockedRepository struct { valueInRepo filetracker.TrackedFile } -func (m mockedRepository) Get(key string) (filetracker.TrackedFile, error) { +func (m mockedRepository) Get(key string) (filetracker.TrackedFile, bool) { switch key { case ShouldMakeRepoFail: - return filetracker.TrackedFile{}, ErrTestError + return filetracker.TrackedFile{}, false default: - return m.valueInRepo, nil + return m.valueInRepo, true } } diff --git a/internal/datastore/filetracker/hasher.go b/internal/datastore/filetracker/hasher.go index 4c7e10de..a51da943 100644 --- a/internal/datastore/filetracker/hasher.go +++ b/internal/datastore/filetracker/hasher.go @@ -8,11 +8,11 @@ import ( "github.com/pierrec/xxHash/xxHash32" ) -// xxHash32Hasher implements a Hasher using xxHash32 package. -type xxHash32Hasher struct{} +// XXHash32Hasher implements a Hasher using xxHash32 package. +type XXHash32Hasher struct{} // Hash returns the xxHash32 of the file specified by filename. -func (h xxHash32Hasher) Hash(filename string) (string, error) { +func (h XXHash32Hasher) Hash(filename string) (string, error) { file, err := os.Open(filename) if err != nil { return "", err diff --git a/internal/datastore/filetracker/hasher_test.go b/internal/datastore/filetracker/hasher_test.go index 300e2f1e..5f78065e 100644 --- a/internal/datastore/filetracker/hasher_test.go +++ b/internal/datastore/filetracker/hasher_test.go @@ -1,12 +1,11 @@ package filetracker_test import ( - "testing" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" + "testing" ) -func TestXxHash32Hasher_Hash(t *testing.T) { +func TestXXHash32Hasher_Hash(t *testing.T) { testCases := []struct { name string input string @@ -17,11 +16,11 @@ func TestXxHash32Hasher_Hash(t *testing.T) { {"Should fail", "testdata/non-existent", "", true}, } - ft := filetracker.New(&mockedRepository{}) + hasher := filetracker.XXHash32Hasher{} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got, err := ft.Hasher.Hash(tc.input) + got, err := hasher.Hash(tc.input) assertExpectedError(t, tc.isErrExpected, err) if tc.want != got { t.Errorf("want: %s, got: %s", tc.want, got) diff --git a/internal/datastore/filetracker/leveldb_repository.go b/internal/datastore/filetracker/leveldb_repository.go index edc09d96..c029dc6f 100644 --- a/internal/datastore/filetracker/leveldb_repository.go +++ b/internal/datastore/filetracker/leveldb_repository.go @@ -13,7 +13,7 @@ type DB interface { Close() error } -// LevelDBRepository implements a Repository using LevelDB. +// LevelDBRepository implements a FileRepository using LevelDB. type LevelDBRepository struct { DB DB } @@ -28,12 +28,12 @@ func NewLevelDBRepository(filename string) (*LevelDBRepository, error) { // Get returns the item specified by key. It returns ErrItemNotFound if the // DB does not contains the key. -func (r LevelDBRepository) Get(key string) (TrackedFile, error) { +func (r LevelDBRepository) Get(key string) (TrackedFile, bool) { val, err := r.DB.Get([]byte(key), nil) if err != nil { - return TrackedFile{}, ErrItemNotFound + return TrackedFile{}, false } - return NewTrackedFile(string(val)), nil + return NewTrackedFile(string(val)), true } // Put stores the item under key. diff --git a/internal/datastore/filetracker/leveldb_repository_test.go b/internal/datastore/filetracker/leveldb_repository_test.go index 7e3263b1..a3c86ef2 100644 --- a/internal/datastore/filetracker/leveldb_repository_test.go +++ b/internal/datastore/filetracker/leveldb_repository_test.go @@ -1,21 +1,20 @@ package filetracker_test import ( + "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" "testing" "github.com/syndtr/goleveldb/leveldb/opt" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/filetracker" ) func TestLevelDBRepository_Get(t *testing.T) { testCases := []struct { - name string - input string - isErrExpected bool + name string + input string + found bool }{ - {"Should success", ShouldSuccess, false}, - {"Should fail", ShouldMakeRepoFail, true}, + {"Should success", ShouldSuccess, true}, + {"Should fail", ShouldMakeRepoFail, false}, } repo := filetracker.LevelDBRepository{ @@ -24,8 +23,10 @@ func TestLevelDBRepository_Get(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := repo.Get(tc.input) - assertExpectedError(t, tc.isErrExpected, err) + _, found := repo.Get(tc.input) + if tc.found != found { + t.Errorf("want: %t, got: %t", tc.found, found) + } }) } } diff --git a/internal/datastore/leveldbstore/store.go b/internal/datastore/leveldbstore/store.go deleted file mode 100644 index 1321a3af..00000000 --- a/internal/datastore/leveldbstore/store.go +++ /dev/null @@ -1,63 +0,0 @@ -// Package leveldbstore provides implementation of LevelDB key/value database. -// -// Create or open a database: -// -// // The returned DB instance is safe for concurrent use. Which mean that all -// // DB's methods may be called concurrently from multiple goroutine. -// db, err := leveldbstore.NewStore("path/to/db") -// ... -// defer db.Close() -// ... -// -// Read or modify the database content: -// -// // Remember that the contents of the returned slice should not be modified. -// data := db.Get(key) -// ... -// db.Put(key), []byte("value")) -// ... -// db.Delete(key) -// ... -package leveldbstore - -import ( - "github.com/syndtr/goleveldb/leveldb" -) - -type LevelDBStore struct { - db *leveldb.DB -} - -// NewStore create a new Store implemented by LevelDB -func NewStore(path string) (*LevelDBStore, error) { - db, err := leveldb.OpenFile(path, nil) - if err != nil { - return nil, err - } - - s := &LevelDBStore{db: db} - return s, err -} - -// Get returns the value corresponding to the given key -func (s *LevelDBStore) Get(key string) []byte { - v, err := s.db.Get([]byte(key), nil) - if err != nil { - return []byte{} - } - return v -} - -// Set stores the url for a given fingerprint -func (s *LevelDBStore) Set(key string, value []byte) { - _ = s.db.Put([]byte(key), value, nil) -} - -func (s *LevelDBStore) Delete(key string) { - _ = s.db.Delete([]byte(key), nil) -} - -// Close closes the service -func (s *LevelDBStore) Close() error { - return s.db.Close() -} diff --git a/internal/datastore/tokenmanager/keyring_repository.go b/internal/datastore/tokenmanager/keyring_repository.go index 06a2572a..c7e67824 100644 --- a/internal/datastore/tokenmanager/keyring_repository.go +++ b/internal/datastore/tokenmanager/keyring_repository.go @@ -2,13 +2,10 @@ package tokenmanager import ( "encoding/json" - "fmt" - "os" - "syscall" - "github.com/99designs/keyring" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/feedback" "golang.org/x/oauth2" - "golang.org/x/term" + "os" ) // KeyringRepository represents a repository provided by different secrets @@ -19,12 +16,12 @@ type KeyringRepository struct { // defaultConfig returns the default configuration from the keyring package. func defaultConfig(keyringDir string) keyring.Config { - const serviceName = "gPhotosUploader" + const serviceName = "GooglePhotosCLI" return keyring.Config{ ServiceName: serviceName, KeychainName: serviceName, - KeychainPasswordFunc: promptFn(&stdInPasswordReader{}), - FilePasswordFunc: promptFn(&stdInPasswordReader{}), + KeychainPasswordFunc: getPassphraseFromEnvOrUserInputFn(), + FilePasswordFunc: getPassphraseFromEnvOrUserInputFn(), FileDir: keyringDir, AllowedBackends: supportedBackendTypes, } @@ -48,8 +45,8 @@ var supportedBackendTypes = []keyring.BackendType{ } // NewKeyringRepository creates a new repository -// backend could be used to select which backed will be used. If it's empty or auto -// the library will select the most suitable depending OS. +// backend could be used to select which backed will be used. If it's empty or auto, +// the library will select the most suitable depending on OS. func NewKeyringRepository(backend string, promptFunc *keyring.PromptFunc, keyringDir string) (*KeyringRepository, error) { keyringConfig := defaultConfig(keyringDir) if backend != "" && backend != "auto" { @@ -80,7 +77,7 @@ func (r *KeyringRepository) Set(email string, token *oauth2.Token) error { }) } -// getToken returns the specified token from the repository. +// Get returns the specified token from the repository. func (r *KeyringRepository) Get(email string) (*oauth2.Token, error) { var nullToken = &oauth2.Token{} @@ -99,32 +96,19 @@ func (r *KeyringRepository) Get(email string) (*oauth2.Token, error) { // Close closes the keyring repository. func (r *KeyringRepository) Close() error { - // in this particular implementation we don't need to do anything. + // in this particular implementation, we don't need to do anything. return nil } -// passwordReader represents a function to read a password. -type passwordReader interface { - ReadPassword() (string, error) -} - -// promptFn returns the key to open the keyring. -// It will read it from an environment var if is set, or read from the terminal otherwise. -func promptFn(pr passwordReader) func(string) (string, error) { +// getPassphraseFromEnvOrUserInputFn returns the key to open the keyring. +// It will read it from an environment var if it's set, or read from the terminal otherwise. +func getPassphraseFromEnvOrUserInputFn() func(string) (string, error) { return func(_ string) (string, error) { + // TODO: Use the configuration package to gather this env var. if key, ok := os.LookupEnv("GPHOTOS_CLI_TOKENSTORE_KEY"); ok { return key, nil } - fmt.Print("Enter the passphrase to open the token store: ") - return pr.ReadPassword() - } -} -// stdInPasswordReader reads a password from the stdin. -type stdInPasswordReader struct{} - -func (pr *stdInPasswordReader) ReadPassword() (string, error) { - pwd, err := term.ReadPassword(syscall.Stdin) - fmt.Println() - return string(pwd), err + return feedback.InputUserField("Enter the passphrase to open the token store: ", true) + } } diff --git a/internal/datastore/tokenmanager/keyring_repository_test.go b/internal/datastore/tokenmanager/keyring_repository_test.go index 51661baa..2b4f9a91 100644 --- a/internal/datastore/tokenmanager/keyring_repository_test.go +++ b/internal/datastore/tokenmanager/keyring_repository_test.go @@ -2,6 +2,7 @@ package tokenmanager import ( "fmt" + "github.com/stretchr/testify/assert" "os" "path/filepath" "reflect" @@ -92,41 +93,36 @@ func TestKeyringRepository_Close(t *testing.T) { }) } -type mockedPasswordReader struct { - value string -} +//type mockedPasswordReader struct { +// value string +//} +// +//func (m *mockedPasswordReader) ReadPassword() (string, error) { +// return m.value, nil +//} -func (m *mockedPasswordReader) ReadPassword() (string, error) { - return m.value, nil -} +func TestGetPassphraseFromEnvOrUserInputFn(t *testing.T) { + //want := "foo" -func TestPromptFn(t *testing.T) { - want := "foo" + t.Run("Should return the passphrase from the user input", func(t *testing.T) { + getPassphraseFromEnvOrUserInputFn := getPassphraseFromEnvOrUserInputFn() + _, err := getPassphraseFromEnvOrUserInputFn("") - t.Run("ReturnKeyFromTerminal", func(t *testing.T) { - promptFn := promptFn(&mockedPasswordReader{value: want}) - got, err := promptFn("") - if err != nil { - t.Errorf("error was not expected: err=%s", err) - } - if got != want { - t.Errorf("want: %s, got: %s", want, got) - } + // It should fail because this is not an interactive terminal. + assert.Error(t, err) }) - t.Run("ReturnKeyFromEnv", func(t *testing.T) { - if err := os.Setenv("GPHOTOS_CLI_TOKENSTORE_KEY", want); err != nil { + t.Run("Should return the passphrase from the env var", func(t *testing.T) { + if err := os.Setenv("GPHOTOS_CLI_TOKENSTORE_KEY", "This-key-comes-from-env-var"); err != nil { t.Fatalf("error was not expected at this stage: err=%s", err) } - promptFn := promptFn(&mockedPasswordReader{value: "dummy"}) - got, err := promptFn("") - if err != nil { - t.Errorf("error was not expected: err=%s", err) - } - if got != want { - t.Errorf("want: %s, got: %s", want, got) - } + getPassphraseFromEnvOrUserInputFn := getPassphraseFromEnvOrUserInputFn() + got, err := getPassphraseFromEnvOrUserInputFn("") + + assert.NoError(t, err) + assert.Equal(t, "This-key-comes-from-env-var", got) + }) } @@ -141,5 +137,5 @@ func getDefaultToken() *oauth2.Token { } func tempDir() string { - return filepath.Join(os.TempDir(), fmt.Sprintf("gphotos-cli.%d", time.Now().UnixNano())) + return filepath.Join(os.TempDir(), fmt.Sprintf("gphotos-uploader-cli.%d", time.Now().UnixNano())) } diff --git a/internal/datastore/upload_tracker/leveldb.go b/internal/datastore/upload_tracker/leveldb.go new file mode 100644 index 00000000..2425796c --- /dev/null +++ b/internal/datastore/upload_tracker/leveldb.go @@ -0,0 +1,44 @@ +// Package upload_tracker provides implementation of [gphotosuploader/google-photos-api-client-go] Store interface +package upload_tracker + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +type LevelDBStore struct { + db *leveldb.DB +} + +// NewStore create a new Store implemented by LevelDB +func NewStore(path string) (*LevelDBStore, error) { + db, err := leveldb.OpenFile(path, nil) + if err != nil { + return nil, err + } + + s := &LevelDBStore{db: db} + return s, err +} + +// Get returns the value corresponding to the given key +func (s *LevelDBStore) Get(key string) (string, bool) { + v, err := s.db.Get([]byte(key), nil) + if err != nil { + return "", false + } + return string(v), true +} + +// Set stores the url for a given fingerprint +func (s *LevelDBStore) Set(key string, value string) { + _ = s.db.Put([]byte(key), []byte(value), nil) +} + +func (s *LevelDBStore) Delete(key string) { + _ = s.db.Delete([]byte(key), nil) +} + +// Close closes the service +func (s *LevelDBStore) Close() { + _ = s.db.Close() +} diff --git a/internal/datastore/upload_tracker/leveldb_test.go b/internal/datastore/upload_tracker/leveldb_test.go new file mode 100644 index 00000000..8e7cab84 --- /dev/null +++ b/internal/datastore/upload_tracker/leveldb_test.go @@ -0,0 +1,107 @@ +package upload_tracker_test + +import ( + "github.com/gphotosuploader/gphotos-uploader-cli/internal/datastore/upload_tracker" + "os" + "testing" +) + +func RemoveDB(path string) { + _ = os.RemoveAll(path) +} + +func TestNewStore(t *testing.T) { + t.Run("Should success when folder is writable", func(t *testing.T) { + name, err := os.MkdirTemp(os.TempDir(), "upload_tracker") + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer RemoveDB(name) + + store, err := upload_tracker.NewStore(name) + if err != nil { + t.Errorf("error was not expected: %v", err) + } + store.Close() + }) + + t.Run("Should fail when folder is not writable", func(t *testing.T) { + name := "/non-existent" + + store, err := upload_tracker.NewStore(name) + if err == nil { + store.Close() + t.Errorf("error was expected but not produced") + } + }) +} + +func TestLevelDBStore_GetSet(t *testing.T) { + t.Run("Should get the value when the key is present", func(t *testing.T) { + name, err := os.MkdirTemp(os.TempDir(), "upload_tracker") + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer RemoveDB(name) + + store, err := upload_tracker.NewStore(name) + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer store.Close() + + store.Set("fooKey", "fooValue") + + got, found := store.Get("fooKey") + + if !found || "fooValue" != got { + t.Errorf("want: %s, got: %s", "fooValue", got) + } + }) + + t.Run("Should return false if the key is not present", func(t *testing.T) { + name, err := os.MkdirTemp(os.TempDir(), "upload_tracker") + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer RemoveDB(name) + + store, err := upload_tracker.NewStore(name) + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer store.Close() + + got, found := store.Get("non-existent") + + if found { + t.Errorf("key was not expected, got: %s", got) + } + }) +} + +func TestLevelDBStore_Delete(t *testing.T) { + t.Run("Should delete a key", func(t *testing.T) { + name, err := os.MkdirTemp(os.TempDir(), "upload_tracker") + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer RemoveDB(name) + + store, err := upload_tracker.NewStore(name) + if err != nil { + t.Fatalf("error was not expected at this time: %v", err) + } + defer store.Close() + + store.Set("fooKey", "fooValue") + + store.Delete("fooKey") + + got, found := store.Get("fooKey") + + if found { + t.Errorf("key was not expected, got: %s", got) + } + }) +} diff --git a/internal/feedback/errorcodes.go b/internal/feedback/errorcodes.go new file mode 100644 index 00000000..d20fe63c --- /dev/null +++ b/internal/feedback/errorcodes.go @@ -0,0 +1,30 @@ +package feedback + +// ExitCode to be used for Fatal. +type ExitCode int + +const ( + // Success (0 is the no-error return code in Unix) + Success ExitCode = iota + + // ErrGeneric Generic error (1 is the reserved "catchall" code in Unix) + ErrGeneric + + _ // (2 Is reserved in Unix) + + // ErrNoConfigFile is returned when the config file is not found (3) + ErrNoConfigFile + + _ // (4 was ErrBadCall and has been removed) + + // ErrNetwork is returned when a network error occurs (5) + ErrNetwork + + // ErrCoreConfig represents an error in the cli core config, for example, some basic + // files shipped with the installation are missing, or cannot create or get basic + // directories vital for the CLI to work. (6) + ErrCoreConfig + + // ErrBadArgument is returned when the arguments are not valid (7) + ErrBadArgument +) diff --git a/internal/feedback/feedback.go b/internal/feedback/feedback.go new file mode 100644 index 00000000..35696f64 --- /dev/null +++ b/internal/feedback/feedback.go @@ -0,0 +1,107 @@ +package feedback + +import ( + "bytes" + "fmt" + "github.com/sirupsen/logrus" + "io" + "os" +) + +var ( + stdOut io.Writer + stdErr io.Writer + feedbackOut io.Writer + feedbackErr io.Writer + bufferOut *bytes.Buffer + bufferErr *bytes.Buffer +) + +func init() { + reset() +} + +// reset resets the feedback package to its initial state, useful for unit testing. +func reset() { + stdOut = os.Stdout + stdErr = os.Stderr + feedbackOut = os.Stdout + feedbackErr = os.Stderr + bufferOut = &bytes.Buffer{} + bufferErr = &bytes.Buffer{} +} + +// Result is anything more complex than a sentence that needs to be printed +// for the user. +type Result interface { + fmt.Stringer + Data() interface{} +} + +// ErrorResult is a result embedding also an error. The error will be printed +// on stderr. +type ErrorResult interface { + Result + ErrorString() string +} + +// SetOut can be used to change the out writer at runtime. +func SetOut(out io.Writer) { + stdOut = out + feedbackOut = io.MultiWriter(bufferOut, stdOut) +} + +// SetErr can be used to change the err writer at runtime. +func SetErr(err io.Writer) { + stdErr = err + feedbackErr = io.MultiWriter(bufferErr, stdErr) +} + +// Printf behaves like fmt.Printf but writes on the out writer and adds a newline. +func Printf(format string, v ...interface{}) { + Print(fmt.Sprintf(format, v...)) +} + +// Print behaves like fmt.Print but writes on the out writer and adds a newline. +func Print(v string) { + fmt.Fprintln(feedbackOut, v) +} + +// Warning outputs a warning message. +func Warning(msg string) { + fmt.Fprintln(feedbackErr, msg) + logrus.Warning(msg) +} + +// FatalError outputs the error and exits with status exitCode. +func FatalError(err error, exitCode ExitCode) { + Fatal(err.Error(), exitCode) +} + +// FatalResult outputs the result and exits with status exitCode. +func FatalResult(res ErrorResult, exitCode ExitCode) { + PrintResult(res) + os.Exit(int(exitCode)) +} + +// Fatal outputs the errorMsg and exits with status exitCode. +func Fatal(errorMsg string, exitCode ExitCode) { + fmt.Fprintln(stdErr, errorMsg) + os.Exit(int(exitCode)) +} + +// PrintResult is a convenient wrapper to provide feedback for complex data. +func PrintResult(res Result) { + var data string + var dataErr string + data = res.String() + if resErr, ok := res.(ErrorResult); ok { + dataErr = resErr.ErrorString() + } + if data != "" { + fmt.Fprintln(stdOut, data) + } + if dataErr != "" { + fmt.Fprintln(stdErr, dataErr) + } +} diff --git a/internal/feedback/feedback_test.go b/internal/feedback/feedback_test.go new file mode 100644 index 00000000..a514169b --- /dev/null +++ b/internal/feedback/feedback_test.go @@ -0,0 +1,22 @@ +package feedback + +import ( + "bytes" + "github.com/stretchr/testify/require" + "testing" +) + +func TestOutputSelection(t *testing.T) { + reset() + + myErr := new(bytes.Buffer) + myOut := new(bytes.Buffer) + SetOut(myOut) + SetErr(myErr) + + Print("Hello Foo!") + require.Equal(t, myOut.String(), "Hello Foo!\n") + + Warning("Hello Bar!") + require.Equal(t, myErr.String(), "Hello Bar!\n") +} diff --git a/internal/feedback/progress.go b/internal/feedback/progress.go new file mode 100644 index 00000000..ff6b4e94 --- /dev/null +++ b/internal/feedback/progress.go @@ -0,0 +1,33 @@ +package feedback + +import "github.com/schollz/progressbar/v3" + +// Progress is a progress bar to show the status of a task. +type Progress struct { + bar *progressbar.ProgressBar +} + +// NewTaskProgressBar returns a progress bar for tasks that outputs to +// the terminal. +func NewTaskProgressBar(desc string, steps int, visibility bool) *Progress { + bar := progressbar.NewOptions(steps, + progressbar.OptionSetWriter(feedbackOut), + progressbar.OptionSetDescription(desc), + progressbar.OptionSetVisibility(visibility), + progressbar.OptionFullWidth(), + progressbar.OptionShowCount(), + progressbar.OptionClearOnFinish(), + ) + + return &Progress{ + bar: bar, + } +} + +func (pb *Progress) Add(num int) { + _ = pb.bar.Add(num) +} + +func (pb *Progress) Finish() { + _ = pb.bar.Finish() +} diff --git a/internal/feedback/terminal.go b/internal/feedback/terminal.go new file mode 100644 index 00000000..8caf6f28 --- /dev/null +++ b/internal/feedback/terminal.go @@ -0,0 +1,34 @@ +package feedback + +import ( + "bufio" + "errors" + "fmt" + "golang.org/x/term" + "os" +) + +func isTerminal() bool { + return term.IsTerminal(int(os.Stdin.Fd())) +} + +// InputUserField prompts the user to input the provided user field. +func InputUserField(prompt string, secret bool) (string, error) { + if !isTerminal() { + return "", errors.New("user input not supported in non interactive mode") + } + + fmt.Fprintf(stdOut, "%s: ", prompt) + + if secret { + // Read and return a password (no character echoed on terminal) + value, err := term.ReadPassword(int(os.Stdin.Fd())) + fmt.Fprintln(stdOut) + return string(value), err + } + + // Read and return an input line + sc := bufio.NewScanner(os.Stdin) + sc.Scan() + return sc.Text(), sc.Err() +} diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index d51074ed..0faa5289 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -1,6 +1,8 @@ package filter_test import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "testing" "github.com/gphotosuploader/gphotos-uploader-cli/internal/filter" @@ -22,8 +24,10 @@ func TestCompile(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { _, err := filter.Compile(tc.allowedList, tc.excludedList) - if err != nil && !tc.errExpected { - t.Errorf("error was not expected, got: %v", err) + if tc.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) } }) } @@ -80,40 +84,31 @@ func TestFilter_AllowDefaultFiles(t *testing.T) { t.Run("ByUsingEmptyPatterns", func(t *testing.T) { f, err := filter.Compile([]string{""}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) t.Run("ByUsingRepeatedEmptyPatterns", func(t *testing.T) { f, err := filter.Compile([]string{"", "", ""}, []string{"", "", ""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) t.Run("ByUsingTaggedPattern", func(t *testing.T) { f, err := filter.Compile([]string{"_IMAGE_EXTENSIONS_"}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) } @@ -138,27 +133,21 @@ func TestFilter_AllowAllFiles(t *testing.T) { t.Run("ByUsingWildCardPattern", func(t *testing.T) { f, err := filter.Compile([]string{"**"}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) t.Run("ByUsingTaggedPattern", func(t *testing.T) { f, err := filter.Compile([]string{"_ALL_FILES_"}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) } @@ -181,14 +170,11 @@ func TestFilter_AllowPNGFiles(t *testing.T) { } f, err := filter.Compile([]string{"**/*.png"}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } } @@ -211,16 +197,11 @@ func TestFilter_AllowPNGAndJPGFiles(t *testing.T) { } f, err := filter.Compile([]string{"**/*.png", "**/*.jpg"}, []string{""}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } - } func TestFilter_AllowImageFilesStartingWithSample(t *testing.T) { @@ -239,16 +220,11 @@ func TestFilter_AllowImageFilesStartingWithSample(t *testing.T) { } f, err := filter.Compile([]string{"**/Sample*"}, []string{"**/*.mp3", "**/*.txt", "**/*.mp4"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } - } func TestFilter_DisallowAllFiles(t *testing.T) { @@ -268,27 +244,19 @@ func TestFilter_DisallowAllFiles(t *testing.T) { t.Run("ByUsingWildcardPattern", func(t *testing.T) { f, err := filter.Compile([]string{"**"}, []string{"**"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) t.Run("ByUsingTaggedPattern", func(t *testing.T) { f, err := filter.Compile([]string{"_ALL_FILES_"}, []string{"_ALL_FILES_"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) @@ -310,16 +278,11 @@ func TestFilter_DisallowFilesStartingWithScreenShot(t *testing.T) { } f, err := filter.Compile([]string{"_ALL_FILES_"}, []string{"**/ScreenShot*"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } - } func TestFilter_DisallowVideos(t *testing.T) { @@ -339,14 +302,10 @@ func TestFilter_DisallowVideos(t *testing.T) { t.Run("ByUsingTaggedPattern", func(t *testing.T) { f, err := filter.Compile([]string{"_ALL_FILES_"}, []string{"_ALL_VIDEO_FILES_"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } }) @@ -366,16 +325,11 @@ func TestFilter_IncludingPNGExceptAFolder(t *testing.T) { } f, err := filter.Compile([]string{"**/*.png"}, []string{"*/folder1/*"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsAllowed(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsAllowed(tc.file)) } - } func TestFilter_ExcludingAFolder(t *testing.T) { @@ -393,14 +347,10 @@ func TestFilter_ExcludingAFolder(t *testing.T) { t.Run("ExcludingFolder1", func(t *testing.T) { f, err := filter.Compile([]string{""}, []string{"**/folder1/*"}) - if err != nil { - t.Fatalf("error was not expected at this point: %v", err) - } + require.NoError(t, err) + for _, tc := range testCases { - got := f.IsExcluded(tc.file) - if tc.out != got { - t.Errorf("Filter result was not expected: file=%s, want %t, got %t", tc.file, tc.out, got) - } + assert.Equal(t, tc.out, f.IsExcluded(tc.file)) } }) diff --git a/internal/filter/patterns_test.go b/internal/filter/patterns_test.go index accac0a4..a366b8b4 100644 --- a/internal/filter/patterns_test.go +++ b/internal/filter/patterns_test.go @@ -1,7 +1,7 @@ package filter import ( - "reflect" + "github.com/stretchr/testify/assert" "testing" ) @@ -18,18 +18,12 @@ func Test_DeleteEmpty(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := deleteEmpty(tc.input) - if !reflect.DeepEqual(tc.want, got) { - t.Errorf("want: %#v, got: %#v", tc.want, got) - } + assert.Equal(t, tc.want, deleteEmpty(tc.input)) }) } t.Run("empty input array", func(t *testing.T) { - got := deleteEmpty([]string{}) - if nil != got { - t.Errorf("want: []string{nil}, got: %#v", got) - } + assert.Nil(t, deleteEmpty([]string{})) }) } @@ -47,9 +41,11 @@ func Test_ValidatePatterns(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := validatePatternList(tc.input) - if got != nil && !tc.errExpected { - t.Errorf("error was not expected, got: %v", got) + err := validatePatternList(tc.input) + if tc.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) } }) } @@ -71,8 +67,10 @@ func Test_Match(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { got, err := match(tc.patterns, tc.input) - if tc.shouldMatch != got || (err != nil && !tc.errExpected) { - t.Errorf("want: %v, got: %v, err: %v", tc.shouldMatch, got, err) + if tc.errExpected { + assert.Error(t, err) + } else { + assert.Equal(t, tc.shouldMatch, got) } }) } diff --git a/internal/mock/filetracker.go b/internal/mock/filetracker.go index 7b178c80..d2bfcf2f 100644 --- a/internal/mock/filetracker.go +++ b/internal/mock/filetracker.go @@ -2,22 +2,22 @@ package mock // FileTracker mocks the service to track already uploaded files. type FileTracker struct { - PutFn func(path string) error - ExistFn func(path string) bool - DeleteFn func(path string) error + MarkAsUploadedFn func(path string) error + IsUploadedFn func(path string) bool + UnmarkAsUploadedFn func(path string) error } -// Put invokes the mock implementation. -func (t *FileTracker) Put(path string) error { - return t.PutFn(path) +// MarkAsUploaded invokes the mock implementation. +func (t *FileTracker) MarkAsUploaded(path string) error { + return t.MarkAsUploadedFn(path) } -// Exist invokes the mock implementation. -func (t *FileTracker) Exist(path string) bool { - return t.ExistFn(path) +// IsUploaded invokes the mock implementation. +func (t *FileTracker) IsUploaded(path string) bool { + return t.IsUploadedFn(path) } -// Delete invokes the mock implementation. -func (t *FileTracker) Delete(path string) error { - return t.DeleteFn(path) +// UnMarkAsUploaded invokes the mock implementation. +func (t *FileTracker) UnmarkAsUploaded(path string) error { + return t.UnmarkAsUploadedFn(path) } diff --git a/internal/task/upload.go b/internal/task/upload.go deleted file mode 100644 index bc10b0a3..00000000 --- a/internal/task/upload.go +++ /dev/null @@ -1,68 +0,0 @@ -package task - -import ( - "context" - - "github.com/gphotosuploader/google-photos-api-client-go/v2/albums" - "github.com/gphotosuploader/google-photos-api-client-go/v2/media_items" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/upload" -) - -type AlbumsService interface { - Create(ctx context.Context, title string) (*albums.Album, error) - GetByTitle(ctx context.Context, title string) (*albums.Album, error) -} - -type UploadsService interface { - UploadFileToAlbum(ctx context.Context, albumId string, filePath string) (media_items.MediaItem, error) -} - -type EnqueuedUpload struct { - Context context.Context - Uploads UploadsService - FileTracker upload.FileTracker - Logger log.Logger - - Path string - AlbumID string - DeleteOnSuccess bool -} - -func (job *EnqueuedUpload) Process() error { - item := upload.NewFileItem(job.Path) - - // Upload the file and add it to PhotosService. - if err := job.addMediaToAlbum(job.AlbumID, item); err != nil { - return err - } - - // Mark the file as uploaded in the FileTracker. - if err := job.FileTracker.Put(job.Path); err != nil { - job.Logger.Warnf("Tracking file as uploaded failed: file=%s, error=%v", job.Path, err) - } - - // If was requested, remove the file after being uploaded. - return job.removeIfItWasRequested(item) -} - -func (job *EnqueuedUpload) ID() string { - return job.Path -} - -func (job *EnqueuedUpload) removeIfItWasRequested(item upload.FileItem) error { - if job.DeleteOnSuccess { - if err := item.Remove(); err != nil { - job.Logger.Errorf("Deletion request failed: file=%s, err=%v", job.Path, err) - } - } - return nil -} - -func (job *EnqueuedUpload) addMediaToAlbum(album string, item upload.FileItem) error { - if _, err := job.Uploads.UploadFileToAlbum(job.Context, album, item.Path); err != nil { - return err - } - return nil -} diff --git a/internal/upload/album_test.go b/internal/upload/album_test.go index b3578a62..793eb218 100644 --- a/internal/upload/album_test.go +++ b/internal/upload/album_test.go @@ -1,6 +1,7 @@ package upload import ( + "github.com/stretchr/testify/assert" "testing" ) @@ -37,15 +38,25 @@ func TestAlbumName(t *testing.T) { job := UploadFolderJob{ CreateAlbums: tt.createAlbums, } - got := job.albumName(tt.in) - if got != tt.want { - t.Errorf("albumName for '%s' failed: expected '%s', got '%s'", tt.in, tt.want, got) - } + + assert.Equal(t, tt.want, job.albumName(tt.in)) }) } } +func TestAlbumNameWithInvalidParameter(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("A Panic was expected but not reached.") + } + }() + job := UploadFolderJob{ + CreateAlbums: "FooBar", + } + _ = job.albumName("/foo/bar/file.jpg") +} + func TestAlbumNameUsingFolderPath(t *testing.T) { var testData = []struct { in string @@ -60,10 +71,7 @@ func TestAlbumNameUsingFolderPath(t *testing.T) { {in: "/foo/bar/", out: "foo_bar"}, } for _, tt := range testData { - got := albumNameUsingFolderPath(tt.in) - if got != tt.out { - t.Errorf("albumNameUsingFolderPath for '%s' failed: expected '%s', got '%s'", tt.in, tt.out, got) - } + assert.Equal(t, tt.out, albumNameUsingFolderPath(tt.in)) } } @@ -81,9 +89,6 @@ func TestAlbumNameUsingFolderName(t *testing.T) { {in: "/foo/bar/", out: "bar"}, } for _, tt := range testData { - got := albumNameUsingFolderName(tt.in) - if got != tt.out { - t.Errorf("albumNameUsingFolderName for '%s' failed: expected '%s', got '%s'", tt.in, tt.out, got) - } + assert.Equal(t, tt.out, albumNameUsingFolderName(tt.in)) } } diff --git a/internal/upload/file_item.go b/internal/upload/file_item.go index e6be0107..1d9c1143 100644 --- a/internal/upload/file_item.go +++ b/internal/upload/file_item.go @@ -8,7 +8,7 @@ import ( ) var ( - // fs represents the filesystem to use. By default it uses functions based on `os` package. + // fs represents the filesystem to use. By default, it uses functions based on the `os` package. // In testing, it uses a memory file system. appFS = afero.NewOsFs() ) @@ -19,43 +19,60 @@ type FileItem struct { AlbumName string } +// NewFileItem creates a new instance of FileItem. func NewFileItem(path string) FileItem { return FileItem{ Path: path, } } -// Open returns a stream. -// Caller should close it finally. -func (m FileItem) Open() (io.ReadSeeker, int64, error) { - f, err := appFS.Stat(m.Path) +// Open opens the file and returns a stream. +// The caller should close it finally. +func (f FileItem) Open() (io.ReadSeeker, int64, error) { + fileInfo, err := appFS.Stat(f.Path) if err != nil { return nil, 0, err } - r, err := appFS.Open(m.Path) + + file, err := appFS.Open(f.Path) if err != nil { return nil, 0, err } - return r, f.Size(), nil + + return file, fileInfo.Size(), nil } // Name returns the filename. -func (m FileItem) Name() string { - return path.Base(m.Path) +func (f FileItem) Name() string { + return path.Base(f.Path) } -func (m FileItem) String() string { - return m.Path +// String returns the string representation of the FileItem. +func (f FileItem) String() string { + return f.Path } -func (m FileItem) Size() int64 { - f, err := appFS.Stat(m.Path) +// Size returns the file size. +func (f FileItem) Size() int64 { + fileInfo, err := appFS.Stat(f.Path) if err != nil { return 0 } - return f.Size() + return fileInfo.Size() +} + +// Remove removes the file from the file system. +func (f FileItem) Remove() error { + return appFS.Remove(f.Path) } -func (m FileItem) Remove() error { - return appFS.Remove(m.Path) +// GroupByAlbum groups FileItem objects by their AlbumName. +func GroupByAlbum(items []FileItem) map[string][]FileItem { + groups := make(map[string][]FileItem) + + for _, item := range items { + groups[item.AlbumName] = append(groups[item.AlbumName], item) + } + + return groups } diff --git a/internal/upload/file_item_test.go b/internal/upload/file_item_test.go index b3b8497e..b7f62aec 100644 --- a/internal/upload/file_item_test.go +++ b/internal/upload/file_item_test.go @@ -1,6 +1,8 @@ package upload import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "testing" "github.com/spf13/afero" @@ -18,9 +20,8 @@ func TestFileItem_Name(t *testing.T) { for _, tc := range testCases { f := NewFileItem(tc.in) - if got := f.Name(); got != tc.want { - t.Errorf("TestCase(%s), want: %s, got: %s", tc.in, tc.want, got) - } + + assert.Equal(t, tc.want, f.Name()) } } @@ -36,9 +37,8 @@ func TestFileItem_String(t *testing.T) { for _, tc := range testCases { f := NewFileItem(tc.in) - if got := f.String(); got != tc.want { - t.Errorf("TestCase(%s), want: %s, got: %s", tc.in, tc.want, got) - } + + assert.Equal(t, tc.want, f.String()) } } @@ -55,23 +55,21 @@ func TestFileItem_Open(t *testing.T) { appFS = afero.NewMemMapFs() // create test files and directories - if err := appFS.MkdirAll("src/", 0755); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } - if err := afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } + err := appFS.MkdirAll("src/", 0755) + require.NoError(t, err) + + err = afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644) + require.NoError(t, err) for _, tc := range testCases { f := NewFileItem(tc.in) _, size, err := f.Open() - switch { - case tc.errExpected && err == nil: - t.Errorf("TestCase(%s), error was expected, but not happened", tc.name) - case !tc.errExpected && err != nil: - t.Errorf("TestCase(%s), error was not expected: err=%s", tc.name, err) - case size != tc.wantSize: - t.Errorf("TestCase(%s), want: %d, got: %d", tc.name, tc.wantSize, size) + + if tc.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.wantSize, size) } } } @@ -88,18 +86,16 @@ func TestFileItem_Size(t *testing.T) { appFS = afero.NewMemMapFs() // create test files and directories - if err := appFS.MkdirAll("src/", 0755); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } - if err := afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } + err := appFS.MkdirAll("src/", 0755) + require.NoError(t, err) + + err = afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644) + require.NoError(t, err) for _, tc := range testCases { f := NewFileItem(tc.in) - if got := f.Size(); got != tc.want { - t.Errorf("Test Case(%s), want: %d, got: %d", tc.in, tc.want, got) - } + + assert.Equal(t, tc.want, f.Size()) } } @@ -115,21 +111,54 @@ func TestFileItem_Remove(t *testing.T) { appFS = afero.NewMemMapFs() // create test files and directories - if err := appFS.MkdirAll("src/", 0755); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } - if err := afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644); err != nil { - t.Fatalf("error was not expected at this point: err=%s", err) - } + err := appFS.MkdirAll("src/", 0755) + require.NoError(t, err) + + err = afero.WriteFile(appFS, "src/existent", []byte("this is content of existing file"), 0644) + require.NoError(t, err) for _, tc := range testCases { f := NewFileItem(tc.in) err := f.Remove() - switch { - case tc.errExpected && err == nil: - t.Errorf("TestCase(%s), error was expected, but not happened", tc.name) - case !tc.errExpected && err != nil: - t.Errorf("TestCase(%s), error was not expected: err=%s", tc.name, err) + + if tc.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) } } } + +func TestFileItem_GroupByAlbum(t *testing.T) { + items := []FileItem{ + {Path: "file1.jpg", AlbumName: "Album 1"}, + {Path: "file2.jpg", AlbumName: "Album 2"}, + {Path: "file3.jpg", AlbumName: "Album 1"}, + {Path: "file4.jpg", AlbumName: "Album 2"}, + {Path: "file5.jpg", AlbumName: "Album 3"}, + } + + expectedGroups := map[string][]FileItem{ + "Album 1": { + {Path: "file1.jpg", AlbumName: "Album 1"}, + {Path: "file3.jpg", AlbumName: "Album 1"}, + }, + "Album 2": { + {Path: "file2.jpg", AlbumName: "Album 2"}, + {Path: "file4.jpg", AlbumName: "Album 2"}, + }, + "Album 3": { + {Path: "file5.jpg", AlbumName: "Album 3"}, + }, + } + + groupedItems := GroupByAlbum(items) + + assert.Len(t, groupedItems, len(expectedGroups)) + + for albumName, expectedItems := range expectedGroups { + + assert.Contains(t, groupedItems, albumName) + assert.Equal(t, expectedItems, groupedItems[albumName]) + } +} diff --git a/internal/upload/testdata/AlreadyUploadedSampleImage.jpg b/internal/upload/testdata/AlreadyUploadedSampleImage.jpg new file mode 100644 index 00000000..e9698d4f Binary files /dev/null and b/internal/upload/testdata/AlreadyUploadedSampleImage.jpg differ diff --git a/internal/upload/testdata/folder1/AlreadyUploadedSampleImage.jpg b/internal/upload/testdata/folder1/AlreadyUploadedSampleImage.jpg new file mode 100644 index 00000000..e9698d4f Binary files /dev/null and b/internal/upload/testdata/folder1/AlreadyUploadedSampleImage.jpg differ diff --git a/internal/upload/types.go b/internal/upload/types.go index 7b8a712f..29e52347 100644 --- a/internal/upload/types.go +++ b/internal/upload/types.go @@ -11,9 +11,9 @@ type UploadFolderJob struct { // FileTracker represents a service to track already uploaded files. type FileTracker interface { - Put(file string) error - Exist(file string) bool - Delete(file string) error + MarkAsUploaded(file string) error + IsUploaded(file string) bool + UnmarkAsUploaded(file string) error } // FileFilterer represents a way to implement include/exclude files filtering. diff --git a/internal/upload/walker.go b/internal/upload/walker.go index 0820c648..874086c3 100644 --- a/internal/upload/walker.go +++ b/internal/upload/walker.go @@ -45,7 +45,7 @@ func (job *UploadFolderJob) getItemToUploadFn(reqs *[]FileItem, logger log.Logge } // check completed uploads db for previous uploads - if job.FileTracker.Exist(fp) { + if job.FileTracker.IsUploaded(fp) { logger.Debugf("Skipping already uploaded file '%s'.", fp) return nil } diff --git a/internal/upload/walker_test.go b/internal/upload/walker_test.go index b9269d80..9609046d 100644 --- a/internal/upload/walker_test.go +++ b/internal/upload/walker_test.go @@ -1,6 +1,9 @@ package upload_test import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" "testing" "github.com/gphotosuploader/gphotos-uploader-cli/internal/filter" @@ -11,100 +14,68 @@ import ( func TestWalker_GetAllFiles(t *testing.T) { var includePatterns = []string{"_ALL_FILES_"} var excludePatterns = []string{""} - - var want = map[string]bool{ - "testdata/SampleAudio.mp3": true, - "testdata/SampleJPGImage.jpg": true, - "testdata/SamplePNGImage.png": true, - "testdata/SampleSVGImage.svg": true, - "testdata/SampleText.txt": true, - "testdata/SampleVideo.mp4": true, - "testdata/ScreenShotJPG.jpg": true, - "testdata/ScreenShotPNG.png": true, - "testdata/folder1/SamplePNGImage.png": true, - "testdata/folder1/SampleJPGImage.jpg": true, - "testdata/folder2/SamplePNGImage.png": true, - "testdata/folder2/SampleJPGImage.jpg": true, - "testdata/folder-symlink/SamplePNGImage.png": true, - "testdata/folder-symlink/SampleJPGImage.jpg": true, + var expected = []string{ + "testdata/SampleAudio.mp3", + "testdata/SampleJPGImage.jpg", + "testdata/SamplePNGImage.png", + "testdata/SampleSVGImage.svg", + "testdata/SampleText.txt", + "testdata/SampleVideo.mp4", + "testdata/ScreenShotJPG.jpg", + "testdata/ScreenShotPNG.png", + "testdata/folder1/SamplePNGImage.png", + "testdata/folder1/SampleJPGImage.jpg", + "testdata/folder2/SamplePNGImage.png", + "testdata/folder2/SampleJPGImage.jpg", + "testdata/folder-symlink/SamplePNGImage.png", + "testdata/folder-symlink/SampleJPGImage.jpg", } - got, err := getScanFolderResult(includePatterns, excludePatterns) - if err != nil { - t.Fatal(err) - } + got, err := getIncludedFilesByScanFolder(includePatterns, excludePatterns) - for i := range want { - if got[i] != want[i] { - t.Errorf("want: %v, got: %v, file: %s", want[i], got[i], i) - } - } + require.NoError(t, err) + assert.ElementsMatch(t, expected, got) } func TestWalker_GetAllPNGFiles(t *testing.T) { var includePatterns = []string{"**/*.png"} var excludePatterns = []string{""} - - var want = map[string]bool{ - "testdata/SampleAudio.mp3": false, - "testdata/SampleJPGImage.jpg": false, - "testdata/SamplePNGImage.png": true, - "testdata/SampleSVGImage.svg": false, - "testdata/SampleText.txt": false, - "testdata/SampleVideo.mp4": false, - "testdata/ScreenShotJPG.jpg": false, - "testdata/ScreenShotPNG.png": true, - "testdata/folder1/SamplePNGImage.png": true, - "testdata/folder1/SampleJPGImage.jpg": false, - "testdata/folder2/SamplePNGImage.png": true, - "testdata/folder2/SampleJPGImage.jpg": false, - "testdata/folder-symlink/SamplePNGImage.png": true, - "testdata/folder-symlink/SampleJPGImage.jpg": false, + var expected = []string{ + "testdata/SamplePNGImage.png", + "testdata/ScreenShotPNG.png", + "testdata/folder1/SamplePNGImage.png", + "testdata/folder2/SamplePNGImage.png", + "testdata/folder-symlink/SamplePNGImage.png", } - got, err := getScanFolderResult(includePatterns, excludePatterns) - if err != nil { - t.Fatal(err) - } + got, err := getIncludedFilesByScanFolder(includePatterns, excludePatterns) - for i := range want { - if got[i] != want[i] { - t.Errorf("want: %v, got: %v, file: %s", want[i], got[i], i) - } - } + require.NoError(t, err) + assert.ElementsMatch(t, expected, got) } func TestWalker_GetAllFilesExcludeFolder1(t *testing.T) { var includePatterns = []string{"_ALL_FILES_"} var excludePatterns = []string{"folder1"} - - var want = map[string]bool{ - "testdata/SampleAudio.mp3": true, - "testdata/SampleJPGImage.jpg": true, - "testdata/SamplePNGImage.png": true, - "testdata/SampleSVGImage.svg": true, - "testdata/SampleText.txt": true, - "testdata/SampleVideo.mp4": true, - "testdata/ScreenShotJPG.jpg": true, - "testdata/ScreenShotPNG.png": true, - "testdata/folder1/SamplePNGImage.png": false, - "testdata/folder1/SampleJPGImage.jpg": false, - "testdata/folder2/SamplePNGImage.png": true, - "testdata/folder2/SampleJPGImage.jpg": true, - "testdata/folder-symlink/SamplePNGImage.png": true, - "testdata/folder-symlink/SampleJPGImage.jpg": true, + var expected = []string{ + "testdata/SampleAudio.mp3", + "testdata/SampleJPGImage.jpg", + "testdata/SamplePNGImage.png", + "testdata/SampleSVGImage.svg", + "testdata/SampleText.txt", + "testdata/SampleVideo.mp4", + "testdata/ScreenShotJPG.jpg", + "testdata/ScreenShotPNG.png", + "testdata/folder2/SamplePNGImage.png", + "testdata/folder2/SampleJPGImage.jpg", + "testdata/folder-symlink/SamplePNGImage.png", + "testdata/folder-symlink/SampleJPGImage.jpg", } - got, err := getScanFolderResult(includePatterns, excludePatterns) - if err != nil { - t.Fatalf("no error was expected at this point: err=%s", err) - } + got, err := getIncludedFilesByScanFolder(includePatterns, excludePatterns) - for i := range want { - if got[i] != want[i] { - t.Errorf("want: %v, got: %v, file: %s", want[i], got[i], i) - } - } + require.NoError(t, err) + assert.ElementsMatch(t, expected, got) } func TestRelativePath(t *testing.T) { @@ -123,25 +94,21 @@ func TestRelativePath(t *testing.T) { {base: "", in: "/foo/bar", want: "/foo/bar"}, {base: "/foo/bar", in: "/abc/def", want: "/abc/def"}, } - for _, tc := range objectsTest { - got := upload.RelativePath(tc.base, tc.in) - if got != tc.want { - t.Errorf("Test Case (%s), basepath '%s': want '%s', got '%s'", tc.base, tc.in, tc.want, got) - } + for _, tc := range objectsTest { + assert.Equal(t, tc.want, upload.RelativePath(tc.base, tc.in)) } } -func getScanFolderResult(includePatterns []string, excludePatterns []string) (map[string]bool, error) { - var results = map[string]bool{} +func getIncludedFilesByScanFolder(includePatterns []string, excludePatterns []string) ([]string, error) { ft := &mock.FileTracker{ - PutFn: func(path string) error { + MarkAsUploadedFn: func(path string) error { return nil }, - ExistFn: func(path string) bool { - return false + IsUploadedFn: func(path string) bool { + return strings.Contains(path, "AlreadyUploaded") }, - DeleteFn: func(path string) error { + UnmarkAsUploadedFn: func(path string) error { return nil }, } @@ -159,8 +126,9 @@ func getScanFolderResult(includePatterns []string, excludePatterns []string) (ma return nil, err } + var results []string for _, i := range foundItems { - results[i.Path] = true + results = append(results, i.Path) } return results, nil diff --git a/internal/worker/queue.go b/internal/worker/queue.go deleted file mode 100644 index 3e6be9d3..00000000 --- a/internal/worker/queue.go +++ /dev/null @@ -1,162 +0,0 @@ -package worker - -import ( - "fmt" - "sync" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" -) - -// Job - interface for job processing -type Job interface { - Process() error - ID() string -} - -// JobResult - the jobResults of a processed Job -type JobResult struct { - ID string - Message string - Err error -} - -// Worker - the worker threads that actually process the jobs -type Worker struct { - id string - done *sync.WaitGroup - readyPool chan chan Job - assignedJobQueue chan Job - - jobResults chan JobResult - quit chan bool - - logger log.Logger -} - -// JobQueue - a queue for enqueueing jobs to be processed -type JobQueue struct { - internalQueue chan Job - readyPool chan chan Job - workers []*Worker - dispatcherStopped sync.WaitGroup - workersStopped *sync.WaitGroup - jobResults chan JobResult - quit chan bool -} - -// NewJobQueue - creates a new job queue -func NewJobQueue(maxWorkers int, logger log.Logger) *JobQueue { - workersStopped := sync.WaitGroup{} - readyPool := make(chan chan Job, maxWorkers) - - // we need to ensure that the results channel is big enough to fulfill workers needs - jobResults := make(chan JobResult, maxWorkers*10) - - // create the pool of workers - workers := make([]*Worker, maxWorkers) - for i := 0; i < maxWorkers; i++ { - workers[i] = NewWorker(fmt.Sprintf("#%d", i+1), readyPool, jobResults, &workersStopped, logger) - } - return &JobQueue{ - internalQueue: make(chan Job), - readyPool: readyPool, - workers: workers, - dispatcherStopped: sync.WaitGroup{}, - workersStopped: &workersStopped, - jobResults: jobResults, - quit: make(chan bool), - } -} - -// ChanJobResults returns the channel where the Job give the results -func (q *JobQueue) ChanJobResults() chan JobResult { - return q.jobResults -} - -// Start - starts the worker routines and dispatcher routine -func (q *JobQueue) Start() { - for i := 0; i < len(q.workers); i++ { - q.workers[i].Start() - } - go q.dispatch() -} - -// Stop - stops the workers and dispatcher routine -func (q *JobQueue) Stop() { - q.quit <- true - q.dispatcherStopped.Wait() -} - -func (q *JobQueue) dispatch() { - q.dispatcherStopped.Add(1) - for { - select { - case job := <-q.internalQueue: // We got something in on our queue - workerChannel := <-q.readyPool // Check out an available worker - workerChannel <- job // Send the request to the channel - case <-q.quit: - for i := 0; i < len(q.workers); i++ { - q.workers[i].Stop() - } - q.workersStopped.Wait() - q.dispatcherStopped.Done() - return - } - } -} - -// Submit - adds a new job to be processed, uses a subroutine to avoid deadlock when the queue is full -func (q *JobQueue) Submit(job Job) { - go func(job Job) { q.internalQueue <- job }(job) -} - -// NewWorker - creates a new worker -func NewWorker(id string, readyPool chan chan Job, result chan JobResult, done *sync.WaitGroup, logger log.Logger) *Worker { - return &Worker{ - id: id, - done: done, - readyPool: readyPool, - assignedJobQueue: make(chan Job), - jobResults: result, - quit: make(chan bool), - logger: logger, - } -} - -// Start - begins the job processing loop for the worker -func (w *Worker) Start() { - go func() { - w.logger.Debugf("Worker %s is starting", w.id) - w.done.Add(1) - for { - w.readyPool <- w.assignedJobQueue // check the job queue in - select { - case job := <-w.assignedJobQueue: // see if anything has been assigned to the queue - w.logger.Debugf("Worker %s processing: %s", w.id, job.ID()) - - r := JobResult{ - ID: job.ID(), - Message: "processed successfully", - Err: job.Process(), - } - - if r.Err != nil { - r.Message = "processed with errors" - w.logger.Error(r.Err) - } - - // send the jobResults of the processed Job - w.jobResults <- r - case <-w.quit: - w.done.Done() - return - } - } - }() -} - -// Stop - stops the worker -func (w *Worker) Stop() { - w.logger.Debugf("Worker %s is stopping", w.id) - w.quit <- true -} diff --git a/internal/worker/queue_test.go b/internal/worker/queue_test.go deleted file mode 100644 index bda52df8..00000000 --- a/internal/worker/queue_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package worker - -import ( - "fmt" - "sync/atomic" - "testing" - - "github.com/gphotosuploader/gphotos-uploader-cli/internal/log" -) - -type TestJob struct { - id int - opsPtr *uint64 -} - -func (t *TestJob) ID() string { return fmt.Sprintf("Job #%d", t.id) } - -func (t *TestJob) Process() error { - for i := 0; i < 1000; i++ { - atomic.AddUint64(t.opsPtr, 1) - } - return nil -} - -func TestQueue(t *testing.T) { - var testData = []struct { - numberOfWorkers int - numberOfJobs int - want uint64 - }{ - {numberOfWorkers: 1, numberOfJobs: 1, want: 1000}, - {numberOfWorkers: 1, numberOfJobs: 2, want: 2000}, - {numberOfWorkers: 1, numberOfJobs: 5, want: 5000}, - {numberOfWorkers: 1, numberOfJobs: 20, want: 20000}, - {numberOfWorkers: 5, numberOfJobs: 5, want: 5000}, - {numberOfWorkers: 5, numberOfJobs: 50, want: 50000}, - {numberOfWorkers: 5, numberOfJobs: 100, want: 100000}, - } - - var logger = &log.DiscardLogger{} - - for _, tt := range testData { - t.Run(fmt.Sprintf("Workers[%d]_Jobs[%d]", tt.numberOfWorkers, tt.numberOfJobs), func(t *testing.T) { - var ops uint64 - - queue := NewJobQueue(tt.numberOfWorkers, logger) - queue.Start() - defer queue.Stop() - - // send jobs to the queue - for i := 0; i < tt.numberOfJobs; i++ { - queue.Submit(&TestJob{id: i + 1, opsPtr: &ops}) - } - - // get results from queue - for i := 0; i < tt.numberOfJobs; i++ { - r := <-queue.ChanJobResults() - want := "processed successfully" - if r.Message != want { - t.Errorf("invalid message: want=%s, got=%s", want, r.Message) - } - } - - if ops != tt.want { - t.Errorf("invalid jobResults: want=%d, got=%d", tt.want, ops) - } - }) - } - -} diff --git a/main.go b/main.go index a5fa3601..5918f03e 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,18 @@ package main import ( + "fmt" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/feedback" "os" - "github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd" + "github.com/gphotosuploader/gphotos-uploader-cli/internal/cli" ) func main() { - cmd.Execute() - os.Exit(0) + gphotosCmd := cli.NewCommand() + if err := gphotosCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Error ocurred: %s", err) + os.Exit(int(feedback.ErrGeneric)) + } + os.Exit(int(feedback.Success)) } diff --git a/version/version.go b/version/version.go new file mode 100644 index 00000000..f2e2391f --- /dev/null +++ b/version/version.go @@ -0,0 +1,46 @@ +package version + +import ( + "fmt" +) + +// VersionInfo contains all info injected during build +var VersionInfo *Info + +// Base version information. +// We use semantic version (see https://semver.org/ for more information). +var ( + // When releasing a new version, Makefile updates the versionString to reflect the new + // version; a git-annotated tag is used to set this version. + versionString = "" // git tag, output of $(git describe --tags --always --dirty) + + // This is the fallback data used when version information from git is not + // provided via go ldflags. It provides an approximation of the application + // version for adhoc builds (e.g. `go build`) that cannot get the version + // information from git + defaultVersionString = "0.0.0-git" +) + +type Info struct { + Application string `json:"Application"` + VersionString string `json:"VersionString"` +} + +func NewInfo() *Info { + return &Info{ + Application: "gphotos-uploader-cli", + VersionString: versionString, + } +} + +func (i *Info) String() string { + return fmt.Sprintf("%s Version: %s", i.Application, i.VersionString) +} + +func init() { + if versionString == "" { + versionString = defaultVersionString + } + + VersionInfo = NewInfo() +} diff --git a/version/version_test.go b/version/version_test.go new file mode 100644 index 00000000..550759cf --- /dev/null +++ b/version/version_test.go @@ -0,0 +1,22 @@ +package version + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +// TestBuildInjectedInfo tests the Info strings passed to the binary at build time +// in order to have this test green launch your testing 'make test' or use: +// +// go test -run TestBuildInjectedInfo -v ./... -ldflags ' +// -X github.com/gphotosuploader/gphotos-uploader-cli/version.versionString=0.0.0-test.preview' +func TestBuildInjectedInfo(t *testing.T) { + goldenInfo := Info{ + Application: "gphotos-uploader-cli", + VersionString: "0.0.0-test.preview", + } + info := NewInfo() + require.Equal(t, goldenInfo.Application, info.Application) + require.Equal(t, goldenInfo.VersionString, info.VersionString) + require.Equal(t, "gphotos-uploader-cli Version: 0.0.0-test.preview", info.String()) +}