diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 499c976..0926b51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,18 @@ name: build -on: [push, pull_request] +on: + push: + paths: + - '.github/workflows/build.yml' + - '**/*.py' + branches: + - "**" + tags: + - "!**" + pull_request: + paths: + - '.github/workflows/build.yml' + - '**/*.py' jobs: sh-checker: @@ -25,6 +37,8 @@ jobs: run: make install - name: Check format run: make format-check + - name: Scan for security vulnerabilities + run: make scan test: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index daa2778..0d51545 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,6 @@ jobs: make install make build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba71dd..fd3d264 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,138 +1,142 @@ # CHANGELOG +## 4.5.1 (2022-09-15) + +- Clarifies default location of github-archive in help output + ## v4.5.0 (2022-02-22) -* Switches from the `threading` package to the `concurrent.futures` package allowing us to return values from each individual thread (in this case, the names of failed git assets) - * Fixes a long-standing bug where git repos that failed to clone would remain in the archive on Windows. This meant that recloning was impossible due to the repo existing and pulling was not possible because the repo wasn't yet fully initialized. The tool now has a proper cleanup step after everything has run that is compatible with both Unix and Windows environments (closes #38) -* Adds `--log_level` to allow the user to specify a custom log level -* Small adjustment to `include` and `exclude` help verbage used to be more clear they are optional filters +- Switches from the `threading` package to the `concurrent.futures` package allowing us to return values from each individual thread (in this case, the names of failed git assets) + - Fixes a long-standing bug where git repos that failed to clone would remain in the archive on Windows. This meant that recloning was impossible due to the repo existing and pulling was not possible because the repo wasn't yet fully initialized. The tool now has a proper cleanup step after everything has run that is compatible with both Unix and Windows environments (closes #38) +- Adds `--log_level` to allow the user to specify a custom log level +- Small adjustment to `include` and `exclude` help verbage used to be more clear they are optional filters ## v4.4.0 (2022-02-18) -* Adds an `--include` and `--exclude` CLI flag that accepts a comma-separated list of repo names to either include or exclude. If neither are passed, no filtering will occur (closes #43) +- Adds an `--include` and `--exclude` CLI flag that accepts a comma-separated list of repo names to either include or exclude. If neither are passed, no filtering will occur (closes #43) ## v4.3.0 (2021-12-16) -* Adds the ability to specify a custom base_url for GitHub (useful for enterprise GitHub users with a custom hostname, closes #41) +- Adds the ability to specify a custom base_url for GitHub (useful for enterprise GitHub users with a custom hostname, closes #41) ## v4.2.2 (2021-11-29) -* Adds `mypy` and fixes typing errors +- Adds `mypy` and fixes typing errors ## v4.2.1 (2021-11-25) -* Bumps `woodchips` to use the new implementation (also fixes a bug where we were creating a new `woodchips.Logger` class each time we called the logger instead of reusing the same logger instance) -* Adds missing `__all__` variable for importing -* Added Python type hinting +- Bumps `woodchips` to use the new implementation (also fixes a bug where we were creating a new `woodchips.Logger` class each time we called the logger instead of reusing the same logger instance) +- Adds missing `__all__` variable for importing +- Added Python type hinting ## v4.2.0 (2021-11-13) -* Uses `woodchips` for logging and removes internal logging logic from the package -* Refactors git commands to not change directories but instead run commands with the `-C` flag to invoke it in the directory we want +- Uses `woodchips` for logging and removes internal logging logic from the package +- Refactors git commands to not change directories but instead run commands with the `-C` flag to invoke it in the directory we want ## v4.1.1 (2021-11-02) -* Fixes a bug that wouldn't allow for gist cloning/pulling because of a bad "forks" check on a gist GitHub object -* Adds a missing check to ensure that at least one CLI argument was passed (closes #40) -* No longer invoke a shell while using the subprocess module. Git operations should now be more stable across operating systems +- Fixes a bug that wouldn't allow for gist cloning/pulling because of a bad "forks" check on a gist GitHub object +- Adds a missing check to ensure that at least one CLI argument was passed (closes #40) +- No longer invoke a shell while using the subprocess module. Git operations should now be more stable across operating systems ## v4.1.0 (2021-09-19) -* Adds a new `--https` flag which will authenticate via HTTPS instead of the default `SSH` -* Fixes a bug where using the `--stars` flag would not properly run due to a missing parameter. This parameter wasn't actually being used anymore and has been removed. Tests were beefed up for this function to protect against this happening again +- Adds a new `--https` flag which will authenticate via HTTPS instead of the default `SSH` +- Fixes a bug where using the `--stars` flag would not properly run due to a missing parameter. This parameter wasn't actually being used anymore and has been removed. Tests were beefed up for this function to protect against this happening again ## v4.0.0 (2021-08-24) ### Breaking Changes -* Reworks the entire app config to use CLI flags solely instead of a mix of CLI flags and env variables, additionally, most flags have changed names and functionality. See the README for new usage instructions or run `github-archive --help` (closes #30) -* Repos or gists that fail to clone or pull will now be completely removed so that they can be retried from scratch on the next run of the tool. This was an especially important change for bad clones as the tool would previously leave an empty initialized git folder even if the clone failed which would not possess the actual git repo yet. In this state, you could not properly pull new changes because the content of the repo hadn't properly been cloned yet -* Bumps required Python version from 3.6 to 3.7 +- Reworks the entire app config to use CLI flags solely instead of a mix of CLI flags and env variables, additionally, most flags have changed names and functionality. See the README for new usage instructions or run `github-archive --help` (closes #30) +- Repos or gists that fail to clone or pull will now be completely removed so that they can be retried from scratch on the next run of the tool. This was an especially important change for bad clones as the tool would previously leave an empty initialized git folder even if the clone failed which would not possess the actual git repo yet. In this state, you could not properly pull new changes because the content of the repo hadn't properly been cloned yet +- Bumps required Python version from 3.6 to 3.7 ### Features -* Adds a new `--users` flag which can be used to clone or pull git assets for a list of comma separated users (closes #20) -* Adds a new `--threads` flag which can specify the number of concurrent threads to run at once, default is `10` (closes #22) -* Adds a new `--view` flag which allows you to "dry run" the application, seeing the entire list of repos and gists based on the input provided (closes #25) -* Adds a new `--stars` flag which you can pass a comma separated list of users to and GitHub Archive will retrieve all of their starred repos which you can then view, clone, or pull (closes #26) -* Adds a new `--forks` flag which will include forks for whatever lists and operations you provide, default is `False` (closes #17)) +- Adds a new `--users` flag which can be used to clone or pull git assets for a list of comma separated users (closes #20) +- Adds a new `--threads` flag which can specify the number of concurrent threads to run at once, default is `10` (closes #22) +- Adds a new `--view` flag which allows you to "dry run" the application, seeing the entire list of repos and gists based on the input provided (closes #25) +- Adds a new `--stars` flag which you can pass a comma separated list of users to and GitHub Archive will retrieve all of their starred repos which you can then view, clone, or pull (closes #26) +- Adds a new `--forks` flag which will include forks for whatever lists and operations you provide, default is `False` (closes #17)) ### Fixes -* Removed verbose logging of skipped actions and "Already up to date" messages. Added additional logging related to API calls -* Added proper validation and type checking of variables and environment on startup -* Various code refactors, bug fixes, and optimizations -* Bumped the default git operation timeout from `180 seconds` to `300 seconds` to assist with cloning or pulling larger repos (closes #22) -* Removes `mock` library in favor of builtin `unittest.mock` library +- Removed verbose logging of skipped actions and "Already up to date" messages. Added additional logging related to API calls +- Added proper validation and type checking of variables and environment on startup +- Various code refactors, bug fixes, and optimizations +- Bumped the default git operation timeout from `180 seconds` to `300 seconds` to assist with cloning or pulling larger repos (closes #22) +- Removes `mock` library in favor of builtin `unittest.mock` library ## v3.1.1 (2021-07-24) -* Removed `branch` flag and functionality as it was causing issues and inconsistencies when cloning/pulling and branches didn't match up. This became especially prevelant when repos started changing from `master` to `main` +- Removed `branch` flag and functionality as it was causing issues and inconsistencies when cloning/pulling and branches didn't match up. This became especially prevelant when repos started changing from `master` to `main` ## v3.1.0 (2020-12-16) -* Changed all classmethods to staticmethods -* Corrected a bug where org names may not have had whitespace stripped properly -* CLI arguments now have an explicit default of `False`, this shouldn't change behavior from previous versions -* CLI default argument for `branch` has been changed from `master` to `None` and is handled via logic now. If no branch is specified, the default repo branch will be used instead of blindly assuming that `master` is the default branch (closes #18) -* Revamped the entire test suite to use conftest, simplified boilerplate, etc +- Changed all classmethods to staticmethods +- Corrected a bug where org names may not have had whitespace stripped properly +- CLI arguments now have an explicit default of `False`, this shouldn't change behavior from previous versions +- CLI default argument for `branch` has been changed from `master` to `None` and is handled via logic now. If no branch is specified, the default repo branch will be used instead of blindly assuming that `master` is the default branch (closes #18) +- Revamped the entire test suite to use conftest, simplified boilerplate, etc ## v3.0.1 (2020-10-01) -* Fixed broken entrypoint after shifting code around +- Fixed broken entrypoint after shifting code around ## v3.0.0 (2020-10-01) -* Refactored the entire codebase to be more pythonic, simpler, DRY, and documented (closes #15) -* Better error handling by raising errors where applicable and switching from a homegrown logger to the built-in Python logger (closes #12) -* Added unit tests and test coverage (closes #14) -* Added various additional configuration options -* Automated releasing on PyPi via Travis -* Various bug fixes throughout -* Better documentation on exactly what is possible with this tool -* Added a Makefile -* Adjusted most of the command and option names to be more uniform and explicit +- Refactored the entire codebase to be more pythonic, simpler, DRY, and documented (closes #15) +- Better error handling by raising errors where applicable and switching from a homegrown logger to the built-in Python logger (closes #12) +- Added unit tests and test coverage (closes #14) +- Added various additional configuration options +- Automated releasing on PyPi via Travis +- Various bug fixes throughout +- Better documentation on exactly what is possible with this tool +- Added a Makefile +- Adjusted most of the command and option names to be more uniform and explicit ## v2.1.2 (2020-08-14) -* Fixed the script's entrypoint (PyPi installs work again) +- Fixed the script's entrypoint (PyPi installs work again) ## v2.1.1 (2020-07-14) -* Fixed the long argument names which had underscores intead of hyphens -* Fixed a bug where threads were not waiting at the end of the script before printing the completion message +- Fixed the long argument names which had underscores intead of hyphens +- Fixed a bug where threads were not waiting at the end of the script before printing the completion message ## v2.1.0 (2020-07-01) -* Replaced `requirements.txt` with `setup.py` -* Removed Launch Agent -* Set the default git pull behavior as `--ff-only` to avoid git message (closes #9) -* Increased default timeout for git operations from `120` seconds to `180` seconds -* Replaced the majority of environment variables with command line args making customization easier as "feature flags" -* Various bug fixes and minor improvements -* Removed log deletion functionality -* Published to Pypi +- Replaced `requirements.txt` with `setup.py` +- Removed Launch Agent +- Set the default git pull behavior as `--ff-only` to avoid git message (closes #9) +- Increased default timeout for git operations from `120` seconds to `180` seconds +- Replaced the majority of environment variables with command line args making customization easier as "feature flags" +- Various bug fixes and minor improvements +- Removed log deletion functionality +- Published to Pypi ## v2.0.1 (2020-05-30) -* Fixed the Python program line in the launch agent plist +- Fixed the Python program line in the launch agent plist ## v2.0.0 (2020-05-18) -* Rewrote the script in Python -* Added concurrency to clone/pull changes incredibly fast for large amounts of repos and gists -* Added even more custimization and control -* Uniform log naming, better logging details -* Added changelog +- Rewrote the script in Python +- Added concurrency to clone/pull changes incredibly fast for large amounts of repos and gists +- Added even more custimization and control +- Uniform log naming, better logging details +- Added changelog ## v1.1.0 (2020) -* Added more customization options -* Allowed organization and gists to be cloned/pulled -* Allowed private repos to be cloned/pulled -* Bug fixes +- Added more customization options +- Allowed organization and gists to be cloned/pulled +- Allowed private repos to be cloned/pulled +- Bug fixes ## v1.0.0 (2019) -* Wrote the script in Bash with Python as dependency -* Added some customization options +- Wrote the script in Bash with Python as dependency +- Added some customization options diff --git a/Makefile b/Makefile index 3b78deb..c7cd998 100644 --- a/Makefile +++ b/Makefile @@ -56,8 +56,12 @@ lint: mypy: $(VIRTUAL_BIN)/mypy $(PROJECT_NAME)/ $(TEST_DIR)/ +## scan - Scans the project for security vulnerabilities +scan: + $(VIRTUAL_BIN)/bandit -r $(PROJECT_NAME)/ + ## test - Test the project test: $(VIRTUAL_BIN)/pytest -.PHONY: help build coverage clean black black-check format format-check install isort isort-check lint mypy test +.PHONY: help build coverage clean black black-check format format-check install isort isort-check lint mypy scan test diff --git a/README.md b/README.md index 1c7ea98..0615d2f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ A powerful tool to concurrently clone or pull user and org repos and gists to cr GitHub Archive is a powerful tool to concurrently clone or pull repositories or gists from GitHub with incredible flexibility. It's the perfect tool for spinning up a new dev environment, keeping a local copy of your GitHub instance, or quickly pulling in projects from your favorite users and organizations. -The power of GitHub Archive comes in its configuration. Maybe you only want to clone or pull your personal public repos or maybe you want to go all out and include private repos from you and all organizations you belong to along with your gists. +The power of GitHub Archive comes in its configuration. Maybe you only want to clone or pull your personal public repos or maybe you want to go all out and include private repos from you and all organizations you belong to along with your gists. GitHub Archive can do it all. ## Install @@ -25,7 +25,7 @@ pip3 install github-archive # Install locally make install -``` +``` ## Usage @@ -53,7 +53,7 @@ Options: -e EXCLUDE, --exclude EXCLUDE Pass a comma separated list of repos to filter what is excluded from the Archive. -l LOCATION, --location LOCATION - The location where you want your GitHub Archive to be stored. + The location where you want your GitHub Archive to be stored. By default, this is /Users/USERNAME/github-archive -ht, --https Use HTTPS URLs instead of SSH. -to TIMEOUT, --timeout TIMEOUT The number of seconds before a git operation times out. diff --git a/github_archive/__init__.py b/github_archive/__init__.py index 2872546..4f3f235 100644 --- a/github_archive/__init__.py +++ b/github_archive/__init__.py @@ -1,4 +1,7 @@ -from github_archive.archive import GithubArchive +from github_archive.archive import ( + GithubArchive, +) + __all__ = [ 'GithubArchive', diff --git a/github_archive/archive.py b/github_archive/archive.py index b5cfe83..7bd2ad6 100644 --- a/github_archive/archive.py +++ b/github_archive/archive.py @@ -1,13 +1,27 @@ import os import shutil import stat -import subprocess -from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait -from datetime import datetime -from typing import List, Optional, Union +import subprocess # nosec +from concurrent.futures import ( + ALL_COMPLETED, + ThreadPoolExecutor, + wait, +) +from datetime import ( + datetime, +) +from typing import ( + List, + Optional, + Union, +) import woodchips -from github import Gist, Github, Repository +from github import ( + Gist, + Github, + Repository, +) from github_archive.constants import ( DEFAULT_BASE_URL, @@ -17,6 +31,7 @@ DEFAULT_TIMEOUT, ) + CLONE_OPERATION = 'clone' PULL_OPERATION = 'pull' @@ -370,7 +385,7 @@ def archive_repo(self, repo: Repository.Repository, repo_path: str, operation: s git_command = commands[operation] try: - subprocess.run( + subprocess.run( # nosec git_command, stdout=subprocess.DEVNULL, stdin=subprocess.DEVNULL, @@ -409,7 +424,7 @@ def archive_gist(self, gist: Gist.Gist, gist_path: str, operation: str) -> Optio git_command = commands[operation] try: - subprocess.run( + subprocess.run( # nosec git_command, stdout=subprocess.DEVNULL, stdin=subprocess.DEVNULL, diff --git a/github_archive/cli.py b/github_archive/cli.py index 353db9b..fd17e39 100644 --- a/github_archive/cli.py +++ b/github_archive/cli.py @@ -1,7 +1,11 @@ import argparse -from typing import get_args +from typing import ( + get_args, +) -from github_archive import GithubArchive +from github_archive import ( + GithubArchive, +) from github_archive.constants import ( DEFAULT_BASE_URL, DEFAULT_LOCATION, @@ -116,7 +120,9 @@ def __init__(self): type=str, required=False, default=DEFAULT_LOCATION, - help='The location where you want your GitHub Archive to be stored.', + help=( + f'The location where you want your GitHub Archive to be stored. By default, this is: {DEFAULT_LOCATION}' + ), ) parser.add_argument( '-ht', diff --git a/github_archive/constants.py b/github_archive/constants.py index 060f0fb..303d4b1 100644 --- a/github_archive/constants.py +++ b/github_archive/constants.py @@ -1,6 +1,9 @@ import os -from typing_extensions import Literal +from typing_extensions import ( + Literal, +) + DEFAULT_BASE_URL = 'https://api.github.com' DEFAULT_LOCATION = os.path.expanduser(os.path.join('~', 'github-archive')) diff --git a/pyproject.toml b/pyproject.toml index 29916b1..01bdf1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,11 @@ skip-string-normalization = true [tool.isort] profile = "black" +line_length = 120 +indent = 4 +force_grid_wrap = true +multi_line_output = 3 +sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +lines_after_imports = 2 +include_trailing_comma = true +use_parentheses = true diff --git a/setup.py b/setup.py index 43845a9..94f9121 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,13 @@ ] DEV_REQUIREMENTS = [ + 'bandit == 1.7.*', 'black == 22.*', - 'build == 0.7.*', + 'build == 0.8.*', 'coveralls == 3.*', - 'flake8 == 4.*', + 'flake8 == 5.*', 'isort == 5.*', - 'mypy == 0.942', + 'mypy == 0.971', 'pytest == 7.*', 'pytest-cov == 3.*', 'twine == 4.*', @@ -23,7 +24,7 @@ setuptools.setup( name='github-archive', - version='4.5.0', + version='4.5.1', description=( 'A powerful tool to concurrently clone or pull user and org repos and gists to create a GitHub archive.' ), diff --git a/test/unit/conftest.py b/test/unit/conftest.py index bdb29b1..8e82923 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -1,4 +1,6 @@ -from unittest.mock import MagicMock +from unittest.mock import ( + MagicMock, +) import pytest diff --git a/test/unit/test_archive.py b/test/unit/test_archive.py index a9a076d..3186d41 100644 --- a/test/unit/test_archive.py +++ b/test/unit/test_archive.py @@ -1,9 +1,13 @@ import subprocess -from unittest.mock import patch +from unittest.mock import ( + patch, +) import pytest -from github_archive import GithubArchive +from github_archive import ( + GithubArchive, +) from github_archive.archive import ( CLONE_OPERATION, GIST_CONTEXT,