From 80731d1b10cbfb545abba79c58412e3f7cc88d38 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:00:58 +0200 Subject: [PATCH 01/56] Changed python formatter to yapf and added working default pylint.rc --- default/pylint.rc | 222 ++++++++++++++++++++++++++++++++++++++++++++++ linter.py | 30 +++---- 2 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 default/pylint.rc diff --git a/default/pylint.rc b/default/pylint.rc new file mode 100644 index 0000000..e2c1a16 --- /dev/null +++ b/default/pylint.rc @@ -0,0 +1,222 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +extension-pkg-whitelist=numpy + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time. +disable=no-name-in-module,import-error,fixme + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +# (Constant dectione is not very reliable, so also allow lower case letters.) +const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +# (Added upper case letters to match coordinate frames, e.g. T_G_I, v_M_I.) +function-rgx=[a-z_][a-zA-Z0-9_]{2,50}$ + +# Regular expression which should only match correct method names +method-rgx=[a-zA-Z_][a-zA-Z0-9_]{0,50}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,50}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-zA-Z_][a-zA-Z0-9_]{0,50}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-zA-Z_][a-zA-Z0-9_]{0,50}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[a-zA-Z_][a-zA-Z0-9_]{0,50}$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent,numpy.uint32,numpy.float32 + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp diff --git a/linter.py b/linter.py index 01d9678..c4995a9 100755 --- a/linter.py +++ b/linter.py @@ -19,13 +19,13 @@ import yaml CLANG_FORMAT_DIFF_EXECUTABLE = "clang-format-diff-3.8" -AUTOPEP8_FORMAT_EXECUTABLE = "autopep8" +YAPF_FORMAT_EXECUTABLE = "yapf" DEFAULT_CONFIG = { 'use_clangformat': True, 'use_cpplint': True, # Disable Python checks by default. - 'use_autopep8': False, + 'use_yapf': False, 'use_pylint': False, # Check all staged files by default. 'whitelist': [] @@ -43,8 +43,8 @@ def read_linter_config(filename): config['use_clangformat'] = parsed_config['clangformat'] if 'cpplint' in parsed_config.keys(): config['use_cpplint'] = parsed_config['cpplint'] - if 'autopep8' in parsed_config.keys(): - config['use_autopep8'] = parsed_config['autopep8'] + if 'yapf' in parsed_config.keys(): + config['use_yapf'] = parsed_config['yapf'] if 'pylint' in parsed_config.keys(): config['use_pylint'] = parsed_config['pylint'] if 'whitelist' in parsed_config.keys(): @@ -233,8 +233,8 @@ def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): return True -def run_autopep8_format(repo_root, staged_files, list_of_changed_staged_files): - """Runs autopep8 format on all python files staged for commit.""" +def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): + """Runs yapf format on all python files staged for commit.""" first_file_formatted = True for staged_file in staged_files: @@ -244,17 +244,15 @@ def run_autopep8_format(repo_root, staged_files, list_of_changed_staged_files): # Check if the file needs formatting by applying the formatting and store # the results into a patch file. - autopep8_format_path = ("/tmp/" + + yapf_format_path = ("/tmp/" + os.path.basename(os.path.normpath(repo_root)) + "_" + datetime.datetime.now().isoformat() + - ".autopep8.patch") - task = (AUTOPEP8_FORMAT_EXECUTABLE + - " -d --aggressive --aggressive --max-line-length=80 " + - "--indent-size=2 --ignore=E24 " + - staged_file + " > " + autopep8_format_path) + ".yapf.patch") + task = (YAPF_FORMAT_EXECUTABLE + " -d " + staged_file + + " > " + yapf_format_path) run_command_in_folder(task, repo_root) - if not os.stat(autopep8_format_path).st_size == 0: + if not os.stat(yapf_format_path).st_size == 0: if first_file_formatted: print("=" * 80) print("Formatted staged python files with autopep8:") @@ -270,7 +268,7 @@ def run_autopep8_format(repo_root, staged_files, list_of_changed_staged_files): exit(1) else: run_command_in_folder( - "git apply -p1 " + autopep8_format_path, repo_root) + "git apply -p0 " + yapf_format_path, repo_root) run_command_in_folder("git add " + staged_file, repo_root) if not first_file_formatted: @@ -399,8 +397,8 @@ def linter_check(repo_root, linter_subfolder): run_clang_format(repo_root, whitelisted_files, list_of_changed_staged_files) - if linter_config['use_autopep8']: - run_autopep8_format(repo_root, whitelisted_files, + if linter_config['use_yapf']: + run_yapf_format(repo_root, whitelisted_files, list_of_changed_staged_files) From 3a22b26721000bdfbd63de10785e73ad94bbe999 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:08:08 +0200 Subject: [PATCH 02/56] Adapted init script --- init-git-hooks.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/init-git-hooks.py b/init-git-hooks.py index 955e505..dfd42c7 100755 --- a/init-git-hooks.py +++ b/init-git-hooks.py @@ -29,7 +29,6 @@ default_cpplint = "modified_cpplint.py" cpplint_url = "" -pylint_url = "https://raw.githubusercontent.com/vinitkumar/googlecl/6dc04b489dba709c53d2f4944473709617506589/googlecl-pylint.rc" clang_format_diff_executable = "clang-format-diff-3.8" @@ -79,7 +78,16 @@ def main(): print("Failed to copy default cpplint.") exit(1) - download_file_from_url(pylint_url, script_directory + "/pylint.rc") + default_pylint_file = os.path.join(script_directory, 'default', 'pylint.rc') + if not os.path.isfile(default_pylint_file): + print("Default pylint.rc wasn't found under", default_pylint_file) + exit(1) + + cp_params = (default_pylint_file + " " + os.path.join(script_directory, 'pylint.rc')) + if subprocess.call("cp " + cp_params, shell=True) != 0: + print("Failed to copy default pylint.rc.") + exit(1) + if not os.path.isfile(script_directory + "/pylint.rc"): print("ERROR: Could not download pylint.rc file!") exit(1) @@ -88,8 +96,8 @@ def main(): print("ERROR: " + clang_format_diff_executable + " is not installed!") exit(1) - if not command_exists("autopep8"): - print("ERROR: autopep8 is not installed! Try: pip install autopep8") + if not command_exists("yapf"): + print("ERROR: yapf is not installed! Try: pip install yapf") exit(1) # Get git root folder of parent repository. From e73a0ccab4228969b2c05f3483fd1953570eec68 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:08:14 +0200 Subject: [PATCH 03/56] Updated readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b8e3065..2e0e0bf 100644 --- a/README.md +++ b/README.md @@ -7,19 +7,18 @@ This repo contains a (C++, python (experimental)) linter and auto formatter pack * **clang-format** Formats your code based on your .clang-format preferences. * **cpplint** Checks your C++ code for style errors and warnings. - * **EXPERIMENTAL - Python** files: + * **Python** files: - **WARNING:** The default linter settings are very strict and might not actually conform with the formatting produced by autopep8. If you want to use this linter for python you will need to do some tweaking. - * **autopep8** Formats your python code. + * **yapf** Formats your python code. * **pylint** Checks your Python code for style errors and warnings. ## Dependencies - * **autopep8** ([Introduction to autopep8](http://avilpage.com/2015/05/automatically-pep8-your-python-code.html)) - * Ubuntu 14.04 / macOS: `pip install autopep8` + * **yapf** + * Ubuntu / macOS: `pip install yapf` * **clang-format** - * Ubuntu 14.04: `sudo apt-get install clang-format-3.X` + * Ubuntu: `sudo apt install clang-format-3.8` * macOS: ``` brew install clang-format From 81bd7fa54184f11ae6c02a0b76e7ac578d2c890d Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:08:39 +0200 Subject: [PATCH 04/56] Enabled python by default --- linter.py | 4 ++-- linterconfig.yaml_example | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/linter.py b/linter.py index c4995a9..6865099 100755 --- a/linter.py +++ b/linter.py @@ -25,8 +25,8 @@ 'use_clangformat': True, 'use_cpplint': True, # Disable Python checks by default. - 'use_yapf': False, - 'use_pylint': False, + 'use_yapf': True, + 'use_pylint': True, # Check all staged files by default. 'whitelist': [] } diff --git a/linterconfig.yaml_example b/linterconfig.yaml_example index 4d15dc6..551b8de 100644 --- a/linterconfig.yaml_example +++ b/linterconfig.yaml_example @@ -1,8 +1,8 @@ # Turn the components of the linter individualy on / off. clangformat: on cpplint: on -autopep8: off -pylint: off +yapf: on +pylint: on # If this whitelist block is present, the linter only operates on the files and # directories listed in this whitelist. From 2a0551f967ba9cc97613ae49300d88c6081ef086 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:09:48 +0200 Subject: [PATCH 05/56] Set yapf style to google --- linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linter.py b/linter.py index 6865099..5f62957 100755 --- a/linter.py +++ b/linter.py @@ -248,7 +248,7 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): os.path.basename(os.path.normpath(repo_root)) + "_" + datetime.datetime.now().isoformat() + ".yapf.patch") - task = (YAPF_FORMAT_EXECUTABLE + " -d " + staged_file + + task = (YAPF_FORMAT_EXECUTABLE + " -style google -d " + staged_file + " > " + yapf_format_path) run_command_in_folder(task, repo_root) From a7bf3ce88f8adbacaf726eb7ccb1d1d9ac9ee34f Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:13:03 +0200 Subject: [PATCH 06/56] Fixed style argument --- linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linter.py b/linter.py index 5f62957..7c07c37 100755 --- a/linter.py +++ b/linter.py @@ -211,7 +211,7 @@ def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): run_command_in_folder("git diff -U0 --cached | " + CLANG_FORMAT_DIFF_EXECUTABLE + - " -style=file -p1 > " + + " --style=file -p1 > " + clang_format_path, repo_root) if not os.stat(clang_format_path).st_size == 0: From f0fc44da78b35df825c9dc9b8b18a3766a0841f2 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 11:13:48 +0200 Subject: [PATCH 07/56] Fixed style in the right place --- linter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linter.py b/linter.py index 7c07c37..73be023 100755 --- a/linter.py +++ b/linter.py @@ -211,7 +211,7 @@ def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): run_command_in_folder("git diff -U0 --cached | " + CLANG_FORMAT_DIFF_EXECUTABLE + - " --style=file -p1 > " + + " -style=file -p1 > " + clang_format_path, repo_root) if not os.stat(clang_format_path).st_size == 0: @@ -248,7 +248,7 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): os.path.basename(os.path.normpath(repo_root)) + "_" + datetime.datetime.now().isoformat() + ".yapf.patch") - task = (YAPF_FORMAT_EXECUTABLE + " -style google -d " + staged_file + + task = (YAPF_FORMAT_EXECUTABLE + " --style google -d " + staged_file + " > " + yapf_format_path) run_command_in_folder(task, repo_root) From 69459642d4e3172d49d0d723ce1ef921d6bfc279 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Fri, 25 May 2018 15:14:49 +0200 Subject: [PATCH 08/56] Changed default python style to pep8 and no longer enforce docstrings everywhere --- default/pylint.rc | 2 +- linter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/default/pylint.rc b/default/pylint.rc index e2c1a16..2b4aaaa 100644 --- a/default/pylint.rc +++ b/default/pylint.rc @@ -31,7 +31,7 @@ extension-pkg-whitelist=numpy # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time. -disable=no-name-in-module,import-error,fixme +disable=no-name-in-module,import-error,fixme,missing-docstring [REPORTS] diff --git a/linter.py b/linter.py index 73be023..54c9f36 100755 --- a/linter.py +++ b/linter.py @@ -248,7 +248,7 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): os.path.basename(os.path.normpath(repo_root)) + "_" + datetime.datetime.now().isoformat() + ".yapf.patch") - task = (YAPF_FORMAT_EXECUTABLE + " --style google -d " + staged_file + + task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + staged_file + " > " + yapf_format_path) run_command_in_folder(task, repo_root) From d0e74dbe199072ccf057acd70def2cc882297ca3 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 10:21:00 +0200 Subject: [PATCH 09/56] Added possibility to install the linter --- README.md | 15 +++++----- .../init_linter_git_hooks | 6 ++-- git-hooks.py | 28 ++++++++++++++----- linter.py | 15 +++++++--- setup_linter.sh | 4 +++ 5 files changed, 48 insertions(+), 20 deletions(-) rename init-git-hooks.py => bin/init_linter_git_hooks (96%) create mode 100755 setup_linter.sh diff --git a/README.md b/README.md index 2e0e0bf..26c0d62 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,17 @@ This repo contains a (C++, python (experimental)) linter and auto formatter pack ## Installation ```bash -cd $YOUR_REPO -git submodule add git@github.com:ethz-asl/linter.git -./linter/init-git-hooks.py +git clone git@github.com:ethz-asl/linter.git +cd linter +echo ". $(realpath setup_linter.sh)" >> ~/.bashrc # Or the matching file for + # your shell. +source ~/.bashrc ``` -You can also add the linter submodule in a subfolder of your repo, e.g.: +The you can install the linter in your repository: ```bash -mkdir $YOUR_REPO/dev_tools -git submodule add git@github.com:ethz-asl/linter.git dev_tools/linter -./dev_tools/linter/init-git-hooks.py +cd $YOUR_REPO +init_linter_git_hooks ``` Define the project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example: diff --git a/init-git-hooks.py b/bin/init_linter_git_hooks similarity index 96% rename from init-git-hooks.py rename to bin/init_linter_git_hooks index dfd42c7..7cb1fb2 100755 --- a/init-git-hooks.py +++ b/bin/init_linter_git_hooks @@ -20,6 +20,8 @@ default_cpplint = "cpplint.py" """ +from __future__ import print_function + import os import requests import shutil @@ -63,7 +65,7 @@ def command_exists(cmd): def main(): """ Download cpplint.py and pylint.py and installs the git hooks""" script_directory = os.path.dirname(sys.argv[0]) - script_directory = os.path.abspath(script_directory) + script_directory = os.path.dirname(os.path.abspath(script_directory)) if cpplint_url != "": # Download linter files. @@ -101,7 +103,7 @@ def main(): exit(1) # Get git root folder of parent repository. - repo_root = get_git_repo_root(script_directory + '/../') + repo_root = get_git_repo_root('.') # Copy git hooks. cp_params = script_directory + "/git-hooks.py " + \ diff --git a/git-hooks.py b/git-hooks.py index c11952c..64e5266 100755 --- a/git-hooks.py +++ b/git-hooks.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +from __future__ import print_function + +import os import subprocess import sys @@ -22,10 +25,21 @@ def get_git_repo_root(some_folder_in_root_repo='./'): some_folder_in_root_repo) -def get_linter_subfolder(root_repo_folder): - """Find the subfolder where this linter is stored.""" - return run_command_in_folder("git submodule | awk '{ print $2 }'" + - " | grep linter", root_repo_folder) +def get_linter_folder(root_repo_folder): + """Find the folder where this linter is stored.""" + linter_folder = run_command_in_folder( + "git submodule | awk '{ print $2 }'" + " | grep linter", + root_repo_folder) + if len(linter_folder) != 0: + return linter_folder + + # Go via environment variable. + try: + return os.environ['LINTER_PATH'] + except KeyError: + print("Cannot find linter because the environment variable " + "LINTER_PATH doesn't exist.") + sys.exit(1) def main(): @@ -33,15 +47,15 @@ def main(): repo_root = get_git_repo_root() # Get linter subfolder - linter_subfolder = get_linter_subfolder(repo_root) + linter_folder = get_linter_folder(repo_root) # Append linter folder to the path so that we can import the linter module. - linter_folder = repo_root + "/" + linter_subfolder + linter_folder = os.path.join(repo_root, linter_folder) sys.path.append(linter_folder) import linter - linter.linter_check(repo_root, linter_subfolder) + linter.linter_check(repo_root, linter_folder) if __name__ == "__main__": diff --git a/linter.py b/linter.py index 54c9f36..630617f 100755 --- a/linter.py +++ b/linter.py @@ -24,7 +24,7 @@ DEFAULT_CONFIG = { 'use_clangformat': True, 'use_cpplint': True, - # Disable Python checks by default. + # Enable Python checks by default. 'use_yapf': True, 'use_pylint': True, # Check all staged files by default. @@ -123,6 +123,13 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): "catkin package that contains: " "{}".format(changed_file)) + # Get relative path to repository root. + common_prefix = os.path.commonprefix([ + os.path.abspath(repo_root), os.path.abspath(package_root)]) + package_root = os.path.relpath(package_root, common_prefix) + + # The package root needs to be relative to the repo root. Otherwise the + # header guard logic will fail!. cpplint._root = package_root + '/include' # pylint: disable=W0212 # Reset error count and messages: @@ -347,9 +354,9 @@ def get_whitelisted_files(repo_root, files, whitelist): def linter_check(repo_root, linter_subfolder): """ Main pre-commit function for calling code checking script. """ - cpplint_file = repo_root + "/" + linter_subfolder + "/cpplint.py" - pylint_file = repo_root + "/" + linter_subfolder + "/pylint.rc" - ascii_art_file = repo_root + "/" + linter_subfolder + "/ascii_art.py" + cpplint_file = os.path.join(linter_subfolder, "cpplint.py") + pylint_file = os.path.join(linter_subfolder, "pylint.rc") + ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") # Read linter config file. linter_config_file = repo_root + '/linterconfig.yaml' diff --git a/setup_linter.sh b/setup_linter.sh new file mode 100755 index 0000000..4338af5 --- /dev/null +++ b/setup_linter.sh @@ -0,0 +1,4 @@ +LINTER_PATH="$( cd "$( dirname "$0" )" && pwd )" +export LINTER_PATH + +export PATH="$PATH:$LINTER_PATH/bin" From fdd5b86fbc1d8486b76f9fc54007cf797d5e8532 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 11:34:03 +0200 Subject: [PATCH 10/56] Enable usage of linter from within submodules --- bin/init_linter_git_hooks | 17 ++++++++++++----- git-hooks.py => git_hooks.py | 1 + linter.py | 24 +++++++++++++++++++----- 3 files changed, 32 insertions(+), 10 deletions(-) rename git-hooks.py => git_hooks.py (96%) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index 7cb1fb2..1f24f55 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -27,6 +27,7 @@ import requests import shutil import subprocess import sys +import yaml default_cpplint = "modified_cpplint.py" @@ -53,7 +54,7 @@ def get_git_repo_root(some_folder_in_root_repo='./'): stdout, _ = get_repo_call.communicate() repo_root = stdout.rstrip() - return repo_root + return repo_root, os.path.basename(repo_root) def command_exists(cmd): @@ -103,14 +104,20 @@ def main(): exit(1) # Get git root folder of parent repository. - repo_root = get_git_repo_root('.') + repo_root, repo_folder_name = get_git_repo_root('.') + hooks_folder = os.path.join(repo_root, '.git/hooks') + if os.path.isfile(os.path.join(repo_root, '.git')): + with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: + git_folder = yaml.load(git_info_file)['gitdir'] + repo_root, _ = get_git_repo_root(os.path.join(repo_root, git_folder)) + hooks_folder = os.path.join(repo_root, git_folder, 'hooks') # Copy git hooks. - cp_params = script_directory + "/git-hooks.py " + \ - repo_root + "/.git/hooks/pre-commit" + cp_params = script_directory + "/git_hooks.py " + \ + hooks_folder + "/pre-commit" if subprocess.call("cp " + cp_params, shell=True) != 0: print("Failed to copy githooks to " - "{}...".format((repo_root + "/.git/hooks/"))) + "{}...".format(hooks_folder)) exit(1) print("Success, githooks initialized!") diff --git a/git-hooks.py b/git_hooks.py similarity index 96% rename from git-hooks.py rename to git_hooks.py index 64e5266..902394e 100755 --- a/git-hooks.py +++ b/git_hooks.py @@ -21,6 +21,7 @@ def run_command_in_folder(command, folder): def get_git_repo_root(some_folder_in_root_repo='./'): """Get the root folder of the current git repository.""" + print('git folder to check:', some_folder_in_root_repo) return run_command_in_folder('git rev-parse --show-toplevel', some_folder_in_root_repo) diff --git a/linter.py b/linter.py index 630617f..9b9a5f1 100755 --- a/linter.py +++ b/linter.py @@ -124,6 +124,20 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): "{}".format(changed_file)) # Get relative path to repository root. + if os.path.isfile(os.path.join(repo_root, '.git')): + # Repo is a submodule, look for parent git repository containing this + # submodule. + search_path = repo_root + found_to_level_git_repo = False + for _ in range(10): + search_path = os.path.dirname(repo_root) + if os.path.isdir(os.path.join(search_path, '.git')): + found_to_level_git_repo = True + break + + assert found_to_level_git_repo + repo_root = search_path + common_prefix = os.path.commonprefix([ os.path.abspath(repo_root), os.path.abspath(package_root)]) package_root = os.path.relpath(package_root, common_prefix) @@ -340,7 +354,7 @@ def get_whitelisted_files(repo_root, files, whitelist): for entry in whitelist: # Add trailing slash if its a directory and no slash is already there. if os.path.isdir(repo_root + '/' + entry): - entry = os.path.join(os.path.normpath(entry), '') + entry = os.path.join(os.path.normpath(entry), '') # Check if the file itself or its parent directory is in the whitelist. if (file == entry @@ -361,10 +375,10 @@ def linter_check(repo_root, linter_subfolder): # Read linter config file. linter_config_file = repo_root + '/linterconfig.yaml' if os.path.isfile(repo_root + '/linterconfig.yaml'): - print("Found repo linter config: {}".format(linter_config_file)) - linter_config = read_linter_config(linter_config_file) + print("Found repo linter config: {}".format(linter_config_file)) + linter_config = read_linter_config(linter_config_file) else: - linter_config = DEFAULT_CONFIG + linter_config = DEFAULT_CONFIG print("Found linter subfolder: {}".format(linter_subfolder)) print("Found ascii art file at: {}".format(ascii_art_file)) @@ -414,7 +428,7 @@ def linter_check(repo_root, linter_subfolder): cpp_lint_success = check_cpp_lint( whitelisted_files, cpplint_file, ascii_art, repo_root) else: - cpp_lint_success = True + cpp_lint_success = True # Use pylint to check for comimpliance with Tensofrflow python style guide. if linter_config['use_pylint']: From 6e511cdf90b9054e723e4264717a3463dd89b86e Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 12:53:13 +0200 Subject: [PATCH 11/56] Cleanup --- README.md | 2 +- bin/init_linter_git_hooks | 8 +++++--- linter.py | 14 +++++++------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 26c0d62..8fcd31a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ echo ". $(realpath setup_linter.sh)" >> ~/.bashrc # Or the matching file for source ~/.bashrc ``` -The you can install the linter in your repository: +Then you can install the linter in your repository: ```bash cd $YOUR_REPO init_linter_git_hooks diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index 1f24f55..c1f1045 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -54,7 +54,7 @@ def get_git_repo_root(some_folder_in_root_repo='./'): stdout, _ = get_repo_call.communicate() repo_root = stdout.rstrip() - return repo_root, os.path.basename(repo_root) + return repo_root def command_exists(cmd): @@ -104,12 +104,14 @@ def main(): exit(1) # Get git root folder of parent repository. - repo_root, repo_folder_name = get_git_repo_root('.') + repo_root = get_git_repo_root('.') hooks_folder = os.path.join(repo_root, '.git/hooks') if os.path.isfile(os.path.join(repo_root, '.git')): + # Git repo is a submodule, git information is in a folder listed in the .git + # file. with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: git_folder = yaml.load(git_info_file)['gitdir'] - repo_root, _ = get_git_repo_root(os.path.join(repo_root, git_folder)) + repo_root = get_git_repo_root(os.path.join(repo_root, git_folder)) hooks_folder = os.path.join(repo_root, git_folder, 'hooks') # Copy git hooks. diff --git a/linter.py b/linter.py index 9b9a5f1..c7057d7 100755 --- a/linter.py +++ b/linter.py @@ -142,8 +142,8 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): os.path.abspath(repo_root), os.path.abspath(package_root)]) package_root = os.path.relpath(package_root, common_prefix) - # The package root needs to be relative to the repo root. Otherwise the - # header guard logic will fail!. + # The package root needs to be relative to the (top-level) repo root. + # Otherwise the header guard logic will fail! cpplint._root = package_root + '/include' # pylint: disable=W0212 # Reset error count and messages: @@ -354,7 +354,7 @@ def get_whitelisted_files(repo_root, files, whitelist): for entry in whitelist: # Add trailing slash if its a directory and no slash is already there. if os.path.isdir(repo_root + '/' + entry): - entry = os.path.join(os.path.normpath(entry), '') + entry = os.path.join(os.path.normpath(entry), '') # Check if the file itself or its parent directory is in the whitelist. if (file == entry @@ -375,10 +375,10 @@ def linter_check(repo_root, linter_subfolder): # Read linter config file. linter_config_file = repo_root + '/linterconfig.yaml' if os.path.isfile(repo_root + '/linterconfig.yaml'): - print("Found repo linter config: {}".format(linter_config_file)) - linter_config = read_linter_config(linter_config_file) + print("Found repo linter config: {}".format(linter_config_file)) + linter_config = read_linter_config(linter_config_file) else: - linter_config = DEFAULT_CONFIG + linter_config = DEFAULT_CONFIG print("Found linter subfolder: {}".format(linter_subfolder)) print("Found ascii art file at: {}".format(ascii_art_file)) @@ -428,7 +428,7 @@ def linter_check(repo_root, linter_subfolder): cpp_lint_success = check_cpp_lint( whitelisted_files, cpplint_file, ascii_art, repo_root) else: - cpp_lint_success = True + cpp_lint_success = True # Use pylint to check for comimpliance with Tensofrflow python style guide. if linter_config['use_pylint']: From 956fdc33d3751322990a99f6bbe781472fa54238 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 14:47:50 +0200 Subject: [PATCH 12/56] Removed a debug print --- git_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/git_hooks.py b/git_hooks.py index 902394e..64e5266 100755 --- a/git_hooks.py +++ b/git_hooks.py @@ -21,7 +21,6 @@ def run_command_in_folder(command, folder): def get_git_repo_root(some_folder_in_root_repo='./'): """Get the root folder of the current git repository.""" - print('git folder to check:', some_folder_in_root_repo) return run_command_in_folder('git rev-parse --show-toplevel', some_folder_in_root_repo) From 1c042545a59a3c7cc3c0722fa7d535d33b49918e Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 16:37:28 +0200 Subject: [PATCH 13/56] Added option to remove linter --- bin/init_linter_git_hooks | 54 ++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index c1f1045..3f44d8d 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -22,6 +22,7 @@ default_cpplint = "cpplint.py" from __future__ import print_function +import argparse import os import requests import shutil @@ -63,7 +64,21 @@ def command_exists(cmd): stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 -def main(): +def get_hooks_folder_for_repo(): + # Get git root folder of parent repository. + repo_root = get_git_repo_root('.') + hooks_folder = os.path.join(repo_root, '.git/hooks') + if os.path.isfile(os.path.join(repo_root, '.git')): + # Git repo is a submodule, git information is in a folder listed in the .git + # file. + with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: + git_folder = yaml.load(git_info_file)['gitdir'] + repo_root = get_git_repo_root(os.path.join(repo_root, git_folder)) + hooks_folder = os.path.join(repo_root, git_folder, 'hooks') + return hooks_folder + + +def install_linter(): """ Download cpplint.py and pylint.py and installs the git hooks""" script_directory = os.path.dirname(sys.argv[0]) script_directory = os.path.dirname(os.path.abspath(script_directory)) @@ -103,18 +118,8 @@ def main(): print("ERROR: yapf is not installed! Try: pip install yapf") exit(1) - # Get git root folder of parent repository. - repo_root = get_git_repo_root('.') - hooks_folder = os.path.join(repo_root, '.git/hooks') - if os.path.isfile(os.path.join(repo_root, '.git')): - # Git repo is a submodule, git information is in a folder listed in the .git - # file. - with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: - git_folder = yaml.load(git_info_file)['gitdir'] - repo_root = get_git_repo_root(os.path.join(repo_root, git_folder)) - hooks_folder = os.path.join(repo_root, git_folder, 'hooks') - # Copy git hooks. + hooks_folder = get_hooks_folder_for_repo() cp_params = script_directory + "/git_hooks.py " + \ hooks_folder + "/pre-commit" if subprocess.call("cp " + cp_params, shell=True) != 0: @@ -125,5 +130,30 @@ def main(): print("Success, githooks initialized!") +def remove_linter(): + hooks_folder = get_hooks_folder_for_repo() + pre_commit_file = os.path.join(hooks_folder, 'pre-commit') + with open(pre_commit_file, 'w+') as out_file: + out_file.write('#!/bin/sh\n') + # subprocess.call('chmod +x' + print("Linter githooks removed!") + + +def main(): + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + '--remove', + '-r', + dest='remove_linter', + action="store_true", + help='Remove the linter from this repository.') + args = arg_parser.parse_args() + + if args.remove_linter: + remove_linter() + else: + install_linter() + + if __name__ == "__main__": main() From 6d416a51776604e4e11d54515facb02679e215b8 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Tue, 29 May 2018 17:58:20 +0200 Subject: [PATCH 14/56] Removed dead code --- bin/init_linter_git_hooks | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index 3f44d8d..e909e36 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -135,7 +135,6 @@ def remove_linter(): pre_commit_file = os.path.join(hooks_folder, 'pre-commit') with open(pre_commit_file, 'w+') as out_file: out_file.write('#!/bin/sh\n') - # subprocess.call('chmod +x' print("Linter githooks removed!") From cd91c540cc62a6e9bd3a58d92e501be6aa854e59 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Wed, 30 May 2018 13:57:06 +0200 Subject: [PATCH 15/56] Fix linter path for bash --- setup_linter.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup_linter.sh b/setup_linter.sh index 4338af5..a621772 100755 --- a/setup_linter.sh +++ b/setup_linter.sh @@ -1,4 +1,10 @@ -LINTER_PATH="$( cd "$( dirname "$0" )" && pwd )" +if [ ! -z $ZSH_NAME ] ; then + LINTER_PATH="$( cd "$( dirname "$0" )" && pwd )" +elif [ ! -z $BASH ] ; then + LINTER_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +else + echo "Linter: Unsupported shell! Only bash and zsh supported at the moment!" +fi export LINTER_PATH export PATH="$PATH:$LINTER_PATH/bin" From a81569c6329988843d5c02b59f2cdc0e85e73578 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Thu, 31 May 2018 15:44:29 +0200 Subject: [PATCH 16/56] Completely removed possiblility to use linter as submodule --- git_hooks.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/git_hooks.py b/git_hooks.py index 64e5266..6b42582 100755 --- a/git_hooks.py +++ b/git_hooks.py @@ -27,13 +27,6 @@ def get_git_repo_root(some_folder_in_root_repo='./'): def get_linter_folder(root_repo_folder): """Find the folder where this linter is stored.""" - linter_folder = run_command_in_folder( - "git submodule | awk '{ print $2 }'" + " | grep linter", - root_repo_folder) - if len(linter_folder) != 0: - return linter_folder - - # Go via environment variable. try: return os.environ['LINTER_PATH'] except KeyError: From 6d61256b4d774f9482132f00a67bf52f34776533 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Mon, 11 Jun 2018 09:27:17 +0200 Subject: [PATCH 17/56] Expanded documentation --- README.md | 18 ++++++++++++++---- bin/init_linter_git_hooks | 1 - 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8fcd31a..cd15303 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ This repo contains a (C++, python (experimental)) linter and auto formatter pack * **C++** files: * **clang-format** Formats your code based on your .clang-format preferences. * **cpplint** Checks your C++ code for style errors and warnings. - + * **Python** files: * **yapf** Formats your python code. * **pylint** Checks your Python code for style errors and warnings. - + ## Dependencies @@ -19,7 +19,7 @@ This repo contains a (C++, python (experimental)) linter and auto formatter pack * Ubuntu / macOS: `pip install yapf` * **clang-format** * Ubuntu: `sudo apt install clang-format-3.8` - * macOS: + * macOS: ``` brew install clang-format ln -s /usr/local/share/clang/clang-format-diff.py /usr/local/bin/clang-format-diff-3.8 @@ -42,7 +42,17 @@ cd $YOUR_REPO init_linter_git_hooks ``` -Define the project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example: +## Uninstallation +To remove the linter from your git repository again, run the following: +```bash +cd $YOUR_REPO +init_linter_git_hooks --remove +``` + +## Linter configuration +To configure the linter, a file named `linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). + +Clang-format can be configured by defining a project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example file: ``` --- diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index e909e36..607563d 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -142,7 +142,6 @@ def main(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument( '--remove', - '-r', dest='remove_linter', action="store_true", help='Remove the linter from this repository.') From 6cf67b55b22e0e0f5359a1200f374d8df847e915 Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Mon, 11 Jun 2018 09:28:35 +0200 Subject: [PATCH 18/56] Fixed grammar --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd15303..05c7134 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ init_linter_git_hooks --remove ``` ## Linter configuration -To configure the linter, a file named `linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). +To configure the linter, add a file named `linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). Clang-format can be configured by defining a project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example file: From de9bf9b8a0cbcd1f7152605fa4971daf907f6a3e Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Mon, 11 Jun 2018 12:23:13 +0200 Subject: [PATCH 19/56] Renamed to .linterconfig.yaml --- README.md | 2 +- linter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 05c7134..418c9a6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ init_linter_git_hooks --remove ``` ## Linter configuration -To configure the linter, add a file named `linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). +To configure the linter, add a file named `.linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). Clang-format can be configured by defining a project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example file: diff --git a/linter.py b/linter.py index c7057d7..4188143 100755 --- a/linter.py +++ b/linter.py @@ -373,8 +373,8 @@ def linter_check(repo_root, linter_subfolder): ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") # Read linter config file. - linter_config_file = repo_root + '/linterconfig.yaml' - if os.path.isfile(repo_root + '/linterconfig.yaml'): + linter_config_file = repo_root + '/.linterconfig.yaml' + if os.path.isfile(repo_root + '/.linterconfig.yaml'): print("Found repo linter config: {}".format(linter_config_file)) linter_config = read_linter_config(linter_config_file) else: From ac5b666002af1a582891688849a95808853fa99e Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Wed, 13 Jun 2018 17:19:58 +0200 Subject: [PATCH 20/56] Disabled two more pylint warnings --- default/pylint.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default/pylint.rc b/default/pylint.rc index 2b4aaaa..8abb46d 100644 --- a/default/pylint.rc +++ b/default/pylint.rc @@ -31,7 +31,7 @@ extension-pkg-whitelist=numpy # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time. -disable=no-name-in-module,import-error,fixme,missing-docstring +disable=no-name-in-module,import-error,fixme,missing-docstring,no-member,wrong-import-position [REPORTS] From b185a75b600eb4e966b40c279711d0b2883fbcfc Mon Sep 17 00:00:00 2001 From: Kevin Egger Date: Thu, 14 Jun 2018 09:34:38 +0200 Subject: [PATCH 21/56] Disabled another pylint check --- default/pylint.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default/pylint.rc b/default/pylint.rc index 8abb46d..8afdb44 100644 --- a/default/pylint.rc +++ b/default/pylint.rc @@ -31,7 +31,7 @@ extension-pkg-whitelist=numpy # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time. -disable=no-name-in-module,import-error,fixme,missing-docstring,no-member,wrong-import-position +disable=no-name-in-module,import-error,fixme,missing-docstring,no-member,wrong-import-position,bad-continuation [REPORTS] From add18a7639e27390163391274fabcf1534d90566 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Fri, 22 Jun 2018 16:23:40 +0200 Subject: [PATCH 22/56] use newest clang-format --- bin/init_linter_git_hooks | 2 +- linter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index 607563d..afd1baf 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -34,7 +34,7 @@ default_cpplint = "modified_cpplint.py" cpplint_url = "" -clang_format_diff_executable = "clang-format-diff-3.8" +clang_format_diff_executable = "clang-format-diff-6.0" def download_file_from_url(url, file_path): diff --git a/linter.py b/linter.py index 4188143..a2942e2 100755 --- a/linter.py +++ b/linter.py @@ -18,7 +18,7 @@ import subprocess import yaml -CLANG_FORMAT_DIFF_EXECUTABLE = "clang-format-diff-3.8" +CLANG_FORMAT_DIFF_EXECUTABLE = "clang-format-diff-6.0" YAPF_FORMAT_EXECUTABLE = "yapf" DEFAULT_CONFIG = { From 1ed08cf522db58cf682dcd9775af97e0bae84b36 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Mon, 25 Jun 2018 10:59:39 +0200 Subject: [PATCH 23/56] automatically search for clang format version, apply linter formatting to linter --- bin/init_linter_git_hooks | 201 +++++----- linter.py | 773 ++++++++++++++++++++------------------ 2 files changed, 511 insertions(+), 463 deletions(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index afd1baf..c62e066 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -1,7 +1,6 @@ #!/usr/bin/env python # Disable pylint filename and missing module member complaints. # pylint: disable=C0103,E1101 - """ Initializes git hooks for the parent git repository. @@ -24,10 +23,11 @@ from __future__ import print_function import argparse import os -import requests import shutil import subprocess import sys + +import requests import yaml default_cpplint = "modified_cpplint.py" @@ -36,122 +36,143 @@ cpplint_url = "" clang_format_diff_executable = "clang-format-diff-6.0" +CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS = [ + "clang-format-diff-6.0", "clang-format-diff-5.0", "clang-format-diff-4.0", + "clang-format-diff-3.9", "clang-format-diff-3.8" +] + def download_file_from_url(url, file_path): - """Download a file from a HTTPS URL. Verification is enabled.""" - request = requests.get(url, verify=True, stream=True) - request.raw.decode_content = True - with open(file_path, 'w') as downloaded_file: - shutil.copyfileobj(request.raw, downloaded_file) + """Download a file from a HTTPS URL. Verification is enabled.""" + request = requests.get(url, verify=True, stream=True) + request.raw.decode_content = True + with open(file_path, 'w') as downloaded_file: + shutil.copyfileobj(request.raw, downloaded_file) def get_git_repo_root(some_folder_in_root_repo='./'): - """Get the root folder of the git repository.""" - get_repo_call = subprocess.Popen("git rev-parse --show-toplevel", - shell=True, - cwd=some_folder_in_root_repo, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + """Get the root folder of the git repository.""" + get_repo_call = subprocess.Popen( + "git rev-parse --show-toplevel", + shell=True, + cwd=some_folder_in_root_repo, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) - stdout, _ = get_repo_call.communicate() - repo_root = stdout.rstrip() - return repo_root + stdout, _ = get_repo_call.communicate() + repo_root = stdout.rstrip() + return repo_root def command_exists(cmd): - """Check if a bash command exists.""" - return subprocess.call("type " + cmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + """Check if a bash command exists.""" + return subprocess.call( + "type " + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0 + + +def find_clang_format_executable(): + for executable in CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS: + if subprocess.call( + "type " + executable, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0: + return executable + + print("ERROR: clang-format-diff is not installed!") + exit(1) def get_hooks_folder_for_repo(): - # Get git root folder of parent repository. - repo_root = get_git_repo_root('.') - hooks_folder = os.path.join(repo_root, '.git/hooks') - if os.path.isfile(os.path.join(repo_root, '.git')): - # Git repo is a submodule, git information is in a folder listed in the .git - # file. - with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: - git_folder = yaml.load(git_info_file)['gitdir'] - repo_root = get_git_repo_root(os.path.join(repo_root, git_folder)) - hooks_folder = os.path.join(repo_root, git_folder, 'hooks') - return hooks_folder + # Get git root folder of parent repository. + repo_root = get_git_repo_root('.') + hooks_folder = os.path.join(repo_root, '.git/hooks') + if os.path.isfile(os.path.join(repo_root, '.git')): + # Git repo is a submodule, git information is in a folder listed in the .git + # file. + with open(os.path.join(repo_root, '.git'), 'r') as git_info_file: + git_folder = yaml.load(git_info_file)['gitdir'] + repo_root = get_git_repo_root(os.path.join(repo_root, git_folder)) + hooks_folder = os.path.join(repo_root, git_folder, 'hooks') + return hooks_folder def install_linter(): - """ Download cpplint.py and pylint.py and installs the git hooks""" - script_directory = os.path.dirname(sys.argv[0]) - script_directory = os.path.dirname(os.path.abspath(script_directory)) - - if cpplint_url != "": - # Download linter files. - download_file_from_url(cpplint_url, script_directory + "/cpplint.py") - if not os.path.isfile(script_directory + "/cpplint.py"): - print("ERROR: Could not download cpplint.py file!") - exit(1) - else: - cp_params = (script_directory + "/default/" + default_cpplint + " " + - script_directory + "/cpplint.py") + """ Download cpplint.py and pylint.py and installs the git hooks""" + script_directory = os.path.dirname(sys.argv[0]) + script_directory = os.path.dirname(os.path.abspath(script_directory)) + + if cpplint_url != "": + # Download linter files. + download_file_from_url(cpplint_url, script_directory + "/cpplint.py") + if not os.path.isfile(script_directory + "/cpplint.py"): + print("ERROR: Could not download cpplint.py file!") + exit(1) + else: + cp_params = (script_directory + "/default/" + default_cpplint + " " + + script_directory + "/cpplint.py") + if subprocess.call("cp " + cp_params, shell=True) != 0: + print("Failed to copy default cpplint.") + exit(1) + + default_pylint_file = os.path.join(script_directory, 'default', + 'pylint.rc') + if not os.path.isfile(default_pylint_file): + print("Default pylint.rc wasn't found under", default_pylint_file) + exit(1) + + cp_params = (default_pylint_file + " " + os.path.join( + script_directory, 'pylint.rc')) if subprocess.call("cp " + cp_params, shell=True) != 0: - print("Failed to copy default cpplint.") - exit(1) + print("Failed to copy default pylint.rc.") + exit(1) - default_pylint_file = os.path.join(script_directory, 'default', 'pylint.rc') - if not os.path.isfile(default_pylint_file): - print("Default pylint.rc wasn't found under", default_pylint_file) - exit(1) + if not os.path.isfile(script_directory + "/pylint.rc"): + print("ERROR: Could not download pylint.rc file!") + exit(1) - cp_params = (default_pylint_file + " " + os.path.join(script_directory, 'pylint.rc')) - if subprocess.call("cp " + cp_params, shell=True) != 0: - print("Failed to copy default pylint.rc.") - exit(1) + find_clang_format_executable() - if not os.path.isfile(script_directory + "/pylint.rc"): - print("ERROR: Could not download pylint.rc file!") - exit(1) - - if not command_exists(clang_format_diff_executable): - print("ERROR: " + clang_format_diff_executable + " is not installed!") - exit(1) + if not command_exists("yapf"): + print("ERROR: yapf is not installed! Try: pip install yapf") + exit(1) - if not command_exists("yapf"): - print("ERROR: yapf is not installed! Try: pip install yapf") - exit(1) - - # Copy git hooks. - hooks_folder = get_hooks_folder_for_repo() - cp_params = script_directory + "/git_hooks.py " + \ - hooks_folder + "/pre-commit" - if subprocess.call("cp " + cp_params, shell=True) != 0: - print("Failed to copy githooks to " - "{}...".format(hooks_folder)) - exit(1) + # Copy git hooks. + hooks_folder = get_hooks_folder_for_repo() + cp_params = script_directory + "/git_hooks.py " + \ + hooks_folder + "/pre-commit" + if subprocess.call("cp " + cp_params, shell=True) != 0: + print("Failed to copy githooks to " "{}...".format(hooks_folder)) + exit(1) - print("Success, githooks initialized!") + print("Success, githooks initialized!") def remove_linter(): - hooks_folder = get_hooks_folder_for_repo() - pre_commit_file = os.path.join(hooks_folder, 'pre-commit') - with open(pre_commit_file, 'w+') as out_file: - out_file.write('#!/bin/sh\n') - print("Linter githooks removed!") + hooks_folder = get_hooks_folder_for_repo() + pre_commit_file = os.path.join(hooks_folder, 'pre-commit') + with open(pre_commit_file, 'w+') as out_file: + out_file.write('#!/bin/sh\n') + print("Linter githooks removed!") def main(): - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument( - '--remove', - dest='remove_linter', - action="store_true", - help='Remove the linter from this repository.') - args = arg_parser.parse_args() + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + '--remove', + dest='remove_linter', + action="store_true", + help='Remove the linter from this repository.') + args = arg_parser.parse_args() - if args.remove_linter: - remove_linter() - else: - install_linter() + if args.remove_linter: + remove_linter() + else: + install_linter() if __name__ == "__main__": - main() + main() diff --git a/linter.py b/linter.py index a2942e2..787f346 100755 --- a/linter.py +++ b/linter.py @@ -3,463 +3,490 @@ # Disable pylint invalid module name complaint. # pylint : disable = C0103 - """ Pre-commit script for invoking linters and auto-formatters.""" # Make sure we use the new print function from __future__ import print_function -from random import randint import datetime import imp import os -import pylint.lint import re import subprocess + +import pylint.lint import yaml -CLANG_FORMAT_DIFF_EXECUTABLE = "clang-format-diff-6.0" +CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS = [ + "clang-format-diff-6.0", "clang-format-diff-5.0", "clang-format-diff-4.0", + "clang-format-diff-3.9", "clang-format-diff-3.8" +] + YAPF_FORMAT_EXECUTABLE = "yapf" DEFAULT_CONFIG = { - 'use_clangformat': True, - 'use_cpplint': True, - # Enable Python checks by default. - 'use_yapf': True, - 'use_pylint': True, - # Check all staged files by default. - 'whitelist': [] + 'use_clangformat': True, + 'use_cpplint': True, + # Enable Python checks by default. + 'use_yapf': True, + 'use_pylint': True, + # Check all staged files by default. + 'whitelist': [] } def read_linter_config(filename): - """Parses yaml config file.""" + """Parses yaml config file.""" - config = DEFAULT_CONFIG - with open(filename, 'r') as ymlfile: - parsed_config = yaml.load(ymlfile) + config = DEFAULT_CONFIG + with open(filename, 'r') as ymlfile: + parsed_config = yaml.load(ymlfile) - if 'clangformat' in parsed_config.keys(): - config['use_clangformat'] = parsed_config['clangformat'] - if 'cpplint' in parsed_config.keys(): - config['use_cpplint'] = parsed_config['cpplint'] - if 'yapf' in parsed_config.keys(): - config['use_yapf'] = parsed_config['yapf'] - if 'pylint' in parsed_config.keys(): - config['use_pylint'] = parsed_config['pylint'] - if 'whitelist' in parsed_config.keys(): - config['whitelist'] = parsed_config['whitelist'] + if 'clangformat' in parsed_config.keys(): + config['use_clangformat'] = parsed_config['clangformat'] + if 'cpplint' in parsed_config.keys(): + config['use_cpplint'] = parsed_config['cpplint'] + if 'yapf' in parsed_config.keys(): + config['use_yapf'] = parsed_config['yapf'] + if 'pylint' in parsed_config.keys(): + config['use_pylint'] = parsed_config['pylint'] + if 'whitelist' in parsed_config.keys(): + config['whitelist'] = parsed_config['whitelist'] - return config + return config def run_command_in_folder(command, folder): - """Run a bash command in a specific folder.""" - run_command = subprocess.Popen(command, - shell=True, - cwd=folder, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - stdout, _ = run_command.communicate() - command_output = stdout.rstrip() - return command_output + """Run a bash command in a specific folder.""" + run_command = subprocess.Popen( + command, + shell=True, + cwd=folder, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, _ = run_command.communicate() + command_output = stdout.rstrip() + return command_output def get_number_of_commits(some_folder_in_root_repo='./'): - command = "git shortlog -s -n --all --author=\"$(git config --get user.email)\" | grep -o '[0-9]\+' | awk '{s+=$1} END {print s}'" - num_commits = run_command_in_folder(command, some_folder_in_root_repo) - if not num_commits: - num_commits = "0" - return num_commits + command = ( + "git shortlog -s -n --all --author=\"$(git config " + + "--get user.email)\" | grep -o '[0-9]\+' | " + # pylint: disable=W1401 + "awk '{s+=$1} END {print s}'") + num_commits = run_command_in_folder(command, some_folder_in_root_repo) + if not num_commits: + num_commits = "0" + return num_commits def get_staged_files(some_folder_in_root_repo='./'): - """Get all staged files from git.""" - output = run_command_in_folder( - "git diff --staged --name-only", some_folder_in_root_repo) - if len(output) == 0: - return [] - else: - return output.split("\n") + """Get all staged files from git.""" + output = run_command_in_folder("git diff --staged --name-only", + some_folder_in_root_repo) + if not output: + return [] + else: + return output.split("\n") def get_unstaged_files(some_folder_in_root_repo='./'): - """Get all unstaged files from git.""" - output = run_command_in_folder( - "git diff --name-only", some_folder_in_root_repo) - return output.split("\n") + """Get all unstaged files from git.""" + output = run_command_in_folder("git diff --name-only", + some_folder_in_root_repo) + return output.split("\n") def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): - """Runs Google's cpplint on all C++ files staged for commit,""" - cpplint = imp.load_source('cpplint', cpplint_file) - cpplint._cpplint_state.SetFilters( - '-legal/copyright,-build/c++11') # pylint: disable=W0212 - - # Prevent cpplint from writing to stdout directly, instead - # the errors will be stored in pplint.output as (line, message) tuples. - cpplint.print_stdout = False - - total_error_count = 0 - for changed_file in staged_files: - if not os.path.isfile(changed_file): - continue - if changed_file.lower().endswith(('.cc', '.h', '.cpp', '.cu', '.cuh')): - # Search iteratively for the root of the catkin package. - package_root = '' - search_dir = os.path.dirname(os.path.abspath(changed_file)) - found_package_root = False - MAX_DEPTH_OF_FILES = 100 - for _ in range(1, MAX_DEPTH_OF_FILES): - if os.path.isfile(search_dir + '/package.xml'): - package_root = search_dir - found_package_root = True - break - # Stop if the root of the git repo is reached. - if os.path.isdir(search_dir + '/.git'): - break - search_dir = os.path.dirname(search_dir) - assert found_package_root, ("Could not find the root of the " - "catkin package that contains: " - "{}".format(changed_file)) - - # Get relative path to repository root. - if os.path.isfile(os.path.join(repo_root, '.git')): - # Repo is a submodule, look for parent git repository containing this - # submodule. - search_path = repo_root - found_to_level_git_repo = False - for _ in range(10): - search_path = os.path.dirname(repo_root) - if os.path.isdir(os.path.join(search_path, '.git')): - found_to_level_git_repo = True - break - - assert found_to_level_git_repo - repo_root = search_path - - common_prefix = os.path.commonprefix([ - os.path.abspath(repo_root), os.path.abspath(package_root)]) - package_root = os.path.relpath(package_root, common_prefix) - - # The package root needs to be relative to the (top-level) repo root. - # Otherwise the header guard logic will fail! - cpplint._root = package_root + '/include' # pylint: disable=W0212 - - # Reset error count and messages: - cpplint.output = [] - cpplint._cpplint_state.ResetErrorCounts() # pylint: disable=W0212 - v_level = cpplint._cpplint_state.verbose_level # pylint: disable=W0212 - - cpplint.ProcessFile(changed_file, v_level) - - error_count = cpplint._cpplint_state.error_count # pylint: disable=W0212 - if error_count > 0: - total_error_count += error_count + """Runs Google's cpplint on all C++ files staged for commit,""" + cpplint = imp.load_source('cpplint', cpplint_file) + cpplint._cpplint_state.SetFilters('-legal/copyright,-build/c++11') # pylint: disable=W0212 + + # Prevent cpplint from writing to stdout directly, instead + # the errors will be stored in pplint.output as (line, message) tuples. + cpplint.print_stdout = False + + total_error_count = 0 + for changed_file in staged_files: + if not os.path.isfile(changed_file): + continue + if changed_file.lower().endswith(('.cc', '.h', '.cpp', '.cu', '.cuh')): + # Search iteratively for the root of the catkin package. + package_root = '' + search_dir = os.path.dirname(os.path.abspath(changed_file)) + found_package_root = False + MAX_DEPTH_OF_FILES = 100 + for _ in range(1, MAX_DEPTH_OF_FILES): + if os.path.isfile(search_dir + '/package.xml'): + package_root = search_dir + found_package_root = True + break + # Stop if the root of the git repo is reached. + if os.path.isdir(search_dir + '/.git'): + break + search_dir = os.path.dirname(search_dir) + assert found_package_root, ("Could not find the root of the " + "catkin package that contains: " + "{}".format(changed_file)) + + # Get relative path to repository root. + if os.path.isfile(os.path.join(repo_root, '.git')): + # Repo is a submodule, look for parent git repository containing + # this submodule. + search_path = repo_root + found_to_level_git_repo = False + for _ in range(10): + search_path = os.path.dirname(repo_root) + if os.path.isdir(os.path.join(search_path, '.git')): + found_to_level_git_repo = True + break + + assert found_to_level_git_repo + repo_root = search_path + + common_prefix = os.path.commonprefix( + [os.path.abspath(repo_root), + os.path.abspath(package_root)]) + package_root = os.path.relpath(package_root, common_prefix) + + # The package root needs to be relative to the (top-level) repo + # root. Otherwise the header guard logic will fail! + cpplint._root = package_root + '/include' # pylint: disable=W0212 + + # Reset error count and messages: + cpplint.output = [] + cpplint._cpplint_state.ResetErrorCounts() # pylint: disable=W0212 + v_level = cpplint._cpplint_state.verbose_level # pylint: disable=W0212 + + cpplint.ProcessFile(changed_file, v_level) + + error_count = cpplint._cpplint_state.error_count # pylint: disable=W0212 + if error_count > 0: + total_error_count += error_count + + print("-" * 80) + print("Found {} errors in : {}".format(error_count, + changed_file)) + print("-" * 80) + for line in cpplint.output: + assert len(line) == 2 + print("line {: >4}:\t {}".format(line[0], line[1])) + + if total_error_count > 0: + print("=" * 80) + print("Found {} cpplint errors".format(total_error_count)) + print("=" * 80) - print("-" * 80) - print("Found {} errors in : {}".format( - error_count, changed_file)) - print("-" * 80) - for line in cpplint.output: - assert len(line) == 2 - print("line {: >4}:\t {}".format(line[0], line[1])) - - if total_error_count > 0: - print("=" * 80) - print("Found {} cpplint errors".format(total_error_count)) - print("=" * 80) - - if total_error_count > 50: - print(ascii_art.AsciiArt.cthulhu) - elif total_error_count > 20: - print(ascii_art.AsciiArt.tiger) - return False - else: - return True + if total_error_count > 50: + print(ascii_art.AsciiArt.cthulhu) + elif total_error_count > 20: + print(ascii_art.AsciiArt.tiger) + return False + else: + return True def check_modified_after_staging(staged_files): - """Checks if one of the staged files was modified after staging.""" - files_changed = get_unstaged_files() - files_changed = filter(None, files_changed) # pylint: disable=W0141 + """Checks if one of the staged files was modified after staging.""" + files_changed = get_unstaged_files() + files_changed = filter(None, files_changed) - staged_files_changed = 0 + staged_files_changed = 0 - staged_files_changed_list = [] + staged_files_changed_list = [] - is_first = True - for changed_file in files_changed: - if changed_file in staged_files: + is_first = True + for changed_file in files_changed: + if changed_file in staged_files: - if is_first: - print("=" * 80) - is_first = False + if is_first: + print("=" * 80) + is_first = False - print("\'{}\' was modified after staging".format(changed_file)) - staged_files_changed_list.append(changed_file) - staged_files_changed += 1 + print("\'{}\' was modified after staging".format(changed_file)) + staged_files_changed_list.append(changed_file) + staged_files_changed += 1 - if staged_files_changed > 0: - print("-" * 80) - print("Found {} files that have been changed after staging".format( - staged_files_changed)) - print("=" * 80) + if staged_files_changed > 0: + print("-" * 80) + print("Found {} files that have been changed after staging".format( + staged_files_changed)) + print("=" * 80) - return staged_files_changed_list + return staged_files_changed_list def check_commit_against_master(repo_root): - """Check if the current commit is intended to for the master branch.""" - current_branch = run_command_in_folder( - "git branch | grep \"*\" | sed \"s/* //\"", repo_root) - print("\nCurrent_branch: {}\n".format(current_branch)) - return current_branch == "master" + """Check if the current commit is intended to for the master branch.""" + current_branch = run_command_in_folder( + "git branch | grep \"*\" | sed \"s/* //\"", repo_root) + print("\nCurrent_branch: {}\n".format(current_branch)) + return current_branch == "master" def check_if_merge_commit(repo_root): - """Check if the current commit is a merge commit.""" - merge_msg_file_path = repo_root + "/.git/MERGE_MSG" - return os.path.isfile(merge_msg_file_path) + """Check if the current commit is a merge commit.""" + merge_msg_file_path = repo_root + "/.git/MERGE_MSG" + return os.path.isfile(merge_msg_file_path) -def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): - """Runs clang format on all cpp files staged for commit.""" +def find_clang_format_executable(): + for executable in CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS: + if subprocess.call( + "type " + executable, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0: + return executable - clang_format_path = ("/tmp/" + os.path.basename( - os.path.normpath(repo_root)) + "_" + - datetime.datetime.now().isoformat() + ".clang.patch") + print("ERROR: clang-format-diff is not installed!") + exit(1) - run_command_in_folder("git diff -U0 --cached | " + - CLANG_FORMAT_DIFF_EXECUTABLE + - " -style=file -p1 > " + - clang_format_path, repo_root) - if not os.stat(clang_format_path).st_size == 0: - if list_of_changed_staged_files: - print("=" * 80) - print("Cannot format your code, because some files are \n" - "only partially staged! Format your code or try \n" - "stashing your unstaged changes...") - print("=" * 80) - exit(1) +def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): + """Runs clang format on all cpp files staged for commit.""" - run_command_in_folder("git apply -p0 " + clang_format_path, repo_root) - run_command_in_folder("git add " + ' '.join(staged_files), repo_root) + clang_format_path = ( + "/tmp/" + os.path.basename(os.path.normpath(repo_root)) + "_" + + datetime.datetime.now().isoformat() + ".clang.patch") - print("=" * 80) - print("Formatted staged C++ files with clang-format.\n" + - "Patch: {}".format(clang_format_path)) - print("=" * 80) - return True + clang_format_diff_executable = find_clang_format_executable() + run_command_in_folder( + "git diff -U0 --cached | " + clang_format_diff_executable + + " -style=file -p1 > " + clang_format_path, repo_root) -def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): - """Runs yapf format on all python files staged for commit.""" - - first_file_formatted = True - for staged_file in staged_files: - if not os.path.isfile(staged_file): - continue - if staged_file.endswith((".py")): - - # Check if the file needs formatting by applying the formatting and store - # the results into a patch file. - yapf_format_path = ("/tmp/" + - os.path.basename(os.path.normpath(repo_root)) + - "_" + datetime.datetime.now().isoformat() + - ".yapf.patch") - task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + staged_file + - " > " + yapf_format_path) - run_command_in_folder(task, repo_root) - - if not os.stat(yapf_format_path).st_size == 0: - if first_file_formatted: - print("=" * 80) - print("Formatted staged python files with autopep8:") - first_file_formatted = False - - print("-> " + staged_file) - - if staged_file in list_of_changed_staged_files: - print("Cannot format your code, because this file is \n" - "only partially staged! Format your code or try \n" - "stashing your unstaged changes...") - print("=" * 80) - exit(1) - else: - run_command_in_folder( - "git apply -p0 " + yapf_format_path, repo_root) - run_command_in_folder("git add " + staged_file, repo_root) - - if not first_file_formatted: - print("=" * 80) - return True + if not os.stat(clang_format_path).st_size == 0: + if list_of_changed_staged_files: + print("=" * 80) + print("Cannot format your code, because some files are \n" + "only partially staged! Format your code or try \n" + "stashing your unstaged changes...") + print("=" * 80) + exit(1) + run_command_in_folder("git apply -p0 " + clang_format_path, repo_root) + run_command_in_folder("git add " + ' '.join(staged_files), repo_root) -def check_python_lint(repo_root, staged_files, pylint_file): - """Runs pylint on all python scripts staged for commit.""" - - class TextReporterBuffer(object): - """Stores the output produced by the pylint TextReporter.""" - - def __init__(self): - """init""" - self.content = [] - - def write(self, input_str): - """write""" - self.content.append(input_str) - - def read(self): - """read""" - return self.content - - # Parse each pylint output line individualy and searches - # for errors in the code. - pylint_errors = [] - for changed_file in staged_files: - if not os.path.isfile(changed_file): - continue - if re.search(r'\.py$', changed_file): - - print("Running pylint on \'{}\'".format(repo_root + "/" + changed_file)) - pylint_output = TextReporterBuffer() - pylint_args = ["--rcfile=" + pylint_file, - "-rn", - repo_root + "/" + changed_file] - from pylint.reporters.text import TextReporter - pylint.lint.Run(pylint_args, - reporter=TextReporter(pylint_output), - exit=False) - - for output_line in pylint_output.read(): - if re.search(r'^(E|C|W):', output_line): - print(changed_file + ": " + output_line) - pylint_errors.append(output_line) - - if len(pylint_errors) > 0: - print("=" * 80) - print("Found {} pylint errors".format(len(pylint_errors))) - print("=" * 80) - return False - else: + print("=" * 80) + print("Formatted staged C++ files with clang-format.\n" + + "Patch: {}".format(clang_format_path)) + print("=" * 80) return True -def get_whitelisted_files(repo_root, files, whitelist): - whitelisted = []; +def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): + """Runs yapf format on all python files staged for commit.""" + + first_file_formatted = True + for staged_file in staged_files: + if not os.path.isfile(staged_file): + continue + if staged_file.endswith((".py")): + + # Check if the file needs formatting by applying the formatting and + # store the results into a patch file. + yapf_format_path = ( + "/tmp/" + os.path.basename(os.path.normpath(repo_root)) + "_" + + datetime.datetime.now().isoformat() + ".yapf.patch") + task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + staged_file + + " > " + yapf_format_path) + run_command_in_folder(task, repo_root) + + if not os.stat(yapf_format_path).st_size == 0: + if first_file_formatted: + print("=" * 80) + print("Formatted staged python files with autopep8:") + first_file_formatted = False + + print("-> " + staged_file) + + if staged_file in list_of_changed_staged_files: + print("Cannot format your code, because this file is \n" + "only partially staged! Format your code or try \n" + "stashing your unstaged changes...") + print("=" * 80) + exit(1) + else: + run_command_in_folder("git apply -p0 " + yapf_format_path, + repo_root) + run_command_in_folder("git add " + staged_file, repo_root) + + if not first_file_formatted: + print("=" * 80) + return True - for file in files: - for entry in whitelist: - # Add trailing slash if its a directory and no slash is already there. - if os.path.isdir(repo_root + '/' + entry): - entry = os.path.join(os.path.normpath(entry), '') - # Check if the file itself or its parent directory is in the whitelist. - if (file == entry - or os.path.commonprefix([file, entry]) == entry): - whitelisted.append(file) - break +def check_python_lint(repo_root, staged_files, pylint_file): + """Runs pylint on all python scripts staged for commit.""" + + class TextReporterBuffer(object): + """Stores the output produced by the pylint TextReporter.""" + + def __init__(self): + """init""" + self.content = [] + + def write(self, input_str): + """write""" + self.content.append(input_str) + + def read(self): + """read""" + return self.content + + # Parse each pylint output line individualy and searches + # for errors in the code. + pylint_errors = [] + for changed_file in staged_files: + if not os.path.isfile(changed_file): + continue + if re.search(r'\.py$', changed_file): + + print("Running pylint on \'{}\'".format(repo_root + "/" + + changed_file)) + pylint_output = TextReporterBuffer() + pylint_args = [ + "--rcfile=" + pylint_file, "-rn", + repo_root + "/" + changed_file + ] + from pylint.reporters.text import TextReporter + pylint.lint.Run( + pylint_args, reporter=TextReporter(pylint_output), exit=False) + + for output_line in pylint_output.read(): + if re.search(r'^(E|C|W):', output_line): + print(changed_file + ": " + output_line) + pylint_errors.append(output_line) + + if pylint_errors: + print("=" * 80) + print("Found {} pylint errors".format(len(pylint_errors))) + print("=" * 80) + return False + else: + return True - return whitelisted +def get_whitelisted_files(repo_root, files, whitelist): + whitelisted = [] -def linter_check(repo_root, linter_subfolder): - """ Main pre-commit function for calling code checking script. """ - - cpplint_file = os.path.join(linter_subfolder, "cpplint.py") - pylint_file = os.path.join(linter_subfolder, "pylint.rc") - ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") - - # Read linter config file. - linter_config_file = repo_root + '/.linterconfig.yaml' - if os.path.isfile(repo_root + '/.linterconfig.yaml'): - print("Found repo linter config: {}".format(linter_config_file)) - linter_config = read_linter_config(linter_config_file) - else: - linter_config = DEFAULT_CONFIG - - print("Found linter subfolder: {}".format(linter_subfolder)) - print("Found ascii art file at: {}".format(ascii_art_file)) - print("Found cpplint file at: {}".format(cpplint_file)) - print("Found pylint file at: {}".format(pylint_file)) - - # Run checks - staged_files = get_staged_files() - - if len(staged_files) == 0: - print("\n") - print("=" * 80) - print("No files staged...") - print("=" * 80) - exit(1) + for file_name in files: + for entry in whitelist: + # Add trailing slash if its a directory and no slash is already + # there. + if os.path.isdir(repo_root + '/' + entry): + entry = os.path.join(os.path.normpath(entry), '') - if len(linter_config['whitelist']) > 0: - whitelisted_files = get_whitelisted_files(repo_root, staged_files, - linter_config['whitelist']) - else: - whitelisted_files = staged_files + # Check if the file itself or its parent directory is in the + # whitelist. + if (file_name == entry + or os.path.commonprefix([file_name, entry]) == entry): + whitelisted.append(file_name) + break - # Load ascii art. - ascii_art = imp.load_source('ascii_art', ascii_art_file) + return whitelisted - if check_commit_against_master(repo_root): - print(ascii_art.AsciiArt.grumpy_cat) - exit(1) - if not check_if_merge_commit(repo_root): - # Do not allow commiting files that were modified after staging. This - # avoids problems such as forgetting to stage fixes of cpplint complaints. - list_of_changed_staged_files = check_modified_after_staging( - whitelisted_files) +def linter_check(repo_root, linter_subfolder): + """ Main pre-commit function for calling code checking script. """ - if linter_config['use_clangformat']: - run_clang_format(repo_root, whitelisted_files, - list_of_changed_staged_files) + cpplint_file = os.path.join(linter_subfolder, "cpplint.py") + pylint_file = os.path.join(linter_subfolder, "pylint.rc") + ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") - if linter_config['use_yapf']: - run_yapf_format(repo_root, whitelisted_files, - list_of_changed_staged_files) + # Read linter config file. + linter_config_file = repo_root + '/.linterconfig.yaml' + if os.path.isfile(repo_root + '/.linterconfig.yaml'): + print("Found repo linter config: {}".format(linter_config_file)) + linter_config = read_linter_config(linter_config_file) + else: + linter_config = DEFAULT_CONFIG + print("Found linter subfolder: {}".format(linter_subfolder)) + print("Found ascii art file at: {}".format(ascii_art_file)) + print("Found cpplint file at: {}".format(cpplint_file)) + print("Found pylint file at: {}".format(pylint_file)) - # Use Google's C++ linter to check for compliance with Google style guide. - if linter_config['use_cpplint']: - cpp_lint_success = check_cpp_lint( - whitelisted_files, cpplint_file, ascii_art, repo_root) - else: - cpp_lint_success = True + # Run checks + staged_files = get_staged_files() - # Use pylint to check for comimpliance with Tensofrflow python style guide. - if linter_config['use_pylint']: - pylint_success = check_python_lint(repo_root, whitelisted_files, pylint_file) - else: - pylint_success = True + if not staged_files: + print("\n") + print("=" * 80) + print("No files staged...") + print("=" * 80) + exit(1) - if not(cpp_lint_success and pylint_success): - print("=" * 80) - print("Commit rejected! Please address the linter errors above.") - print("=" * 80) - exit(1) + if linter_config['whitelist']: + whitelisted_files = get_whitelisted_files(repo_root, staged_files, + linter_config['whitelist']) else: + whitelisted_files = staged_files + + # Load ascii art. + ascii_art = imp.load_source('ascii_art', ascii_art_file) + + if check_commit_against_master(repo_root): + print(ascii_art.AsciiArt.grumpy_cat) + exit(1) + + if not check_if_merge_commit(repo_root): + # Do not allow commiting files that were modified after staging. This + # avoids problems such as forgetting to stage fixes of cpplint + # complaints. + list_of_changed_staged_files = check_modified_after_staging( + whitelisted_files) + + if linter_config['use_clangformat']: + run_clang_format(repo_root, whitelisted_files, + list_of_changed_staged_files) + + if linter_config['use_yapf']: + run_yapf_format(repo_root, whitelisted_files, + list_of_changed_staged_files) + + # Use Google's C++ linter to check for compliance with Google style + # guide. + if linter_config['use_cpplint']: + cpp_lint_success = check_cpp_lint(whitelisted_files, cpplint_file, + ascii_art, repo_root) + else: + cpp_lint_success = True - commit_number = get_number_of_commits(repo_root) - lucky_commit = ((int(commit_number) + 1) % 42 == 0) - - if lucky_commit: - print(ascii_art.AsciiArt.story) + # Use pylint to check for comimpliance with Tensofrflow python + # style guide. + if linter_config['use_pylint']: + pylint_success = check_python_lint(repo_root, whitelisted_files, + pylint_file) + else: + pylint_success = True - print("=" * 80) - print("Commit accepted, well done! This is your {}th commit!".format( - commit_number)) - print("This is a lucky commit! Please enjoy this free sheep story.") - print("=" * 80) - else: - print(ascii_art.AsciiArt.commit_success) + if not (cpp_lint_success and pylint_success): + print("=" * 80) + print("Commit rejected! Please address the linter errors above.") + print("=" * 80) + exit(1) + else: - print("=" * 80) - print("Commit accepted, well done! This is the {}th commit!".format( - commit_number)) - print("=" * 80) - else: - print(ascii_art.AsciiArt.homer_woohoo) + commit_number = get_number_of_commits(repo_root) + lucky_commit = ((int(commit_number) + 1) % 42 == 0) + + if lucky_commit: + print(ascii_art.AsciiArt.story) + + print("=" * 80) + print("Commit accepted, well done! This is your {}th commit!". + format(commit_number)) + print("This is a lucky commit! " + + "Please enjoy this free sheep story.") + print("=" * 80) + else: + print(ascii_art.AsciiArt.commit_success) + + print("=" * 80) + print("Commit accepted, well done! This is the {}th commit!". + format(commit_number)) + print("=" * 80) + else: + print(ascii_art.AsciiArt.homer_woohoo) From 671121736561cd5b98f4ea109add02f715066c11 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Mon, 25 Jun 2018 11:07:17 +0200 Subject: [PATCH 24/56] add generic clang format executable --- bin/init_linter_git_hooks | 4 ++-- linter.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index c62e066..c6f8383 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -37,8 +37,8 @@ cpplint_url = "" clang_format_diff_executable = "clang-format-diff-6.0" CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS = [ - "clang-format-diff-6.0", "clang-format-diff-5.0", "clang-format-diff-4.0", - "clang-format-diff-3.9", "clang-format-diff-3.8" + "clang-format-diff", "clang-format-diff-6.0", "clang-format-diff-5.0", + "clang-format-diff-4.0", "clang-format-diff-3.9", "clang-format-diff-3.8" ] diff --git a/linter.py b/linter.py index 787f346..e77fdab 100755 --- a/linter.py +++ b/linter.py @@ -18,8 +18,8 @@ import yaml CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS = [ - "clang-format-diff-6.0", "clang-format-diff-5.0", "clang-format-diff-4.0", - "clang-format-diff-3.9", "clang-format-diff-3.8" + "clang-format-diff", "clang-format-diff-6.0", "clang-format-diff-5.0", + "clang-format-diff-4.0", "clang-format-diff-3.9", "clang-format-diff-3.8" ] YAPF_FORMAT_EXECUTABLE = "yapf" From 7b5d274bafc3f7d3d94eecdfb7a512a7013b869d Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Mon, 25 Jun 2018 11:08:15 +0200 Subject: [PATCH 25/56] rm unused line --- bin/init_linter_git_hooks | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index c6f8383..76aa206 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -34,8 +34,6 @@ default_cpplint = "modified_cpplint.py" cpplint_url = "" -clang_format_diff_executable = "clang-format-diff-6.0" - CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS = [ "clang-format-diff", "clang-format-diff-6.0", "clang-format-diff-5.0", "clang-format-diff-4.0", "clang-format-diff-3.9", "clang-format-diff-3.8" From 242ca01238ba66b5f73aa610df4c3725bc103e22 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Mon, 25 Jun 2018 12:19:45 +0200 Subject: [PATCH 26/56] Adapt README --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 418c9a6..0fa9718 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # linter -This repo contains a (C++, python (experimental)) linter and auto formatter package that can be included into your repository as a submodule. It provides the following git hooks: +This repo contains a (C++, python) linter and auto formatter package that can be conveniently installed into your repositories using git hooks. It provides the following git hooks: * **General** * Prevent commits to master. * **C++** files: - * **clang-format** Formats your code based on your .clang-format preferences. + * + * **clang-format** Formats your code based on your `.clang-format` preferences. * **cpplint** Checks your C++ code for style errors and warnings. * **Python** files: @@ -18,11 +19,11 @@ This repo contains a (C++, python (experimental)) linter and auto formatter pack * **yapf** * Ubuntu / macOS: `pip install yapf` * **clang-format** - * Ubuntu: `sudo apt install clang-format-3.8` + * Compatible with `clang-format-3.8 - 6.0` + * Ubuntu: `sudo apt install clang-format-${VERSION}` * macOS: ``` brew install clang-format - ln -s /usr/local/share/clang/clang-format-diff.py /usr/local/bin/clang-format-diff-3.8 ``` @@ -50,9 +51,14 @@ init_linter_git_hooks --remove ``` ## Linter configuration + +**General** + To configure the linter, add a file named `.linterconfig.yaml` in your repository root. An example file is given under [`linterconfig.yaml_example`](https://github.com/ethz-asl/linter/blob/master/linterconfig.yaml_example). -Clang-format can be configured by defining a project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example file: +**C++** + +clang-format can be configured by defining a project-specific C++ format by adding a file `.clang-format` to your projects root folder. Example file: ``` --- @@ -73,6 +79,10 @@ IncludeCategories: ... ``` +**Python** + +Currently there it is not possible to configure the python formatter on a per-repository basis. + #### ASCII-Art Sources * [www.retrojunkie.com (accessed through web.archive.org)](https://web.archive.org/web/20150831003349/http://www.retrojunkie.com:80/asciiart/) From c7bc193378b45f1c978052b32e73cd099ca247f9 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Mon, 25 Jun 2018 13:32:05 +0200 Subject: [PATCH 27/56] Fix README for OSX, add section about disabling linter functions --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0fa9718..92292c7 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ This repo contains a (C++, python) linter and auto formatter package that can be ## Dependencies + * **pylint** + * macOS: + ``` + pip install pylint + ``` * **yapf** * Ubuntu / macOS: `pip install yapf` * **clang-format** @@ -24,6 +29,7 @@ This repo contains a (C++, python) linter and auto formatter package that can be * macOS: ``` brew install clang-format + ln -s /usr/local/share/clang/clang-format-diff.py /usr/local/bin/clang-format-diff ``` @@ -34,8 +40,7 @@ git clone git@github.com:ethz-asl/linter.git cd linter echo ". $(realpath setup_linter.sh)" >> ~/.bashrc # Or the matching file for # your shell. -source ~/.bashrc -``` +source ~ Then you can install the linter in your repository: ```bash @@ -83,6 +88,41 @@ IncludeCategories: Currently there it is not possible to configure the python formatter on a per-repository basis. -#### ASCII-Art Sources + +## Disable Linter Functionalities for a Specific Line + + * **C++ Linter (`cpplint`):** + ```cpp + void your_awful_function(int & result) // NOLINT + ``` + * **C++ Formatting (`clang-format`):** + ```cpp + // clang-format off + ... + // clang-format on + ``` + * **Python Linter (`pylint`)** + + For whole file: + ```python + #!/usr/bin/env python + # pylint: disable=C0103,E1101 + ... + ``` + For a line: + ```Python + your_awful_function('-legal/copyright,-build/c++11') # pylint: disable=W0212 + ``` + The full list of pylint warnings and errors can be found [here](http://pylint-messages.wikidot.com/all-messages) + + * **Python Formatting (`yapf`)** + ```python + # yapf: disable + ... + # yapf: enable + ``` + + +## ASCII-Art Sources * [www.retrojunkie.com (accessed through web.archive.org)](https://web.archive.org/web/20150831003349/http://www.retrojunkie.com:80/asciiart/) From 6bd2994b37db39d37a58c89a2274668bb1fefd4d Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Tue, 31 Jul 2018 16:03:27 +0200 Subject: [PATCH 28/56] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 92292c7..88e30c4 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ git clone git@github.com:ethz-asl/linter.git cd linter echo ". $(realpath setup_linter.sh)" >> ~/.bashrc # Or the matching file for # your shell. -source ~ +bash +``` Then you can install the linter in your repository: ```bash From 27a9e4141e096f4faf2418b2fc83340d4fd43af7 Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Thu, 30 May 2019 13:19:45 -0400 Subject: [PATCH 29/56] Removed exit(1) on failure of linter check --- linter.py | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/linter.py b/linter.py index e77fdab..3967537 100755 --- a/linter.py +++ b/linter.py @@ -58,12 +58,11 @@ def read_linter_config(filename): def run_command_in_folder(command, folder): """Run a bash command in a specific folder.""" - run_command = subprocess.Popen( - command, - shell=True, - cwd=folder, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + run_command = subprocess.Popen(command, + shell=True, + cwd=folder, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) stdout, _ = run_command.communicate() command_output = stdout.rstrip() return command_output @@ -232,11 +231,10 @@ def check_if_merge_commit(repo_root): def find_clang_format_executable(): for executable in CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS: - if subprocess.call( - "type " + executable, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) == 0: + if subprocess.call("type " + executable, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0: return executable print("ERROR: clang-format-diff is not installed!") @@ -246,9 +244,9 @@ def find_clang_format_executable(): def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): """Runs clang format on all cpp files staged for commit.""" - clang_format_path = ( - "/tmp/" + os.path.basename(os.path.normpath(repo_root)) + "_" + - datetime.datetime.now().isoformat() + ".clang.patch") + clang_format_path = ("/tmp/" + + os.path.basename(os.path.normpath(repo_root)) + "_" + + datetime.datetime.now().isoformat() + ".clang.patch") clang_format_diff_executable = find_clang_format_executable() @@ -286,11 +284,12 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): # Check if the file needs formatting by applying the formatting and # store the results into a patch file. - yapf_format_path = ( - "/tmp/" + os.path.basename(os.path.normpath(repo_root)) + "_" + - datetime.datetime.now().isoformat() + ".yapf.patch") - task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + staged_file - + " > " + yapf_format_path) + yapf_format_path = ("/tmp/" + + os.path.basename(os.path.normpath(repo_root)) + + "_" + datetime.datetime.now().isoformat() + + ".yapf.patch") + task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + + staged_file + " > " + yapf_format_path) run_command_in_folder(task, repo_root) if not os.stat(yapf_format_path).st_size == 0: @@ -351,8 +350,9 @@ def read(self): repo_root + "/" + changed_file ] from pylint.reporters.text import TextReporter - pylint.lint.Run( - pylint_args, reporter=TextReporter(pylint_output), exit=False) + pylint.lint.Run(pylint_args, + reporter=TextReporter(pylint_output), + exit=False) for output_line in pylint_output.read(): if re.search(r'^(E|C|W):', output_line): @@ -464,9 +464,12 @@ def linter_check(repo_root, linter_subfolder): if not (cpp_lint_success and pylint_success): print("=" * 80) - print("Commit rejected! Please address the linter errors above.") + print( + "Commit not up to standards! Please address the linter errors above." + ) + print("All of these linter errors must be resolved before merge.") print("=" * 80) - exit(1) + else: commit_number = get_number_of_commits(repo_root) From f3f6f04a1fa666b5398abad53bcb6b73947f1b3c Mon Sep 17 00:00:00 2001 From: AutonomousHansen <50837800+AutonomousHansen@users.noreply.github.com> Date: Thu, 30 May 2019 13:30:43 -0400 Subject: [PATCH 30/56] Added docstrings required, check formatting --- default/pylint.rc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default/pylint.rc b/default/pylint.rc index 8afdb44..255e4bb 100644 --- a/default/pylint.rc +++ b/default/pylint.rc @@ -16,7 +16,7 @@ persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=pylint.extensions.docparams extension-pkg-whitelist=numpy @@ -31,7 +31,7 @@ extension-pkg-whitelist=numpy # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time. -disable=no-name-in-module,import-error,fixme,missing-docstring,no-member,wrong-import-position,bad-continuation +disable=no-name-in-module,import-error,fixme,no-member,wrong-import-position,bad-continuation [REPORTS] From 612ec9cd27a09a5a7c19362cb876e77bb32d92ef Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 08:20:48 -0400 Subject: [PATCH 31/56] tripping linter --- .linterconfig.yaml | 7 +++++++ linter.py | 8 ++++++++ linterconfig.yaml_example | 1 + 3 files changed, 16 insertions(+) create mode 100644 .linterconfig.yaml diff --git a/.linterconfig.yaml b/.linterconfig.yaml new file mode 100644 index 0000000..0fb4f69 --- /dev/null +++ b/.linterconfig.yaml @@ -0,0 +1,7 @@ +# Turn the components of the linter individualy on / off. +clangformat: on +cpplint: on +yapf: on +pylint: on +block-commits: off + diff --git a/linter.py b/linter.py index 3967537..9f63e5b 100755 --- a/linter.py +++ b/linter.py @@ -30,6 +30,8 @@ # Enable Python checks by default. 'use_yapf': True, 'use_pylint': True, + # Block commits that don't pass by default + 'block-commits': True, # Check all staged files by default. 'whitelist': [] } @@ -50,6 +52,8 @@ def read_linter_config(filename): config['use_yapf'] = parsed_config['yapf'] if 'pylint' in parsed_config.keys(): config['use_pylint'] = parsed_config['pylint'] + if 'block-commits' in parsed_config.keys(): + config['block-commits'] = parsed_config['block-commits'] if 'whitelist' in parsed_config.keys(): config['whitelist'] = parsed_config['whitelist'] @@ -469,7 +473,11 @@ def linter_check(repo_root, linter_subfolder): ) print("All of these linter errors must be resolved before merge.") print("=" * 80) + if linter_config['block-commits']: + exit(1) + else: + print('However, you have chosen to commit anyway.') else: commit_number = get_number_of_commits(repo_root) diff --git a/linterconfig.yaml_example b/linterconfig.yaml_example index 551b8de..1925a30 100644 --- a/linterconfig.yaml_example +++ b/linterconfig.yaml_example @@ -3,6 +3,7 @@ clangformat: on cpplint: on yapf: on pylint: on +block-commits: on # If this whitelist block is present, the linter only operates on the files and # directories listed in this whitelist. From 6b16c922e152afc74969d6b75019566d5c02e41f Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 08:31:13 -0400 Subject: [PATCH 32/56] trip linter --- .linterconfig.yaml | 1 - ascii_art.py | 35 +++++++++++++++++++++++++++++------ linter.py | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.linterconfig.yaml b/.linterconfig.yaml index 0fb4f69..037a6b5 100644 --- a/.linterconfig.yaml +++ b/.linterconfig.yaml @@ -4,4 +4,3 @@ cpplint: on yapf: on pylint: on block-commits: off - diff --git a/ascii_art.py b/ascii_art.py index bde17c2..852f39f 100644 --- a/ascii_art.py +++ b/ascii_art.py @@ -4,7 +4,7 @@ class AsciiArt(object): - tiger = r""" + tiger = r""" _ ,',\ ( / \\ @@ -37,7 +37,7 @@ class AsciiArt(object): """ - cthulhu = r""" + cthulhu = r""" _.---._ @@ -75,7 +75,7 @@ class AsciiArt(object): """ - story = r""" + story = r""" ______ _ ______ _ \___ \____ | | \__ \____ | | | | _/\__ \ | |__ | | _/\__ \ | |__ @@ -134,7 +134,7 @@ class AsciiArt(object): """ - commit_success = r""" + commit_success = r""" d888888b d888888b d888 8888b d888888 888b @@ -155,7 +155,7 @@ class AsciiArt(object): """ - grumpy_cat = r""" + grumpy_cat = r""" : sdMMm+ /mMMMMMMM+ @@ -203,7 +203,7 @@ class AsciiArt(object): """ - homer_woohoo = r""" + homer_woohoo = r""" ,#M]RRD`"RW ,R,qB*""``"*Rw, @@ -239,3 +239,26 @@ class AsciiArt(object): Woohoo, no formatter or linter for merge commits! """ + + devil = r""" + ,-. + ___,---.__ /'|`\ __,---,___ + ,-' \` `-.____,-' | `-.____,-' // `-. + ,' | ~'\ /`~ | `. + / ___// `. ,' , , \___ \ + | ,-' `-.__ _ | , __,-' `-. | + | / /\_ ` . | , _/\ \ | + \ | \ \`-.___ \ | / ___,-'/ / | / + \ \ | `._ `\\ | //' _,' | / / + `-.\ /' _ `---'' , . ``---' _ `\ /,-' + `` / \ ,='/ \`=. / \ '' + |__ /|\_,--.,-.--,--._/|\ __| + / `./ \\`\ | | | /,//' \,' \ + eViL / / ||--+--|--+-/-| \ \ + | | /'\_\_\ | /_/_/`\ | | + \ \__, \_ `~' _/ .__/ / + `-._,-' `-._______,-' `-._,-' + + 'However, you have chosen to commit anyway.' + + """ \ No newline at end of file diff --git a/linter.py b/linter.py index 9f63e5b..f691f3f 100755 --- a/linter.py +++ b/linter.py @@ -477,7 +477,7 @@ def linter_check(repo_root, linter_subfolder): exit(1) else: - print('However, you have chosen to commit anyway.') + print(ascii_art.AsciiArt.devil) else: commit_number = get_number_of_commits(repo_root) From a27fb4bb818fd28734cb18595a8d2bb8656c51a7 Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 08:40:02 -0400 Subject: [PATCH 33/56] trip linter --- ascii_art.py | 55 +++++++++++++++++++++++++++++++--------------------- linter.py | 3 ++- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/ascii_art.py b/ascii_art.py index 852f39f..1c2a995 100644 --- a/ascii_art.py +++ b/ascii_art.py @@ -240,25 +240,36 @@ class AsciiArt(object): """ - devil = r""" - ,-. - ___,---.__ /'|`\ __,---,___ - ,-' \` `-.____,-' | `-.____,-' // `-. - ,' | ~'\ /`~ | `. - / ___// `. ,' , , \___ \ - | ,-' `-.__ _ | , __,-' `-. | - | / /\_ ` . | , _/\ \ | - \ | \ \`-.___ \ | / ___,-'/ / | / - \ \ | `._ `\\ | //' _,' | / / - `-.\ /' _ `---'' , . ``---' _ `\ /,-' - `` / \ ,='/ \`=. / \ '' - |__ /|\_,--.,-.--,--._/|\ __| - / `./ \\`\ | | | /,//' \,' \ - eViL / / ||--+--|--+-/-| \ \ - | | /'\_\_\ | /_/_/`\ | | - \ \__, \_ `~' _/ .__/ / - `-._,-' `-._______,-' `-._,-' - - 'However, you have chosen to commit anyway.' - - """ \ No newline at end of file + yoda = r""" + ____ + _.' : `._ + .-.'`. ; .'`.-. + __ / : ___\ ; /___ ; \ __ + ,'_ ""--.:__;".-.";: :".-.":__;.--"" _`, + :' `.t""--.. '<@.`;_ ',@>` ..--""j.' `; + `:-.._J '-.-'L__ `-- ' L_..-;' + "-.__ ; .-" "-. : __.-" + L ' /.------.\ ' J + "-. "--" .-" + __.l"-:_JL_;-";.__ + .-j/'.; ;"""" / .'\"-. + .' /:`. "-.: .-" .'; `. + .-" / ; "-. "-..-" .-" : "-. + .+"-. : : "-.__.-" ;-._ \ + ; \ `.; ; : : "+. ; + : ; ; ; : ; : \: + : `."-; ; ; : ; ,/; + ; -: ; : ; : .-"' : + :\ \ : ; : \.-" : + ;`. \ ; : ;.'_..-- / ; + : "-. "-: ; :/." .' : + \ .-`.\ /t-"" ":-+. : + `. .-" `l __/ /`. : ; ; \ ; + \ .-" .-"-.-" .' .'j \ / ;/ + \ / .-" /. .'.' ;_:' ; + :-""-.`./-.' / `.___.' + \ `t ._ / bug :F_P: + "-.t-._:' + Careful be, you must. Unformatted code + to the dark side leads. + """ \ No newline at end of file diff --git a/linter.py b/linter.py index f691f3f..5adc1c9 100755 --- a/linter.py +++ b/linter.py @@ -477,7 +477,8 @@ def linter_check(repo_root, linter_subfolder): exit(1) else: - print(ascii_art.AsciiArt.devil) + print("However, you've chosen to commit anyway.") + print(ascii_art.AsciiArt.yoda) else: commit_number = get_number_of_commits(repo_root) From a4a36ad0309e9b9293b275571f601b7efa13bcdd Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 08:41:25 -0400 Subject: [PATCH 34/56] trip linter --- ascii_art.py | 7 ++++--- linter.py | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ascii_art.py b/ascii_art.py index 1c2a995..4b64828 100644 --- a/ascii_art.py +++ b/ascii_art.py @@ -241,7 +241,7 @@ class AsciiArt(object): """ yoda = r""" - ____ + ____ _.' : `._ .-.'`. ; .'`.-. __ / : ___\ ; /___ ; \ __ @@ -252,7 +252,7 @@ class AsciiArt(object): L ' /.------.\ ' J "-. "--" .-" __.l"-:_JL_;-";.__ - .-j/'.; ;"""" / .'\"-. + .-j/'.; ; / .'\-. .' /:`. "-.: .-" .'; `. .-" / ; "-. "-..-" .-" : "-. .+"-. : : "-.__.-" ;-._ \ @@ -271,5 +271,6 @@ class AsciiArt(object): \ `t ._ / bug :F_P: "-.t-._:' Careful be, you must. Unformatted code - to the dark side leads. + to the dark side leads + """ \ No newline at end of file diff --git a/linter.py b/linter.py index 5adc1c9..3d02c10 100755 --- a/linter.py +++ b/linter.py @@ -468,9 +468,8 @@ def linter_check(repo_root, linter_subfolder): if not (cpp_lint_success and pylint_success): print("=" * 80) - print( - "Commit not up to standards! Please address the linter errors above." - ) + print("Commit not up to standards!") + print("Please address the linter errors above.") print("All of these linter errors must be resolved before merge.") print("=" * 80) if linter_config['block-commits']: From be45d4e3babe846eba8697f2684cd194e8be7e68 Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 13:26:12 -0400 Subject: [PATCH 35/56] trip linter --- linter.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/linter.py b/linter.py index 3d02c10..0f78193 100755 --- a/linter.py +++ b/linter.py @@ -13,6 +13,8 @@ import os import re import subprocess +import distutils +import sys import pylint.lint import yaml @@ -37,6 +39,21 @@ } +def get_user_confirmation(default=False): + """Requests confirmation string from user""" + sys.stdin = open('/dev/tty') + while (True): + resp = raw_input() + try: + if distutils.util.strtobool(resp): + return True + else: + return False + except ValueError: + print('{} is not a valid response.'.format(resp)) + print('Please answer with y(es) or n(o)') + + def read_linter_config(filename): """Parses yaml config file.""" @@ -470,14 +487,23 @@ def linter_check(repo_root, linter_subfolder): print("=" * 80) print("Commit not up to standards!") print("Please address the linter errors above.") - print("All of these linter errors must be resolved before merge.") print("=" * 80) if linter_config['block-commits']: exit(1) else: - print("However, you've chosen to commit anyway.") - print(ascii_art.AsciiArt.yoda) + print("However, you may commit anyway, if you must.") + print( + "All of these linter errors must be resolved before merge." + ) + print("Would you like to commit anyway? yN") + + if get_user_confirmation(): + print(ascii_art.AsciiArt.yoda) + + else: + exit(1) + else: commit_number = get_number_of_commits(repo_root) From 317b9c6bb7fe8124f2b0546f65d0175d35348b7a Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 13:29:00 -0400 Subject: [PATCH 36/56] Real commit, added option to choose to commit at the end --- linter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/linter.py b/linter.py index 0f78193..3daab28 100755 --- a/linter.py +++ b/linter.py @@ -45,7 +45,9 @@ def get_user_confirmation(default=False): while (True): resp = raw_input() try: - if distutils.util.strtobool(resp): + if resp == '': + return default + elif distutils.util.strtobool(resp): return True else: return False From bf00832cbe384b2e6f76e713b9ca84d1cfb7afe6 Mon Sep 17 00:00:00 2001 From: Hunter Hansen Date: Fri, 31 May 2019 13:34:39 -0400 Subject: [PATCH 37/56] Removed .linterconfig.yaml --- .linterconfig.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .linterconfig.yaml diff --git a/.linterconfig.yaml b/.linterconfig.yaml deleted file mode 100644 index 037a6b5..0000000 --- a/.linterconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Turn the components of the linter individualy on / off. -clangformat: on -cpplint: on -yapf: on -pylint: on -block-commits: off From a64865d74c68bf12359db4535744e6ac7b2619e9 Mon Sep 17 00:00:00 2001 From: Igor Gilitschenski Date: Fri, 31 May 2019 15:58:19 -0400 Subject: [PATCH 38/56] Added ascii art credit. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 88e30c4..063294e 100644 --- a/README.md +++ b/README.md @@ -126,4 +126,5 @@ Currently there it is not possible to configure the python formatter on a per-re ## ASCII-Art Sources + * [www.asciiart.eu](https://www.asciiart.eu) * [www.retrojunkie.com (accessed through web.archive.org)](https://web.archive.org/web/20150831003349/http://www.retrojunkie.com:80/asciiart/) From a2230c7309c453add713c7e71585cf67b95b1ee6 Mon Sep 17 00:00:00 2001 From: Igor Gilitschenski Date: Sat, 1 Jun 2019 13:33:29 -0400 Subject: [PATCH 39/56] Reverse docstring related changes for PR to upstream --- default/pylint.rc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default/pylint.rc b/default/pylint.rc index 255e4bb..8afdb44 100644 --- a/default/pylint.rc +++ b/default/pylint.rc @@ -16,7 +16,7 @@ persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins=pylint.extensions.docparams +load-plugins= extension-pkg-whitelist=numpy @@ -31,7 +31,7 @@ extension-pkg-whitelist=numpy # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time. -disable=no-name-in-module,import-error,fixme,no-member,wrong-import-position,bad-continuation +disable=no-name-in-module,import-error,fixme,missing-docstring,no-member,wrong-import-position,bad-continuation [REPORTS] From 5c16bde18335f6777b7775a5383625ac0f9b00d5 Mon Sep 17 00:00:00 2001 From: Igor Gilitschenski Date: Mon, 3 Jun 2019 11:48:58 -0400 Subject: [PATCH 40/56] Added option for local pylint config. --- linter.py | 39 ++++++++++++++++++++++++++++----------- linterconfig.yaml_example | 10 +++++++++- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/linter.py b/linter.py index 3daab28..8d72a6a 100755 --- a/linter.py +++ b/linter.py @@ -27,13 +27,15 @@ YAPF_FORMAT_EXECUTABLE = "yapf" DEFAULT_CONFIG = { + # Enable code checks and formatting by default. 'use_clangformat': True, 'use_cpplint': True, - # Enable Python checks by default. 'use_yapf': True, 'use_pylint': True, + # Use the config files of the linter by default. + 'pylint_config': 'global', # Block commits that don't pass by default - 'block-commits': True, + 'block_commits': True, # Check all staged files by default. 'whitelist': [] } @@ -71,8 +73,10 @@ def read_linter_config(filename): config['use_yapf'] = parsed_config['yapf'] if 'pylint' in parsed_config.keys(): config['use_pylint'] = parsed_config['pylint'] - if 'block-commits' in parsed_config.keys(): - config['block-commits'] = parsed_config['block-commits'] + if 'pylint_config' in parsed_config.keys(): + config['pylint_config'] = parsed_config['pylint_config'] + if 'block_commits' in parsed_config.keys(): + config['block_commits'] = parsed_config['block_commits'] if 'whitelist' in parsed_config.keys(): config['whitelist'] = parsed_config['whitelist'] @@ -414,10 +418,6 @@ def get_whitelisted_files(repo_root, files, whitelist): def linter_check(repo_root, linter_subfolder): """ Main pre-commit function for calling code checking script. """ - cpplint_file = os.path.join(linter_subfolder, "cpplint.py") - pylint_file = os.path.join(linter_subfolder, "pylint.rc") - ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") - # Read linter config file. linter_config_file = repo_root + '/.linterconfig.yaml' if os.path.isfile(repo_root + '/.linterconfig.yaml'): @@ -426,10 +426,27 @@ def linter_check(repo_root, linter_subfolder): else: linter_config = DEFAULT_CONFIG + + cpplint_file = os.path.join(linter_subfolder, "cpplint.py") + ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") + + if linter_config['pylint_config'] == "global": + pylintrc_file = os.path.join(linter_subfolder, "pylint.rc") + elif linter_config['pylint_config'] == "local": + if not os.path.isfile(os.path.join(repo_root, ".pylintrc")): + print("A .pylintrc file is required in the repository root") + exit(1) + pylintrc_file = os.path.join(repo_root, ".pylintrc") + else: + print("{} is not a valid setting for pylint_config".format( + linter_config['pylint_config'])) + exit(1) + + print("Found linter subfolder: {}".format(linter_subfolder)) print("Found ascii art file at: {}".format(ascii_art_file)) print("Found cpplint file at: {}".format(cpplint_file)) - print("Found pylint file at: {}".format(pylint_file)) + print("Found pylint config file at: {}".format(pylintrc_file)) # Run checks staged_files = get_staged_files() @@ -481,7 +498,7 @@ def linter_check(repo_root, linter_subfolder): # style guide. if linter_config['use_pylint']: pylint_success = check_python_lint(repo_root, whitelisted_files, - pylint_file) + pylintrc_file) else: pylint_success = True @@ -490,7 +507,7 @@ def linter_check(repo_root, linter_subfolder): print("Commit not up to standards!") print("Please address the linter errors above.") print("=" * 80) - if linter_config['block-commits']: + if linter_config['block_commits']: exit(1) else: diff --git a/linterconfig.yaml_example b/linterconfig.yaml_example index 1925a30..3cab654 100644 --- a/linterconfig.yaml_example +++ b/linterconfig.yaml_example @@ -3,7 +3,15 @@ clangformat: on cpplint: on yapf: on pylint: on -block-commits: on + +# If set to global, the linter uses the pylint.rc from the "default" folder. If +# set to local, the linter uses .pylintrc from the repository root. +pylint_config: global + +# If on, the linter blocks commits after detecting an error. Otherwise, the +# user is asked whether to proceed +# with the commit. +block_commits: on # If this whitelist block is present, the linter only operates on the files and # directories listed in this whitelist. From dab9bd6d3137ecc366d7b35e913e0efb4df7048b Mon Sep 17 00:00:00 2001 From: Igor Gilitschenski Date: Wed, 5 Jun 2019 11:16:34 -0400 Subject: [PATCH 41/56] Addressed comments by @mfehr --- README.md | 3 ++- linter.py | 15 ++------------- linterconfig.yaml_example | 7 +------ 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 063294e..f4906b1 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ IncludeCategories: **Python** -Currently there it is not possible to configure the python formatter on a per-repository basis. +Currently there it is not possible to configure the python formatter on a per-repository basis. The linter can be configured on a per-repository bais by adding a `.pylintrc` file to your repositorie's root folder. + ## Disable Linter Functionalities for a Specific Line diff --git a/linter.py b/linter.py index 8d72a6a..895f8a9 100755 --- a/linter.py +++ b/linter.py @@ -32,8 +32,6 @@ 'use_cpplint': True, 'use_yapf': True, 'use_pylint': True, - # Use the config files of the linter by default. - 'pylint_config': 'global', # Block commits that don't pass by default 'block_commits': True, # Check all staged files by default. @@ -73,8 +71,6 @@ def read_linter_config(filename): config['use_yapf'] = parsed_config['yapf'] if 'pylint' in parsed_config.keys(): config['use_pylint'] = parsed_config['pylint'] - if 'pylint_config' in parsed_config.keys(): - config['pylint_config'] = parsed_config['pylint_config'] if 'block_commits' in parsed_config.keys(): config['block_commits'] = parsed_config['block_commits'] if 'whitelist' in parsed_config.keys(): @@ -430,17 +426,10 @@ def linter_check(repo_root, linter_subfolder): cpplint_file = os.path.join(linter_subfolder, "cpplint.py") ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") - if linter_config['pylint_config'] == "global": - pylintrc_file = os.path.join(linter_subfolder, "pylint.rc") - elif linter_config['pylint_config'] == "local": - if not os.path.isfile(os.path.join(repo_root, ".pylintrc")): - print("A .pylintrc file is required in the repository root") - exit(1) + if os.path.isfile(os.path.join(repo_root, ".pylintrc")): pylintrc_file = os.path.join(repo_root, ".pylintrc") else: - print("{} is not a valid setting for pylint_config".format( - linter_config['pylint_config'])) - exit(1) + pylintrc_file = os.path.join(linter_subfolder, "pylint.rc") print("Found linter subfolder: {}".format(linter_subfolder)) diff --git a/linterconfig.yaml_example b/linterconfig.yaml_example index 3cab654..1ae3d34 100644 --- a/linterconfig.yaml_example +++ b/linterconfig.yaml_example @@ -4,13 +4,8 @@ cpplint: on yapf: on pylint: on -# If set to global, the linter uses the pylint.rc from the "default" folder. If -# set to local, the linter uses .pylintrc from the repository root. -pylint_config: global - # If on, the linter blocks commits after detecting an error. Otherwise, the -# user is asked whether to proceed -# with the commit. +# user is asked whether to proceed with the commit. block_commits: on # If this whitelist block is present, the linter only operates on the files and From c2b19b3e7d0bdcb8de20013c33fbc64a123e06d5 Mon Sep 17 00:00:00 2001 From: Marius Fehr Date: Thu, 19 Dec 2019 15:06:40 +0100 Subject: [PATCH 42/56] Update ascii_art.py --- ascii_art.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ascii_art.py b/ascii_art.py index 4b64828..07b3ff6 100644 --- a/ascii_art.py +++ b/ascii_art.py @@ -150,7 +150,7 @@ class AsciiArt(object): 88 _(<_ / )_ 88 d88b (__\_\_|_/__) d88b - Your happiness shall not depend on what you have or what your are, + Your happiness shall not depend on what you have or what you are, it shall solely depend on the beauty of your code. """ @@ -273,4 +273,4 @@ class AsciiArt(object): Careful be, you must. Unformatted code to the dark side leads - """ \ No newline at end of file + """ From 47de7346aeba90382ee61ffcd90ff6dd3b40f996 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Thu, 23 Jul 2020 13:08:29 +0200 Subject: [PATCH 43/56] add run on whole repo function --- bin/linter_check_all | 55 +++++++++++++++ linter.py | 163 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 200 insertions(+), 18 deletions(-) create mode 100755 bin/linter_check_all diff --git a/bin/linter_check_all b/bin/linter_check_all new file mode 100755 index 0000000..787d10f --- /dev/null +++ b/bin/linter_check_all @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import subprocess +import sys + + +def run_command_in_folder(command, folder): + """Run a bash command in a specific folder.""" + run_command = subprocess.Popen(command, + shell=True, + cwd=folder, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, _ = run_command.communicate() + command_output = stdout.rstrip() + return command_output + + +def get_git_repo_root(some_folder_in_root_repo='./'): + """Get the root folder of the current git repository.""" + return run_command_in_folder('git rev-parse --show-toplevel', + some_folder_in_root_repo) + + +def get_linter_folder(root_repo_folder): + """Find the folder where this linter is stored.""" + try: + return os.environ['LINTER_PATH'] + except KeyError: + print("Cannot find linter because the environment variable " + "LINTER_PATH doesn't exist.") + sys.exit(1) + + +def main(): + # Get git root folder. + repo_root = get_git_repo_root() + + # Get linter subfolder + linter_folder = get_linter_folder(repo_root) + + # Append linter folder to the path so that we can import the linter module. + linter_folder = os.path.join(repo_root, linter_folder) + sys.path.append(linter_folder) + + import linter + + linter.linter_check_all(repo_root, linter_folder) + + +if __name__ == "__main__": + main() diff --git a/linter.py b/linter.py index 895f8a9..73b8173 100755 --- a/linter.py +++ b/linter.py @@ -24,6 +24,11 @@ "clang-format-diff-4.0", "clang-format-diff-3.9", "clang-format-diff-3.8" ] +CLANG_FORMAT_EXECUTABLE_VERSIONS = [ + "clang-format", "clang-format-6.0", "clang-format-5.0", "clang-format-4.0", + "clang-format-3.9", "clang-format-3.8" +] + YAPF_FORMAT_EXECUTABLE = "yapf" DEFAULT_CONFIG = { @@ -38,11 +43,16 @@ 'whitelist': [] } +# Files containing these in name or path will not be checked by get_all_files() +ALL_FILES_BLACKLISTED_NAMES = ['cmake-build-debug', '3rd_party', 'third_party'] + +CPP_SUFFIXES = ['.cpp', '.cc', '.cu', '.cuh', '.h', '.hpp', '.hxx'] + def get_user_confirmation(default=False): """Requests confirmation string from user""" sys.stdin = open('/dev/tty') - while (True): + while True: resp = raw_input() try: if resp == '': @@ -119,8 +129,24 @@ def get_unstaged_files(some_folder_in_root_repo='./'): return output.split("\n") +def get_all_files(repo_root): + """Get all files from in this repo.""" + output = [] + + for root, _, files in os.walk(repo_root): + for f in files: + if f.lower().endswith(tuple(CPP_SUFFIXES + ['.py'])): + full_name = os.path.join(root, f)[len(repo_root) + 1:] + if not any(n in full_name + for n in ALL_FILES_BLACKLISTED_NAMES): + output.append(full_name) + + return output + + def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): - """Runs Google's cpplint on all C++ files staged for commit,""" + """Runs Google's cpplint on all C++ files staged for commit, + return success and number of errors""" cpplint = imp.load_source('cpplint', cpplint_file) cpplint._cpplint_state.SetFilters('-legal/copyright,-build/c++11') # pylint: disable=W0212 @@ -132,7 +158,7 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): for changed_file in staged_files: if not os.path.isfile(changed_file): continue - if changed_file.lower().endswith(('.cc', '.h', '.cpp', '.cu', '.cuh')): + if changed_file.lower().endswith(tuple(CPP_SUFFIXES)): # Search iteratively for the root of the catkin package. package_root = '' search_dir = os.path.dirname(os.path.abspath(changed_file)) @@ -203,9 +229,9 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): print(ascii_art.AsciiArt.cthulhu) elif total_error_count > 20: print(ascii_art.AsciiArt.tiger) - return False + return False, total_error_count else: - return True + return True, 0 def check_modified_after_staging(staged_files): @@ -252,7 +278,7 @@ def check_if_merge_commit(repo_root): return os.path.isfile(merge_msg_file_path) -def find_clang_format_executable(): +def find_clang_format_diff_executable(): for executable in CLANG_FORMAT_DIFF_EXECUTABLE_VERSIONS: if subprocess.call("type " + executable, shell=True, @@ -264,6 +290,18 @@ def find_clang_format_executable(): exit(1) +def find_clang_format_executable(): + for executable in CLANG_FORMAT_EXECUTABLE_VERSIONS: + if subprocess.call("type " + executable, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0: + return executable + + print("ERROR: clang-format is not installed!") + exit(1) + + def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): """Runs clang format on all cpp files staged for commit.""" @@ -271,7 +309,7 @@ def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): os.path.basename(os.path.normpath(repo_root)) + "_" + datetime.datetime.now().isoformat() + ".clang.patch") - clang_format_diff_executable = find_clang_format_executable() + clang_format_diff_executable = find_clang_format_diff_executable() run_command_in_folder( "git diff -U0 --cached | " + clang_format_diff_executable + @@ -296,6 +334,23 @@ def run_clang_format(repo_root, staged_files, list_of_changed_staged_files): return True +def run_clang_format_on_all(repo_root, files): + """Runs clang format on all cpp files in repo.""" + + clang_format_executable = find_clang_format_executable() + counter = 0 + for f in files: + if f.lower().endswith(tuple(CPP_SUFFIXES)): + run_command_in_folder(clang_format_executable + " -i " + f, + repo_root) + counter = counter + 1 + + print("=" * 80) + print("Formatted all (%i) C++ files in repo with clang-format." % counter) + print("=" * 80) + return True + + def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): """Runs yapf format on all python files staged for commit.""" @@ -340,11 +395,10 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): def check_python_lint(repo_root, staged_files, pylint_file): - """Runs pylint on all python scripts staged for commit.""" - + """Runs pylint on all python scripts staged for commit. + Return success and number of errors.""" class TextReporterBuffer(object): """Stores the output produced by the pylint TextReporter.""" - def __init__(self): """init""" self.content = [] @@ -386,9 +440,9 @@ def read(self): print("=" * 80) print("Found {} pylint errors".format(len(pylint_errors))) print("=" * 80) - return False + return False, pylint_errors else: - return True + return True, 0 def get_whitelisted_files(repo_root, files, whitelist): @@ -422,7 +476,6 @@ def linter_check(repo_root, linter_subfolder): else: linter_config = DEFAULT_CONFIG - cpplint_file = os.path.join(linter_subfolder, "cpplint.py") ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") @@ -431,7 +484,6 @@ def linter_check(repo_root, linter_subfolder): else: pylintrc_file = os.path.join(linter_subfolder, "pylint.rc") - print("Found linter subfolder: {}".format(linter_subfolder)) print("Found ascii art file at: {}".format(ascii_art_file)) print("Found cpplint file at: {}".format(cpplint_file)) @@ -478,16 +530,17 @@ def linter_check(repo_root, linter_subfolder): # Use Google's C++ linter to check for compliance with Google style # guide. if linter_config['use_cpplint']: - cpp_lint_success = check_cpp_lint(whitelisted_files, cpplint_file, - ascii_art, repo_root) + cpp_lint_success, _ = check_cpp_lint(whitelisted_files, + cpplint_file, ascii_art, + repo_root) else: cpp_lint_success = True # Use pylint to check for comimpliance with Tensofrflow python # style guide. if linter_config['use_pylint']: - pylint_success = check_python_lint(repo_root, whitelisted_files, - pylintrc_file) + pylint_success, _ = check_python_lint(repo_root, whitelisted_files, + pylintrc_file) else: pylint_success = True @@ -535,3 +588,77 @@ def linter_check(repo_root, linter_subfolder): print("=" * 80) else: print(ascii_art.AsciiArt.homer_woohoo) + + +def linter_check_all(repo_root, linter_subfolder): + """ Run linter check on all files in repo. """ + + # Read linter config file. + linter_config_file = repo_root + '/.linterconfig.yaml' + if os.path.isfile(repo_root + '/.linterconfig.yaml'): + print("Found repo linter config: {}".format(linter_config_file)) + linter_config = read_linter_config(linter_config_file) + else: + linter_config = DEFAULT_CONFIG + + cpplint_file = os.path.join(linter_subfolder, "cpplint.py") + ascii_art_file = os.path.join(linter_subfolder, "ascii_art.py") + + if os.path.isfile(os.path.join(repo_root, ".pylintrc")): + pylintrc_file = os.path.join(repo_root, ".pylintrc") + else: + pylintrc_file = os.path.join(linter_subfolder, "pylint.rc") + + print("Found linter subfolder: {}".format(linter_subfolder)) + print("Found ascii art file at: {}".format(ascii_art_file)) + print("Found cpplint file at: {}".format(cpplint_file)) + print("Found pylint config file at: {}".format(pylintrc_file)) + + # Run checks + files = get_all_files(repo_root) + + # Load ascii art. + ascii_art = imp.load_source('ascii_art', ascii_art_file) + + if linter_config['use_clangformat']: + run_clang_format_on_all(repo_root, files) + + if linter_config['use_yapf']: + run_yapf_format(repo_root, files, []) + + # Use Google's C++ linter to check for compliance with Google style + # guide. + cpp_errors = 0 + if linter_config['use_cpplint']: + cpp_lint_success, cpp_errors = check_cpp_lint(files, cpplint_file, + ascii_art, repo_root) + else: + cpp_lint_success = True + + # Use pylint to check for comimpliance with Tensofrflow python + # style guide. + py_errors = 0 + if linter_config['use_pylint']: + pylint_success, py_errors = check_python_lint(repo_root, files, + pylintrc_file) + else: + pylint_success = True + + n_python = len([f for f in files if f.lower().endswith('.py')]) + checked_files_msg = "Found %i errors in %i C++ files and %i erros in %i " \ + "Python files." % (cpp_errors, len(files) - n_python, + py_errors, n_python) + + if not (cpp_lint_success and pylint_success): + print("=" * 80) + print("Code not up to standards!") + print(checked_files_msg) + print("Please address the linter errors above.") + print("=" * 80) + else: + print(ascii_art.AsciiArt.commit_success) + + print("=" * 80) + print("Code is up to standards, well done!") + print(checked_files_msg) + print("=" * 80) From 1ae28c4b110ccb7c9450d6eb68a2f323cac466ee Mon Sep 17 00:00:00 2001 From: Schmluk Date: Thu, 23 Jul 2020 13:14:07 +0200 Subject: [PATCH 44/56] clearer printing summary --- linter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/linter.py b/linter.py index 73b8173..e7ca0a6 100755 --- a/linter.py +++ b/linter.py @@ -644,10 +644,10 @@ def linter_check_all(repo_root, linter_subfolder): else: pylint_success = True - n_python = len([f for f in files if f.lower().endswith('.py')]) - checked_files_msg = "Found %i errors in %i C++ files and %i erros in %i " \ - "Python files." % (cpp_errors, len(files) - n_python, - py_errors, n_python) + n_py = len([f for f in files if f.lower().endswith('.py')]) + checked_files_msg = "Found %i errors checking %i C++ files and " \ + "%i erros checking %i Python files." \ + % (cpp_errors, len(files) - n_py, py_errors, n_py) if not (cpp_lint_success and pylint_success): print("=" * 80) From a33f2c4d3a0883eb7950b2deb037cb1221ac11a7 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Thu, 23 Jul 2020 13:32:43 +0200 Subject: [PATCH 45/56] fix pylint error count --- linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linter.py b/linter.py index e7ca0a6..0a68f5b 100755 --- a/linter.py +++ b/linter.py @@ -440,7 +440,7 @@ def read(self): print("=" * 80) print("Found {} pylint errors".format(len(pylint_errors))) print("=" * 80) - return False, pylint_errors + return False, len(pylint_errors) else: return True, 0 From c425a88a85b2c338b4253adf25611cdbfa176705 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Thu, 23 Jul 2020 15:13:05 +0200 Subject: [PATCH 46/56] pretty printing for commit numbers --- bin/linter_check_all | 5 +++++ linter.py | 24 +++++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/bin/linter_check_all b/bin/linter_check_all index 787d10f..213f6d0 100755 --- a/bin/linter_check_all +++ b/bin/linter_check_all @@ -1,5 +1,10 @@ #!/usr/bin/env python +""" +This script loads the linter and auto-formatters and runs it on all project +files in the current repo. +""" + from __future__ import print_function import os diff --git a/linter.py b/linter.py index 0a68f5b..0e45733 100755 --- a/linter.py +++ b/linter.py @@ -38,7 +38,7 @@ 'use_yapf': True, 'use_pylint': True, # Block commits that don't pass by default - 'block_commits': True, + 'block_commits': False, # Check all staged files by default. 'whitelist': [] } @@ -570,12 +570,26 @@ def linter_check(repo_root, linter_subfolder): commit_number = get_number_of_commits(repo_root) lucky_commit = ((int(commit_number) + 1) % 42 == 0) + # Pretty printing for the commit number + commit_number_text = str(commit_number) + if commit_number_text[-1:] == "1" \ + and commit_number_text[-2:] != "11": + commit_number_text = commit_number_text + "st" + elif commit_number_text[-1:] == "2" \ + and commit_number_text[-2:] != "12": + commit_number_text = commit_number_text + "nd" + elif commit_number_text[-1:] == "3" \ + and commit_number_text[-2:] != "13": + commit_number_text = commit_number_text + "rd" + else: + commit_number_text = commit_number_text + "th" + if lucky_commit: print(ascii_art.AsciiArt.story) print("=" * 80) - print("Commit accepted, well done! This is your {}th commit!". - format(commit_number)) + print("Commit accepted, well done! This is your %s commit!" % + commit_number_text) print("This is a lucky commit! " + "Please enjoy this free sheep story.") print("=" * 80) @@ -583,8 +597,8 @@ def linter_check(repo_root, linter_subfolder): print(ascii_art.AsciiArt.commit_success) print("=" * 80) - print("Commit accepted, well done! This is the {}th commit!". - format(commit_number)) + print("Commit accepted, well done! This is your %s commit!" % + commit_number_text) print("=" * 80) else: print(ascii_art.AsciiArt.homer_woohoo) From 684e464ca2805bc8c2eefb00189ebffb54eec53b Mon Sep 17 00:00:00 2001 From: Schmluk Date: Sat, 25 Jul 2020 21:55:20 +0200 Subject: [PATCH 47/56] fix linter_check_all, prettier printing for pylint --- linter.py | 170 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 142 insertions(+), 28 deletions(-) diff --git a/linter.py b/linter.py index 0e45733..e1be613 100755 --- a/linter.py +++ b/linter.py @@ -40,7 +40,9 @@ # Block commits that don't pass by default 'block_commits': False, # Check all staged files by default. - 'whitelist': [] + 'whitelist': [], + # Prevents pylint from printing the config XX times. + 'allow_pylint_stderr': False } # Files containing these in name or path will not be checked by get_all_files() @@ -213,8 +215,12 @@ def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root): total_error_count += error_count print("-" * 80) + name_to_print = changed_file + if name_to_print[:len(repo_root)] == repo_root: + name_to_print = changed_file[len(repo_root) + 1:] + print("Found {} errors in : {}".format(error_count, - changed_file)) + name_to_print)) print("-" * 80) for line in cpplint.output: assert len(line) == 2 @@ -339,16 +345,32 @@ def run_clang_format_on_all(repo_root, files): clang_format_executable = find_clang_format_executable() counter = 0 + formatted_counter = 0 for f in files: + f_long = os.path.join(repo_root, f) + if not os.path.isfile(f_long): + continue if f.lower().endswith(tuple(CPP_SUFFIXES)): + counter = counter + 1 + stat = os.stat(f_long) run_command_in_folder(clang_format_executable + " -i " + f, repo_root) - counter = counter + 1 + if os.stat(f_long) != stat: + if formatted_counter == 0: + print("=" * 80) + print("Formatted C++ files with clang-format:") - print("=" * 80) - print("Formatted all (%i) C++ files in repo with clang-format." % counter) - print("=" * 80) - return True + print("-> " + f) + formatted_counter = formatted_counter + 1 + + if formatted_counter > 0: + print("Formatted %i of %i checked C++ files in this repo." % + (formatted_counter, counter)) + else: + print("=" * 80) + print("All %i checked C++ files in this repo are in agreement with " + "clang-format." % counter) + return formatted_counter def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): @@ -359,7 +381,6 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): if not os.path.isfile(staged_file): continue if staged_file.endswith((".py")): - # Check if the file needs formatting by applying the formatting and # store the results into a patch file. yapf_format_path = ("/tmp/" + @@ -394,7 +415,47 @@ def run_yapf_format(repo_root, staged_files, list_of_changed_staged_files): return True -def check_python_lint(repo_root, staged_files, pylint_file): +def run_yapf_format_on_all(repo_root, files): + """Runs yapf format on all python files.""" + + # Check if the file needs formatting by applying the formatting and + # store the results into a patch file. + yapf_format_path = ("/tmp/" + + os.path.basename(os.path.normpath(repo_root)) + "_" + + datetime.datetime.now().isoformat() + ".yapf.patch") + + counter = 0 + formatted_counter = 0 + for f in files: + if not os.path.isfile(os.path.join(repo_root, f)): + continue + if f.endswith((".py")): + counter = counter + 1 + task = (YAPF_FORMAT_EXECUTABLE + " --style pep8 -d " + f + " > " + + yapf_format_path) + run_command_in_folder(task, repo_root) + + if not os.stat(yapf_format_path).st_size == 0: + if formatted_counter == 0: + print("=" * 80) + print("Formatted python files with autopep8:") + + print("-> " + f) + run_command_in_folder("git apply -p0 " + yapf_format_path, + repo_root) + formatted_counter = formatted_counter + 1 + + if formatted_counter > 0: + print("Formatted %i of %i checked python files in this repo." % + (formatted_counter, counter)) + else: + print("=" * 80) + print("All %i checked python files in this repo are in agreement with " + "autopep8." % counter) + return formatted_counter + + +def check_python_lint(repo_root, staged_files, pylint_file, allow_stderr=True): """Runs pylint on all python scripts staged for commit. Return success and number of errors.""" class TextReporterBuffer(object): @@ -411,6 +472,8 @@ def read(self): """read""" return self.content + print("Running pylint...") + # Parse each pylint output line individualy and searches # for errors in the code. pylint_errors = [] @@ -418,24 +481,40 @@ def read(self): if not os.path.isfile(changed_file): continue if re.search(r'\.py$', changed_file): + name_to_print = changed_file + if name_to_print[:len(repo_root)] == repo_root: + name_to_print = changed_file[len(repo_root) + 1:] - print("Running pylint on \'{}\'".format(repo_root + "/" + - changed_file)) pylint_output = TextReporterBuffer() pylint_args = [ "--rcfile=" + pylint_file, "-rn", - repo_root + "/" + changed_file + repo_root + "/" + name_to_print ] + + # Prevent pylint from printing the config XX times + prev_stderr = sys.stderr + if not allow_stderr: + sys.stderr = open(os.devnull, "w") + from pylint.reporters.text import TextReporter pylint.lint.Run(pylint_args, reporter=TextReporter(pylint_output), exit=False) + sys.stderr = prev_stderr + errors = [] for output_line in pylint_output.read(): if re.search(r'^(E|C|W):', output_line): - print(changed_file + ": " + output_line) + errors.append(output_line) pylint_errors.append(output_line) + if errors: + print("-" * 80) + print("Found {} errors in : {}".format(len(errors), + name_to_print)) + print("-" * 80) + print("\n".join(errors)) + if pylint_errors: print("=" * 80) print("Found {} pylint errors".format(len(pylint_errors))) @@ -470,10 +549,12 @@ def linter_check(repo_root, linter_subfolder): # Read linter config file. linter_config_file = repo_root + '/.linterconfig.yaml' + print("=" * 80) if os.path.isfile(repo_root + '/.linterconfig.yaml'): print("Found repo linter config: {}".format(linter_config_file)) linter_config = read_linter_config(linter_config_file) else: + print("Using default linter config.") linter_config = DEFAULT_CONFIG cpplint_file = os.path.join(linter_subfolder, "cpplint.py") @@ -539,8 +620,11 @@ def linter_check(repo_root, linter_subfolder): # Use pylint to check for comimpliance with Tensofrflow python # style guide. if linter_config['use_pylint']: + allow_stderr = True + if 'allow_pylint_stderr' in linter_config: + allow_stderr = linter_config['allow_pylint_stderr'] pylint_success, _ = check_python_lint(repo_root, whitelisted_files, - pylintrc_file) + pylintrc_file, allow_stderr) else: pylint_success = True @@ -609,10 +693,12 @@ def linter_check_all(repo_root, linter_subfolder): # Read linter config file. linter_config_file = repo_root + '/.linterconfig.yaml' + print("=" * 80) if os.path.isfile(repo_root + '/.linterconfig.yaml'): print("Found repo linter config: {}".format(linter_config_file)) linter_config = read_linter_config(linter_config_file) else: + print("Using default linter config.") linter_config = DEFAULT_CONFIG cpplint_file = os.path.join(linter_subfolder, "cpplint.py") @@ -634,18 +720,27 @@ def linter_check_all(repo_root, linter_subfolder): # Load ascii art. ascii_art = imp.load_source('ascii_art', ascii_art_file) + cpp_formatted = 0 if linter_config['use_clangformat']: - run_clang_format_on_all(repo_root, files) + cpp_formatted = run_clang_format_on_all(repo_root, files) + py_formatted = 0 if linter_config['use_yapf']: - run_yapf_format(repo_root, files, []) + py_formatted = run_yapf_format_on_all(repo_root, files) + + # Otherwise the linter os.isfile fails when calling the linters. + files = [os.path.join(repo_root, f) for f in files] # Use Google's C++ linter to check for compliance with Google style # guide. cpp_errors = 0 if linter_config['use_cpplint']: + print("=" * 80) cpp_lint_success, cpp_errors = check_cpp_lint(files, cpplint_file, ascii_art, repo_root) + if cpp_errors == 0: + print("Found 0 cpplint errors.") + print("=" * 80) else: cpp_lint_success = True @@ -653,26 +748,45 @@ def linter_check_all(repo_root, linter_subfolder): # style guide. py_errors = 0 if linter_config['use_pylint']: + allow_stderr = True + if 'allow_pylint_stderr' in linter_config: + allow_stderr = linter_config['allow_pylint_stderr'] pylint_success, py_errors = check_python_lint(repo_root, files, - pylintrc_file) + pylintrc_file, + allow_stderr) + if py_errors == 0: + print("=" * 80) + print("Found 0 pylint errors.") + print("=" * 80) else: pylint_success = True + # Summary + if cpp_lint_success and pylint_success: + print(ascii_art.AsciiArt.commit_success) + print("=" * 80) + n_py = len([f for f in files if f.lower().endswith('.py')]) - checked_files_msg = "Found %i errors checking %i C++ files and " \ - "%i erros checking %i Python files." \ - % (cpp_errors, len(files) - n_py, py_errors, n_py) + n_cpp = 0 + if linter_config['use_cpplint']: + n_cpp = len(files) - n_py + if not linter_config['use_pylint']: + n_py = 0 + + print("Summary: %s C++ files checked." + " %s python files checked.\n" + " %s C++ files reformatted." + " %s python files reformatted.\n" + " %s C++ linter errors found." + " %s python linter errors found." % + (str(n_cpp).ljust(3), str(n_py).ljust(3), + str(cpp_formatted).ljust(3), str(py_formatted).ljust(3), + str(cpp_errors).ljust(3), str(py_errors).ljust(3))) + print("=" * 80) if not (cpp_lint_success and pylint_success): - print("=" * 80) print("Code not up to standards!") - print(checked_files_msg) print("Please address the linter errors above.") - print("=" * 80) else: - print(ascii_art.AsciiArt.commit_success) - - print("=" * 80) print("Code is up to standards, well done!") - print(checked_files_msg) - print("=" * 80) + print("=" * 80) From 3fe27b44309c660ec59176eb93cf8e827511726a Mon Sep 17 00:00:00 2001 From: Schmluk Date: Sat, 25 Jul 2020 22:02:20 +0200 Subject: [PATCH 48/56] update readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f4906b1..3047f12 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,11 @@ Currently there it is not possible to configure the python formatter on a per-re ... # yapf: enable ``` + +## Tools +* **Cheking an entire repository** + + Type `linter_check_all` to get run the linter on all files within the current repository. ## ASCII-Art Sources From 04bf8888d3629f377808effac791fd7b589de126 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Mon, 27 Jul 2020 10:30:35 +0200 Subject: [PATCH 49/56] minor fix to readme and default config --- README.md | 2 +- linter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3047f12..983b80a 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Currently there it is not possible to configure the python formatter on a per-re ## Tools * **Cheking an entire repository** - Type `linter_check_all` to get run the linter on all files within the current repository. + Type `linter_check_all` to run the linters and formatters on all files within the current repository. ## ASCII-Art Sources diff --git a/linter.py b/linter.py index e1be613..4654c41 100755 --- a/linter.py +++ b/linter.py @@ -38,7 +38,7 @@ 'use_yapf': True, 'use_pylint': True, # Block commits that don't pass by default - 'block_commits': False, + 'block_commits': True, # Check all staged files by default. 'whitelist': [], # Prevents pylint from printing the config XX times. From 72d054dece1c438341c20fa9968b363aca3e9bcf Mon Sep 17 00:00:00 2001 From: Schmluk Date: Mon, 10 Aug 2020 17:20:27 +0200 Subject: [PATCH 50/56] add pylint warning filtering so other warnings pass --- linter.py | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/linter.py b/linter.py index 4654c41..d7b31b3 100755 --- a/linter.py +++ b/linter.py @@ -16,6 +16,7 @@ import distutils import sys +from cStringIO import StringIO import pylint.lint import yaml @@ -42,7 +43,7 @@ # Check all staged files by default. 'whitelist': [], # Prevents pylint from printing the config XX times. - 'allow_pylint_stderr': False + 'filter_pylint_stderr': True } # Files containing these in name or path will not be checked by get_all_files() @@ -455,7 +456,10 @@ def run_yapf_format_on_all(repo_root, files): return formatted_counter -def check_python_lint(repo_root, staged_files, pylint_file, allow_stderr=True): +def check_python_lint(repo_root, + staged_files, + pylint_file, + filter_pylint_stderr=False): """Runs pylint on all python scripts staged for commit. Return success and number of errors.""" class TextReporterBuffer(object): @@ -472,9 +476,10 @@ def read(self): """read""" return self.content - print("Running pylint...") + if filter_pylint_stderr: + print("Running pylint...") - # Parse each pylint output line individualy and searches + # Parse each pylint output line individually and searches # for errors in the code. pylint_errors = [] for changed_file in staged_files: @@ -484,7 +489,6 @@ def read(self): name_to_print = changed_file if name_to_print[:len(repo_root)] == repo_root: name_to_print = changed_file[len(repo_root) + 1:] - pylint_output = TextReporterBuffer() pylint_args = [ "--rcfile=" + pylint_file, "-rn", @@ -493,14 +497,24 @@ def read(self): # Prevent pylint from printing the config XX times prev_stderr = sys.stderr - if not allow_stderr: - sys.stderr = open(os.devnull, "w") + if filter_pylint_stderr: + stderr_buffer = StringIO() + sys.stderr = stderr_buffer + # Run the linter from pylint.reporters.text import TextReporter pylint.lint.Run(pylint_args, reporter=TextReporter(pylint_output), exit=False) + + # Reset stderr and print filtered warnings. sys.stderr = prev_stderr + if filter_pylint_stderr: + warnings = stderr_buffer.getvalue().splitlines() + for warning in warnings: + if warning[:18] != 'Using config file ': + sys.stderr.write(warning) + sys.stderr.flush() errors = [] for output_line in pylint_output.read(): @@ -617,14 +631,15 @@ def linter_check(repo_root, linter_subfolder): else: cpp_lint_success = True - # Use pylint to check for comimpliance with Tensofrflow python + # Use pylint to check for compliance with Tensorflow python # style guide. if linter_config['use_pylint']: - allow_stderr = True - if 'allow_pylint_stderr' in linter_config: - allow_stderr = linter_config['allow_pylint_stderr'] + filter_pylint_stderr = False + if 'filter_pylint_stderr' in linter_config: + filter_pylint_stderr = linter_config['filter_pylint_stderr'] pylint_success, _ = check_python_lint(repo_root, whitelisted_files, - pylintrc_file, allow_stderr) + pylintrc_file, + filter_pylint_stderr) else: pylint_success = True @@ -648,9 +663,7 @@ def linter_check(repo_root, linter_subfolder): else: exit(1) - else: - commit_number = get_number_of_commits(repo_root) lucky_commit = ((int(commit_number) + 1) % 42 == 0) @@ -747,13 +760,14 @@ def linter_check_all(repo_root, linter_subfolder): # Use pylint to check for comimpliance with Tensofrflow python # style guide. py_errors = 0 + print("Files:", files) if linter_config['use_pylint']: - allow_stderr = True - if 'allow_pylint_stderr' in linter_config: - allow_stderr = linter_config['allow_pylint_stderr'] + filter_pylint_stderr = False + if 'filter_pylint_stderr' in linter_config: + filter_pylint_stderr = linter_config['filter_pylint_stderr'] pylint_success, py_errors = check_python_lint(repo_root, files, pylintrc_file, - allow_stderr) + filter_pylint_stderr) if py_errors == 0: print("=" * 80) print("Found 0 pylint errors.") From 4ca8884a8a4a3760a56e11a38ec4ae6bb46381f0 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Mon, 10 Aug 2020 17:22:24 +0200 Subject: [PATCH 51/56] lucky commit on 42nd and not 41st commit :) --- linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linter.py b/linter.py index d7b31b3..271436f 100755 --- a/linter.py +++ b/linter.py @@ -665,7 +665,7 @@ def linter_check(repo_root, linter_subfolder): exit(1) else: commit_number = get_number_of_commits(repo_root) - lucky_commit = ((int(commit_number) + 1) % 42 == 0) + lucky_commit = (int(commit_number) % 42 == 0) # Pretty printing for the commit number commit_number_text = str(commit_number) From 2007a9de1684cf7abec022d0e791c72c54ecbad1 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Mon, 17 Aug 2020 22:31:23 +0200 Subject: [PATCH 52/56] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 983b80a..00be9fd 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Currently there it is not possible to configure the python formatter on a per-re ``` ## Tools -* **Cheking an entire repository** +* **Checking an entire repository** Type `linter_check_all` to run the linters and formatters on all files within the current repository. From a6561ade620eb804ffb132ef6085b0acff1bdb18 Mon Sep 17 00:00:00 2001 From: Schmluk Date: Wed, 19 Aug 2020 09:13:26 +0200 Subject: [PATCH 53/56] remove printing --- linter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/linter.py b/linter.py index 271436f..048c6a0 100755 --- a/linter.py +++ b/linter.py @@ -760,7 +760,6 @@ def linter_check_all(repo_root, linter_subfolder): # Use pylint to check for comimpliance with Tensofrflow python # style guide. py_errors = 0 - print("Files:", files) if linter_config['use_pylint']: filter_pylint_stderr = False if 'filter_pylint_stderr' in linter_config: From e944d49501a91217e6cb4fdba90241b592d6563f Mon Sep 17 00:00:00 2001 From: Lukas Bernreiter Date: Tue, 5 Jul 2022 17:33:38 +0200 Subject: [PATCH 54/56] fix for repo root --- bin/init_linter_git_hooks | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/init_linter_git_hooks b/bin/init_linter_git_hooks index 76aa206..25029ce 100755 --- a/bin/init_linter_git_hooks +++ b/bin/init_linter_git_hooks @@ -55,7 +55,8 @@ def get_git_repo_root(some_folder_in_root_repo='./'): shell=True, cwd=some_folder_in_root_repo, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) stdout, _ = get_repo_call.communicate() repo_root = stdout.rstrip() From 09f2264cda80a7d5eb75a11acaaf853fd976b633 Mon Sep 17 00:00:00 2001 From: Lukas Bernreiter Date: Tue, 5 Jul 2022 17:44:54 +0200 Subject: [PATCH 55/56] more ports to py3 --- bin/linter_check_all | 3 ++- git_hooks.py | 3 ++- linter.py | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bin/linter_check_all b/bin/linter_check_all index 213f6d0..b41d19a 100755 --- a/bin/linter_check_all +++ b/bin/linter_check_all @@ -18,7 +18,8 @@ def run_command_in_folder(command, folder): shell=True, cwd=folder, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) stdout, _ = run_command.communicate() command_output = stdout.rstrip() return command_output diff --git a/git_hooks.py b/git_hooks.py index 6b42582..f8078fd 100755 --- a/git_hooks.py +++ b/git_hooks.py @@ -13,7 +13,8 @@ def run_command_in_folder(command, folder): shell=True, cwd=folder, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) stdout, _ = run_command.communicate() command_output = stdout.rstrip() return command_output diff --git a/linter.py b/linter.py index 048c6a0..07b548b 100755 --- a/linter.py +++ b/linter.py @@ -16,7 +16,11 @@ import distutils import sys -from cStringIO import StringIO +if sys.version_info > (3,0): + from io import StringIO +else: + from cStringIO import StringIO + import pylint.lint import yaml @@ -98,7 +102,8 @@ def run_command_in_folder(command, folder): shell=True, cwd=folder, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) stdout, _ = run_command.communicate() command_output = stdout.rstrip() return command_output From 9e3a9b0eb1888f7375c0f77339d07ced40c4ce3e Mon Sep 17 00:00:00 2001 From: Lukas Bernreiter Date: Wed, 6 Jul 2022 14:25:48 +0200 Subject: [PATCH 56/56] added requests install to readme --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 00be9fd..0274a61 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,11 @@ This repo contains a (C++, python) linter and auto formatter package that can be brew install clang-format ln -s /usr/local/share/clang/clang-format-diff.py /usr/local/bin/clang-format-diff ``` - + * **requests** + * Ubuntu: + ``` + pip install requests + ``` ## Installation