From cb4c7ff2bc7ec59568f6a7947823e34a9d5dbf75 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Tue, 8 Dec 2020 18:35:34 -0500 Subject: [PATCH 01/12] Modify 'git ls-files' to pick up PROJECT_ROOT Also, add PROJECT_ROOT to the results of 'git ls-files' before using the file list. --- Makefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 94c49d09..42843fde 100644 --- a/Makefile +++ b/Makefile @@ -178,31 +178,33 @@ cfn/lint: | guard/program/cfn-lint ## Runs eclint against the project eclint/lint: | guard/program/eclint guard/program/git eclint/lint: HAS_UNTRACKED_CHANGES ?= $(shell cd $(PROJECT_ROOT) && git status -s) -eclint/lint: ECLINT_FILES ?= git ls-files -z +eclint/lint: ECLINT_FILES ?= git -C $(PROJECT_ROOT) ls-files -z eclint/lint: @ echo "[$@]: Running eclint..." cd $(PROJECT_ROOT) && \ [ -z "$(HAS_UNTRACKED_CHANGES)" ] || (echo "untracked changes detected!" && exit 1) cd $(PROJECT_ROOT) && \ - $(ECLINT_FILES) | grep -zv ".bats" | xargs -0 -I {} eclint check {} + $(ECLINT_FILES) | grep -zv ".bats" | \ + $(XARGS) -0 eclint check $(PROJECT_ROOT)/{} @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= git ls-files --cached --others --exclude-standard '*.py' +python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - $(PYTHON_FILES) | xargs black --check + $(PYTHON_FILES) | xargs printf "$(PROJECT_ROOT)/%s " | xargs black --check $(PYTHON_FILES) | $(XARGS) -n1 pylint -rn -sn \ - --msg-template="{path}:{line} [{symbol}] {msg}" {} + --msg-template="{path}:{line} [{symbol}] {msg}" \ + $(PROJECT_ROOT)/{} @ echo "[$@]: Python files PASSED lint test!" ## Formats Python files. python/format: | guard/program/black guard/program/git python/format: @ echo "[$@]: Formatting Python files..." - $(PYTHON_FILES) | xargs black + $(PYTHON_FILES) | xargs printf "$(PROJECT_ROOT)/%s " | xargs black @ echo "[$@]: Successfully formatted Python files!" ## Lints terraform files From 044ae49422ec1fb5ecb6b6e649b98cd198cd95c3 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Wed, 9 Dec 2020 11:23:07 -0500 Subject: [PATCH 02/12] Add $PROJECT_ROOT to PYTHON_FILES not commands Loren requested that PYTHON_FILES reflect the $PROJECT_ROOT path. This means users who redefine PYTHON_FILES must be aware of the tardigrade-ci directory structure. --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 42843fde..79def73f 100644 --- a/Makefile +++ b/Makefile @@ -188,23 +188,22 @@ eclint/lint: $(XARGS) -0 eclint check $(PROJECT_ROOT)/{} @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' +python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)/%s\n" ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - $(PYTHON_FILES) | xargs printf "$(PROJECT_ROOT)/%s " | xargs black --check + $(PYTHON_FILES) | xargs black --check $(PYTHON_FILES) | $(XARGS) -n1 pylint -rn -sn \ - --msg-template="{path}:{line} [{symbol}] {msg}" \ - $(PROJECT_ROOT)/{} + --msg-template="{path}:{line} [{symbol}] {msg}" {} @ echo "[$@]: Python files PASSED lint test!" ## Formats Python files. python/format: | guard/program/black guard/program/git python/format: @ echo "[$@]: Formatting Python files..." - $(PYTHON_FILES) | xargs printf "$(PROJECT_ROOT)/%s " | xargs black + $(PYTHON_FILES) | xargs black @ echo "[$@]: Successfully formatted Python files!" ## Lints terraform files From f7675498bbe2cb1f8362d3bc7243091eb3159b60 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Wed, 9 Dec 2020 12:00:31 -0500 Subject: [PATCH 03/12] Assume PROJECT_ROOT has a trailing slash That appears to be the case if you look at the Makefile. Not sure it's consistently treated in that way. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 79def73f..50c48b86 100644 --- a/Makefile +++ b/Makefile @@ -185,10 +185,10 @@ eclint/lint: [ -z "$(HAS_UNTRACKED_CHANGES)" ] || (echo "untracked changes detected!" && exit 1) cd $(PROJECT_ROOT) && \ $(ECLINT_FILES) | grep -zv ".bats" | \ - $(XARGS) -0 eclint check $(PROJECT_ROOT)/{} + $(XARGS) -0 eclint check $(PROJECT_ROOT){} @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)/%s\n" +python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)%s\n" ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git From 9d1a8fd543cf5e41c8f583148241be942d617cf4 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Wed, 9 Dec 2020 19:16:33 -0500 Subject: [PATCH 04/12] Modify ECLINT_FILES to allow user override --- Makefile | 6 ++---- tests/make/eclint_lint_success.bats | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 50c48b86..8726a1d5 100644 --- a/Makefile +++ b/Makefile @@ -178,14 +178,12 @@ cfn/lint: | guard/program/cfn-lint ## Runs eclint against the project eclint/lint: | guard/program/eclint guard/program/git eclint/lint: HAS_UNTRACKED_CHANGES ?= $(shell cd $(PROJECT_ROOT) && git status -s) -eclint/lint: ECLINT_FILES ?= git -C $(PROJECT_ROOT) ls-files -z +eclint/lint: ECLINT_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files -z | grep -zv ".bats" | xargs -0) eclint/lint: @ echo "[$@]: Running eclint..." cd $(PROJECT_ROOT) && \ [ -z "$(HAS_UNTRACKED_CHANGES)" ] || (echo "untracked changes detected!" && exit 1) - cd $(PROJECT_ROOT) && \ - $(ECLINT_FILES) | grep -zv ".bats" | \ - $(XARGS) -0 eclint check $(PROJECT_ROOT){} + ls $(ECLINT_FILES) | $(XARGS) eclint check {} @ echo "[$@]: Project PASSED eclint test!" python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)%s\n" diff --git a/tests/make/eclint_lint_success.bats b/tests/make/eclint_lint_success.bats index 3d98d82e..76a7a533 100644 --- a/tests/make/eclint_lint_success.bats +++ b/tests/make/eclint_lint_success.bats @@ -18,11 +18,11 @@ done git add "$TEST_DIR/." git commit -m 'eclint success testing' - } @test "eclint/lint: success" { - run make eclint/lint + ECLINT_FILES=$(find "${TEST_DIR}" -type f | xargs echo) + run make eclint/lint ECLINT_FILES="${ECLINT_FILES}" [ "$status" -eq 0 ] } From 454fb3ed2c08c1c07e5ad18c50b48e3f0bcc7ceb Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Wed, 9 Dec 2020 23:21:30 -0500 Subject: [PATCH 05/12] Make PYTHON_FILES, ECLINT_FILES files, not pipes --- Makefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8726a1d5..aef26734 100644 --- a/Makefile +++ b/Makefile @@ -183,25 +183,27 @@ eclint/lint: @ echo "[$@]: Running eclint..." cd $(PROJECT_ROOT) && \ [ -z "$(HAS_UNTRACKED_CHANGES)" ] || (echo "untracked changes detected!" && exit 1) - ls $(ECLINT_FILES) | $(XARGS) eclint check {} + eclint check $(ECLINT_FILES) @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)%s\n" +python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)%s ") ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - $(PYTHON_FILES) | xargs black --check - $(PYTHON_FILES) | $(XARGS) -n1 pylint -rn -sn \ - --msg-template="{path}:{line} [{symbol}] {msg}" {} + black --check $(PYTHON_FILES) + for python_file in $(PYTHON_FILES); do \ + pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ + -rn -sn $$python_file; \ + done @ echo "[$@]: Python files PASSED lint test!" ## Formats Python files. python/format: | guard/program/black guard/program/git python/format: @ echo "[$@]: Formatting Python files..." - $(PYTHON_FILES) | xargs black + black $(PYTHON_FILES) @ echo "[$@]: Successfully formatted Python files!" ## Lints terraform files From 179ec8948519fc1aee94f9296bf51d6191b74176 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Wed, 9 Dec 2020 23:44:32 -0500 Subject: [PATCH 06/12] Handle case where there are no python files --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index aef26734..186c477a 100644 --- a/Makefile +++ b/Makefile @@ -192,11 +192,13 @@ python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --oth python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - black --check $(PYTHON_FILES) - for python_file in $(PYTHON_FILES); do \ - pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ - -rn -sn $$python_file; \ - done + @ [[ ! -z "$(findstring .py, $(PYTHON_FILES))" ]] && \ + black --check $(PYTHON_FILES) && \ + for python_file in $(PYTHON_FILES); do \ + pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ + -rn -sn $$python_file; \ + done || \ + (echo "No python files to lint."; exit 0) @ echo "[$@]: Python files PASSED lint test!" ## Formats Python files. From c69d85a304449117fa7118842f1e0e7d6a9efd23 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 12:35:30 -0500 Subject: [PATCH 07/12] Pylint errors should cause make to return error The previous logic would always return success whether pylint found an error or not. Reverted to shell-like logic to filter out any non-Python files that could be passed in by the user though the PYTHON_FILES variable or when there are no python files and PYTHON_FILES is set to $(PROJECT_ROOT) (shame on printf's handling of null). --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 186c477a..b6bf53fc 100644 --- a/Makefile +++ b/Makefile @@ -192,14 +192,14 @@ python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --oth python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - @ [[ ! -z "$(findstring .py, $(PYTHON_FILES))" ]] && \ - black --check $(PYTHON_FILES) && \ - for python_file in $(PYTHON_FILES); do \ + black --check $(PYTHON_FILES) + for python_file in $(PYTHON_FILES); do \ + if [[ $$python_file = *.py ]]; then \ pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ -rn -sn $$python_file; \ - done || \ - (echo "No python files to lint."; exit 0) - @ echo "[$@]: Python files PASSED lint test!" + fi \ + done + echo "[$@]: Python files PASSED lint test!" ## Formats Python files. python/format: | guard/program/black guard/program/git From fd8b4f2f55adc930b580717fbd62bbcb5143046e Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 12:42:03 -0500 Subject: [PATCH 08/12] Remove trailing space in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b6bf53fc..8d65b996 100644 --- a/Makefile +++ b/Makefile @@ -192,7 +192,7 @@ python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --oth python/lint: | guard/program/pylint guard/program/black guard/program/git python/lint: @ echo "[$@]: Linting Python files..." - black --check $(PYTHON_FILES) + black --check $(PYTHON_FILES) for python_file in $(PYTHON_FILES); do \ if [[ $$python_file = *.py ]]; then \ pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ From f8f6c8085fd69c1c8a6772de616f15c3f8dcb7b2 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 13:07:49 -0500 Subject: [PATCH 09/12] Add a test that sets PYTHON_FILES --- tests/make/python_format_success.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/make/python_format_success.bats b/tests/make/python_format_success.bats index 1575e9ef..b8e131b0 100644 --- a/tests/make/python_format_success.bats +++ b/tests/make/python_format_success.bats @@ -21,6 +21,10 @@ done run make python/format [ "$status" -eq 0 ] [[ "$output" == *"[python/format]: Successfully formatted Python files!"* ]] + + run make python/format PYTHON_FILES="$TEST_DIR/nested/*.py" + [ "$status" -eq 0 ] + [[ "$output" == *"1 file left unchanged"* ]] } function teardown() { From cd99c6d7bbec4d9e42986406b0f2117b6aa32073 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 13:24:17 -0500 Subject: [PATCH 10/12] Workaround PROJECT_ROOT's missing trailing slash This is a circular problem with the docker make actions. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8d65b996..54795be2 100644 --- a/Makefile +++ b/Makefile @@ -186,7 +186,7 @@ eclint/lint: eclint check $(ECLINT_FILES) @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)%s ") +python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)/%s ") ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git From 0c4cc604f5e990934dd8009cadcc88a69ef1a289 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 16:10:19 -0500 Subject: [PATCH 11/12] Don't exclude non-'*.py' files Also added the --no-run-if-empty argument to the pipeline that generates the list of python files. This removes the problem when there are no python files in the pipeline and printf then adds $PROJECT_NAME to an empty space anyway. --- Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 54795be2..7a138a97 100644 --- a/Makefile +++ b/Makefile @@ -186,7 +186,7 @@ eclint/lint: eclint check $(ECLINT_FILES) @ echo "[$@]: Project PASSED eclint test!" -python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs printf "$(PROJECT_ROOT)/%s ") +python/%: PYTHON_FILES ?= $(shell git -C $(PROJECT_ROOT) ls-files --cached --others --exclude-standard '*.py' | xargs --no-run-if-empty printf "$(PROJECT_ROOT)/%s ") ## Checks format and lints Python files. Runs pylint on each individual ## file and uses a custom format for the lint messages. python/lint: | guard/program/pylint guard/program/black guard/program/git @@ -194,10 +194,8 @@ python/lint: @ echo "[$@]: Linting Python files..." black --check $(PYTHON_FILES) for python_file in $(PYTHON_FILES); do \ - if [[ $$python_file = *.py ]]; then \ - pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ - -rn -sn $$python_file; \ - fi \ + pylint --msg-template="{path}:{line} [{symbol}] {msg}" \ + -rn -sn $$python_file; \ done echo "[$@]: Python files PASSED lint test!" From 927d3b66735d3aca3ef6c9916aa79558a7bdfb77 Mon Sep 17 00:00:00 2001 From: Karri Balk Date: Thu, 10 Dec 2020 16:44:38 -0500 Subject: [PATCH 12/12] Bumps version to 0.3.0 --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 997fc850..e52bfb36 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.3.0 commit = True message = Bumps version to {new_version} tag = False