Add commit hook tooling. (#356)

* Add commit hook tooling. (#346)
* Make pre commit code changes (#345)
* Refactor exception function using accurate type specs.
* Upgrade the mypy version in order to use TypeAlias-es.
* Upgrade the mypy version in the worfklow. Fix bug introduced by using str's instead of Optional[str]'s
* Address code review comments: Remove integration test command since there are multiple ways we can run tests in spark.
* Add changelog entry
* Altering names of dev_requirements references.

.bumpversion.cfg

@@ -4,7 +4,7 @@ parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
 	((?P<prerelease>a|b|rc)(?P<num>\d+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}{prerelease}{num}
 	{major}.{minor}.{patch}
 commit = False
@@ -13,7 +13,7 @@ tag = False
 [bumpversion:part:prerelease]
 first_value = a
 optional_value = final
-values = 
+values =
 	a
 	b
 	rc
@@ -25,4 +25,3 @@ first_value = 1
 [bumpversion:file:setup.py]
 
 [bumpversion:file:dbt/adapters/spark/__version__.py]
-

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+select =
+    E
+    W
+    F
+ignore =
+    W503 # makes Flake8 work like black
+    W504
+    E203 # makes Flake8 work like black
+    E741
+    E501
+exclude = test

.github/ISSUE_TEMPLATE/dependabot.yml

@@ -5,4 +5,4 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
-    rebase-strategy: "disabled"
+    rebase-strategy: "disabled"

.github/ISSUE_TEMPLATE/release.md

@@ -7,4 +7,4 @@ assignees: ''
 
 ---
 
-### TBD
+### TBD

.github/pull_request_template.md

@@ -18,4 +18,4 @@ resolves #
 - [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
 - [ ] I have run this code in development and it appears to resolve the stated issue
 - [ ] This PR includes tests, or tests are not required/relevant for this PR
-- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-spark next" section.
+- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-spark next" section.

.github/workflows/jira-creation.yml

@@ -13,7 +13,7 @@ name: Jira Issue Creation
 on:
   issues:
     types: [opened, labeled]
-    
+
 permissions:
   issues: write
 

.github/workflows/jira-label.yml

@@ -13,7 +13,7 @@ name: Jira Label Mirroring
 on:
   issues:
     types: [labeled, unlabeled]
-    
+
 permissions:
   issues: read
 
@@ -24,4 +24,3 @@ jobs:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
       JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-

.github/workflows/jira-transition.yml

@@ -21,4 +21,4 @@ jobs:
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
-      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}

.github/workflows/main.yml

@@ -37,19 +37,10 @@ defaults:
 
 jobs:
   code-quality:
-    name: ${{ matrix.toxenv }}
+    name: code-quality
 
     runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        toxenv: [flake8]
-
-    env:
-      TOXENV: ${{ matrix.toxenv }}
-      PYTEST_ADDOPTS: "-v --color=yes"
-
     steps:
       - name: Check out the repository
         uses: actions/checkout@v2
@@ -58,18 +49,19 @@ jobs:
 
       - name: Set up Python
         uses: actions/setup-python@v2
-        with: 
+        with:
           python-version: '3.8'
 
       - name: Install python dependencies
         run: |
           sudo apt-get install libsasl2-dev
           pip install --user --upgrade pip
-          pip install tox
-          pip --version
-          tox --version
-      - name: Run tox
-        run: tox
+          pip install -r dev-requirements.txt
+          pre-commit --version
+          mypy --version
+          dbt --version
+      - name: pre-commit hooks
+        run: pre-commit run --all-files --show-diff-on-failure
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -153,7 +145,7 @@ jobs:
       - name: Check wheel contents
         run: |
           check-wheel-contents dist/*.whl --ignore W007,W008
-          
+
       - name: Check if this is an alpha version
         id: check-is-alpha
         run: |

.github/workflows/release.yml

@@ -116,4 +116,3 @@ jobs:
           pip install twine wheel setuptools
           python setup.py sdist bdist_wheel
           twine upload --non-interactive dist/dbt_spark-${{env.version_number}}-py3-none-any.whl dist/dbt-spark-${{env.version_number}}.tar.gz
-

.github/workflows/stale.yml

@@ -13,5 +13,3 @@ jobs:
           stale-pr-message: "This PR has been marked as Stale because it has been open for 180 days with no activity. If you would like the PR to remain open, please remove the stale label or comment on the PR, or it will be closed in 7 days."
           # mark issues/PRs stale when they haven't seen activity in 180 days
           days-before-stale: 180
-          # ignore checking issues with the following labels
-          exempt-issue-labels: "epic, discussion"

.gitignore

@@ -1,18 +1,47 @@
-.hive-metastore/
-.spark-warehouse/
-*.egg-info
-env/
-*.pyc
+# Byte-compiled / optimized / DLL files
 __pycache__
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+env*/
+dbt_env/
+dist/
+*.egg-info
+logs/
+
+
+# Unit test
 .tox/
 .env
+test.env
+
+
+# Django stuff
+*.log
+
+# Mypy
+*.pytest_cache/
+
+# Vim
+*.sw*
+
+# Pyenv
+.python-version
+
+# pycharm
 .idea/
-build/
-dist/
-dbt-integration-tests
-test/integration/.user.yml
+
+# MacOS
 .DS_Store
-test.env
+
+# vscode
 .vscode
-*.log
-logs/
+
+# other
+.hive-metastore/
+.spark-warehouse/
+dbt-integration-tests
+test/integration/.user.yml

.pre-commit-config.yaml

@@ -0,0 +1,66 @@
+# For more on configuring pre-commit hooks (see https://pre-commit.com/)
+
+# TODO: remove global exclusion of tests when testing overhaul is complete
+exclude: '^tests/.*'
+
+# Force all unspecified python hooks to run python 3.8
+default_language_version:
+  python: python3.8
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+  - id: check-yaml
+    args: [--unsafe]
+  - id: check-json
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-case-conflict
+- repo: https://github.com/psf/black
+  rev: 21.12b0
+  hooks:
+  - id: black
+    additional_dependencies: ['click==8.0.4']
+    args:
+    - "--line-length=99"
+    - "--target-version=py38"
+  - id: black
+    alias: black-check
+    stages: [manual]
+    additional_dependencies: ['click==8.0.4']
+    args:
+    - "--line-length=99"
+    - "--target-version=py38"
+    - "--check"
+    - "--diff"
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 4.0.1
+  hooks:
+  - id: flake8
+  - id: flake8
+    alias: flake8-check
+    stages: [manual]
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.950
+  hooks:
+  - id: mypy
+    # N.B.: Mypy is... a bit fragile.
+    #
+    # By using `language: system` we run this hook in the local
+    # environment instead of a pre-commit isolated one.  This is needed
+    # to ensure mypy correctly parses the project.
+
+    # It may cause trouble in that it adds environmental variables out
+    # of our control to the mix.  Unfortunately, there's nothing we can
+    # do about per pre-commit's author.
+    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+    args: [--show-error-codes, --ignore-missing-imports]
+    files: ^dbt/adapters/.*
+    language: system
+  - id: mypy
+    alias: mypy-check
+    stages: [manual]
+    args: [--show-error-codes, --pretty, --ignore-missing-imports]
+    files: ^dbt/adapters
+    language: system

CHANGELOG.md

@@ -5,6 +5,7 @@
 - rename file to match reference to dbt-core ([#344](https://github.com/dbt-labs/dbt-spark/pull/344))
 
 ### Under the hood
+- Add precommit tooling to this repo ([#356](https://github.com/dbt-labs/dbt-spark/pull/356))
 - Use dbt.tests.adapter.basic in test suite ([#298](https://github.com/dbt-labs/dbt-spark/issues/298), [#299](https://github.com/dbt-labs/dbt-spark/pull/299))
 - Make internal macros use macro dispatch to be overridable in child adapters ([#319](https://github.com/dbt-labs/dbt-spark/issues/319), [#320](https://github.com/dbt-labs/dbt-spark/pull/320))
 - Override adapter method 'run_sql_for_tests' ([#323](https://github.com/dbt-labs/dbt-spark/issues/323), [#324](https://github.com/dbt-labs/dbt-spark/pull/324))

MANIFEST.in

@@ -1 +1 @@
-recursive-include dbt/include *.sql *.yml *.md
+recursive-include dbt/include *.sql *.yml *.md

Makefile

@@ -0,0 +1,56 @@
+.DEFAULT_GOAL:=help
+
+.PHONY: dev
+dev: ## Installs adapter in develop mode along with development depedencies
+	@\
+	pip install -r dev-requirements.txt && pre-commit install
+
+.PHONY: mypy
+mypy: ## Runs mypy against staged changes for static type checking.
+	@\
+	pre-commit run --hook-stage manual mypy-check | grep -v "INFO"
+
+.PHONY: flake8
+flake8: ## Runs flake8 against staged changes to enforce style guide.
+	@\
+	pre-commit run --hook-stage manual flake8-check | grep -v "INFO"
+
+.PHONY: black
+black: ## Runs black  against staged changes to enforce style guide.
+	@\
+	pre-commit run --hook-stage manual black-check -v | grep -v "INFO"
+
+.PHONY: lint
+lint: ## Runs flake8 and mypy code checks against staged changes.
+	@\
+	pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
+
+.PHONY: linecheck
+linecheck: ## Checks for all Python lines 100 characters or more
+	@\
+	find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \;
+
+.PHONY: unit
+unit: ## Runs unit tests with py38.
+	@\
+	tox -e py38
+
+.PHONY: test
+test: ## Runs unit tests with py38 and code checks against staged changes.
+	@\
+	tox -p -e py38; \
+	pre-commit run black-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
+
+.PHONY: clean
+	@echo "cleaning repo"
+	@git clean -f -X
+
+.PHONY: help
+help: ## Show this help message.
+	@echo 'usage: make [target]'
+	@echo
+	@echo 'targets:'
+	@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

dbt/adapters/spark/__init__.py

@@ -8,6 +8,5 @@
 from dbt.include import spark
 
 Plugin = AdapterPlugin(
-    adapter=SparkAdapter,
-    credentials=SparkCredentials,
-    include_path=spark.PACKAGE_PATH)
+    adapter=SparkAdapter, credentials=SparkCredentials, include_path=spark.PACKAGE_PATH
+)