Implement hatch

.github/workflows/build.yml

@@ -61,31 +61,5 @@ jobs:
         run: |
           hatch build
 
-      - name: "Show distributions"
-        run: ls -lh dist/
-
-      - name: "Check distribution descriptions"
-        run: |
-          twine check dist/*
-
-      - name: "Check wheel contents"
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
-
-      - name: "Install wheel distributions"
-        run: |
-          find ./dist/dbt_common-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-
-      # TODO: how to validate here?  we did dbt --version previously. this checks it's there, but not that it can do anything. maybe it's enough?
-      - name: "Check wheel distributions"
-        run: |
-          pip freeze | grep dbt-common
-
-      - name: "Install source distributions"
-        run: |
-          find ./dist/dbt_common-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-
-      # TODO: how to validate here?  we did dbt --version previously. this checks it's there, but not that it can do anything. maybe it's enough?
-      - name: "Check source distributions"
-        run: |
-          pip freeze | grep dbt-common
+      - name: "Check build"
+        run: hatch run build:check-all

.github/workflows/ci_code_quality.yml

@@ -51,5 +51,9 @@ jobs:
         shell: bash
         run: pip3 install hatch
 
+      - name: Install pre-commit
+        shell: bash
+        run: pip3 install pre-commit
+
       - name: Run Pre-commit Hooks
-        run: hatch run dev-env:pre-commit run --show-diff-on-failure --color=always --all-files
+        run: pre-commit run --show-diff-on-failure --color=always --all-files

.github/workflows/ci_tests.yml

@@ -57,7 +57,7 @@ jobs:
         run: pip3 install hatch
 
       - name: "Run Tests"
-        run: hatch run dev-env:pytest tests
+        run: hatch run test:unit
 
       - name: "Get current date"
         if: always()

.pre-commit-config.yaml

@@ -10,51 +10,34 @@ repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v3.2.0
   hooks:
+  - id: debug-statements
   - id: check-yaml
     args: [--unsafe]
   - id: end-of-file-fixer
   - id: trailing-whitespace
     exclude_types:
       - "markdown"
   - id: check-case-conflict
-- repo: https://github.com/psf/black
-  rev: 22.3.0
+- repo: local
   hooks:
   - id: black
-  - id: black
-    alias: black-check
-    stages: [manual]
-    args:
-    - "--check"
-    - "--diff"
-- repo: https://github.com/pycqa/flake8
-  rev: 4.0.1
-  hooks:
+    name: black
+    entry: hatch run lint:black
+    language: system
+    types: [python]
+    pass_filenames: false
+    verbose: true
   - id: flake8
-  - id: flake8
-    alias: flake8-check
-    stages: [manual]
-# - repo: https://github.com/pre-commit/mirrors-mypy
-#   rev: v1.4.1
-#   hooks:
-#   - id: mypy
-#     # N.B.: Mypy is... a bit fragile.
-#     #
-#     # By using `language: system` we run this hook in the local
-#     # environment instead of a pre-commit isolated one.  This is needed
-#     # to ensure mypy correctly parses the project.
-
-#     # It may cause trouble
-#     # in that it adds environmental variables out of our control to the
-#     # mix.  Unfortunately, there's nothing we can do about per pre-commit's
-#     # author.
-#     # See https://github.com/pre-commit/pre-commit/issues/730 for details.
-#     args: [--show-error-codes]
-#     files: ^dbt_common/
-#     language: system
-#   - id: mypy
-#     alias: mypy-check
-#     stages: [manual]
-#     args: [--show-error-codes, --pretty]
-#     files: ^dbt_common
-#     language: system
+    name: flake8
+    entry: hatch run lint:flake8
+    language: system
+    types: [python]
+    pass_filenames: false
+    verbose: true
+  - id: mypy
+    name: mypy
+    entry: hatch run lint:mypy
+    language: system
+    types: [python]
+    pass_filenames: false
+    verbose: true

CONTRIBUTING.md

@@ -54,6 +54,7 @@ There are some tools that will be helpful to you in developing locally. While th
 
 These are the tools used in `dbt-common` development and testing:
 
+- [`hatch`](https://hatch.pypa.io/latest/) for project management
 - [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
 - [`black`](https://github.com/psf/black) for code formatting
 - [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking
@@ -62,31 +63,15 @@ These are the tools used in `dbt-common` development and testing:
 
 A deep understanding of these tools in not required to effectively contribute to `dbt-common`, but we recommend checking out the attached documentation if you're interested in learning more about each one.
 
-#### Virtual environments
-
-We strongly recommend using virtual environments when developing code in `dbt-common`. We recommend creating this virtualenv
-in the root of the `dbt-common` repository. To create a new virtualenv, run:
-```sh
-python3 -m venv env
-source env/bin/activate
-```
-
-This will create and activate a new Python virtual environment.
-
 ## Running `dbt-common` in development
 
 ### Installation
 
-First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment).  Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-common` (and its dependencies):
-
-```sh
-git
-pre-commit install
-```
+Ensure you have the latest version of pip installed with `pip install --upgrade pip` as well as [hatch](https://hatch.pypa.io/latest/install/).
 
 ### Running `dbt-common`
 
-This repository is just a template and cannot be run.
+This repository cannot be run on its own.
 
 ## Testing
 
@@ -98,29 +83,36 @@ Once you're able to manually test that your code change is working as expected,
 
 ### Initial setup
 
-None needed.
+- [Install pre-commit](https://pre-commit.com/#usage)
+- [Install hatch](https://hatch.pypa.io/1.7/install/#pip)
 
-### Test commands
+- Nothing needed to set up your environments.  hatch will create your environment as defined in the `pyproject.toml` when you run.
 
-No tests included.
+- set up pre-commit to use hatch: `hatch run setup-pre-commit`
 
+### Hatch Commands
 
-### Unit, Integration, Functional?
+See the pyproject.toml for a complete list of custom commands.  See the h[atch docs](https://hatch.pypa.io/latest/cli/reference/) for a description of built in commands and flags.  These are the most useful custom commands to use while developing.
 
-Here are some general rules for adding tests:
-* unit tests (`tests/unit`) don’t need to access a database; "pure Python" tests should be written as unit tests
-* functional tests (`tests/functional`) cover anything that interacts with a database, namely adapter
+|Type|Command|Description|
+|---|---|---|
+|Utility|`hatch run proto`|regenerate protobuf definitions|
+|Testing|`hatch run test:unit`|run all tests|
+|Testing|`hatch shell test`|Drops you into a shell env set up for manual testing|
+|Code Quality|`hatch run lint:all`|run black, flake8 and mypy checks|
+|Code Quality|`hatch run lint:black`|run black|
+|Code Quality|`hatch run lint:flake8`|run flake8|
+|Code Quality|`hatch run lint:mypy`|run mypy|
+|Testing|`hatch shell`|Drops you into a shell env with all dependencies + dev dependencies installed.  Use `exit` to leave the shell, _not_ `deactivate`.|
 
 ## Debugging
 
-1. The logs for a `dbt run` have stack traces and other information for debugging errors (in `logs/dbt.log` in your project directory).
-2. Try using a debugger, like `ipdb`. For pytest: `--pdb --pdbcls=IPython.terminal.debugger:pdb`
-3. 
+1. Try using a debugger, like `ipdb`. For pytest: `--pdb --pdbcls=IPython.terminal.debugger:pdb`
+2. 
 
 ### Assorted development tips
 * Append `# type: ignore` to the end of a line if you need to disable `mypy` on that line.
 * Sometimes flake8 complains about lines that are actually fine, in which case you can put a comment on the line such as: # noqa or # noqa: ANNN, where ANNN is the error code that flake8 issues.
-* To collect output for `CProfile`, run dbt with the `-r` option and the name of an output file, i.e. `dbt -r dbt.cprof run`. If you just want to profile parsing, you can do: `dbt -r dbt.cprof parse`. `pip` install `snakeviz` to view the output. Run `snakeviz dbt.cprof` and output will be rendered in a browser window.
 
 ## Adding or modifying a CHANGELOG Entry
 

dbt_common/clients/_jinja_blocks.py

@@ -28,7 +28,11 @@ def __init__(self, contents: str) -> None:
 
 class BlockTag:
     def __init__(
-        self, block_type_name: str, block_name: str, contents: Optional[str] = None, full_block: Optional[str] = None
+        self,
+        block_type_name: str,
+        block_name: str,
+        contents: Optional[str] = None,
+        full_block: Optional[str] = None,
     ) -> None:
         self.block_type_name = block_type_name
         self.block_name = block_name
@@ -106,7 +110,9 @@ def __init__(self, text: str) -> None:
         self.pos: int = 0
 
     def linepos(self, end: Optional[int] = None) -> str:
-        """Given an absolute position in the input text, return a pair of
+        """Return relative position in line.
+
+        Given an absolute position in the input data, return a pair of
         line number + relative position to the start of the line.
         """
         end_val: int = self.pos if end is None else end
@@ -148,7 +154,9 @@ def _expect_match(self, expected_name: str, *patterns) -> re.Match:  # type: ign
         return match
 
     def handle_expr(self, match: re.Match) -> None:
-        """Handle an expression. At this point we're at a string like:
+        """Handle an expression.
+
+        At this point we're at a string like:
             {{ 1 + 2 }}
             ^ right here
 
@@ -180,6 +188,7 @@ def handle_comment(self, match: re.Match) -> None:
 
     def _expect_block_close(self) -> None:
         """Search for the tag close marker.
+
         To the right of the type name, there are a few possiblities:
            - a name (handled by the regex's 'block_name')
            - any number of: `=`, `(`, `)`, strings, etc (arguments)
@@ -191,7 +200,9 @@ def _expect_block_close(self) -> None:
         are quote and `%}` - nothing else can hide the %} and be valid jinja.
         """
         while True:
-            end_match = self._expect_match('tag close ("%}")', QUOTE_START_PATTERN, TAG_CLOSE_PATTERN)
+            end_match = self._expect_match(
+                'tag close ("%}")', QUOTE_START_PATTERN, TAG_CLOSE_PATTERN
+            )
             self.advance(end_match.end())
             if end_match.groupdict().get("tag_close") is not None:
                 return
@@ -207,7 +218,9 @@ def handle_raw(self) -> int:
         return match.end()
 
     def handle_tag(self, match: re.Match) -> Tag:
-        """The tag could be one of a few things:
+        """Determine tag type.
+
+        The tag could be one of a few things:
 
             {% mytag %}
             {% mytag x = y %}
@@ -229,11 +242,15 @@ def handle_tag(self, match: re.Match) -> Tag:
         else:
             self.advance(match.end())
             self._expect_block_close()
-        return Tag(block_type_name=block_type_name, block_name=block_name, start=start_pos, end=self.pos)
+        return Tag(
+            block_type_name=block_type_name, block_name=block_name, start=start_pos, end=self.pos
+        )
 
     def find_tags(self) -> Iterator[Tag]:
         while True:
-            match = self._first_match(BLOCK_START_PATTERN, COMMENT_START_PATTERN, EXPR_START_PATTERN)
+            match = self._first_match(
+                BLOCK_START_PATTERN, COMMENT_START_PATTERN, EXPR_START_PATTERN
+            )
             if match is None:
                 break
 
@@ -253,7 +270,8 @@ def find_tags(self) -> Iterator[Tag]:
                 yield self.handle_tag(match)
             else:
                 raise DbtInternalError(
-                    "Invalid regex match in next_block, expected block start, " "expr start, or comment start"
+                    "Invalid regex match in next_block, expected block start, "
+                    "expr start, or comment start"
                 )
 
     def __iter__(self) -> Iterator[Tag]:
@@ -349,4 +367,6 @@ def find_blocks(
     def lex_for_blocks(
         self, allowed_blocks: Optional[Set[str]] = None, collect_raw_data: bool = True
     ) -> List[Union[BlockData, BlockTag]]:
-        return list(self.find_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data))
+        return list(
+            self.find_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data)
+        )