Support dbt-core==1.0.0 (Issue #13) (#14)

* fix for dbt v1.0.0

* updating workflow

* allow multiple lines in log result

Co-authored-by: robastel <[email protected]>

* bump versions

* hardcoding json log format

* missing and

* linter reformats

* Run workflow on pull request synchronize

* fixing python version for test in workflow

* Exclude Python 3.9 dbt 0.18.x test combo

* Update README.md

Co-authored-by: robastel <[email protected]>
Co-authored-by: Jan Nitschke <[email protected]>

.github/workflows/test.yml

@@ -7,6 +7,7 @@ on:
     types:
       - opened
       - reopened
+      - synchronize
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -29,10 +30,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.x]
-        dbt-version: [0.18.x, 0.19.x]
+        python-version: [3.7, 3.9]
+        dbt-version: [0.18.x, 0.19.x, 1.x]
         exclude:
-          - python-version: 3.x
+          - python-version: 3.9
             dbt-version: 0.18.x
     steps:
       - uses: actions/checkout@v2
@@ -58,4 +59,4 @@ jobs:
         env:
           TARGET: default
         run: |
-          python tests/test.py
+          python tests/test.py

README.md

@@ -190,7 +190,7 @@ dbt-invoke properties.delete <options>
   - This may be partially remedied by increasing the value of the `--threads` 
     option in `dbt-invoke properties.update`.
 - dbt-invoke is tested against:
-  - Python 3.7 as well as the latest version Python 3.x at the time of merge
-  - dbt 0.18 and 0.19
+  - Python 3.7 and 3.9
+  - dbt 0.18, 0.19, and 1.0
   - macos-latest, windows-latest, ubuntu-latest
 - dbt-invoke has not been tested across different types of data warehouses.

dbt_invoke/internal/_utils.py

@@ -236,6 +236,10 @@ def dbt_run_operation(
         'bypass_cache': bypass_cache,
     }
     dbt_cli_kwargs = get_cli_kwargs(**dbt_kwargs)
+
+    dbt_global_kwargs = {'log-format': 'json'}
+    dbt_global_cli_kwargs = get_cli_kwargs(**dbt_global_kwargs)
+
     macro_kwargs = json.dumps(kwargs, sort_keys=False)
     if platform.system().lower().startswith('win'):
         # Format YAML string for Windows Command Prompt
@@ -249,12 +253,12 @@ def dbt_run_operation(
         macro_kwargs = macro_kwargs.replace("'", """'"'"'""")
         macro_kwargs = f"'{macro_kwargs}'"
     command = (
-        f"dbt run-operation {dbt_cli_kwargs}"
+        f"dbt {dbt_global_cli_kwargs} run-operation {dbt_cli_kwargs}"
         f" {macro_name} --args {macro_kwargs}"
     )
     logger.debug(f'Running command: {command}')
     result = ctx.run(command, hide=hide)
-    result_lines = result.stdout.splitlines()[1:]
+    result_lines = [json.loads(data) for data in result.stdout.splitlines()]
     return result_lines
 
 

dbt_invoke/internal/_version.py

@@ -1 +1 @@
-__version__ = '0.1.2'
+__version__ = '0.1.3'

dbt_invoke/properties.py

@@ -6,6 +6,7 @@
 from invoke import task
 
 from dbt_invoke.internal import _utils
+import ast
 
 _LOGGER = _utils.get_logger('dbt-invoke')
 _MACRO_NAME = '_log_columns_list'
@@ -502,13 +503,21 @@ def _get_columns(ctx, resource_location, resource_dict, **kwargs):
         result_lines = _utils.dbt_run_operation(
             ctx, _MACRO_NAME, hide=True, logger=_LOGGER, sql=sql, **kwargs
         )
-    result_list_strings = [
-        s for s in result_lines if s.startswith('[') and s.endswith(']')
-    ]
-    # Take the last line that started with '[' and ended with ']'
-    # (just in case there is multi-line output)
-    columns_string = result_list_strings[-1].strip('][').split(', ')
-    columns = [col.strip("'") for col in columns_string]
+
+    # from dbt-core v1.0.0 onwards, run_operations INFO logs have code M011 and messages have key 'msg'
+    # https://github.com/dbt-labs/dbt-core/blob/22b1a09aa218e8152b0c2dd261abe2503ea15ddb/core/dbt/events/types.py#L401
+    relevant_lines = list(
+        filter(lambda x: x.get('code') == 'M011', result_lines)
+    )
+    if len(relevant_lines) >= 1:
+        columns = relevant_lines[-1].get('msg')
+    else:
+        # for older dbt-core versions, we need to cross fingers a little harder
+        relevant_lines = result_lines[1:]
+        # also, the message key is different
+        columns = relevant_lines[-1].get('message')
+        # columns are not passed as valid json but as a string representation of a list
+        columns = ast.literal_eval(columns)
     return columns
 
 

requirements/requirements_dbt_1.x.txt

@@ -0,0 +1,7 @@
+agate<1.6.2
+dbt-core~=1.0.0
+dbt-sqlite~=0.2.2
+invoke>=1.4.1
+pytz<2021.0
+PyYAML>=5.1
+-e .

tests/dbt_project_files/dbt_project.yml

@@ -0,0 +1,18 @@
+name: test_dbt_project
+version: 1.0.0
+config-version: 2
+
+profile: dbt-sqlite
+
+seed-paths: ["data"]
+
+clean-targets:
+  - target
+  - dbt_packages
+  - logs
+
+analysis-paths:
+  - analyses
+
+snapshots:
+  +target_schema: main

tests/dbt_project_files/dbt_project_pre_dbt_v1.yml

@@ -0,0 +1,16 @@
+name: test_dbt_project
+version: 1.0.0
+config-version: 2
+
+profile: dbt-sqlite
+
+clean-targets:
+  - target
+  - dbt_modules
+  - logs
+
+analysis-paths:
+  - analyses
+
+snapshots:
+  +target_schema: main

tests/test.py

@@ -9,6 +9,9 @@
 from dbt_invoke import properties
 from dbt_invoke.internal import _utils
 
+import pkg_resources
+import shutil
+
 PARENT_DIR = Path(__file__).parent
 
 
@@ -23,6 +26,25 @@ def setUpClass(cls):
         cls.logger = _utils.get_logger('dbt-invoke', level='DEBUG')
         cls.config_path = Path(PARENT_DIR, 'test_config.yml')
         cls.config = _utils.parse_yaml(cls.config_path)
+
+        # for backward compatibility, select the correct dbt_project.yml file
+        if pkg_resources.get_distribution("dbt-core").version >= '1.0.0':
+            shutil.copy(
+                Path(PARENT_DIR, 'dbt_project_files/dbt_project.yml'),
+                Path(
+                    PARENT_DIR, cls.config['project_name'], 'dbt_project.yml'
+                ),
+            )
+        else:
+            shutil.copy(
+                Path(
+                    PARENT_DIR, 'dbt_project_files/dbt_project_pre_dbt_v1.yml'
+                ),
+                Path(
+                    PARENT_DIR, cls.config['project_name'], 'dbt_project.yml'
+                ),
+            )
+
         cls.project_dir = Path(PARENT_DIR, cls.config['project_name'])
         cls.profiles_dir = Path(PARENT_DIR, cls.config['project_name'])
         cls.expected_properties = cls.config['expected_properties']