From ce474811003a323ea882df51b2945edabd9f5f20 Mon Sep 17 00:00:00 2001 From: Ayokunle Bamisile-Aje Date: Thu, 13 Aug 2020 19:21:49 -0400 Subject: [PATCH 1/9] Add dbt operator to run dbt docs generate --- airflow_dbt/operators/dbt_operator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 04d32f1..233b45d 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -89,3 +89,13 @@ def __init__(self, profiles_dir=None, target=None, *args, **kwargs): def execute(self, context): self.create_hook().run_cli('test') + + +class DbtDocsGenerateOperator(DbtBaseOperator): + @apply_defaults + def __init__(self, profiles_dir=None, target=None, *args, **kwargs): + super(DbtDocsGenerateOperator, self).__init__( + profiles_dir=profiles_dir, target=target, *args, **kwargs + ) + def execute(self, context): + self.hook.run_cli("docs", "generate") \ No newline at end of file From 1f2e98fa5d5cc93c2dc3d9a15fe18c04224a0784 Mon Sep 17 00:00:00 2001 From: Ayokunle Bamisile-Aje Date: Thu, 13 Aug 2020 20:05:18 -0400 Subject: [PATCH 2/9] Import DbtDocsGenerateOperator in __init__.py --- airflow_dbt/__init__.py | 2 +- airflow_dbt/operators/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_dbt/__init__.py b/airflow_dbt/__init__.py index 5ee30a6..b5cfede 100644 --- a/airflow_dbt/__init__.py +++ b/airflow_dbt/__init__.py @@ -1,2 +1,2 @@ from .hooks import DbtCliHook -from .operators import DbtRunOperator, DbtTestOperator +from .operators import DbtRunOperator, DbtTestOperator, DbtDocsGenerateOperator diff --git a/airflow_dbt/operators/__init__.py b/airflow_dbt/operators/__init__.py index 772a4be..226bb88 100644 --- a/airflow_dbt/operators/__init__.py +++ b/airflow_dbt/operators/__init__.py @@ -1 +1 @@ -from .dbt_operator import DbtRunOperator, DbtTestOperator +from .dbt_operator import DbtRunOperator, DbtTestOperator, DbtDocsGenerateOperator From 5bed674e14951ff12ab90bbe7b7bc5ab93e2a9be Mon Sep 17 00:00:00 2001 From: Ayokunle Bamisile-Aje Date: Fri, 14 Aug 2020 18:04:36 -0400 Subject: [PATCH 3/9] Refactor DbtDocsGenerateOperator --- airflow_dbt/operators/dbt_operator.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 233b45d..86fcf4c 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -94,8 +94,6 @@ def execute(self, context): class DbtDocsGenerateOperator(DbtBaseOperator): @apply_defaults def __init__(self, profiles_dir=None, target=None, *args, **kwargs): - super(DbtDocsGenerateOperator, self).__init__( - profiles_dir=profiles_dir, target=target, *args, **kwargs - ) + super(DbtDocsGenerateOperator, self).__init__(profiles_dir=profiles_dir, target=target, *args, **kwargs) def execute(self, context): - self.hook.run_cli("docs", "generate") \ No newline at end of file + self.create_hook().run_cli('docs', 'generate') \ No newline at end of file From 50352935bf0762c74cad9708057ae0bb48db2c2d Mon Sep 17 00:00:00 2001 From: Ayokunle Bamisile-Aje Date: Fri, 14 Aug 2020 20:34:40 -0400 Subject: [PATCH 4/9] Refactor DbtDocsGenerateOperator --- airflow_dbt/operators/dbt_operator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 86fcf4c..809bb48 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -95,5 +95,6 @@ class DbtDocsGenerateOperator(DbtBaseOperator): @apply_defaults def __init__(self, profiles_dir=None, target=None, *args, **kwargs): super(DbtDocsGenerateOperator, self).__init__(profiles_dir=profiles_dir, target=target, *args, **kwargs) + def execute(self, context): self.create_hook().run_cli('docs', 'generate') \ No newline at end of file From bc4bd9d860e36b049e45bbf39eda5e0b59575fc6 Mon Sep 17 00:00:00 2001 From: abamisileaje-tc <43453144+abamisileaje-tc@users.noreply.github.com> Date: Fri, 21 Aug 2020 15:57:40 -0400 Subject: [PATCH 5/9] Add schema and data options to test operator (#2) --- airflow_dbt/hooks/dbt_hook.py | 10 ++++++++++ airflow_dbt/operators/dbt_operator.py | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 418aa40..debf63c 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -39,6 +39,8 @@ def __init__(self, dir='.', vars=None, full_refresh=False, + data_test=False, + schema_test=False, models=None, exclude=None, dbt_bin='dbt', @@ -49,6 +51,8 @@ def __init__(self, self.target = target self.vars = vars self.full_refresh = full_refresh + self.data_test = data_test + self.schema_test = schema_test self.models = models self.exclude = exclude self.dbt_bin = dbt_bin @@ -92,6 +96,12 @@ def run_cli(self, *command): if self.full_refresh: dbt_cmd.extend(['--full-refresh']) + if self.data_test: + dbt_cmd.extend(['--data']) + + if self.schema_test: + dbt_cmd.extend(['--schema']) + sp = subprocess.Popen( dbt_cmd, stdout=subprocess.PIPE, diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 809bb48..118d007 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -43,6 +43,8 @@ def __init__(self, dbt_bin='dbt', verbose=True, full_refresh=False, + data_test=False, + schema_test=False, *args, **kwargs): super(DbtBaseOperator, self).__init__(*args, **kwargs) @@ -53,6 +55,8 @@ def __init__(self, self.vars = vars self.models = models self.full_refresh = full_refresh + self.data_test = data_test + self.schema_test = schema_test self.exclude = exclude self.dbt_bin = dbt_bin self.verbose = verbose @@ -65,6 +69,8 @@ def create_hook(self): dir=self.dir, vars=self.vars, full_refresh=self.full_refresh, + data_test=self.data_test, + schema_test=self.schema_test, models=self.models, exclude=self.exclude, dbt_bin=self.dbt_bin, From 842da105422f068a9ce4ec4005bbc50f736c7f7f Mon Sep 17 00:00:00 2001 From: Ayo Bamisile-Aje Date: Fri, 21 Aug 2020 17:46:12 -0400 Subject: [PATCH 6/9] Fix extension order of dbt_cmd --- airflow_dbt/hooks/dbt_hook.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index debf63c..088271c 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -83,6 +83,12 @@ def run_cli(self, *command): if self.vars is not None: dbt_cmd.extend(['--vars', self._dump_vars()]) + + if self.data_test: + dbt_cmd.extend(['--data']) + + if self.schema_test: + dbt_cmd.extend(['--schema']) if self.models is not None: dbt_cmd.extend(['--models', self.models]) @@ -96,12 +102,6 @@ def run_cli(self, *command): if self.full_refresh: dbt_cmd.extend(['--full-refresh']) - if self.data_test: - dbt_cmd.extend(['--data']) - - if self.schema_test: - dbt_cmd.extend(['--schema']) - sp = subprocess.Popen( dbt_cmd, stdout=subprocess.PIPE, From 5be8320307904d4456414387cd44812d29becfdd Mon Sep 17 00:00:00 2001 From: "ayo@github" <36321243+ayobamshy@users.noreply.github.com> Date: Wed, 23 Sep 2020 03:56:43 -0400 Subject: [PATCH 7/9] Update __init__.py Remove merge conflict markers --- airflow_dbt/operators/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/airflow_dbt/operators/__init__.py b/airflow_dbt/operators/__init__.py index 4224cd9..13bbb7b 100644 --- a/airflow_dbt/operators/__init__.py +++ b/airflow_dbt/operators/__init__.py @@ -1,10 +1,7 @@ -<<<<<<< HEAD -from .dbt_operator import DbtRunOperator, DbtTestOperator, DbtDocsGenerateOperator -======= from .dbt_operator import ( DbtSeedOperator, DbtSnapshotOperator, DbtRunOperator, - DbtTestOperator + DbtTestOperator, + DbtDocsGenerateOperator ) ->>>>>>> upstream/master From 671998bfb96435b7dd4957cc7668f8f69cb8abb4 Mon Sep 17 00:00:00 2001 From: ayobamshy Date: Wed, 23 Sep 2020 09:57:21 -0400 Subject: [PATCH 8/9] Change instance names in DbtBaseOperator data_test and schema_test changed to data and schema respectively --- airflow_dbt/hooks/dbt_hook.py | 2 +- airflow_dbt/operators/dbt_operator.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 0c24840..2db767d 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -87,7 +87,7 @@ def run_cli(self, *command): if self.vars is not None: dbt_cmd.extend(['--vars', self._dump_vars()]) - + if self.data_test: dbt_cmd.extend(['--data']) diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 3b5c704..fcbf65e 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -46,8 +46,8 @@ def __init__(self, dbt_bin='dbt', verbose=True, full_refresh=False, - data_test=False, - schema_test=False, + data=False, + schema=False, *args, **kwargs): super(DbtBaseOperator, self).__init__(*args, **kwargs) @@ -58,8 +58,8 @@ def __init__(self, self.vars = vars self.models = models self.full_refresh = full_refresh - self.data_test = data_test - self.schema_test = schema_test + self.data = data + self.schema = schema self.exclude = exclude self.select = select self.dbt_bin = dbt_bin @@ -73,8 +73,8 @@ def create_hook(self): dir=self.dir, vars=self.vars, full_refresh=self.full_refresh, - data_test=self.data_test, - schema_test=self.schema_test, + data=self.data, + schema=self.schema, models=self.models, exclude=self.exclude, select=self.select, @@ -105,10 +105,13 @@ def execute(self, context): class DbtDocsGenerateOperator(DbtBaseOperator): @apply_defaults def __init__(self, profiles_dir=None, target=None, *args, **kwargs): - super(DbtDocsGenerateOperator, self).__init__(profiles_dir=profiles_dir, target=target, *args, **kwargs) - + super(DbtDocsGenerateOperator, self).__init__(profiles_dir=profiles_dir, target=target, *args, + **kwargs) + def execute(self, context): self.create_hook().run_cli('docs', 'generate') + + class DbtSnapshotOperator(DbtBaseOperator): @apply_defaults def __init__(self, profiles_dir=None, target=None, *args, **kwargs): From 38e5a0e6d106e7908e0957405935c7a0f67731da Mon Sep 17 00:00:00 2001 From: ayobamshy Date: Wed, 23 Sep 2020 10:42:45 -0400 Subject: [PATCH 9/9] Change instance names in DbtCliHook data_test and schema_test changed to data and schema respectively --- airflow_dbt/hooks/dbt_hook.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 2db767d..349a5fc 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -41,8 +41,8 @@ def __init__(self, dir='.', vars=None, full_refresh=False, - data_test=False, - schema_test=False, + data=False, + schema=False, models=None, exclude=None, select=None, @@ -54,8 +54,8 @@ def __init__(self, self.target = target self.vars = vars self.full_refresh = full_refresh - self.data_test = data_test - self.schema_test = schema_test + self.data = data + self.schema = schema self.models = models self.exclude = exclude self.select = select @@ -88,10 +88,10 @@ def run_cli(self, *command): if self.vars is not None: dbt_cmd.extend(['--vars', self._dump_vars()]) - if self.data_test: + if self.data: dbt_cmd.extend(['--data']) - if self.schema_test: + if self.schema: dbt_cmd.extend(['--schema']) if self.models is not None: