From 5573c9803c0710b6e7db8b22dbe7abf7ca26df3b Mon Sep 17 00:00:00 2001 From: TimmyCarbone Date: Thu, 17 Sep 2020 00:28:07 -0400 Subject: [PATCH 1/4] Handle --select parameter in 'dbt snapshot' --- airflow_dbt/hooks/dbt_hook.py | 5 +++++ airflow_dbt/operators/dbt_operator.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 4f5b9d6..16962d3 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -41,6 +41,7 @@ def __init__(self, full_refresh=False, models=None, exclude=None, + select=None, dbt_bin='dbt', output_encoding='utf-8', verbose=True): @@ -51,6 +52,7 @@ def __init__(self, self.full_refresh = full_refresh self.models = models self.exclude = exclude + self.select = select self.dbt_bin = dbt_bin self.verbose = verbose self.output_encoding = output_encoding @@ -86,6 +88,9 @@ def run_cli(self, *command): if self.exclude is not None: dbt_cmd.extend(['--exclude', self.exclude]) + if self.select is not None: + dbt_cmd.extend(['--select', self.exclude]) + if self.full_refresh: dbt_cmd.extend(['--full-refresh']) diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index 8f705fc..cb213da 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -40,6 +40,7 @@ def __init__(self, vars=None, models=None, exclude=None, + select=None, dbt_bin='dbt', verbose=True, full_refresh=False, @@ -54,6 +55,7 @@ def __init__(self, self.models = models self.full_refresh = full_refresh self.exclude = exclude + self.select = select self.dbt_bin = dbt_bin self.verbose = verbose self.create_hook() @@ -67,6 +69,7 @@ def create_hook(self): full_refresh=self.full_refresh, models=self.models, exclude=self.exclude, + select=self.select, dbt_bin=self.dbt_bin, verbose=self.verbose) From ecb6e69414422a42057f34501de7ea7b3fac6ebc Mon Sep 17 00:00:00 2001 From: TimmyCarbone Date: Thu, 17 Sep 2020 00:29:08 -0400 Subject: [PATCH 2/4] Add documentation for select parameter --- airflow_dbt/hooks/dbt_hook.py | 2 ++ airflow_dbt/operators/dbt_operator.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 16962d3..991e7d3 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -25,6 +25,8 @@ class DbtCliHook(BaseHook): :type models: str :param exclude: If set, passed as the `--exclude` argument to the `dbt` command :type exclude: str + :param select: If set, passed as the `--select` argument to the `dbt` command + :type select: str :param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH` :type dbt_bin: str :param output_encoding: Output encoding of bash command. Defaults to utf-8 diff --git a/airflow_dbt/operators/dbt_operator.py b/airflow_dbt/operators/dbt_operator.py index cb213da..ad0d2fd 100644 --- a/airflow_dbt/operators/dbt_operator.py +++ b/airflow_dbt/operators/dbt_operator.py @@ -22,6 +22,8 @@ class DbtBaseOperator(BaseOperator): :type models: str :param exclude: If set, passed as the `--exclude` argument to the `dbt` command :type exclude: str + :param select: If set, passed as the `--select` argument to the `dbt` command + :type select: str :param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH` :type dbt_bin: str :param verbose: The operator will log verbosely to the Airflow logs From 7d510a067f14a352a858cddefa1f88a861a37246 Mon Sep 17 00:00:00 2001 From: TimmyCarbone Date: Thu, 17 Sep 2020 12:04:21 -0400 Subject: [PATCH 3/4] Fix reference to select parameter --- airflow_dbt/hooks/dbt_hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_dbt/hooks/dbt_hook.py b/airflow_dbt/hooks/dbt_hook.py index 991e7d3..47590b0 100644 --- a/airflow_dbt/hooks/dbt_hook.py +++ b/airflow_dbt/hooks/dbt_hook.py @@ -91,7 +91,7 @@ def run_cli(self, *command): dbt_cmd.extend(['--exclude', self.exclude]) if self.select is not None: - dbt_cmd.extend(['--select', self.exclude]) + dbt_cmd.extend(['--select', self.select]) if self.full_refresh: dbt_cmd.extend(['--full-refresh']) From d7de44d9cb5a89485d4d214c836a7485a17be5ba Mon Sep 17 00:00:00 2001 From: TimmyCarbone Date: Thu, 17 Sep 2020 12:28:37 -0400 Subject: [PATCH 4/4] Add documentation for the --select parameter --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d6706e7..86df748 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a collection of [Airflow](https://airflow.apache.org/) operators to prov ```py from airflow import DAG -from airflow_dbt.operators.dbt_operator import DbtRunOperator, DbtTestOperator +from airflow_dbt.operators.dbt_operator import DbtSnapshotOperator, DbtRunOperator, DbtTestOperator from airflow.utils.dates import days_ago default_args = { @@ -67,6 +67,8 @@ Each of the above operators accept the following arguments: * If set, passed as the `--models` argument to the `dbt` command * `exclude` * If set, passed as the `--exclude` argument to the `dbt` command +* `select` + * If set, passed as the `--select` argument to the `dbt` command * `dbt_bin` * The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH` * `verbose`