From bae8a56f0d8f16f1e9fdd4ff228abd461800d5bd Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 3 Nov 2023 09:24:15 -0600 Subject: [PATCH 01/12] Add @p.profile and @p.target to the list of "global" CLI flags --- core/dbt/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 7d4560a7910..bbb7b4a6b29 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -139,11 +139,13 @@ def global_flags(func): @p.populate_cache @p.print @p.printer_width + @p.profile @p.quiet @p.record_timing_info @p.send_anonymous_usage_stats @p.single_threaded @p.static_parser + @p.target @p.use_colors @p.use_colors_file @p.use_experimental_parser From 87203ac3b08ea2d461cc1789d5f34d2448bf8d7e Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 3 Nov 2023 09:25:57 -0600 Subject: [PATCH 02/12] Add env vars (DBT_PROFILE, DBT_TARGET) to the params --- core/dbt/cli/params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 1898815a724..8653cb1b6f9 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -335,7 +335,7 @@ profile = click.option( "--profile", - envvar=None, + envvar="DBT_TARGET", help="Which existing profile to load. Overrides setting in dbt_project.yml.", ) @@ -566,7 +566,7 @@ target = click.option( "--target", "-t", - envvar=None, + envvar="DBT_PROFILE", help="Which target to load for the given profile", ) From 653d0343dcd7361e33b112cb5fe4682bcf78e14b Mon Sep 17 00:00:00 2001 From: fraserbarton Date: Tue, 14 Nov 2023 17:41:49 +0000 Subject: [PATCH 03/12] Add unit test --- tests/unit/test_cli_flags.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index 83c0e251deb..b954266d8ad 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -349,6 +349,40 @@ def test_log_file_settings_from_config(self): assert flags.USE_COLORS is True assert flags.USE_COLORS_FILE is False + @pytest.mark.parametrize( + "cli_profile, flag_profile", + [ + (None, None), + ("profile", "profile"), + ], + ) + @pytest.mark.parametrize( + "cli_target, flag_target", + [ + (None, None), + ("target", "target"), + ], + ) + def test_profile_settings_interaction( + self, cli_profile, cli_target, flag_profile, flag_target + ): + cli_params = [] + + if cli_profile is not None: + cli_params += ["--profile", cli_profile] + + if cli_target is not None: + cli_params += ["--target", cli_target] + + cli_params += ["run"] + + context = self.make_dbt_context("run", cli_params) + + flags = Flags(context, None) + + assert flags.PROFILE == flag_profile + assert flags.TARGET == flag_target + def test_duplicate_flags_raises_error(self): parent_context = self.make_dbt_context("parent", ["--version-check"]) context = self.make_dbt_context("child", ["--version-check"], parent_context) From 9ce2b93a84964bdf8b91b80828c00d84b5c7db71 Mon Sep 17 00:00:00 2001 From: fraserbarton Date: Tue, 14 Nov 2023 17:44:33 +0000 Subject: [PATCH 04/12] Simplify unit test --- tests/unit/test_cli_flags.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index b954266d8ad..c589ab70987 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -366,15 +366,13 @@ def test_log_file_settings_from_config(self): def test_profile_settings_interaction( self, cli_profile, cli_target, flag_profile, flag_target ): - cli_params = [] - - if cli_profile is not None: - cli_params += ["--profile", cli_profile] + cli_params = ["run"] - if cli_target is not None: - cli_params += ["--target", cli_target] + if cli_profile: + cli_params = ["--profile", cli_profile] + cli_params - cli_params += ["run"] + if cli_target: + cli_params = ["--target", cli_target] + cli_params context = self.make_dbt_context("run", cli_params) From d5536fa43dcbc3e8f8213f68a2a019767b52b9fd Mon Sep 17 00:00:00 2001 From: fraserbarton Date: Wed, 15 Nov 2023 09:20:25 +0000 Subject: [PATCH 05/12] changie --- .changes/unreleased/Features-20231115-092005.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20231115-092005.yaml diff --git a/.changes/unreleased/Features-20231115-092005.yaml b/.changes/unreleased/Features-20231115-092005.yaml new file mode 100644 index 00000000000..614b069903c --- /dev/null +++ b/.changes/unreleased/Features-20231115-092005.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Move --target and --profile command line flags to global config. Also, allow + these flags to be set by environment variable (DBT_TARGET, and DBT_PROFILE). +time: 2023-11-15T09:20:05.12461Z +custom: + Author: barton996 + Issue: "7798" From 51b0a20e3b29a9c56ff63eb977b8178a3ca557d2 Mon Sep 17 00:00:00 2001 From: barton996 <57592874+barton996@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:49:10 +0000 Subject: [PATCH 06/12] Update .changes/unreleased/Features-20231115-092005.yaml Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- .changes/unreleased/Features-20231115-092005.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.changes/unreleased/Features-20231115-092005.yaml b/.changes/unreleased/Features-20231115-092005.yaml index 614b069903c..6f156764aff 100644 --- a/.changes/unreleased/Features-20231115-092005.yaml +++ b/.changes/unreleased/Features-20231115-092005.yaml @@ -1,6 +1,5 @@ kind: Features -body: Move --target and --profile command line flags to global config. Also, allow - these flags to be set by environment variable (DBT_TARGET, and DBT_PROFILE). +body: Global config for --target and --profile CLI flags and DBT_TARGET and DBT_PROFILE environment variables. time: 2023-11-15T09:20:05.12461Z custom: Author: barton996 From 2eaae5d126d719c3fa27892c3d16f6120794ce7c Mon Sep 17 00:00:00 2001 From: fraserbarton Date: Mon, 20 Nov 2023 16:01:18 +0000 Subject: [PATCH 07/12] Fix incorrect envvar names --- core/dbt/cli/params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 8653cb1b6f9..6c0a90b4edb 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -335,7 +335,7 @@ profile = click.option( "--profile", - envvar="DBT_TARGET", + envvar="DBT_PROFILE", help="Which existing profile to load. Overrides setting in dbt_project.yml.", ) @@ -566,7 +566,7 @@ target = click.option( "--target", "-t", - envvar="DBT_PROFILE", + envvar="DBT_TARGET", help="Which target to load for the given profile", ) From 9af262ce78f97040ed530f21a0e42e536a7c4b34 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Mon, 20 Nov 2023 09:08:58 -0700 Subject: [PATCH 08/12] Realign environment variable names --- core/dbt/cli/params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 8653cb1b6f9..6c0a90b4edb 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -335,7 +335,7 @@ profile = click.option( "--profile", - envvar="DBT_TARGET", + envvar="DBT_PROFILE", help="Which existing profile to load. Overrides setting in dbt_project.yml.", ) @@ -566,7 +566,7 @@ target = click.option( "--target", "-t", - envvar="DBT_PROFILE", + envvar="DBT_TARGET", help="Which target to load for the given profile", ) From d404c9de4b8d40ec11105654eb0bae453f943f30 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Mon, 20 Nov 2023 10:09:47 -0700 Subject: [PATCH 09/12] Remove from specific subcommands --- core/dbt/cli/main.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index bbb7b4a6b29..ce35c0f9765 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -190,7 +190,6 @@ def cli(ctx, **kwargs): @p.full_refresh @p.include_saved_query @p.indirect_selection -@p.profile @p.profiles_dir @p.project_dir @p.resource_type @@ -201,7 +200,6 @@ def cli(ctx, **kwargs): @p.defer_state @p.deprecated_state @p.store_failures -@p.target @p.target_path @p.threads @p.vars @@ -229,10 +227,8 @@ def build(ctx, **kwargs): @click.pass_context @global_flags @p.clean_project_files_only -@p.profile @p.profiles_dir @p.project_dir -@p.target @p.target_path @p.vars @requires.postflight @@ -266,7 +262,6 @@ def docs(ctx, **kwargs): @p.exclude @p.favor_state @p.deprecated_favor_state -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -276,7 +271,6 @@ def docs(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -305,10 +299,8 @@ def docs_generate(ctx, **kwargs): @global_flags @p.browser @p.port -@p.profile @p.profiles_dir @p.project_dir -@p.target @p.target_path @p.vars @requires.postflight @@ -341,7 +333,6 @@ def docs_serve(ctx, **kwargs): @p.show_output_format @p.indirect_selection @p.introspect -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -351,7 +342,6 @@ def docs_serve(ctx, **kwargs): @p.defer_state @p.deprecated_state @p.compile_inject_ephemeral_ctes -@p.target @p.target_path @p.threads @p.vars @@ -389,7 +379,6 @@ def compile(ctx, **kwargs): @p.show_limit @p.indirect_selection @p.introspect -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -398,7 +387,6 @@ def compile(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -428,10 +416,8 @@ def show(ctx, **kwargs): @global_flags @p.debug_connection @p.config_dir -@p.profile @p.profiles_dir_exists_false @p.project_dir -@p.target @p.vars @requires.postflight @requires.preflight @@ -452,10 +438,8 @@ def debug(ctx, **kwargs): @cli.command("deps") @click.pass_context @global_flags -@p.profile @p.profiles_dir_exists_false @p.project_dir -@p.target @p.vars @p.source @p.dry_run @@ -502,11 +486,9 @@ def deps(ctx, **kwargs): @global_flags # for backwards compatibility, accept 'project_name' as an optional positional argument @click.argument("project_name", required=False) -@p.profile @p.profiles_dir_exists_false @p.project_dir @p.skip_profile_setup -@p.target @p.vars @requires.postflight @requires.preflight @@ -528,7 +510,6 @@ def init(ctx, **kwargs): @p.models @p.output @p.output_keys -@p.profile @p.profiles_dir @p.project_dir @p.resource_type @@ -537,7 +518,6 @@ def init(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.vars @requires.postflight @@ -569,10 +549,8 @@ def list(ctx, **kwargs): @cli.command("parse") @click.pass_context @global_flags -@p.profile @p.profiles_dir @p.project_dir -@p.target @p.target_path @p.threads @p.vars @@ -598,7 +576,6 @@ def parse(ctx, **kwargs): @p.deprecated_favor_state @p.exclude @p.full_refresh -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -606,7 +583,6 @@ def parse(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -636,8 +612,6 @@ def run(ctx, **kwargs): @p.project_dir @p.profiles_dir @p.vars -@p.profile -@p.target @p.state @p.threads @requires.postflight @@ -666,14 +640,12 @@ def retry(ctx, **kwargs): @p.defer_state @p.exclude @p.full_refresh -@p.profile @p.profiles_dir @p.project_dir @p.resource_type @p.select @p.selector @p.state # required -@p.target @p.target_path @p.threads @p.vars @@ -702,10 +674,8 @@ def clone(ctx, **kwargs): @global_flags @click.argument("macro") @p.args -@p.profile @p.profiles_dir @p.project_dir -@p.target @p.target_path @p.threads @p.vars @@ -734,7 +704,6 @@ def run_operation(ctx, **kwargs): @global_flags @p.exclude @p.full_refresh -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -743,7 +712,6 @@ def run_operation(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -774,7 +742,6 @@ def seed(ctx, **kwargs): @p.exclude @p.favor_state @p.deprecated_favor_state -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -782,7 +749,6 @@ def seed(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -819,7 +785,6 @@ def source(ctx, **kwargs): @global_flags @p.exclude @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -827,7 +792,6 @@ def source(ctx, **kwargs): @p.state @p.defer_state @p.deprecated_state -@p.target @p.target_path @p.threads @p.vars @@ -866,7 +830,6 @@ def freshness(ctx, **kwargs): @p.favor_state @p.deprecated_favor_state @p.indirect_selection -@p.profile @p.profiles_dir @p.project_dir @p.select @@ -875,7 +838,6 @@ def freshness(ctx, **kwargs): @p.defer_state @p.deprecated_state @p.store_failures -@p.target @p.target_path @p.threads @p.vars From e925d2b0879c2116a8511a70870e5281ded735f7 Mon Sep 17 00:00:00 2001 From: fraserbarton Date: Mon, 20 Nov 2023 19:29:30 +0000 Subject: [PATCH 10/12] Add test_global_flags_not_on_subcommands --- tests/unit/test_cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a55767585c9..a4a1d587c4a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -52,3 +52,14 @@ def test_commands_in_enum_and_dict(self): continue cmd = Command.from_str(command.name) command_args(cmd) + + def test_global_flags_not_on_subcommands(self): + def run_test(command): + # For each subcommand + for command_name, command in command.commands.items(): + # Get string representation of all parameter options for this subcommand e.g. "['-t', '--target']" + parameter_options = [str(sorted(param.opts)) for param in command.params] + # Check there are no parameter options that have been applied twice + assert sorted(list(set(parameter_options))) == sorted(parameter_options) + + run_test(cli) From 89358c2816eb18c9b63dc49fd0b35691d66ef4b5 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:23:51 -0600 Subject: [PATCH 11/12] Remove one unnecessary test case --- tests/unit/test_cli_flags.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index 3171d947f74..dcc8f800659 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -346,38 +346,6 @@ def test_log_file_settings_from_config(self): assert flags.USE_COLORS is True assert flags.USE_COLORS_FILE is False - @pytest.mark.parametrize( - "cli_profile, flag_profile", - [ - (None, None), - ("profile", "profile"), - ], - ) - @pytest.mark.parametrize( - "cli_target, flag_target", - [ - (None, None), - ("target", "target"), - ], - ) - def test_profile_settings_interaction( - self, cli_profile, cli_target, flag_profile, flag_target - ): - cli_params = ["run"] - - if cli_profile: - cli_params = ["--profile", cli_profile] + cli_params - - if cli_target: - cli_params = ["--target", cli_target] + cli_params - - context = self.make_dbt_context("run", cli_params) - - flags = Flags(context, None) - - assert flags.PROFILE == flag_profile - assert flags.TARGET == flag_target - def test_duplicate_flags_raises_error(self): parent_context = self.make_dbt_context("parent", ["--version-check"]) context = self.make_dbt_context("child", ["--version-check"], parent_context) From 873af90a6e512de783a71f202d17b5d5e5c5363f Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:24:54 -0600 Subject: [PATCH 12/12] Remove other unnecessary test case --- tests/unit/test_cli.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a4a1d587c4a..a55767585c9 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -52,14 +52,3 @@ def test_commands_in_enum_and_dict(self): continue cmd = Command.from_str(command.name) command_args(cmd) - - def test_global_flags_not_on_subcommands(self): - def run_test(command): - # For each subcommand - for command_name, command in command.commands.items(): - # Get string representation of all parameter options for this subcommand e.g. "['-t', '--target']" - parameter_options = [str(sorted(param.opts)) for param in command.params] - # Check there are no parameter options that have been applied twice - assert sorted(list(set(parameter_options))) == sorted(parameter_options) - - run_test(cli)