From b287a78eab53c0a59614c6fafde4d203767e2031 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 14 Feb 2023 18:16:13 -0500 Subject: [PATCH 01/11] CT-1844: Add flags and flag-interaciton tests for more granular log control. --- core/dbt/cli/flags.py | 11 ++++ core/dbt/cli/main.py | 4 ++ core/dbt/cli/params.py | 35 +++++++++++- tests/unit/test_cli_flags.py | 106 +++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index e7f8f06d036..0e42808df30 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -130,6 +130,11 @@ def assign_params(ctx, params_assigned_from_default): object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) + # Apply the lead/follow relationship between some parameters + self._override_if_set("USE_COLORS", "USE_COLORS_FILE", params_assigned_from_default) + self._override_if_set("LOG_LEVEL", "LOG_LEVEL_FILE", params_assigned_from_default) + self._override_if_set("LOG_FORMAT", "LOG_FORMAT_FILE", params_assigned_from_default) + # Default LOG_PATH from PROJECT_DIR, if available. if getattr(self, "LOG_PATH", None) is None: log_path = "logs" @@ -166,6 +171,12 @@ def assign_params(ctx, params_assigned_from_default): for param in params: object.__setattr__(self, param.lower(), getattr(self, param)) + # If the value of the lead parameter was set explicitly, apply the value to follow, + # unless follow was also set explicitly. + def _override_if_set(self, lead: str, follow: str, defaulted: set[str]) -> None: + if lead.lower() not in defaulted and follow.lower() in defaulted: + object.__setattr__(self, follow.upper(), getattr(self, lead.upper(), None)) + def __str__(self) -> str: return str(pf(self.__dict__)) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 8d1b91667f8..5b0e52a47ca 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -83,6 +83,9 @@ def handle_and_check(args): @p.fail_fast @p.log_cache_events @p.log_format +@p.log_format_file +@p.log_level +@p.log_level_file @p.log_path @p.macro_debugging @p.partial_parse @@ -93,6 +96,7 @@ def handle_and_check(args): @p.single_threaded @p.static_parser @p.use_colors +@p.use_colors_file @p.use_experimental_parser @p.version @p.version_check diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index ec3c1107fa9..bef6618eb75 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -121,10 +121,34 @@ "--log-format", envvar="DBT_LOG_FORMAT", help="Specify the log format, overriding the command's default.", - type=click.Choice(["text", "json", "default"], case_sensitive=False), + type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False), default="default", ) +log_format_file = click.option( + "--log-format-file", + envvar="DBT_LOG_FORMAT_FILE", + help="Specify the file log format, overriding the command's default and the value of --log-format.", + type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False), + default="debug", +) + +log_level = click.option( + "--log-level", + envvar="DBT_LOG_LEVEL", + help="Specify the minimum severity of events that are logged.", + type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False), + default="info", +) + +log_level_file = click.option( + "--log-level-file", + envvar="DBT_LOG_LEVEL_FILE", + help="Specify the minimum severity of events that are logged to file, overriding the value of --log-level-file.", + type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False), + default="debug", +) + log_path = click.option( "--log-path", envvar="DBT_LOG_PATH", @@ -365,7 +389,14 @@ use_colors = click.option( "--use-colors/--no-use-colors", envvar="DBT_USE_COLORS", - help="Output is colorized by default and may also be set in a profile or at the command line.", + help="Specify whether log output is colorized.", + default=True, +) + +use_colors_file = click.option( + "--use-colors-file/--no-use-colors-file", + envvar="DBT_USE_COLORS_FILE", + help="Specify whether log file output is colorized overriding --use-colors/--no-use-colors.", default=True, ) diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index 5b10b3f6f08..14c4b17a9ef 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -193,3 +193,109 @@ def test_mutually_exclusive_options_from_user_config_and_envvar( with pytest.raises(click.BadOptionUsage): Flags(context, user_config) + + @pytest.mark.parametrize( + "cli_colors,cli_colors_file,flag_colors,flag_colors_file", + [ + (None, None, True, True), + (True, None, True, True), + (None, True, True, True), + (False, None, False, False), + (None, False, True, False), + (True, True, True, True), + (False, False, False, False), + (True, False, True, False), + (False, True, False, True), + ], + ) + def test_no_color_interaction( + self, cli_colors, cli_colors_file, flag_colors, flag_colors_file + ): + cli_params = [] + + if cli_colors is not None: + cli_params.append("--use-colors" if cli_colors else "--no-use-colors") + + if cli_colors_file is not None: + cli_params.append("--use-colors-file" if cli_colors_file else "--no-use-colors-file") + + cli_params.append("run") + + context = self.make_dbt_context("run", cli_params) + + flags = Flags(context, None) + + assert flags.USE_COLORS == flag_colors + assert flags.USE_COLORS_FILE == flag_colors_file + + @pytest.mark.parametrize( + "cli_log_level,cli_log_level_file,flag_log_level,flag_log_level_file", + [ + (None, None, "info", "debug"), + ("error", None, "error", "error"), # explicit level overrides file level... + ("info", None, "info", "info"), # ...but file level doesn't change console level + ( + "debug", + "warn", + "debug", + "warn", + ), # still, two separate explicit levels are applied independently + ], + ) + def test_log_level_interaction( + self, cli_log_level, cli_log_level_file, flag_log_level, flag_log_level_file + ): + cli_params = [] + + if cli_log_level is not None: + cli_params.append("--log-level") + cli_params.append(cli_log_level) + + if cli_log_level_file is not None: + cli_params.append("--log-level-file") + cli_params.append(cli_log_level_file) + + cli_params.append("run") + + context = self.make_dbt_context("run", cli_params) + + flags = Flags(context, None) + + assert flags.LOG_LEVEL == flag_log_level + assert flags.LOG_LEVEL_FILE == flag_log_level_file + + @pytest.mark.parametrize( + "cli_log_format,cli_log_format_file,flag_log_format,flag_log_format_file", + [ + (None, None, "default", "debug"), + ("json", None, "json", "json"), # explicit format overrides file format... + (None, "json", "default", "json"), # ...but file format doesn't change console format + ( + "debug", + "text", + "debug", + "text", + ), # still, two separate explicit formats are applied independently + ], + ) + def test_log_format_interaction( + self, cli_log_format, cli_log_format_file, flag_log_format, flag_log_format_file + ): + cli_params = [] + + if cli_log_format is not None: + cli_params.append("--log-format") + cli_params.append(cli_log_format) + + if cli_log_format_file is not None: + cli_params.append("--log-format-file") + cli_params.append(cli_log_format_file) + + cli_params.append("run") + + context = self.make_dbt_context("run", cli_params) + + flags = Flags(context, None) + + assert flags.LOG_FORMAT == flag_log_format + assert flags.LOG_FORMAT_FILE == flag_log_format_file From f6f733ceb46ec9c4e225cf6c1387a8a30b6f3e18 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 16 Feb 2023 10:49:33 -0500 Subject: [PATCH 02/11] CT-1844: Apply parameter settings to log configuration --- core/dbt/cli/requires.py | 7 +- core/dbt/events/functions.py | 112 ++++++++++++++++++----------- core/dbt/tests/fixtures/project.py | 12 +++- 3 files changed, 84 insertions(+), 47 deletions(-) diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index b8fcec358b9..36cb98f3384 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -30,12 +30,7 @@ def wrapper(*args, **kwargs): # Logging # N.B. Legacy logger is not supported - setup_event_logger( - flags.LOG_PATH, - flags.LOG_FORMAT, - flags.USE_COLORS, - flags.DEBUG, - ) + setup_event_logger(flags) # Profiling if flags.RECORD_TIMING_INFO: diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 28211dd6b91..91c7c5c779f 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -19,89 +19,121 @@ metadata_vars: Optional[Dict[str, str]] = None -def setup_event_logger(log_path: str, log_format: str, use_colors: bool, debug: bool): +def setup_event_logger(flags): cleanup_event_logger() - make_log_dir_if_missing(log_path) + make_log_dir_if_missing(flags.LOG_PATH) if get_flags().ENABLE_LEGACY_LOGGER: - EVENT_MANAGER.add_logger(_get_logbook_log_config(debug)) + EVENT_MANAGER.add_logger(_get_logbook_log_config(flags.DEBUG, flags.USE_COLORS)) else: - EVENT_MANAGER.add_logger(_get_stdout_config(log_format, debug, use_colors)) - - if _CAPTURE_STREAM: - # Create second stdout logger to support test which want to know what's - # being sent to stdout. - # debug here is true because we need to capture debug events, and we pass in false in main - capture_config = _get_stdout_config(log_format, True, use_colors) - capture_config.output_stream = _CAPTURE_STREAM - EVENT_MANAGER.add_logger(capture_config) - - # create and add the file logger to the event manager - EVENT_MANAGER.add_logger( - _get_logfile_config(os.path.join(log_path, "dbt.log"), use_colors, log_format) - ) - - -def _get_stdout_config(log_format: str, debug: bool, use_colors: bool) -> LoggerConfig: - fmt = LineFormat.PlainText - if log_format == "json": - fmt = LineFormat.Json - elif debug: - fmt = LineFormat.DebugText - level = EventLevel.DEBUG if debug else EventLevel.INFO + if flags.LOG_LEVEL != "none": + log_format = ( + LineFormat.DebugText + if flags.DEBUG + else _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) + ) + log_level = EventLevel(flags.LOG_LEVEL) + EVENT_MANAGER.add_logger( + _get_stdout_config(log_format, flags.DEBUG, flags.USE_COLORS, log_level) + ) + + if _CAPTURE_STREAM: + # Create second stdout logger to support test which want to know what's + # being sent to stdout. + # debug here is true because we need to capture debug events, and we pass in false in main + capture_config = _get_stdout_config( + flags.LOG_FORMAT, True, flags.USE_COLORS, log_level + ) + capture_config.output_stream = _CAPTURE_STREAM + EVENT_MANAGER.add_logger(capture_config) + + if flags.LOG_LEVEL_FILE != "none": + # create and add the file logger to the event manager + log_file = os.path.join(flags.LOG_PATH, "dbt.log") + log_file_format = _line_format_from_str(flags.LOG_FORMAT_FILE, LineFormat.DebugText) + log_level_file = EventLevel(flags.LOG_LEVEL_FILE) + EVENT_MANAGER.add_logger( + _get_logfile_config( + log_file, flags.USE_COLORS_FILE, log_file_format, log_level_file + ) + ) + + +def _line_format_from_str(format_str: str, default: LineFormat) -> LineFormat: + if format_str == "text": + return LineFormat.PlainText + elif format_str == "debug": + return LineFormat.DebugText + elif format_str == "json": + return LineFormat.Json + + return default + + +def _get_stdout_config( + line_format: LineFormat, debug: bool, use_colors: bool, level: EventLevel +) -> LoggerConfig: return LoggerConfig( name="stdout_log", level=level, use_colors=use_colors, - line_format=fmt, + line_format=line_format, scrubber=env_scrubber, filter=partial( _stdout_filter, bool(flags_module.LOG_CACHE_EVENTS), debug, bool(flags_module.QUIET), - log_format, + line_format, ), output_stream=sys.stdout, ) def _stdout_filter( - log_cache_events: bool, debug_mode: bool, quiet_mode: bool, log_format: str, msg: EventMsg + log_cache_events: bool, + debug_mode: bool, + quiet_mode: bool, + line_format: LineFormat, + msg: EventMsg, ) -> bool: return ( not isinstance(msg.data, NoStdOut) and (not isinstance(msg.data, Cache) or log_cache_events) and (EventLevel(msg.info.level) != EventLevel.DEBUG or debug_mode) and (EventLevel(msg.info.level) == EventLevel.ERROR or not quiet_mode) - and not (log_format == "json" and type(msg.data) == Formatting) + and not (line_format == LineFormat.Json and type(msg.data) == Formatting) ) -def _get_logfile_config(log_path: str, use_colors: bool, log_format: str) -> LoggerConfig: +def _get_logfile_config( + log_path: str, use_colors: bool, line_format: LineFormat, level: EventLevel +) -> LoggerConfig: return LoggerConfig( name="file_log", - line_format=LineFormat.Json if log_format == "json" else LineFormat.DebugText, + line_format=line_format, use_colors=use_colors, - level=EventLevel.DEBUG, # File log is *always* debug level + level=level, # File log is *always* debug level scrubber=env_scrubber, - filter=partial(_logfile_filter, bool(get_flags().LOG_CACHE_EVENTS), log_format), + filter=partial(_logfile_filter, bool(get_flags().LOG_CACHE_EVENTS), line_format), output_file_name=log_path, ) -def _logfile_filter(log_cache_events: bool, log_format: str, msg: EventMsg) -> bool: +def _logfile_filter(log_cache_events: bool, line_format: LineFormat, msg: EventMsg) -> bool: return ( not isinstance(msg.data, NoFile) and not (isinstance(msg.data, Cache) and not log_cache_events) - and not (log_format == "json" and type(msg.data) == Formatting) + and not (line_format == LineFormat.Json and type(msg.data) == Formatting) ) -def _get_logbook_log_config(debug: bool) -> LoggerConfig: +def _get_logbook_log_config(debug: bool, use_colors: bool) -> LoggerConfig: # use the default one since this code should be removed when we remove logbook flags = get_flags() - config = _get_stdout_config("", debug, bool(flags.USE_COLORS)) + config = _get_stdout_config( + LineFormat.PlainText, debug, use_colors, EventLevel.DEBUG if debug else EventLevel.INFO + ) config.name = "logbook_log" config.filter = NoFilter if flags.LOG_CACHE_EVENTS else lambda e: not isinstance(e.data, Cache) config.logger = GLOBAL_LOGGER @@ -125,9 +157,9 @@ def cleanup_event_logger(): # create a default configuration with default settings and no file output. EVENT_MANAGER: EventManager = EventManager() EVENT_MANAGER.add_logger( - _get_logbook_log_config(flags_module.DEBUG) # type: ignore + _get_logbook_log_config(False, True) # type: ignore if flags_module.ENABLE_LEGACY_LOGGER - else _get_stdout_config(flags_module.LOG_FORMAT, flags_module.DEBUG, flags_module.USE_COLORS) # type: ignore + else _get_stdout_config(LineFormat.PlainText, False, True, EventLevel.INFO) ) # This global, and the following two functions for capturing stdout logs are diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 45bb9c99210..0d586d09e80 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -474,7 +474,17 @@ def project( # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. # This _only_ works for tests in `tests/` that use the project fixture. warnings.filterwarnings("ignore", category=DeprecationWarning, module="logbook") - setup_event_logger(logs_dir, "json", False, False) + log_flags = Namespace( + LOG_PATH=logs_dir, + LOG_FORMAT="json", + LOG_FORMAT_FILE="json", + USE_COLORS=False, + USE_COLORS_FILE=False, + LOG_LEVEL="info", + LOG_LEVEL_FILE="debug", + DEBUG=False, + ) + setup_event_logger(log_flags) orig_cwd = os.getcwd() os.chdir(project_root) # Return whatever is needed later in tests but can only come from fixtures, so we can keep From 6ddde180034dbbf88183dcc15f8b618ffb226545 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 16 Feb 2023 13:47:19 -0500 Subject: [PATCH 03/11] CT-1844: Fixes per testing --- core/dbt/events/functions.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 91c7c5c779f..072b8449c83 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -27,12 +27,8 @@ def setup_event_logger(flags): EVENT_MANAGER.add_logger(_get_logbook_log_config(flags.DEBUG, flags.USE_COLORS)) else: if flags.LOG_LEVEL != "none": - log_format = ( - LineFormat.DebugText - if flags.DEBUG - else _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) - ) - log_level = EventLevel(flags.LOG_LEVEL) + log_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) + log_level = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL) EVENT_MANAGER.add_logger( _get_stdout_config(log_format, flags.DEBUG, flags.USE_COLORS, log_level) ) @@ -42,7 +38,7 @@ def setup_event_logger(flags): # being sent to stdout. # debug here is true because we need to capture debug events, and we pass in false in main capture_config = _get_stdout_config( - flags.LOG_FORMAT, True, flags.USE_COLORS, log_level + log_format, flags.DEBUG, flags.USE_COLORS, log_level ) capture_config.output_stream = _CAPTURE_STREAM EVENT_MANAGER.add_logger(capture_config) @@ -51,7 +47,7 @@ def setup_event_logger(flags): # create and add the file logger to the event manager log_file = os.path.join(flags.LOG_PATH, "dbt.log") log_file_format = _line_format_from_str(flags.LOG_FORMAT_FILE, LineFormat.DebugText) - log_level_file = EventLevel(flags.LOG_LEVEL_FILE) + log_level_file = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL_FILE) EVENT_MANAGER.add_logger( _get_logfile_config( log_file, flags.USE_COLORS_FILE, log_file_format, log_level_file From ab952c9ec9c7c7593d83afc606d7a69c471de559 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 16 Feb 2023 14:16:57 -0500 Subject: [PATCH 04/11] CT-1844: Type annotation fix. --- core/dbt/cli/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index d725da49c9d..94f2058be8a 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -159,7 +159,7 @@ def assign_params(ctx, params_assigned_from_default): # If the value of the lead parameter was set explicitly, apply the value to follow, # unless follow was also set explicitly. - def _override_if_set(self, lead: str, follow: str, defaulted: set[str]) -> None: + def _override_if_set(self, lead: str, follow: str, defaulted: Set[str]) -> None: if lead.lower() not in defaulted and follow.lower() in defaulted: object.__setattr__(self, follow.upper(), getattr(self, lead.upper(), None)) From bf7bee090f6fc95116a7e28544610a7ebf9685cc Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 16 Feb 2023 19:19:04 +0000 Subject: [PATCH 05/11] Add generated CLI API docs --- .../docs/build/doctrees/environment.pickle | Bin 207366 -> 207366 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/core/dbt/docs/build/doctrees/environment.pickle b/core/dbt/docs/build/doctrees/environment.pickle index 6db282f704c5e84979cee697573f0377d3d84e94..8b3ec81ac4d7c290dfe847f89c5ff595976c424b 100644 GIT binary patch delta 32 mcmZp>!qawzXG60*>w?Trmi)~f^6ee+j6lq^y+fY4jtcO delta 32 mcmZp>!qawzXG60*t3$Taec|Q~`SuQZMj&R|-XYIi#{~e Date: Thu, 16 Feb 2023 14:46:02 -0500 Subject: [PATCH 06/11] CT-1844: Add changelog entry. --- .changes/unreleased/Features-20230216-144534.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230216-144534.yaml diff --git a/.changes/unreleased/Features-20230216-144534.yaml b/.changes/unreleased/Features-20230216-144534.yaml new file mode 100644 index 00000000000..bdcc96b3f7c --- /dev/null +++ b/.changes/unreleased/Features-20230216-144534.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Implemented new log cli parameters for finer-grained control. +time: 2023-02-16T14:45:34.038453-05:00 +custom: + Author: peterallenwebb + Issue: "6639" From 11717962ea2dc1808ad6cd60c8722bd20e72a55f Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 21 Feb 2023 16:30:25 +0000 Subject: [PATCH 07/11] Add generated CLI API docs --- .../docs/build/doctrees/environment.pickle | Bin 207366 -> 207366 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/core/dbt/docs/build/doctrees/environment.pickle b/core/dbt/docs/build/doctrees/environment.pickle index 8b3ec81ac4d7c290dfe847f89c5ff595976c424b..b6a2c0c918ce607a920cf5b8fc6ba5e23c8f9d04 100644 GIT binary patch delta 33 ncmZp>!qawzXG4=b+h3auaqq{?9rEoR@{B;tw7o-~xsD3}>)H%_ delta 33 ncmZp>!qawzXG4=b+wTRLpDg*CJLKCt Date: Tue, 21 Feb 2023 15:16:23 -0500 Subject: [PATCH 08/11] CT-1844: Add new logging parameters to UserConfig --- core/dbt/cli/flags.py | 2 +- core/dbt/contracts/project.py | 4 ++++ tests/unit/test_cli_flags.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 15ebf723a76..6fe8a2c926b 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -161,7 +161,7 @@ def assign_params(ctx, params_assigned_from_default): version_check = getattr(self, "VERSION_CHECK", True) object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check)) - # Support console DO NOT TRACK initiave + # Support console DO NOT TRACK initiative if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"): object.__setattr__(self, "SEND_ANONYMOUS_USAGE_STATS", False) diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 6b93708409b..3041b1bd4b9 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -245,12 +245,16 @@ def validate(cls, data): class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract): send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS use_colors: Optional[bool] = None + use_colors_file: Optional[bool] = None partial_parse: Optional[bool] = None printer_width: Optional[int] = None write_json: Optional[bool] = None warn_error: Optional[bool] = None warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None log_format: Optional[str] = None + log_format_file: Optional[str] = None + log_level: Optional[str] = None + log_level_file: Optional[str] = None debug: Optional[bool] = None version_check: Optional[bool] = None fail_fast: Optional[bool] = None diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index b9a276f59de..a3fb76a1a8b 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -300,3 +300,34 @@ def test_log_format_interaction( assert flags.LOG_FORMAT == flag_log_format assert flags.LOG_FORMAT_FILE == flag_log_format_file + + # Test that values set in UserConfig for log settings will set flags as expected + def test_log_settings_from_config(self): + context = self.make_dbt_context("run", ["run"]) + + config = UserConfig(log_format="json", log_level="warn", use_colors=False) + + flags = Flags(context, config) + + assert flags.LOG_FORMAT == "json" + assert flags.LOG_FORMAT_FILE == "json" + assert flags.LOG_LEVEL == "warn" + assert flags.LOG_LEVEL_FILE == "warn" + assert flags.USE_COLORS is False + assert flags.USE_COLORS_FILE is False + + # Test that values set in UserConfig for log *file* settings will set flags as expected, + # leaving the console logging flags with their default values + def test_log_file_settings_from_config(self): + context = self.make_dbt_context("run", ["run"]) + + config = UserConfig(log_format_file="json", log_level_file="warn", use_colors_file=False) + + flags = Flags(context, config) + + assert flags.LOG_FORMAT == "default" + assert flags.LOG_FORMAT_FILE == "json" + assert flags.LOG_LEVEL == "info" + assert flags.LOG_LEVEL_FILE == "warn" + assert flags.USE_COLORS is True + assert flags.USE_COLORS_FILE is False From 9a1549065dea055369c5fd6a6ec4b22c84688631 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 21 Feb 2023 20:19:07 +0000 Subject: [PATCH 09/11] Add generated CLI API docs --- .../docs/build/doctrees/environment.pickle | Bin 207366 -> 207366 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/core/dbt/docs/build/doctrees/environment.pickle b/core/dbt/docs/build/doctrees/environment.pickle index b6a2c0c918ce607a920cf5b8fc6ba5e23c8f9d04..4201afc09eb4d854b6c6f51ff8023e914a281493 100644 GIT binary patch delta 32 mcmZp>!qawzXG60*tLM4pUsg4D$hUXMGXgQw_6~XGIxYb2LJaZ% delta 32 mcmZp>!qawzXG60*t4)Tu_v7Xc`SuQZMj&R|-XYIi#{~e;{R@8p From 7a7b476af6f8bfb9a2bbb4264b7a39681e16175b Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 21 Feb 2023 17:24:04 -0500 Subject: [PATCH 10/11] CT-1844: Code review fixes --- core/dbt/events/functions.py | 32 +++++++++++--------------------- tests/unit/test_cli_flags.py | 6 +++--- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index be82decdc5b..a4e69710483 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -30,33 +30,23 @@ def setup_event_logger(flags) -> None: ) else: if flags.LOG_LEVEL != "none": - log_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) + line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) log_level = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL) - EVENT_MANAGER.add_logger( - _get_stdout_config( - log_format, - flags.DEBUG, - flags.USE_COLORS, - log_level, - flags.LOG_CACHE_EVENTS, - flags.QUIET, - ) + console_config = _get_stdout_config( + line_format, + flags.DEBUG, + flags.USE_COLORS, + log_level, + flags.LOG_CACHE_EVENTS, + flags.QUIET, ) + EVENT_MANAGER.add_logger(console_config) if _CAPTURE_STREAM: # Create second stdout logger to support test which want to know what's # being sent to stdout. - # debug here is true because we need to capture debug events, and we pass in false in main - capture_config = _get_stdout_config( - log_format, - flags.DEBUG, - flags.USE_COLORS, - log_level, - flags.LOG_CACHE_EVENTS, - flags.QUIET, - ) - capture_config.output_stream = _CAPTURE_STREAM - EVENT_MANAGER.add_logger(capture_config) + console_config.output_stream = _CAPTURE_STREAM + EVENT_MANAGER.add_logger(console_config) if flags.LOG_LEVEL_FILE != "none": # create and add the file logger to the event manager diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index a3fb76a1a8b..bf58703a1c0 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -301,8 +301,8 @@ def test_log_format_interaction( assert flags.LOG_FORMAT == flag_log_format assert flags.LOG_FORMAT_FILE == flag_log_format_file - # Test that values set in UserConfig for log settings will set flags as expected def test_log_settings_from_config(self): + """Test that values set in UserConfig for log settings will set flags as expected""" context = self.make_dbt_context("run", ["run"]) config = UserConfig(log_format="json", log_level="warn", use_colors=False) @@ -316,9 +316,9 @@ def test_log_settings_from_config(self): assert flags.USE_COLORS is False assert flags.USE_COLORS_FILE is False - # Test that values set in UserConfig for log *file* settings will set flags as expected, - # leaving the console logging flags with their default values def test_log_file_settings_from_config(self): + """Test that values set in UserConfig for log *file* settings will set flags as expected, leaving the console + logging flags with their default values""" context = self.make_dbt_context("run", ["run"]) config = UserConfig(log_format_file="json", log_level_file="warn", use_colors_file=False) From cc52342107979c63d6a9962e2a0ae4716f265474 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 23 Feb 2023 20:41:48 +0000 Subject: [PATCH 11/11] Add generated CLI API docs --- .../docs/build/doctrees/environment.pickle | Bin 207366 -> 207366 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/core/dbt/docs/build/doctrees/environment.pickle b/core/dbt/docs/build/doctrees/environment.pickle index 4201afc09eb4d854b6c6f51ff8023e914a281493..45eeb69e648060a873b75fb7b0fabd51ef3f11bd 100644 GIT binary patch delta 32 mcmZp>!qawzXG60*>*q)JPkT3a$hUXMGXgQw_6~XGIxYbDDGgu% delta 32 mcmZp>!qawzXG60*tLM4pUsg4D$hUXMGXgQw_6~XGIxYb2LJaZ%