From 4040bdf02ae197d03e5a2782dbc83c16ebe4f38a Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 3 Nov 2023 23:04:13 -0700 Subject: [PATCH 1/5] metadata --- scripts/train/train.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/train/train.py b/scripts/train/train.py index e29f2c9a47..7dad2a2dfa 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -408,6 +408,11 @@ def main(cfg: DictConfig) -> Trainer: 'compile_config', must_exist=False, default_value=None) + mosaicml_metadata: Optional[Dict[str, str]] = pop_config(cfg, + 'mosaicml_metadata', + must_exist=False, + default_value=None, + convert=True) # Enable autoresume from model checkpoints if possible autoresume_default: bool = False if logged_cfg.get('run_name', None) is not None \ @@ -485,6 +490,15 @@ def main(cfg: DictConfig) -> Trainer: mosaicml_logger = MosaicMLLogger() loggers.append(mosaicml_logger) + if mosaicml_metadata is not None: + if mosaicml_logger is None: + raise ValueError( + 'mosaicml_metadata was provided but no MosaicML logger was found.' + + ' mosaicml_metadata can only be used on the MosaicML platform.' + ) + mosaicml_logger.log_metrics(mosaicml_metadata) + mosaicml_logger._flush_metadata(force_flush=True) + # Profiling profiler: Optional[Profiler] = None profiler_cfg: Optional[DictConfig] = pop_config(cfg, From 3db75c40ac8f60b397a081c2b94b43104a5edaaf Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 3 Nov 2023 23:04:48 -0700 Subject: [PATCH 2/5] precommit --- scripts/train/train.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/train/train.py b/scripts/train/train.py index 7dad2a2dfa..379d6aa389 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -408,11 +408,12 @@ def main(cfg: DictConfig) -> Trainer: 'compile_config', must_exist=False, default_value=None) - mosaicml_metadata: Optional[Dict[str, str]] = pop_config(cfg, - 'mosaicml_metadata', - must_exist=False, - default_value=None, - convert=True) + mosaicml_metadata: Optional[Dict[str, + str]] = pop_config(cfg, + 'mosaicml_metadata', + must_exist=False, + default_value=None, + convert=True) # Enable autoresume from model checkpoints if possible autoresume_default: bool = False if logged_cfg.get('run_name', None) is not None \ @@ -494,8 +495,8 @@ def main(cfg: DictConfig) -> Trainer: if mosaicml_logger is None: raise ValueError( 'mosaicml_metadata was provided but no MosaicML logger was found.' - + ' mosaicml_metadata can only be used on the MosaicML platform.' - ) + + + ' mosaicml_metadata can only be used on the MosaicML platform.') mosaicml_logger.log_metrics(mosaicml_metadata) mosaicml_logger._flush_metadata(force_flush=True) From 5f3f0045bfe7615d1ce3c061a5eb84750a55f4d9 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 3 Nov 2023 23:37:40 -0700 Subject: [PATCH 3/5] add to config for other exp trackers --- scripts/train/train.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts/train/train.py b/scripts/train/train.py index 379d6aa389..f28c6391ed 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -408,9 +408,8 @@ def main(cfg: DictConfig) -> Trainer: 'compile_config', must_exist=False, default_value=None) - mosaicml_metadata: Optional[Dict[str, - str]] = pop_config(cfg, - 'mosaicml_metadata', + metadata: Optional[Dict[str, str]] = pop_config(cfg, + 'metadata', must_exist=False, default_value=None, convert=True) @@ -491,14 +490,11 @@ def main(cfg: DictConfig) -> Trainer: mosaicml_logger = MosaicMLLogger() loggers.append(mosaicml_logger) - if mosaicml_metadata is not None: - if mosaicml_logger is None: - raise ValueError( - 'mosaicml_metadata was provided but no MosaicML logger was found.' - + - ' mosaicml_metadata can only be used on the MosaicML platform.') - mosaicml_logger.log_metrics(mosaicml_metadata) - mosaicml_logger._flush_metadata(force_flush=True) + if metadata is not None: + logged_cfg.update(metadata, merge=True) + if mosaicml_logger is not None: + mosaicml_logger.log_metrics(mosaicml_metadata) + mosaicml_logger._flush_metadata(force_flush=True) # Profiling profiler: Optional[Profiler] = None From 0d2ae71f3156276eeed84c3aa8c08cbec34266b0 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 3 Nov 2023 23:38:27 -0700 Subject: [PATCH 4/5] fix --- scripts/train/train.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/train/train.py b/scripts/train/train.py index f28c6391ed..df01bd510a 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -409,10 +409,10 @@ def main(cfg: DictConfig) -> Trainer: must_exist=False, default_value=None) metadata: Optional[Dict[str, str]] = pop_config(cfg, - 'metadata', - must_exist=False, - default_value=None, - convert=True) + 'metadata', + must_exist=False, + default_value=None, + convert=True) # Enable autoresume from model checkpoints if possible autoresume_default: bool = False if logged_cfg.get('run_name', None) is not None \ @@ -493,7 +493,7 @@ def main(cfg: DictConfig) -> Trainer: if metadata is not None: logged_cfg.update(metadata, merge=True) if mosaicml_logger is not None: - mosaicml_logger.log_metrics(mosaicml_metadata) + mosaicml_logger.log_metrics(metadata) mosaicml_logger._flush_metadata(force_flush=True) # Profiling From 7fde8bc17a98c6f9c9348ced19b3767de52802b7 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Sat, 4 Nov 2023 00:06:25 -0700 Subject: [PATCH 5/5] pop off of config --- scripts/train/train.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/train/train.py b/scripts/train/train.py index df01bd510a..86fb2471db 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -413,6 +413,7 @@ def main(cfg: DictConfig) -> Trainer: must_exist=False, default_value=None, convert=True) + # Enable autoresume from model checkpoints if possible autoresume_default: bool = False if logged_cfg.get('run_name', None) is not None \ @@ -491,6 +492,8 @@ def main(cfg: DictConfig) -> Trainer: loggers.append(mosaicml_logger) if metadata is not None: + # Flatten the metadata for logging + logged_cfg.pop('metadata', None) logged_cfg.update(metadata, merge=True) if mosaicml_logger is not None: mosaicml_logger.log_metrics(metadata)