From a066bfc1ed73dec786ec3574f6db97cb589c7c9a Mon Sep 17 00:00:00 2001 From: v-chen_data Date: Wed, 5 Jun 2024 13:05:51 -0700 Subject: [PATCH 01/18] bump version --- llmfoundry/__init__.py | 2 +- llmfoundry/utils/config_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llmfoundry/__init__.py b/llmfoundry/__init__.py index c9666566bf..5e2795f9c9 100644 --- a/llmfoundry/__init__.py +++ b/llmfoundry/__init__.py @@ -71,4 +71,4 @@ 'utils', ] -__version__ = '0.9.0.dev0' +__version__ = '0.10.0.dev0' diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index b6a5acf6d9..1885f75ab9 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -289,8 +289,8 @@ def make_dataclass_and_log_config( unstructured_config['variables'] = {} for key in extraneous_keys: - warnings.warn( - f'Unused parameter {key} found in cfg. Please check your yaml to ensure this parameter is necessary. Interpreting {key} as a variable for logging purposes. Top-level variables are deprecated and will not be supported in future releases. Please place any variables under the `variables` key.', + raise ValueError( + f'Unused parameter {key} found in cfg. Please check your yaml to ensure this parameter is necessary. Please place any variables under the `variables` key.', category=DeprecationWarning, ) unstructured_config['variables'][key] = unstructured_config.pop(key) From 7dfb6f8c755a2b68bf8f9611fe6d40f0652c1d7c Mon Sep 17 00:00:00 2001 From: v-chen_data Date: Wed, 5 Jun 2024 13:24:57 -0700 Subject: [PATCH 02/18] typo --- llmfoundry/utils/config_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index 1885f75ab9..065927bcc8 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -291,9 +291,7 @@ def make_dataclass_and_log_config( for key in extraneous_keys: raise ValueError( f'Unused parameter {key} found in cfg. Please check your yaml to ensure this parameter is necessary. Please place any variables under the `variables` key.', - category=DeprecationWarning, ) - unstructured_config['variables'][key] = unstructured_config.pop(key) dataclass_dict_config: DictConfig = om.structured( dataclass_constructor(**unstructured_config), From ae0a14c59f4e63106a42342a0451fd2f5662edd4 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 16:50:10 -0400 Subject: [PATCH 03/18] Update config_utils.py These changes are necessary as the deprecation broke compatibility with `update_batch_size`. --- llmfoundry/utils/config_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index 065927bcc8..a0524fe14a 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -85,6 +85,12 @@ class EvalConfig: # Variables to ignore variables: Optional[Dict[str, Any]] = None + # Fields created by `update_batch_size` + n_gpus: Optional[int] = None + device_train_batch_size: Optional[int] = None + device_train_microbatch_size: Optional[int] = None + device_train_grad_accum: Optional[int] = None + EVAL_CONFIG_KEYS = {field.name for field in fields(EvalConfig)} @@ -180,6 +186,12 @@ class TrainConfig: # Variables to ignore variables: Optional[Dict[str, Any]] = None + # Fields created by `update_batch_size` + n_gpus: Optional[int] = None + device_train_batch_size: Optional[int] = None + device_train_microbatch_size: Optional[int] = None + device_train_grad_accum: Optional[int] = None + TRAIN_CONFIG_KEYS = {field.name for field in fields(TrainConfig)} From e1b2c59bd6cd92b182f639624629977a9769537e Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 17:00:21 -0400 Subject: [PATCH 04/18] Update config_utils.py fix typo --- llmfoundry/utils/config_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index a0524fe14a..cc301a2d9a 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -86,10 +86,10 @@ class EvalConfig: variables: Optional[Dict[str, Any]] = None # Fields created by `update_batch_size` - n_gpus: Optional[int] = None - device_train_batch_size: Optional[int] = None - device_train_microbatch_size: Optional[int] = None - device_train_grad_accum: Optional[int] = None + n_gpus: int = MISSING + device_train_batch_size: int = MISSING + device_train_microbatch_size: int = MISSING + device_train_grad_accum: str = MISSING EVAL_CONFIG_KEYS = {field.name for field in fields(EvalConfig)} @@ -187,10 +187,10 @@ class TrainConfig: variables: Optional[Dict[str, Any]] = None # Fields created by `update_batch_size` - n_gpus: Optional[int] = None - device_train_batch_size: Optional[int] = None - device_train_microbatch_size: Optional[int] = None - device_train_grad_accum: Optional[int] = None + n_gpus: int = MISSING + device_train_batch_size: int = MISSING + device_train_microbatch_size: int = MISSING + device_train_grad_accum: str = MISSING TRAIN_CONFIG_KEYS = {field.name for field in fields(TrainConfig)} From 30603effe1caa35b281ed1545e244d56c3a729c8 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 17:10:18 -0400 Subject: [PATCH 05/18] typo --- llmfoundry/utils/config_utils.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index cc301a2d9a..a8a11e930b 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -85,12 +85,6 @@ class EvalConfig: # Variables to ignore variables: Optional[Dict[str, Any]] = None - # Fields created by `update_batch_size` - n_gpus: int = MISSING - device_train_batch_size: int = MISSING - device_train_microbatch_size: int = MISSING - device_train_grad_accum: str = MISSING - EVAL_CONFIG_KEYS = {field.name for field in fields(EvalConfig)} @@ -186,10 +180,9 @@ class TrainConfig: # Variables to ignore variables: Optional[Dict[str, Any]] = None - # Fields created by `update_batch_size` + # Fields created by `update_batch_size_info` n_gpus: int = MISSING device_train_batch_size: int = MISSING - device_train_microbatch_size: int = MISSING device_train_grad_accum: str = MISSING From 0fc514827deba4ce244cb3d1e3b8298d8931c66f Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 17:32:07 -0400 Subject: [PATCH 06/18] typo I --- llmfoundry/utils/config_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index a8a11e930b..6b63cc669f 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -67,6 +67,7 @@ class EvalConfig: # Logging parameters python_log_level: Optional[str] = 'debug' loggers: Optional[Dict[str, Any]] = None + console_log_interval: Union[int, str] = '1ba' log_config: bool = True # Model/run parameters From f9f8ab2a624f8cc0ca399ecda20a8f7ce7907081 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 21:39:57 +0000 Subject: [PATCH 07/18] update tests --- tests/a_scripts/eval/test_eval_inputs.py | 9 +++------ tests/a_scripts/train/test_train_inputs.py | 8 ++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/a_scripts/eval/test_eval_inputs.py b/tests/a_scripts/eval/test_eval_inputs.py index 98b15743b3..3e1fc83b09 100644 --- a/tests/a_scripts/eval/test_eval_inputs.py +++ b/tests/a_scripts/eval/test_eval_inputs.py @@ -42,12 +42,13 @@ def test_mispelled_mandatory_params_fail(self, cfg: DictConfig) -> None: omegaconf.errors.InterpolationKeyError, omegaconf.errors.MissingMandatoryValue, TypeError, + ValueError, )): cfg[p + '-mispelled'] = cfg.pop(p) main(cfg) cfg[p] = cfg.pop(p + '-mispelled') - def test_optional_mispelled_params_raise_warning( + def test_optional_mispelled_params_raise_error( self, cfg: DictConfig, ) -> None: @@ -67,15 +68,11 @@ def test_optional_mispelled_params_raise_warning( orig_value = cfg.pop(param, None) updated_param = param + '-mispelling' cfg[updated_param] = orig_value - with warnings.catch_warnings(record=True) as warning_list: + with pytest.raises(ValueError): try: main(cfg) except: pass - assert any( - f'Unused parameter {updated_param} found in cfg.' in - str(warning.message) for warning in warning_list - ) # restore configs. cfg = copy.deepcopy(old_cfg) diff --git a/tests/a_scripts/train/test_train_inputs.py b/tests/a_scripts/train/test_train_inputs.py index 5a3b21dc3b..6806772534 100644 --- a/tests/a_scripts/train/test_train_inputs.py +++ b/tests/a_scripts/train/test_train_inputs.py @@ -89,7 +89,7 @@ def test_missing_mandatory_parameters_fail(self, cfg: DictConfig) -> None: main(cfg) cfg[param] = orig_param - def test_optional_misspelled_params_raise_warning( + def test_optional_misspelled_params_raise_error( self, cfg: DictConfig, ) -> None: @@ -113,15 +113,11 @@ def test_optional_misspelled_params_raise_warning( orig_value = cfg.pop(param, None) updated_param = param + '-misspelling' cfg[updated_param] = orig_value - with warnings.catch_warnings(record=True) as warning_list: + with pytest.raises(ValueError): try: main(cfg) except: pass - assert any( - f'Unused parameter {updated_param} found in cfg.' in - str(warning.message) for warning in warning_list - ) # restore configs. cfg = copy.deepcopy(old_cfg) From 6ed76b5af6ba4a42b388cda6a696756dffbba6eb Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 21:43:20 +0000 Subject: [PATCH 08/18] typo II --- tests/a_scripts/eval/test_eval_inputs.py | 1 - tests/a_scripts/train/test_train_inputs.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/a_scripts/eval/test_eval_inputs.py b/tests/a_scripts/eval/test_eval_inputs.py index 3e1fc83b09..b1e88b2443 100644 --- a/tests/a_scripts/eval/test_eval_inputs.py +++ b/tests/a_scripts/eval/test_eval_inputs.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import copy import os -import warnings import omegaconf import pytest diff --git a/tests/a_scripts/train/test_train_inputs.py b/tests/a_scripts/train/test_train_inputs.py index 6806772534..bfd7cf849a 100644 --- a/tests/a_scripts/train/test_train_inputs.py +++ b/tests/a_scripts/train/test_train_inputs.py @@ -3,7 +3,6 @@ import copy import json import os -import warnings import omegaconf import pytest From ef3f6577dd3d2ef0f006091596002407ade61c17 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Wed, 5 Jun 2024 22:01:36 +0000 Subject: [PATCH 09/18] typo III --- tests/a_scripts/eval/test_eval_inputs.py | 5 +---- tests/a_scripts/train/test_train_inputs.py | 9 ++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/a_scripts/eval/test_eval_inputs.py b/tests/a_scripts/eval/test_eval_inputs.py index b1e88b2443..0ca5765a26 100644 --- a/tests/a_scripts/eval/test_eval_inputs.py +++ b/tests/a_scripts/eval/test_eval_inputs.py @@ -68,10 +68,7 @@ def test_optional_mispelled_params_raise_error( updated_param = param + '-mispelling' cfg[updated_param] = orig_value with pytest.raises(ValueError): - try: - main(cfg) - except: - pass + main(cfg) # restore configs. cfg = copy.deepcopy(old_cfg) diff --git a/tests/a_scripts/train/test_train_inputs.py b/tests/a_scripts/train/test_train_inputs.py index bfd7cf849a..5901d53e94 100644 --- a/tests/a_scripts/train/test_train_inputs.py +++ b/tests/a_scripts/train/test_train_inputs.py @@ -62,7 +62,9 @@ def cfg(self, foundry_dir: str) -> DictConfig: def test_misspelled_mandatory_params_fail(self, cfg: DictConfig) -> None: """Check that mandatory misspelled inputs fail to train.""" cfg.trai_loader = cfg.pop('train_loader') - with pytest.raises((omegaconf.errors.MissingMandatoryValue, TypeError)): + with pytest.raises( + (omegaconf.errors.MissingMandatoryValue, TypeError, ValueError), + ): main(cfg) def test_missing_mandatory_parameters_fail(self, cfg: DictConfig) -> None: @@ -113,10 +115,7 @@ def test_optional_misspelled_params_raise_error( updated_param = param + '-misspelling' cfg[updated_param] = orig_value with pytest.raises(ValueError): - try: - main(cfg) - except: - pass + main(cfg) # restore configs. cfg = copy.deepcopy(old_cfg) From 28372948e6c0b87b27c36c0dc576e959c450ea19 Mon Sep 17 00:00:00 2001 From: v-chen_data Date: Wed, 5 Jun 2024 16:06:12 -0700 Subject: [PATCH 10/18] bump composer version --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 78182976d4..0556050de9 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ ] install_requires = [ - 'mosaicml[libcloud,wandb,oci,gcs]>=0.22.0,<0.23', + 'mosaicml[libcloud,wandb,oci,gcs]>=0.23.0,<0.24', 'mlflow>=2.12.1,<2.13', 'accelerate>=0.25,<0.26', # for HF inference `device_map` 'transformers>=4.40,<4.41', @@ -92,14 +92,14 @@ ] extra_deps['databricks'] = [ - 'mosaicml[databricks]>=0.22.0,<0.23', + 'mosaicml[databricks]>=0.23.0,<0.24', 'databricks-sql-connector>=3,<4', 'databricks-connect==14.1.0', 'lz4>=4,<5', ] extra_deps['tensorboard'] = [ - 'mosaicml[tensorboard]>=0.22.0,<0.23', + 'mosaicml[tensorboard]>=0.23.0,<0.24', ] # Flash 2 group kept for backwards compatibility @@ -110,7 +110,7 @@ extra_deps['gpu'] = copy.deepcopy(extra_deps['gpu-flash2']) extra_deps['peft'] = [ - 'mosaicml[peft]>=0.22.0,<0.23', + 'mosaicml[peft]>=0.23.0,<0.24', ] extra_deps['openai'] = [ From 0a2f2a85300b6d481a81d8f920a81865dd3f898b Mon Sep 17 00:00:00 2001 From: v-chen_data Date: Wed, 5 Jun 2024 16:13:01 -0700 Subject: [PATCH 11/18] undo composer bump for seperate pr --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0556050de9..78182976d4 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ ] install_requires = [ - 'mosaicml[libcloud,wandb,oci,gcs]>=0.23.0,<0.24', + 'mosaicml[libcloud,wandb,oci,gcs]>=0.22.0,<0.23', 'mlflow>=2.12.1,<2.13', 'accelerate>=0.25,<0.26', # for HF inference `device_map` 'transformers>=4.40,<4.41', @@ -92,14 +92,14 @@ ] extra_deps['databricks'] = [ - 'mosaicml[databricks]>=0.23.0,<0.24', + 'mosaicml[databricks]>=0.22.0,<0.23', 'databricks-sql-connector>=3,<4', 'databricks-connect==14.1.0', 'lz4>=4,<5', ] extra_deps['tensorboard'] = [ - 'mosaicml[tensorboard]>=0.23.0,<0.24', + 'mosaicml[tensorboard]>=0.22.0,<0.23', ] # Flash 2 group kept for backwards compatibility @@ -110,7 +110,7 @@ extra_deps['gpu'] = copy.deepcopy(extra_deps['gpu-flash2']) extra_deps['peft'] = [ - 'mosaicml[peft]>=0.23.0,<0.24', + 'mosaicml[peft]>=0.22.0,<0.23', ] extra_deps['openai'] = [ From 1d8d104bdef5b972c963b59ad61aadcdea587673 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 00:39:50 +0000 Subject: [PATCH 12/18] fix test --- llmfoundry/utils/config_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index 6b63cc669f..a2a8767b63 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -76,6 +76,9 @@ class EvalConfig: run_name: Optional[str] = None metadata: Optional[Dict[str, str]] = None + # Dataloader + global_train_batch_size: Optional[int] = None + # Distributed parameters dist_timeout: Union[float, int] = 600.0 fsdp_config: Optional[Dict[str, Any]] = None From cf1d31fba6724b15716d118bfae4af5b8bef7429 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 01:57:53 +0000 Subject: [PATCH 13/18] fix tests II --- llmfoundry/utils/config_utils.py | 3 --- tests/a_scripts/eval/test_eval.py | 10 +++++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index a2a8767b63..6b63cc669f 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -76,9 +76,6 @@ class EvalConfig: run_name: Optional[str] = None metadata: Optional[Dict[str, str]] = None - # Dataloader - global_train_batch_size: Optional[int] = None - # Distributed parameters dist_timeout: Union[float, int] = 600.0 fsdp_config: Optional[Dict[str, Any]] = None diff --git a/tests/a_scripts/eval/test_eval.py b/tests/a_scripts/eval/test_eval.py index a56778538c..01f3760d26 100644 --- a/tests/a_scripts/eval/test_eval.py +++ b/tests/a_scripts/eval/test_eval.py @@ -13,7 +13,7 @@ from llmfoundry.utils import build_tokenizer from llmfoundry.utils.builders import build_composer_model -from llmfoundry.utils.config_utils import to_dict_container +from llmfoundry.utils.config_utils import EVAL_CONFIG_KEYS, to_dict_container from scripts.eval.eval import main # noqa: E402 from tests.data_utils import create_c4_dataset_xxsmall, gpt_tiny_cfg @@ -134,6 +134,14 @@ def test_loader_eval( test_cfg.eval_interval = '1ba' test_cfg.loggers = om.DictConfig({'inmemory': om.DictConfig({})}) + # This test uses a training yaml with training-only keys present. + # We exclude these keys before calling `main` from the eval script. + allowed_keys = EVAL_CONFIG_KEYS + present_keys = set(test_cfg.keys()) + keys_to_pop = present_keys.difference(allowed_keys) + + [test_cfg.pop(key) for key in keys_to_pop] + trainers, eval_gauntlet_df = main(test_cfg) assert eval_gauntlet_df is None From 649b6ff2ee3439e3be5ffa0ce2f386b003092964 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 14:42:25 +0000 Subject: [PATCH 14/18] yolo --- llmfoundry/utils/config_utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index 6b63cc669f..e5298ee46c 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -236,6 +236,22 @@ def to_container( return cfg # type: ignore (dicts and lists are already in the correct format) +def move_variables_to_section(cfg: Dict[str, Any]) -> Dict[str, Any]: + """Moves variables from the root of the config to the variables section. + + This function is used to move variables from the root of the config to the + variables section. This is necessary because the variables section is + required for interpolation. + """ + if 'variables' not in cfg: + cfg['variables'] = {} + for key in list(cfg.keys()): + if 'variables.' in key: + _, suffix = key.split('.', 1) + cfg['variables'][suffix] = cfg.pop(key) + return cfg + + T = TypeVar('T') @@ -249,6 +265,8 @@ def make_dataclass_and_log_config( ) -> Tuple[Dict[str, Any], T]: """Converts a DictConfig to a dataclass and creates a logged config.""" # Resolve all interpolation variables as early as possible + # before resolving, convert keys of the form `variables.key` to `variables[key]`. + cfg = move_variables_to_section(cfg) unstructured_config = om.to_container(cfg, resolve=True) assert isinstance(unstructured_config, dict) assert all(isinstance(k, str) for k in unstructured_config.keys()) From 7d23716993909f3b6324259d7f38cbc78ffd7cca Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 14:58:12 +0000 Subject: [PATCH 15/18] tye-o --- llmfoundry/utils/config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index e5298ee46c..48a01904b0 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -236,7 +236,7 @@ def to_container( return cfg # type: ignore (dicts and lists are already in the correct format) -def move_variables_to_section(cfg: Dict[str, Any]) -> Dict[str, Any]: +def move_variables_to_section(cfg: DictConfig) -> Dict[str, Any]: """Moves variables from the root of the config to the variables section. This function is used to move variables from the root of the config to the From 3ab437438bd5c68c1272ffbb182fb2d45cf647d7 Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 15:50:51 +0000 Subject: [PATCH 16/18] pyrite --- llmfoundry/utils/config_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index 48a01904b0..c0d6fda057 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -236,7 +236,7 @@ def to_container( return cfg # type: ignore (dicts and lists are already in the correct format) -def move_variables_to_section(cfg: DictConfig) -> Dict[str, Any]: +def move_variables_to_section(cfg: DictConfig) -> DictConfig: """Moves variables from the root of the config to the variables section. This function is used to move variables from the root of the config to the @@ -246,6 +246,8 @@ def move_variables_to_section(cfg: DictConfig) -> Dict[str, Any]: if 'variables' not in cfg: cfg['variables'] = {} for key in list(cfg.keys()): + if not isinstance(key, str): + continue if 'variables.' in key: _, suffix = key.split('.', 1) cfg['variables'][suffix] = cfg.pop(key) From 47d0e811236e74e425395f963dbb03d9776f51bf Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 17:19:07 +0000 Subject: [PATCH 17/18] we resolve later --- scripts/train/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/train/train.py b/scripts/train/train.py index c9e2d67bf4..eb5135113a 100644 --- a/scripts/train/train.py +++ b/scripts/train/train.py @@ -553,6 +553,5 @@ def main(cfg: DictConfig) -> Trainer: yaml_cfg = om.load(f) cli_cfg = om.from_cli(args_list) cfg = om.merge(yaml_cfg, cli_cfg) - om.resolve(cfg) assert isinstance(cfg, DictConfig) main(cfg) From 27db09d69c8ec8ab8cc84587cfa3cd279cc7bc5d Mon Sep 17 00:00:00 2001 From: Milo Cress Date: Thu, 6 Jun 2024 19:37:34 +0000 Subject: [PATCH 18/18] revert new . syntax --- llmfoundry/utils/config_utils.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py index c0d6fda057..5ab148bbe8 100644 --- a/llmfoundry/utils/config_utils.py +++ b/llmfoundry/utils/config_utils.py @@ -236,24 +236,6 @@ def to_container( return cfg # type: ignore (dicts and lists are already in the correct format) -def move_variables_to_section(cfg: DictConfig) -> DictConfig: - """Moves variables from the root of the config to the variables section. - - This function is used to move variables from the root of the config to the - variables section. This is necessary because the variables section is - required for interpolation. - """ - if 'variables' not in cfg: - cfg['variables'] = {} - for key in list(cfg.keys()): - if not isinstance(key, str): - continue - if 'variables.' in key: - _, suffix = key.split('.', 1) - cfg['variables'][suffix] = cfg.pop(key) - return cfg - - T = TypeVar('T') @@ -266,9 +248,6 @@ def make_dataclass_and_log_config( icl_tasks_required: bool = False, ) -> Tuple[Dict[str, Any], T]: """Converts a DictConfig to a dataclass and creates a logged config.""" - # Resolve all interpolation variables as early as possible - # before resolving, convert keys of the form `variables.key` to `variables[key]`. - cfg = move_variables_to_section(cfg) unstructured_config = om.to_container(cfg, resolve=True) assert isinstance(unstructured_config, dict) assert all(isinstance(k, str) for k in unstructured_config.keys())