diff --git a/notebooks/validate_and_tokenize_data.ipynb b/notebooks/validate_and_tokenize_data.ipynb index f070da0c43..367cf5701e 100644 --- a/notebooks/validate_and_tokenize_data.ipynb +++ b/notebooks/validate_and_tokenize_data.ipynb @@ -232,7 +232,25 @@ "**Temporary Data Path Configuration:**\n", "\n", "- temporary_jsonl_data_path: Defines a filesystem path where temporary data related to the training process will be stored.\n", - "- Environment variables for Hugging Face caches (HF_DATASETS_CACHE) are set to '/tmp/', directing dataset caching to a temporary directory." + "- Environment variables for Hugging Face caches (HF_DATASETS_CACHE) are set to '/tmp/', directing dataset caching to a temporary directory.\n", + "\n", + "**[Supported Models by FT API](https://docs.mosaicml.com/projects/mcli/en/latest/finetuning/finetuning.html#supported-models):**. \n", + "\n", + "You need to specify context length based on the model mapping below.\n", + "```\n", + "ft_models = {\n", + " 'mosaicml/mpt-7b-8k': 8192, \n", + " 'mosaicml/mpt-7b': 2048,\n", + " 'mosaicml/mpt-30b': 8192,\n", + " 'meta-llama/Llama-2-13b-hf': 4096,\n", + " 'meta-llama/Llama-2-7b-hf': 4096,\n", + " 'meta-llama/Llama-2-70b-hf': 4096,\n", + " 'codellama/CodeLlama-7b-hf': 16384,\n", + " 'codellama/CodeLlama-13b-hf': 16384,\n", + " 'codellama/CodeLlama-34b-hf': 16384,\n", + " 'mistralai/Mistral-7B-v0.1': 32768,\n", + "}\n", + "```" ] }, {