From cc0cbca5d21112f8ad5d5e2c9de5ba261440bce0 Mon Sep 17 00:00:00 2001 From: huseinzol05 Date: Fri, 3 Jan 2025 01:08:56 +0800 Subject: [PATCH] tested --- session/small-malaysian-reasoning/3b.sh | 24 + session/small-malaysian-reasoning/README.md | 10 + session/small-malaysian-reasoning/train.py | 482 ++++++++++++++++++++ 3 files changed, 516 insertions(+) create mode 100644 session/small-malaysian-reasoning/3b.sh create mode 100644 session/small-malaysian-reasoning/README.md create mode 100644 session/small-malaysian-reasoning/train.py diff --git a/session/small-malaysian-reasoning/3b.sh b/session/small-malaysian-reasoning/3b.sh new file mode 100644 index 00000000..14fb3be7 --- /dev/null +++ b/session/small-malaysian-reasoning/3b.sh @@ -0,0 +1,24 @@ +WANDB_PROJECT="lora-embedding-256-llama3.2-3b-small-malaysian-reasoning" \ +CUDA_VISIBLE_DEVICES="2" \ +TORCH_DISTRIBUTED_DEBUG="info" \ +torchrun --nproc_per_node 1 \ +-m train \ +--model_name_or_path unsloth/Llama-3.2-3B-Instruct \ +--per_device_train_batch_size 2 \ +--gradient_accumulation_steps 6 \ +--output_dir lora-embedding-256-llama3.2-3b-small-malaysian-reasoning \ +--bf16 --do_train --do_eval false --num_train_epochs 5 \ +--train_file packing-4k \ +--logging_steps 1 \ +--learning_rate 2e-5 \ +--weight_decay 0.01 \ +--block_size 24576 \ +--save_steps 100 \ +--save_total_limit 3 \ +--gradient_checkpointing true \ +--neftune_noise_alpha 5.0 \ +--torch_dtype bfloat16 \ +--rank 256 \ +--ddp_find_unused_parameters false \ +--dataloader_num_workers 3 \ +--dataloader_prefetch_factor 4 \ No newline at end of file diff --git a/session/small-malaysian-reasoning/README.md b/session/small-malaysian-reasoning/README.md new file mode 100644 index 00000000..d138735a --- /dev/null +++ b/session/small-malaysian-reasoning/README.md @@ -0,0 +1,10 @@ +# Small Malaysian Reasoning + +## how to + +1. Install necessary libraries, + +```bash +pip3 install transformers==4.47.0 accelerate==1.1.1 +pip3 install git+https://github.com/mesolitica/ml-cross-entropy-lora-embedding +``` \ No newline at end of file diff --git a/session/small-malaysian-reasoning/train.py b/session/small-malaysian-reasoning/train.py new file mode 100644 index 00000000..4a27ad12 --- /dev/null +++ b/session/small-malaysian-reasoning/train.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=text-generation +""" +# You can also adapt this script on your own causal language modeling +# task. Pointers for this are left as comments. + +import torch + +torch._dynamo.config.optimize_ddp=False + +import logging +import math +import os +import sys +import warnings +from dataclasses import dataclass, field +from itertools import chain +from typing import Optional + +import datasets +import evaluate +from datasets import load_dataset + +import transformers +import random +from transformers import ( + CONFIG_MAPPING, + MODEL_FOR_CAUSAL_LM_MAPPING, + AutoConfig, + AutoModelForCausalLM, + AutoTokenizer, + HfArgumentParser, + Trainer, + TrainingArguments, + default_data_collator, + DataCollatorWithPadding, + DataCollatorForLanguageModeling, + is_torch_tpu_available, + set_seed, +) +from transformers.testing_utils import CaptureLogger +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import check_min_version, send_example_telemetry +from transformers.utils.versions import require_version +from transformers.models.llama.modeling_llama import LlamaForCausalLM +import streaming +import json +import numpy as np +from streaming import LocalDataset +from streaming.base.format.mds.encodings import Encoding, _encodings +from peft import LoraConfig, get_peft_model +from cut_cross_entropy import linear_cross_entropy + +require_version( + "datasets>=1.8.0", + "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") + +logger = logging.getLogger(__name__) + + +MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + rank: int = field( + default=256, + metadata={ + "help": "lora rank" + }, + ) + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={ + "help": "If training from scratch, pass a model type from the list: " + + ", ".join(MODEL_TYPES)}, + ) + config_overrides: Optional[str] = field( + default=None, metadata={ + "help": ( + "Override some existing default config settings when a model is trained from scratch. Example: " + "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index")}, ) + config_name: Optional[str] = field( + default=None, metadata={ + "help": "Pretrained config name or path if not the same as model_name"}) + tokenizer_name: Optional[str] = field( + default=None, metadata={ + "help": "Pretrained tokenizer name or path if not the same as model_name"}) + cache_dir: Optional[str] = field( + default=None, metadata={ + "help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, ) + use_fast_tokenizer: bool = field( + default=True, metadata={ + "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, ) + model_revision: str = field( + default="main", metadata={ + "help": "The specific model version to use (can be a branch name, tag name or commit id)."}, ) + token: str = field( + default=None, + metadata={ + "help": ( + "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " + "generated when running `huggingface-cli login` (stored in `~/.huggingface`)." + ) + }, + ) + use_auth_token: bool = field( + default=None, + metadata={ + "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`." + }, + ) + trust_remote_code: bool = field( + default=False, metadata={ + "help": ( + "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option" + "should only be set to `True` for repositories you trust and in which you have read the code, as it will" + "execute code present on the Hub on your local machine.")}, ) + torch_dtype: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " + "dtype will be automatically derived from the model's weights."), + "choices": [ + "auto", + "bfloat16", + "float16", + "float32"], + }, + ) + low_cpu_mem_usage: bool = field( + default=False, + metadata={ + "help": ( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "set True will benefit LLM loading time and RAM consumption." + ) + }, + ) + + def __post_init__(self): + if self.config_overrides is not None and ( + self.config_name is not None or self.model_name_or_path is not None): + raise ValueError( + "--config_overrides can't be used in combination with --config_name or --model_name_or_path" + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={ + "help": "The name of the dataset to use (via the datasets library)."}) + dataset_config_name: Optional[str] = field( + default=None, metadata={ + "help": "The configuration name of the dataset to use (via the datasets library)."}) + train_file: Optional[str] = field( + default=None, metadata={ + "help": "The input training data file (a text file)."}) + validation_file: Optional[str] = field( + default=None, metadata={ + "help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, ) + max_train_samples: Optional[int] = field( + default=None, metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set.")}, ) + max_eval_samples: Optional[int] = field( + default=None, metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set.")}, ) + streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"}) + block_size: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + validation_split_percentage: Optional[int] = field( + default=5, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + keep_linebreaks: bool = field( + default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."} + ) + +def print_trainable_parameters(model): + """ + Prints the number of trainable parameters in the model. + """ + trainable_params = 0 + all_param = 0 + for _, param in model.named_parameters(): + all_param += param.numel() + if param.requires_grad: + trainable_params += param.numel() + print( + f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" + ) + +def block_diagonal_concat_inverted(*masks, dtype=torch.bfloat16): + total_size = sum(mask.size(0) for mask in masks) + combined_mask = torch.zeros(total_size, total_size, dtype=dtype) + + current_pos = 0 + + for mask in masks: + size = mask.size(0) + combined_mask[current_pos:current_pos + size, current_pos:current_pos + size] = mask + current_pos += size + + min_value = torch.finfo(dtype).min if dtype.is_floating_point else torch.iinfo(dtype).min + inverted_mask = torch.where(combined_mask == 1, torch.tensor(0, dtype=dtype), min_value) + return inverted_mask.unsqueeze(0) + +class Llama(LlamaForCausalLM): + def __init__(self, config): + super().__init__(config) + + def forward(self, input_ids, position_ids, attention_mask, labels = None, **kwargs): + super_out = super().forward( + input_ids = input_ids, + position_ids = position_ids, + attention_mask = attention_mask, + output_hidden_states = True, + ) + if labels is not None: + embeddings = super_out.hidden_states[-1] + auto_shift_loss = linear_cross_entropy(embeddings, self.lm_head, labels, shift=True, impl = 'torch_compile') + return {'loss': auto_shift_loss} + return super_out + +def main(): + + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file( + json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + if model_args.use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v4.34.", + FutureWarning) + if model_args.token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + model_args.token = model_args.use_auth_token + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_clm", model_args, data_args) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level + # at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}") + logger.info(f"Training/evaluation parameters {training_args}") + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir( + training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Load pretrained model and tokenizer + # + # Distributed training: + # The .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + + config_kwargs = { + "cache_dir": model_args.cache_dir, + "revision": model_args.model_revision, + "token": model_args.token, + "trust_remote_code": model_args.trust_remote_code, + } + + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + if model_args.config_overrides is not None: + logger.info(f"Overriding config: {model_args.config_overrides}") + config.update_from_string(model_args.config_overrides) + logger.info(f"New config: {config}") + + tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path) + + class UInt32(Encoding): + def encode(self, obj) -> bytes: + return obj.tobytes() + + def decode(self, data: bytes): + return np.frombuffer(data, np.uint32) + + _encodings['uint32'] = UInt32 + + class DatasetFixed(torch.utils.data.Dataset): + def __init__(self, local): + self.dataset = LocalDataset(local=local) + + def __getitem__(self, idx): + data = self.dataset[idx] + data['labels'] = data["input_ids"].copy() + masking = data.pop('attention_mask') + + data.pop('token_type_ids', None) + for k in data.keys(): + data[k] = data[k].astype(np.int64) + + masks = [] + for m in masking: + masks.append(torch.tril(torch.ones(m, m))) + attention_mask = block_diagonal_concat_inverted(*masks) + data['attention_mask'] = attention_mask + + return data + + def __len__(self): + return len(self.dataset) + + dataset = DatasetFixed(data_args.train_file) + print('dataset', len(dataset), dataset[0]['attention_mask'].shape) + + torch_dtype = ( + model_args.torch_dtype + if model_args.torch_dtype in ["auto", None] + else getattr(torch, model_args.torch_dtype) + ) + + model = Llama.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + token=model_args.token, + trust_remote_code=model_args.trust_remote_code, + torch_dtype=torch_dtype, + low_cpu_mem_usage=model_args.low_cpu_mem_usage, + attn_implementation = 'sdpa', + ) + print(model) + + peft_config = LoraConfig( + lora_alpha=model_args.rank * 2, + lora_dropout=0.0, + r=model_args.rank, + bias="none", + task_type="CAUSAL_LM", + target_modules=["q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj", "embed_tokens", "lm_head"], + ) + + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + model = get_peft_model(model, peft_config) + print_trainable_parameters(model) + + trainer = Trainer( + model=model, + args=training_args, + train_dataset=dataset, + eval_dataset=None, + tokenizer=tokenizer, + data_collator=default_data_collator, + compute_metrics=None, + preprocess_logits_for_metrics=None, + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + trainer.save_state() + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main()