From a9f6aa8dc6b36a889102292505c10976f462888f Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 7 Feb 2024 18:30:20 +0100 Subject: [PATCH] draft CI --- .github/workflows/test_model.yml | 28 +++++ tests/models/__init__.py | 0 tests/models/llama/__init__.py | 0 tests/models/llama/test_inference_llama.py | 39 ++++++ tests/models/llama/test_train_llama.py | 18 +++ tests/models/test_model.py | 138 +++++++++++++++++++++ tests/models/vit/__init__.py | 0 tests/models/vit/test_training_vit.py | 50 ++++++++ 8 files changed, 273 insertions(+) create mode 100644 .github/workflows/test_model.yml create mode 100644 tests/models/__init__.py create mode 100644 tests/models/llama/__init__.py create mode 100644 tests/models/llama/test_inference_llama.py create mode 100644 tests/models/llama/test_train_llama.py create mode 100644 tests/models/test_model.py create mode 100644 tests/models/vit/__init__.py create mode 100644 tests/models/vit/test_training_vit.py diff --git a/.github/workflows/test_model.yml b/.github/workflows/test_model.yml new file mode 100644 index 00000000..2e169e2c --- /dev/null +++ b/.github/workflows/test_model.yml @@ -0,0 +1,28 @@ +name: Model Test + +on: + push: + branches: + - draft_ci + +env: + HF_HOME: /mnt/cache + # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. + # This token is created under the bot `hf-transformers-bot`. + HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} + +jobs: + github_repo: + name: github repo + runs-on: [single-gpu, nvidia-gpu, a10, ci] + container: + # TODO: make this $ {{ inputs.image }} to use GCP's DLC images + image: huggingface/transformers-all-latest-gpu + options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: run tests + run: | + python3 -m pytest -v tests/models diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/models/llama/__init__.py b/tests/models/llama/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/models/llama/test_inference_llama.py b/tests/models/llama/test_inference_llama.py new file mode 100644 index 00000000..f41fcc8b --- /dev/null +++ b/tests/models/llama/test_inference_llama.py @@ -0,0 +1,39 @@ +import os +import unittest +import torch + +from transformers import AutoModelForCausalLM, AutoTokenizer +from ..test_model import ModelInferenceTestMixin + +torch.backends.cudnn.deterministic = True +torch.backends.cuda.matmul.allow_tf32 = False +torch.backends.cudnn.allow_tf32 = False + +device = "cuda" + + +class LLaMaInferenceTest(ModelInferenceTestMixin, unittest.TestCase): + + def test_inference(self): + ckpt = "meta-llama/Llama-2-7b-hf" + hf_token = token=os.getenv("HF_HUB_READ_TOKEN", None) + + tokenizer = AutoTokenizer.from_pretrained(ckpt, token=hf_token) + + prompt = "Hey, are you conscious? Can you talk to me?" + inputs = tokenizer(prompt, return_tensors="pt").to(device) + + model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16, token=hf_token) + model.to(device) + + # To make generation's sample deterministic + torch.manual_seed(1) + + # Generate + with torch.no_grad(): + generate_ids = model.generate(inputs.input_ids, max_length=30) + output = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + + expected_output = "Hey, are you conscious? Can you talk to me?\nI'm not sure if you can hear me, but I'm talking" + + assert output == expected_output diff --git a/tests/models/llama/test_train_llama.py b/tests/models/llama/test_train_llama.py new file mode 100644 index 00000000..3d6005fe --- /dev/null +++ b/tests/models/llama/test_train_llama.py @@ -0,0 +1,18 @@ +import unittest + +from ..test_model import ModelTrainingTestMixin, TestCasePlus + + +class LLamaModelTrainingTest(ModelTrainingTestMixin, TestCasePlus): + + # WIP + def get_training_script(self): + raise NotImplementedError + + @unittest.skip("WIP") + def prepare_training_command(self, **kwargs): + output_dir = kwargs.get("output_dir", "my_dir") + + testargs = f"""""" + + return testargs diff --git a/tests/models/test_model.py b/tests/models/test_model.py new file mode 100644 index 00000000..6531433e --- /dev/null +++ b/tests/models/test_model.py @@ -0,0 +1,138 @@ +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from transformers.utils import is_accelerate_available +from transformers.testing_utils import TestCasePlus + + +if is_accelerate_available(): + from accelerate.state import AcceleratorState, PartialState + + +class ModelTrainingTestMixin: + + def get_training_script(self): + raise NotImplementedError + + def prepare_training_command(self, **kwargs): + raise NotImplementedError + + def get_results(self, output_dir): + path = os.path.join(output_dir, "all_results.json") + if os.path.exists(path): + with open(path, "r") as f: + results = json.load(f) + else: + raise ValueError(f"can't find {path}") + return results + + def test_training(self): + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = self.prepare_training_command(output_dir=tmp_dir).split() + + with patch.object(sys, "argv", testargs): + self.get_training_script().main() + result = self.get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.8) + + +class ModelInferenceTestMixin: + def test_inference(self): + raise NotImplementedError + + +# Copied from https://github.com/huggingface/transformers/blob/308d2b90049b4979a949a069aa4f43b2788254d6/src/transformers/testing_utils.py#L1335 # noqa +# (with minimal set of methods and their contents) +class TestCasePlus(unittest.TestCase): + + def setUp(self): + # get_auto_remove_tmp_dir feature: + self.teardown_tmp_dirs = [] + + def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None): + """ + Args: + tmp_dir (`string`, *optional*): + if `None`: + + - a unique temporary path will be created + - sets `before=True` if `before` is `None` + - sets `after=True` if `after` is `None` + else: + + - `tmp_dir` will be created + - sets `before=True` if `before` is `None` + - sets `after=False` if `after` is `None` + before (`bool`, *optional*): + If `True` and the `tmp_dir` already exists, make sure to empty it right away if `False` and the + `tmp_dir` already exists, any existing files will remain there. + after (`bool`, *optional*): + If `True`, delete the `tmp_dir` at the end of the test if `False`, leave the `tmp_dir` and its contents + intact at the end of the test. + + Returns: + tmp_dir(`string`): either the same value as passed via *tmp_dir* or the path to the auto-selected tmp dir + """ + if tmp_dir is not None: + # defining the most likely desired behavior for when a custom path is provided. + # this most likely indicates the debug mode where we want an easily locatable dir that: + # 1. gets cleared out before the test (if it already exists) + # 2. is left intact after the test + if before is None: + before = True + if after is None: + after = False + + # using provided path + path = Path(tmp_dir).resolve() + + # to avoid nuking parts of the filesystem, only relative paths are allowed + if not tmp_dir.startswith("./"): + raise ValueError( + f"`tmp_dir` can only be a relative path, i.e. `./some/path`, but received `{tmp_dir}`" + ) + + # ensure the dir is empty to start with + if before is True and path.exists(): + shutil.rmtree(tmp_dir, ignore_errors=True) + + path.mkdir(parents=True, exist_ok=True) + + else: + # defining the most likely desired behavior for when a unique tmp path is auto generated + # (not a debug mode), here we require a unique tmp dir that: + # 1. is empty before the test (it will be empty in this situation anyway) + # 2. gets fully removed after the test + if before is None: + before = True + if after is None: + after = True + + # using unique tmp dir (always empty, regardless of `before`) + tmp_dir = tempfile.mkdtemp() + + if after is True: + # register for deletion + self.teardown_tmp_dirs.append(tmp_dir) + + return tmp_dir + + def tearDown(self): + # get_auto_remove_tmp_dir feature: remove registered temp dirs + for path in self.teardown_tmp_dirs: + shutil.rmtree(path, ignore_errors=True) + self.teardown_tmp_dirs = [] + if is_accelerate_available(): + AcceleratorState._reset_state() + PartialState._reset_state() + + # delete all the env variables having `ACCELERATE` in them + for k in list(os.environ.keys()): + if "ACCELERATE" in k: + del os.environ[k] diff --git a/tests/models/vit/__init__.py b/tests/models/vit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/models/vit/test_training_vit.py b/tests/models/vit/test_training_vit.py new file mode 100644 index 00000000..dcb7cb44 --- /dev/null +++ b/tests/models/vit/test_training_vit.py @@ -0,0 +1,50 @@ +import os +import sys + +from ..test_model import ModelTrainingTestMixin, TestCasePlus + + +# So far, the image we use has a `/transformers` directory. +# TODO: Find a way to get the installed `transformers` directory. +SRC_DIRS = [ + os.path.join("/transformers/examples/pytorch", dirname) + for dirname in [ + "image-classification", + ] +] +sys.path.extend(SRC_DIRS) + + +if SRC_DIRS is not None: + import run_image_classification + + +class ViTTrainingTest(ModelTrainingTestMixin, TestCasePlus): + + def get_training_script(self): + return run_image_classification + + def prepare_training_command(self, **kwargs): + output_dir = kwargs.get("output_dir", "my_dir") + + testargs = f""" + run_image_classification.py + --output_dir {output_dir} + --model_name_or_path google/vit-base-patch16-224-in21k + --dataset_name hf-internal-testing/cats_vs_dogs_sample + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --dataloader_num_workers 16 + --metric_for_best_model accuracy + --max_steps 10 + --train_val_split 0.1 + --seed 42 + --label_column_name labels + """ + + return testargs