Skip to content

Commit

Permalink
push [push-ci-image]
Browse files Browse the repository at this point in the history
  • Loading branch information
@ArthurZucker
ArthurZucker committed Apr 3, 2024
1 parent 3cccbaf commit 770b603
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 60 deletions.
118 changes: 59 additions & 59 deletions .circleci/create_circleci_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ def job_name(self):
# JOBS
torch_and_tf_job = CircleCIJob(
"torch_and_tf",
docker_image=[{"image":"huggingface/transformers-torch-tf-light"}], # TODO
docker_image=[{"image":"huggingface/transformers-torch-tf-light"}],
additional_env={"RUN_PT_TF_CROSS_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
Expand Down Expand Up @@ -510,8 +510,8 @@ def create_circleci_config(folder=None):
all_test_list = f.read()
else:
all_test_list = []
# if len(all_test_list) > 0:
# jobs.extend(PIPELINE_TESTS)
if len(all_test_list) > 0:
jobs.extend(PIPELINE_TESTS)

test_file = os.path.join(folder, "filtered_test_list.txt")
if os.path.exists(test_file):
Expand All @@ -520,62 +520,62 @@ def create_circleci_config(folder=None):
else:
test_list = []
if len(test_list) > 0:
jobs.extend([torch_job, custom_tokenizers_job, tf_job, exotic_models_job])

# extended_tests_to_run = set(test_list.split())
# # Extend the test files for cross test jobs
# for job in jobs:
# if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
# for test_path in copy.copy(extended_tests_to_run):
# dir_path, fn = os.path.split(test_path)
# if fn.startswith("test_modeling_tf_"):
# fn = fn.replace("test_modeling_tf_", "test_modeling_")
# elif fn.startswith("test_modeling_flax_"):
# fn = fn.replace("test_modeling_flax_", "test_modeling_")
# else:
# if job.job_name == "test_torch_and_tf":
# fn = fn.replace("test_modeling_", "test_modeling_tf_")
# elif job.job_name == "test_torch_and_flax":
# fn = fn.replace("test_modeling_", "test_modeling_flax_")
# new_test_file = str(os.path.join(dir_path, fn))
# if os.path.isfile(new_test_file):
# if new_test_file not in extended_tests_to_run:
# extended_tests_to_run.add(new_test_file)
# extended_tests_to_run = sorted(extended_tests_to_run)
# for job in jobs:
# if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
# job.tests_to_run = extended_tests_to_run
# fn = "filtered_test_list_cross_tests.txt"
# f_path = os.path.join(folder, fn)
# with open(f_path, "w") as fp:
# fp.write(" ".join(extended_tests_to_run))

# example_file = os.path.join(folder, "examples_test_list.txt")
# if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
# with open(example_file, "r", encoding="utf-8") as f:
# example_tests = f.read()
# for job in EXAMPLES_TESTS:
# framework = job.name.replace("examples_", "").replace("torch", "pytorch")
# if example_tests == "all":
# job.tests_to_run = [f"examples/{framework}"]
# else:
# job.tests_to_run = [f for f in example_tests.split(" ") if f.startswith(f"examples/{framework}")]

# if len(job.tests_to_run) > 0:
# jobs.append(job)

# doctest_file = os.path.join(folder, "doctest_list.txt")
# if os.path.exists(doctest_file):
# with open(doctest_file) as f:
# doctest_list = f.read()
# else:
# doctest_list = []
# if len(doctest_list) > 0:
# jobs.extend(DOC_TESTS)

# repo_util_file = os.path.join(folder, "test_repo_utils.txt")
# if os.path.exists(repo_util_file) and os.path.getsize(repo_util_file) > 0:
# jobs.extend(REPO_UTIL_TESTS)
jobs.extend(REGULAR_TESTS)

extended_tests_to_run = set(test_list.split())
# Extend the test files for cross test jobs
for job in jobs:
if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
for test_path in copy.copy(extended_tests_to_run):
dir_path, fn = os.path.split(test_path)
if fn.startswith("test_modeling_tf_"):
fn = fn.replace("test_modeling_tf_", "test_modeling_")
elif fn.startswith("test_modeling_flax_"):
fn = fn.replace("test_modeling_flax_", "test_modeling_")
else:
if job.job_name == "test_torch_and_tf":
fn = fn.replace("test_modeling_", "test_modeling_tf_")
elif job.job_name == "test_torch_and_flax":
fn = fn.replace("test_modeling_", "test_modeling_flax_")
new_test_file = str(os.path.join(dir_path, fn))
if os.path.isfile(new_test_file):
if new_test_file not in extended_tests_to_run:
extended_tests_to_run.add(new_test_file)
extended_tests_to_run = sorted(extended_tests_to_run)
for job in jobs:
if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
job.tests_to_run = extended_tests_to_run
fn = "filtered_test_list_cross_tests.txt"
f_path = os.path.join(folder, fn)
with open(f_path, "w") as fp:
fp.write(" ".join(extended_tests_to_run))

example_file = os.path.join(folder, "examples_test_list.txt")
if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
with open(example_file, "r", encoding="utf-8") as f:
example_tests = f.read()
for job in EXAMPLES_TESTS:
framework = job.name.replace("examples_", "").replace("torch", "pytorch")
if example_tests == "all":
job.tests_to_run = [f"examples/{framework}"]
else:
job.tests_to_run = [f for f in example_tests.split(" ") if f.startswith(f"examples/{framework}")]

if len(job.tests_to_run) > 0:
jobs.append(job)

doctest_file = os.path.join(folder, "doctest_list.txt")
if os.path.exists(doctest_file):
with open(doctest_file) as f:
doctest_list = f.read()
else:
doctest_list = []
if len(doctest_list) > 0:
jobs.extend(DOC_TESTS)

repo_util_file = os.path.join(folder, "test_repo_utils.txt")
if os.path.exists(repo_util_file) and os.path.getsize(repo_util_file) > 0:
jobs.extend(REPO_UTIL_TESTS)

if len(jobs) == 0:
jobs = [EmptyJob()]
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-ci-docker-images.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
strategy:
matrix:
file: ["quality", "consistency", "custom-tokenizers", "torch-light", "tf-light", "exotic-models"] # Add your file names here
file: ["quality", "consistency", "custom-tokenizers", "torch-light", "tf-light", "exotic-models", "torch-tf-light"] # Add your file names here

runs-on: ubuntu-22.04

Expand Down

0 comments on commit 770b603

Please sign in to comment.