Run Scheduled Events Docker #167
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Scheduled Events Docker | |
| permissions: | |
| actions: write | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 10 * *' | |
| jobs: | |
| run-scheduled-events: | |
| uses: Cemberk/fork-maintenance-system/.github/workflows/fork-maintenance-action.yml@artifacts | |
| with: | |
| platform: 'gfx90a' | |
| upstream_repo: 'https://github.com/huggingface/transformers' | |
| pr_branch_prefix: 'scheduled-merge' | |
| requirements_command: | | |
| rm -rf $(pip show numpy | grep Location: | awk '{print $2}')/numpy* && | |
| sudo sed -i 's/torchaudio//g' examples/pytorch/_tests_requirements.txt && | |
| pip install -r examples/pytorch/_tests_requirements.txt && | |
| git restore examples/pytorch/_tests_requirements.txt && | |
| pip install --no-cache-dir GPUtil azureml azureml-core tokenizers ninja cerberus sympy sacremoses sacrebleu==1.5.1 sentencepiece scipy scikit-learn urllib3 && pip install huggingface_hub datasets && | |
| pip install parameterized && | |
| pip install -e . | |
| #unit_test_command: cd tests; folders=$$(python3 -c 'import os; tests = os.getcwd(); models = "models"; model_tests = os.listdir(os.path.join(tests, models)); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [os.path.join(models, x) for x in model_tests]))); d1.remove(models); d = d2 + d1; print(" ".join(d[:5]))'); cd ..; for folder in \${folders[@]}; do pytest tests/\${folder} -v --make-reports=huggingface_unit_tests_\${machine_type}_run_models_gpu_\${folder} -rfEs --continue-on-collection-errors -m \"not not_device_test\" -p no:cacheprovider; done; allstats=\$(find reports -name stats.txt); for stat in \${allstats[@]}; do echo \$stat; cat \$stat; done | |
| unit_test_command: folders=\$(python3 -c 'import os; print(\"hello\")'); echo \$folders; cd tests; pwd; ls; exit 1; | |
| #cd tests; folders=\$(python3 -c \import os; tests = os.getcwd(); models = \"models\"; model_tests = os.listdir(os.path.join(tests, models)); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [os.path.join(models, x) for x in model_tests]))); d1.remove(models); d = d2 + d1; print(\" \".join(d[:5]))' ); cd ..; for folder in \${folders[@]}; do pytest tests/\${folder} -v --make-reports=huggingface_unit_tests_\${machine_type}_run_models_gpu_\${folder} -rfEs --continue-on-collection-errors -m \"not not_device_test\" -p no:cacheprovider; done; allstats=\$(find reports -name stats.txt); for stat in \${allstats[@]}; do echo \$stat; cat \$stat; done | |
| #unit_test_command: cd tests; pwd; ls; folders=$(python3 -c 'import os; tests = os.getcwd(); repo_root = os.path.dirname(tests); models_dir = os.path.join(repo_root, "models"); import sys; sys.path.append(repo_root); model_tests = os.listdir(models_dir); d1 = sorted([d for d in os.listdir(tests) if os.path.isdir(os.path.join(tests, d)) and d != "models"]); d2 = sorted([os.path.join("models", x) for x in model_tests if os.path.isdir(os.path.join(models_dir, x))]); d = d2 + d1; print(" ".join(d[:5]))'); cd ..; for folder in ${folders[@]}; do pytest tests/${folder} -v --make-reports=huggingface_unit_tests_${machine_type}_run_models_gpu_${folder} -rfEs --continue-on-collection-errors -m "not not_device_test" -p no:cacheprovider; done; allstats=$(find reports -name stats.txt); for stat in ${allstats[@]}; do echo $stat; cat $stat; done | |
| # unit_test_command: | | |
| # set -x # Enable shell debugging | |
| # echo "Running unit tests inside Docker..." | |
| # echo "Current directory before changing to tests: $$(pwd)" | |
| # cd tests || { echo "Failed to change directory to 'tests'"; exit 1; } | |
| # echo "Current directory after changing to tests: $$(pwd)" | |
| # Write Python code to a temporary script | |
| # Write Python code to a temporary script | |
| # cat << 'EOF' > get_folders.py | |
| # import os | |
| # tests = os.getcwd() | |
| # models = "models" | |
| # Debug: Print current working directory | |
| # print(f"Current working directory: {tests}") | |
| # Get list of model tests | |
| # model_tests_path = os.path.join(tests, models) | |
| # if not os.path.exists(model_tests_path): | |
| # print(f"Models path does not exist: {model_tests_path}") | |
| # exit(1) | |
| # model_tests = os.listdir(model_tests_path) | |
| # Debug: Print model tests | |
| # print(f"Model tests: {model_tests}") | |
| # d1 = sorted([d for d in os.listdir(tests) if os.path.isdir(os.path.join(tests, d))]) | |
| # d2 = sorted([d for d in model_tests if os.path.isdir(os.path.join(models, d))]) | |
| # Debug: Print directories | |
| # print(f"d1 directories: {d1}") | |
| # print(f"d2 directories: {d2}") | |
| # if models in d1: | |
| # d1.remove(models) | |
| # d = d2 + d1 | |
| # Debug: Print combined directory list | |
| # print(f"Combined directories (d): {d}") | |
| # Print the first 5 directories | |
| #print(" ".join(d[:5])) | |
| #EOF | |
| # echo "Contents of get_folders.py:" | |
| # cat get_folders.py | |
| # Execute the Python script and capture the output and errors | |
| # folders=$(python3 get_folders.py 2>&1) | |
| # exit_code=$? | |
| # Optionally, you can print the output and exit code for debugging | |
| # echo "Folders Output:" | |
| # echo "$folders" | |
| # echo "Exit Code: $exit_code" | |
| # echo "Python script exit code: $exit_code" | |
| # echo "Output from get_folders.py:" | |
| # echo "$folders" | |
| # if [ "$exit_code" -ne 0 ]; then | |
| # echo "Python script failed with exit code $exit_code" | |
| # exit $exit_code | |
| # fi | |
| # cd .. | |
| # Convert folders string into an array | |
| # IFS=' ' read -r -a folder_array <<< "$folders" | |
| # echo "Folders obtained as array:" | |
| # printf "'%s'\n" "${folder_array[@]}" | |
| # if [ "${#folder_array[@]}" -eq 0 ]; then | |
| # echo "No folders found to run tests on." | |
| # exit 1 | |
| # fi | |
| # Iterate over each folder and run pytest | |
| # for folder in "${folder_array[@]}"; do | |
| # echo "Running pytest on folder: $folder" | |
| # pytest tests/$folder -v \ | |
| # --make-reports=huggingface_unit_tests_${machine_type}_run_models_gpu_$folder \ | |
| # -rfEs --continue-on-collection-errors \ | |
| # -m "not not_device_test" -p no:cacheprovider | |
| # done | |
| # Find and display stats | |
| # allstats=$(find reports -name stats.txt) | |
| # echo "All stats files found:" | |
| # echo "$allstats" | |
| # for stat in $$allstats; do | |
| # echo "Stat file: $$stat" | |
| # cat "$$stat" | |
| # done | |
| performance_test_command: 'echo \"python examples/pytorch/language-modeling/run_mlm.py --model_name_or_path bert-base-uncased --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train --do_eval --output_dir /tmp/test-mlm --per_device_train_batch_size 8 --per_device_eval_batch_size 8 --max_steps 500\"' | |
| docker_image: 'rocm/pytorch:latest' | |
| docker_options: '--device=/dev/kfd --device=/dev/dri --group-add video --shm-size 16G --network=host' | |
| secrets: | |
| GIT_TOKEN: ${{ secrets.CRED_TOKEN }} | |
| schedule_json: ${{ secrets.SCHEDULE_CONFIG }} |