Run Scheduled Events Docker #102
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Scheduled Events | |
| permissions: | |
| actions: write | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| run-scheduled-events: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fork Maintenance System | |
| uses: Cemberk/Fork-Maintenance-System@conda | |
| with: | |
| github_token: ${{ secrets.CRED_TOKEN }} | |
| upstream_repo: "https://github.com/huggingface/transformers" | |
| schedule_json: | | |
| { | |
| "upstream_main_branch": "main", | |
| "upstream_release_branch": "v4.43-release", | |
| "downstream_main_branch": "upstream_sync", | |
| "downstream_testing_branch": "rocm6.3_testing_rel4.43_testing", | |
| "downstream_develop_branch": "develop", | |
| "commits": [ | |
| "731f0308bb0a2796e28b68664f4ed050eab6dbfd", | |
| "0c1b311bff616a4faf214461584a758fbab14a6f", | |
| "5b2d4fec5abb8ffe755fe66ce0856fc066c561a6" | |
| ] | |
| } | |
| pr_branch_prefix: "scheduled-merge" | |
| conda_environment: | | |
| name: my_new_env | |
| channels: | |
| - defaults | |
| dependencies: | |
| - _libgcc_mutex=0.1=main | |
| - _openmp_mutex=5.1=1_gnu | |
| - ca-certificates=2024.7.2=h06a4308_0 | |
| - ld_impl_linux-64=2.38=h1181459_1 | |
| - libffi=3.4.4=h6a678d5_1 | |
| - libgcc-ng=11.2.0=h1234567_1 | |
| - libgomp=11.2.0=h1234567_1 | |
| - libstdcxx-ng=11.2.0=h1234567_1 | |
| - ncurses=6.4=h6a678d5_0 | |
| - openssl=3.0.14=h5eee18b_0 | |
| - pip=24.2=py39h06a4308_0 | |
| - python=3.9.19=h955ad1f_1 | |
| - readline=8.2=h5eee18b_0 | |
| - setuptools=72.1.0=py39h06a4308_0 | |
| - sqlite=3.45.3=h5eee18b_0 | |
| - tk=8.6.14=h39e8969_0 | |
| - wheel=0.43.0=py39h06a4308_0 | |
| - xz=5.4.6=h5eee18b_1 | |
| - zlib=1.2.13=h5eee18b_1 | |
| - pip: | |
| - absl-py==2.1.0 | |
| - accelerate==0.33.0 | |
| - adal==1.2.7 | |
| - aiohappyeyeballs==2.3.4 | |
| - aiohttp==3.10.0 | |
| - aiosignal==1.3.1 | |
| - albucore==0.0.14 | |
| - albumentations==1.4.14 | |
| - altair==5.4.1 | |
| - annotated-types==0.7.0 | |
| - antlr4-python3-runtime==4.9.3 | |
| - argcomplete==3.5.0 | |
| - array-record==0.5.1 | |
| - arrow==1.3.0 | |
| - asgiref==3.8.1 | |
| - astunparse==1.6.3 | |
| - async-timeout==4.0.3 | |
| - attrs==23.2.0 | |
| - audioread==3.0.1 | |
| - av==9.2.0 | |
| - azure-common==1.1.28 | |
| - azure-core==1.30.2 | |
| - azure-graphrbac==0.61.1 | |
| - azure-mgmt-authorization==4.0.0 | |
| - azure-mgmt-containerregistry==10.3.0 | |
| - azure-mgmt-core==1.4.0 | |
| - azure-mgmt-keyvault==10.3.1 | |
| - azure-mgmt-network==26.0.0 | |
| - azure-mgmt-resource==23.1.1 | |
| - azure-mgmt-storage==21.2.1 | |
| - azureml==0.2.7 | |
| - azureml-core | |
| - backports-tempfile==1.0 | |
| - backports-weakref==1.0.post1 | |
| - bcrypt==4.2.0 | |
| - beautifulsoup4==4.12.3 | |
| - binaryornot==0.4.4 | |
| - black==24.8.0 | |
| - blinker==1.8.2 | |
| - boto3==1.26.0 | |
| - botocore==1.29.8 | |
| - cachetools==5.4.0 | |
| - cerberus==1.3.5 | |
| - certifi==2024.7.4 | |
| - cffi==1.16.0 | |
| - chardet==5.2.0 | |
| - charset-normalizer==3.3.2 | |
| - clang==17.0.6 | |
| - click==8.1.7 | |
| - cloudpickle==3.0.0 | |
| - colorama==0.4.6 | |
| - conllu==5.0.1 | |
| - contextlib2==21.6.0 | |
| - contourpy==1.3.0 | |
| - cookiecutter==1.7.3 | |
| - coremltools==5.0b5 | |
| - cryptography==43.0.0 | |
| - cycler==0.12.1 | |
| - cython==3.0.10 | |
| - datasets==2.21.0 | |
| - decorator==5.1.1 | |
| - decord==0.6.0 | |
| - dill==0.3.4 | |
| - django==4.2.14 | |
| - dm-tree==0.1.8 | |
| - docker==7.1.0 | |
| - elastic-transport | |
| - elasticsearch==8.15.0 | |
| - etils==1.5.2 | |
| - eval-type-backport==0.2.0 | |
| - evaluate==0.4.2 | |
| - exceptiongroup==1.2.2 | |
| - execnet==2.1.1 | |
| - expecttest==0.1.6 | |
| - faiss-cpu==1.8.0.post1 | |
| - filelock==3.13.1 | |
| - fire==0.6.0 | |
| - flatbuffers==2.0 | |
| - fonttools==4.53.1 | |
| - frozenlist==1.4.1 | |
| - fsspec==2024.6.1 | |
| - future==1.0.0 | |
| - fvcore==0.1.5.post20221221 | |
| - geojson==2.5.0 | |
| - ghstack==0.8.0 | |
| - git-python==1.0.3 | |
| - gitdb==4.0.11 | |
| - gitpython==3.1.18 | |
| - google-auth==2.32.0 | |
| - google-auth-oauthlib==1.0.0 | |
| - gputil==1.4.0 | |
| - grpcio==1.65.4 | |
| - huggingface-hub==0.24.6 | |
| - humanfriendly==10.0 | |
| - hydra-core==1.3.2 | |
| - hypothesis==5.35.1 | |
| - idna==3.7 | |
| - image==1.5.33 | |
| - imageio==2.34.2 | |
| - importlib-metadata==8.2.0 | |
| - importlib-resources==6.4.4 | |
| - iniconfig==2.0.0 | |
| - iopath==0.1.9 | |
| - isodate==0.6.1 | |
| - itsdangerous==2.0.1 | |
| - jeepney==0.8.0 | |
| - jinja2==3.1.3 | |
| - jinja2-time==0.2.0 | |
| - jiwer==3.0.4 | |
| - jmespath==0.10.0 | |
| - joblib==1.4.2 | |
| - jsonpickle==3.2.2 | |
| - jsonschema==4.23.0 | |
| - jsonschema-specifications==2023.12.1 | |
| - junitparser==2.1.1 | |
| - kiwisolver==1.4.5 | |
| - knack==0.11.0 | |
| - lark==0.12.0 | |
| - lazy-loader==0.4 | |
| - librosa==0.10.2.post1 | |
| - lightning-utilities==0.11.6 | |
| - lintrunner==0.10.7 | |
| - llvmlite==0.43.0 | |
| - lxml==5.0.0 | |
| - markdown==3.6 | |
| - markdown-it-py==3.0.0 | |
| - markupsafe==2.1.5 | |
| - matplotlib==3.9.2 | |
| - mdurl==0.1.2 | |
| - mpmath==1.3.0 | |
| - msal==1.30.0 | |
| - msal-extensions==1.2.0 | |
| - msgpack==1.0.8 | |
| - msrest==0.7.1 | |
| - msrestazure==0.6.4.post1 | |
| - multidict==6.0.5 | |
| - multiprocess==0.70.12.2 | |
| - mypy==1.9.0 | |
| - mypy-extensions==1.0.0 | |
| - narwhals==1.5.5 | |
| - ndg-httpsclient==0.5.1 | |
| - networkx==2.8.8 | |
| - ninja==1.11.1.1 | |
| - nltk==3.9.1 | |
| - numba==0.60.0 | |
| - numpy==1.26.4 | |
| - oauthlib==3.2.2 | |
| - omegaconf==2.3.0 | |
| - opencv-python-headless==4.10.0.84 | |
| - opt-einsum==3.3.0 | |
| - optionloop==1.0.7 | |
| - optree==0.11.0 | |
| - packaging==24.1 | |
| - pandas==2.2.2 | |
| - parameterized==0.9.0 | |
| - paramiko==3.4.1 | |
| - pathspec==0.12.1 | |
| - pillow==10.2.0 | |
| - pkginfo==1.11.1 | |
| - platformdirs==4.2.2 | |
| - pluggy==1.5.0 | |
| - pooch==1.8.2 | |
| - portalocker==2.0.0 | |
| - poyo==0.5.0 | |
| - promise==2.3 | |
| - protobuf==3.20.3 | |
| - psutil==6.0.0 | |
| - pyarrow==17.0.0 | |
| - pyasn1==0.6.0 | |
| - pyasn1-modules==0.4.0 | |
| - pycocotools==2.0.8 | |
| - pycparser==2.22 | |
| - pydantic==2.8.2 | |
| - pydantic-core==2.20.1 | |
| - pydeck==0.9.1 | |
| - pygments==2.15.0 | |
| - pyjwt==2.9.0 | |
| - pynacl==1.5.0 | |
| - pyopenssl==24.2.1 | |
| - pyparsing==3.1.4 | |
| - pysocks==1.7.1 | |
| - pytesseract==0.3.13 | |
| - pytest==7.3.2 | |
| - pytest-cpp==2.3.0 | |
| - pytest-flakefinder==1.1.0 | |
| - pytest-rerunfailures==14.0 | |
| - pytest-rich==0.1.1 | |
| - pytest-timeout==2.3.1 | |
| - pytest-xdist==3.3.1 | |
| - python-dateutil==2.9.0.post0 | |
| - python-slugify==8.0.4 | |
| - pytz==2024.1 | |
| - pywavelets==1.4.1 | |
| - pyyaml==6.0.2 | |
| - rapidfuzz==3.9.6 | |
| - referencing==0.35.1 | |
| - regex==2024.7.24 | |
| - requests==2.32.2 | |
| - requests-oauthlib==2.0.0 | |
| - rich==13.8.0 | |
| - rjieba==0.1.11 | |
| - rockset==1.0.3 | |
| - rouge-score==0.1.2 | |
| - rpds-py==0.20.0 | |
| - rsa==4.9 | |
| - ruff==0.4.4 | |
| - s3transfer==0.5.2 | |
| - sacrebleu==1.5.1 | |
| - sacremoses==0.1.1 | |
| - safetensors==0.4.4 | |
| - scikit-image==0.24.0 | |
| - scikit-learn==1.5.1 | |
| - scipy==1.13.1 | |
| - secretstorage==3.3.3 | |
| - sentencepiece==0.2.0 | |
| - seqeval==1.2.2 | |
| - six==1.16.0 | |
| - smmap==5.0.1 | |
| - sortedcontainers==2.4.0 | |
| - soundfile==0.12.1 | |
| - soupsieve==2.6 | |
| - soxr==0.4.0 | |
| - sqlparse==0.5.1 | |
| - streamlit==1.38.0 | |
| - sympy==1.12.1 | |
| - tabulate==0.9.0 | |
| - tb-nightly==2.13.0a20230426 | |
| - tenacity==8.5.0 | |
| - tensorboard==2.13.0 | |
| - tensorboard-data-server==0.7.2 | |
| - tensorflow-datasets==4.9.3 | |
| - tensorflow-metadata==1.15.0 | |
| - termcolor==2.4.0 | |
| - text-unidecode==1.3 | |
| - threadpoolctl==3.5.0 | |
| - tifffile==2024.7.24 | |
| - timeout-decorator==0.5.0 | |
| - timm==1.0.9 | |
| - tokenizers | |
| - toml==0.10.2 | |
| - tomli==2.0.1 | |
| - torch==2.4.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1 | |
| - torchaudio==2.4.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1 | |
| - torchvision==0.19.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1 | |
| - tornado==6.4.1 | |
| - tqdm==4.66.4 | |
| - transformers==4.43.0 | |
| - triton==3.0.0 | |
| - types-python-dateutil==2.9.0.20240821 | |
| - typing-extensions==4.12.2 | |
| - tzdata==2024.1 | |
| - unittest-xml-reporting==3.2.0 | |
| - urllib3==1.26 | |
| - watchdog==4.0.2 | |
| - werkzeug==3.0.3 | |
| - wrapt==1.16.0 | |
| - xdoctest==1.1.0 | |
| - xxhash==3.5.0 | |
| - yacs==0.1.8 | |
| - yarl==1.9.4 | |
| - z3-solver==4.12.2.0 | |
| - zipp==3.19.2 | |
| prefix: /opt/conda/envs/my_new_env | |
| unit_test_command: > | |
| cd tests; folders=($(python3 -c 'import os; tests = os.getcwd(); model_tests = os.listdir(os.path.join(tests, "models")); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [f"models/{x}" for x in model_tests]))); d1.remove("models"); d = d2 + d1; print("\n".join(d))')); cd ..; for folder in "${folders[@]}"; do pytest tests/${folder} -v --make-reports="huggingface_unit_tests_run_models_gpu_${folder}" -rfEsx --continue-on-collection-errors -m "not not_device_test" -p no:cacheprovider; done; allstats=($(find reports -name stats.txt)); for stat in "${allstats[@]}"; do echo "$stat"; cat $stat; done | |
| performance_test_command: > | |
| python examples/pytorch/language-modeling/run_mlm.py --model_name_or_path bert-base-uncased --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train --do_eval --output_dir /tmp/test-mlm --per_device_train_batch_size 8 --per_device_eval_batch_size 8 --max_steps 500 | |
| extra_install_command: > | |
| pip install tokenizers==0.19 | |