From 6cbeb42270bf6c8d97af36352465decc2b0a718d Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Wed, 6 Dec 2023 21:04:17 +0100 Subject: [PATCH] Updated cruft template --- .cookiecutter.json | 1 - .cruft.json | 5 +- .github/dependabot.yml | 16 - .github/workflows/check_for_rej.yml | 27 -- .github/workflows/cruft.yml | 66 ---- .github/workflows/dependabot_automerge.yml | 30 -- .github/workflows/lint.yml | 28 ++ .github/workflows/pre-commit.yml | 79 ---- .github/workflows/static_type_checks.yml | 71 +--- .github/workflows/tests.yml | 44 +-- .pre-commit-config.yaml | 31 -- CONTRIBUTING.md | 27 +- README.md | 4 +- makefile | 39 ++ pyproject.toml | 43 +-- tasks.py | 427 --------------------- training/main/requirements.txt | 3 - 17 files changed, 102 insertions(+), 839 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/check_for_rej.yml delete mode 100644 .github/workflows/cruft.yml delete mode 100644 .github/workflows/dependabot_automerge.yml create mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pre-commit.yml delete mode 100644 .pre-commit-config.yaml create mode 100644 makefile delete mode 100644 tasks.py diff --git a/.cookiecutter.json b/.cookiecutter.json index 2665f074..9d03ac9f 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -8,7 +8,6 @@ "email": "kennethcenevoldsen@gmail.com", "friendly_name": "DaCy", "github_user": "centre-for-humanities-computing", - "license": "MIT", "package_name": "dacy", "project_name": "dacy", "version": "2.4.2" diff --git a/.cruft.json b/.cruft.json index a732c277..a14abc9a 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/KennethEnevoldsen/swift-python-cookiecutter", - "commit": "7fdb02999e8596c525377c208ca902645d134f97", + "commit": "e96eb05162a0e45a8ad5aa446c72229372e79cdb", "checkout": null, "context": { "cookiecutter": { @@ -12,7 +12,6 @@ "github_user": "centre-for-humanities-computing", "version": "2.4.2", "copyright_year": "2023", - "license": "MIT", "_copy_without_render": [ "*.github" ], @@ -20,4 +19,4 @@ } }, "directory": null -} \ No newline at end of file +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index ee72a897..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Configuration: https://dependabot.com/docs/config-file/ -# Docs: https://docs.github.com/en/github/administering-a-repository/keeping-your-dependencies-updated-automatically - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" - day: "monday" - time: "13:00" - timezone: "Europe/Copenhagen" - open-pull-requests-limit: 20 - commit-message: - prefix: "deps:" - include: "scope" diff --git a/.github/workflows/check_for_rej.yml b/.github/workflows/check_for_rej.yml deleted file mode 100644 index ed200fcc..00000000 --- a/.github/workflows/check_for_rej.yml +++ /dev/null @@ -1,27 +0,0 @@ -# .rej files occur when cruft update could not merge two files. -# They need to be handled, but are easy to miss if there's no CI -name: Check for .rej files - -on: - pull_request: - types: [opened, synchronize] - -jobs: - check-for-rej-files: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Check for .rej files - run: | - files=`find . -type f -name "*.rej"` - count=`echo $files | grep -o "\.rej" | wc -l` - if [[ $count != 0 ]]; then - echo "Found .rej files in the repository." - echo $files | - exit 1 - else - echo "No .rej files found in the repository." - fi diff --git a/.github/workflows/cruft.yml b/.github/workflows/cruft.yml deleted file mode 100644 index 714de363..00000000 --- a/.github/workflows/cruft.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Cruft Check - -on: - pull_request: - branches: - - main - -jobs: - cruft-check: - runs-on: ubuntu-latest - permissions: - pull-requests: write - - steps: - # Avoid infinite loop where main - # Feature PR -> cruft check from main -> - # Cruft update PR -> cruft check from main -> - # Cruft update PR ... - - name: Check if pull request is from a fork - run: | - if [ "${{ github.event.pull_request.head.repo.fork }}" = "true" ]; then - echo "Pull request is from a fork and does not have permissions for PR creation. Exiting gracefully." - exit 0 - elif [ "${{github.event.pull_request.title}}" == "ci - update cruft" ]; then - echo "Pull request is already a cruft update. Exiting gracefully." - exit 0 - else - echo "Pull request is not from a fork, continuing." - fi - - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: main - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install Cruft - run: pip install cruft - - - name: Update cruft - id: cruft_check - run: | - cruft_output=$(cruft update --skip-apply-ask) - if echo "$cruft_output" | grep -q "Good work!"; then - echo "$cruft_output" - echo "cruft_updated=true" >> $GITHUB_OUTPUT - else - echo "$cruft_output" - echo "cruft_updated=false" >> $GITHUB_OUTPUT - fi - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 - if: ${{ steps.cruft_check.outputs.cruft_updated == 'true' && github.event.pull_request.title != 'ci - update cruft' }} - continue-on-error: true - with: - title: "ci - update cruft" - branch: "update-cruft" - body: "🌲 Cruft updates" - token: ${{ secrets.PAT }} - commit-message: "ci: update cruft" - labels: "dependencies" # This makes the PR exempt from the stale bot diff --git a/.github/workflows/dependabot_automerge.yml b/.github/workflows/dependabot_automerge.yml deleted file mode 100644 index 22d2ecd7..00000000 --- a/.github/workflows/dependabot_automerge.yml +++ /dev/null @@ -1,30 +0,0 @@ -# GitHub action to automerge dependabot PRs. Only merges if tests passes the -# branch protections in the repository settings. -# You can set branch protections in the repository under Settings > Branches > Add rule -name: automerge-bot-prs - -on: pull_request - -permissions: - contents: write - pull-requests: write - -jobs: - dependabot-automerge: - runs-on: ubuntu-latest - # if actor is dependabot or pre-commit-ci[bot] then run - if: ${{ github.actor == 'dependabot[bot]' }} - - steps: - # Checkout action is required for token to persist - - name: Enable auto-merge for Dependabot PRs - run: gh pr merge --auto --merge "$PR_URL" # Use Github CLI to merge automatically the PR - env: - PR_URL: ${{github.event.pull_request.html_url}} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Auto approve dependabot PRs - if: ${{ github.actor == 'dependabot[bot]' }} - uses: hmarr/auto-approve-action@v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..d27e4d45 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +# GitHub action to run linting + +name: run-pre-commit + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + cache: "pip" + + - name: Install pre-commit + run: make install + + - name: Lint + id: lint + run: | + make lint diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 79671c4a..00000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,79 +0,0 @@ -# GitHub action to check if pre-commit has been run. Runs from .pre-commit-config.yaml, where the pre-commit actions are. - -name: run-pre-commit - -on: - pull_request: - branches: [main] - push: - branches: [main] - -jobs: - pre-commit: - permissions: - pull-requests: write - concurrency: - group: "${{ github.workflow }} @ ${{ github.ref }}" - cancel-in-progress: true - if: ${{ github.actor != 'dependabot[bot]' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - token: ${{ secrets.PAT }} - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install pre-commit - run: pip install pre-commit - - - name: Run pre-commit - id: pre_commit - continue-on-error: true - run: | - if pre-commit run --color always --all-files; then - echo "Pre-commit check passed" - echo "pre_commit_failed=0" >> $GITHUB_OUTPUT - else - echo "Pre-commit check failed" - echo "pre_commit_failed=1" >> $GITHUB_OUTPUT - exit 1 - fi - - # Have this step before commit in case the PR is from a fork. In this case, we want the - # add-pr-comment to fail, because it makes it means that the contributer is directed here, - # and are given the informative error message, instead of directed to a "could not commit error message". - - uses: mshick/add-pr-comment@v2 - if: ${{ steps.pre_commit.outputs.pre_commit_failed == 1 && github.event_name == 'pull_request' }} - id: add_comment - with: - message: | - Looks like some formatting rules failed. - - ✨ The action has attempted automatic fixes ✨ - - If any were succesful, they were committed to the branch. - We suggest using `git pull --rebase` to apply them locally. - - If some errors could not be fixed automatically, you can: - - 🏎️ Get results locally by running `pre-commit run --all-files` - πŸ•΅οΈ Examine the results in the `Run pre-commit` section of this workflow `pre-commit` - - We also recommend setting up the `ruff` and `black` extensions to auto-format on save in your chosen editor. - - - name: Commit formatting - if: ${{ steps.pre_commit.outputs.pre_commit_failed == 1 && github.event_name == 'pull_request' }} - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git commit -am "style: linting" - git push --no-verify - - - name: Fail workflow - if: ${{ steps.pre_commit.outputs.pre_commit_failed == 1 && github.event_name == 'pull_request' }} - run: exit 1 diff --git a/.github/workflows/static_type_checks.yml b/.github/workflows/static_type_checks.yml index a1baa9c2..04faedde 100644 --- a/.github/workflows/static_type_checks.yml +++ b/.github/workflows/static_type_checks.yml @@ -1,6 +1,3 @@ -# We do not include static_type_checks as a pre-commit hook because pre-commit hooks -# are installed in their own virtual environment, so static_type_checks cannot -# use stubs from imports name: static_type_checks on: @@ -12,11 +9,6 @@ on: jobs: static_type_checks: runs-on: ubuntu-latest - permissions: - pull-requests: write - concurrency: - group: "${{ github.workflow }} @ ${{ github.ref }}" - cancel-in-progress: true strategy: matrix: os: [ubuntu-latest] @@ -24,74 +16,19 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache tox - uses: actions/cache@v3.2.6 - id: cache_tox - with: - path: | - .tox - key: ${{ runner.os }}-${{ matrix.python-version }}-static-type-checks - - name: Set up Python uses: actions/setup-python@v4 id: setup_python with: python-version: ${{ matrix.python-version}} + cache: pip - name: Install dependencies shell: bash run: | - pip install invoke tox pyright + make install - name: Run static type checker - id: pyright - continue-on-error: true - run: | - if inv static-type-checks; then - echo "pyright check passed" - echo "pyright_failed=0" >> $GITHUB_OUTPUT - else - echo "pyright check failed" - echo "pyright_failed=1" >> $GITHUB_OUTPUT - fi - - - name: Find Comment - uses: peter-evans/find-comment@v2 - id: find_comment - if: ${{github.event_name == 'pull_request'}} - continue-on-error: true - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: ✨ Looks like pyright failed ✨ - - - uses: mshick/add-pr-comment@v2 - if: ${{ steps.pyright.outputs.pyright_failed == 1 && github.event_name == 'pull_request'}} - id: add_comment - with: - message: | - ✨ Looks like pyright failed ✨ - - If you want to fix this, we recommend doing it locally by either: - - a) Enabling pyright in VSCode and going through the errors in the problems tab - - `VSCode settings > Python > Analysis: Type checking mode > "basic"` - - b) Debugging via the command line - - 1. Installing pyright, which is included in the dev dependencies: `pip install -e ".[dev]"` - 2. Diagnosing the errors by running `pyright .` - - - uses: mshick/add-pr-comment@v2 - if: ${{ steps.pyright.outputs.pyright_failed == 0 && steps.find_comment.outputs.comment-id != '' && github.event_name == 'pull_request'}} - with: - message-id: ${{ steps.find_comment.outputs.comment-id }} - message: | - 🌟 pyright succeeds! 🌟 - - - name: Show pyright output - id: fail_run - if: ${{steps.pyright.outputs.pyright_failed == 1}} + shell: bash run: | - inv static-type-checks # Rerunning pyright isn't optimal computationally, but typically takes no more than a couple of seconds, and this ensures that the errors are in the failing step + make static-type-checks diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8931ad8d..3a1814c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,7 @@ -# This workflow will install Python dependencies, run pytests and run notebooks -# then it will in python 3.9 (ubuntu-latest) create a badge with the coverage -# and add it to the PR. This badge will be updated if the PR is updated. +# This workflow will: +# 1) install Python dependencies +# 2) run make test + name: Tests on: @@ -30,15 +31,6 @@ jobs: steps: - uses: actions/checkout@v3 - - - name: Cache tox - uses: actions/cache@v3.2.6 - id: cache_tox - with: - path: | - .tox - key: ${{ runner.os }}-${{ matrix.python-version }}-tests-1 - - name: Set up Python uses: actions/setup-python@v4 with: @@ -48,31 +40,9 @@ jobs: - name: Install dependencies shell: bash run: | - pip install invoke tox + make install - - name: Run and write pytest + - name: Run tests shell: bash run: | - # Specifying two sets of "--pytest-args" is required for invoke to parse it as a list - export DACY_CACHE_DIR=/tmp/dacy_cache - inv test --pytest-args="--durations=0" --pytest-args="--junitxml=pytest.xml --cov-report=term-missing --cov=src/" - - - - name: Test report on failures - uses: EnricoMi/publish-unit-test-result-action@v2 - id: test_report_with_annotations - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' && github.actor != 'dependabot[bot]' && github.event_name == 'pull_request' && (success() || failure()) }} # Do not run for dependabot, run whether tests failed or succeeded - with: - comment_mode: "failures" - files: | - pytest.xml - - - name: Pytest coverage comment - id: coverage-comment - uses: MishaKav/pytest-coverage-comment@main - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' && github.actor != 'dependabot[bot]' && github.event_name == 'pull_request' && (success() || failure()) }} - with: - create-new-comment: false - report-only-changed-files: false - pytest-coverage-path: pytest-coverage.txt - junitxml-path: ./pytest.xml + make test \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index b9887b16..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -default_stages: [commit] - -repos: - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.265 - hooks: - - id: ruff - args: - [ - "--extend-select", - "F401", - "--extend-select", - "F841", - "--fix", - "--exit-non-zero-on-fix", - ] - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - - - repo: https://github.com/repo-helper/pyproject-parser - rev: v0.9.0b2 - hooks: - - id: check-pyproject diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b65d3ab..7237cfe6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Here is a list of important resources for contributors: - [Issue Tracker] - [Code of Conduct] -[mit license]: https://opensource.org/licenses/MIT +[Apache-2.0 license]: https://opensource.org/license/apache-2-0/ [source code]: https://github.com/centre-for-humanities-computing/dacy [documentation]: https://dacy.readthedocs.io/ [issue tracker]: https://github.com/centre-for-humanities-computing/dacy/issues @@ -37,24 +37,22 @@ Request features on the [Issue Tracker]. ## How to set up your development environment -Install the package with development requirements: +To install all the development dependencies, you can use the [make] command: ```console -$ pip install -e ."[dev,tests]" +$ make install ``` + ## How to test the project Run the full test suite: ```console -$ pytest +$ make test ``` -Unit tests are located in the _tests_ directory, -and are written using the [pytest] testing framework. - -[pytest]: https://pytest.readthedocs.io/ +Unit tests are located in the _tests_ directory. ## How to submit changes @@ -62,23 +60,24 @@ Open a [pull request] to submit changes to this project. Your pull request needs to meet the following guidelines for acceptance: -- The Nox test suite must pass without errors and warnings. -- Include unit tests. This project maintains 100% code coverage. +- The test suite should ideally pass without errors and warnings. +- Ideally add tests for your changes. - If your changes add functionality, update the documentation accordingly. Feel free to submit early, thoughβ€”we can always iterate on this. -To run linting and code formatting checks before committing your change, you can install pre-commit as a Git hook by running the following command: +To run linting and code formatting checks before committing your change, you can run the following [make] command: ```console -$ nox --session=pre-commit -- install +$ make lint ``` -It is recommended to open an issue before starting work on anything. +It is recommended to open an issue before starting work on any major changes. This will allow a chance to talk it over with the owners and validate your approach. [pull request]: https://github.com/centre-for-humanities-computing/dacy/pulls +[make]: https://makefiletutorial.com -[code of conduct]: CODE_OF_CONDUCT.md +[code of conduct]: CODE_OF_CONDUCT.md \ No newline at end of file diff --git a/README.md b/README.md index 85e5896e..8744f1ef 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ [![PyPI](https://img.shields.io/pypi/v/dacy.svg)][pypi status] [![pip downloads](https://img.shields.io/pypi/dm/dacy.svg)](https://pypi.org/project/dacy/) [![Python Version](https://img.shields.io/pypi/pyversions/dacy)][pypi status] -[![Black](https://img.shields.io/badge/code%20style-black-000000.svg)][black] +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)]([ruff]) [![documentation](https://github.com/centre-for-humanities-computing/dacy/actions/workflows/documentation.yml/badge.svg)][documentation] [![Tests](https://github.com/centre-for-humanities-computing/dacy/actions/workflows/tests.yml/badge.svg)][tests] [pypi status]: https://pypi.org/project/dacy/ [documentation]: https://centre-for-humanities-computing.github.io/DaCy/ [tests]: https://github.com/centre-for-humanities-computing/dacy/actions?workflow=Tests -[black]: https://github.com/psf/black +[ruff]: https://github.com/astral-sh/ruff diff --git a/makefile b/makefile new file mode 100644 index 00000000..e94def5b --- /dev/null +++ b/makefile @@ -0,0 +1,39 @@ +install: + @echo "--- πŸš€ Installing project ---" + pip install -e ".[dev, docs, tests]" + +static-type-check: + @echo "--- πŸ” Running static type check ---" + pyright . + +lint: + @echo "--- 🧹 Running linters ---" + pyproject-parser check pyproject.toml # check pyproject.toml + ruff format . # running ruff formatting + ruff . --fix # running ruff linting + +test: + @echo "--- πŸ§ͺ Running tests ---" + pytest tests/ + +pr: + @echo "--- πŸš€ Running PR checks ---" + make lint + make static-type-check + make test + @echo "Ready to make a PR" + +build-docs: + @echo "--- πŸ“š Building docs ---" + @echo "Builds the docs and puts them in the 'site' folder" + mkdocs build + +view-docs: + @echo "--- πŸ‘€ Viewing docs ---" + mkdocs serve + +update-from-template: + @echo "--- πŸ”„ Updating from template ---" + @echo "This will update the project from the template, make sure to resolve any .rej files" + cruft update + diff --git a/pyproject.toml b/pyproject.toml index 3370978d..818c224e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,11 +50,9 @@ name = "Apache License 2.0" [project.optional-dependencies] dev = [ "cruft>=2.0.0", - "pyright==1.1.305", - "pyright-polite>=0.0.1", - "pre-commit>=2.20.0", + "pyright==1.1.328", "ruff>=0.0.262", - "black[jupyter]>=23.3.0", + "pyproject-parser[cli, readme]>=0.9.1", ] tests = ["pytest>=7.1.2", "pytest-cov>=3.0.0", "pytest-instafail>=0.4.2"] docs = [ @@ -112,6 +110,7 @@ exclude = [".*venv*", ".tox"] pythonPlatform = "Darwin" [tool.ruff] +extend-include = ["*.ipynb"] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. select = [ "A", @@ -150,6 +149,7 @@ ignore = [ "F841", "RET504", "ANN202", + "COM812", ] ignore-init-module-imports = true # Allow autofix for all enabled rules (when `--fix`) is provided. @@ -190,6 +190,8 @@ exclude = [ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" target-version = "py38" +tool.ruff.lint.pydocstyle] +convention = "google" [tool.ruff.flake8-annotations] mypy-init-return = true suppress-none-returning = true @@ -206,35 +208,4 @@ version_toml = ["pyproject.toml:project.version"] build_command = "python -m pip install build; python -m build" [tool.setuptools] -include-package-data = true - - -[tool.tox] -legacy_tox_ini = """ -[tox] -envlist = py{39,310} - -[testenv] -description: run unit tests -extras = tests -use_develop = true -commands = - pytest {posargs:test} - -[testenv:type] -allowlist_externals = pyright -description: run type checks -extras = tests, dev -basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version -use_develop = true -commands = - pyright src/ - -[testenv:docs] -description: build docs -extras = docs -basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version -use_develop = true -commands = - sphinx-build -b html docs docs/_build/html -""" +include-package-data = true \ No newline at end of file diff --git a/tasks.py b/tasks.py deleted file mode 100644 index 20b84722..00000000 --- a/tasks.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -This project uses Invoke (pyinvoke.org) for task management. -Install it via: - -``` -pip install invoke -``` - -And then run: - -``` -inv --list -``` - -If you do not wish to use invoke you can simply delete this file. -""" - - -import platform -import re -import shutil -from pathlib import Path -from typing import List, Optional - -from invoke import Context, Result, task - -# Extract supported python versions from the pyproject.toml classifiers key -SUPPORTED_PYTHON_VERSIONS = [ - line.split("::")[-1].strip().replace('"', "").replace(",", "") - for line in Path("pyproject.toml").read_text().splitlines() - if "Programming Language :: Python ::" in line -] - -NOT_WINDOWS = platform.system() != "Windows" - - -def echo_header(msg: str): - print(f"\n--- {msg} ---") - - -class MsgType: - # Emojis have to be encoded as bytes to not break the terminal on Windows - @property - def DOING(self) -> str: - return b"\xf0\x9f\xa4\x96".decode() if NOT_WINDOWS else "DOING:" - - @property - def GOOD(self) -> str: - return b"\xe2\x9c\x85".decode() if NOT_WINDOWS else "DONE:" - - @property - def FAIL(self) -> str: - return b"\xf0\x9f\x9a\xa8".decode() if NOT_WINDOWS else "FAILED:" - - @property - def WARN(self) -> str: - return b"\xf0\x9f\x9a\xa7".decode() if NOT_WINDOWS else "WARNING:" - - @property - def SYNC(self) -> str: - return b"\xf0\x9f\x9a\x82".decode() if NOT_WINDOWS else "SYNCING:" - - @property - def PY(self) -> str: - return b"\xf0\x9f\x90\x8d".decode() if NOT_WINDOWS else "" - - @property - def CLEAN(self) -> str: - return b"\xf0\x9f\xa7\xb9".decode() if NOT_WINDOWS else "CLEANING:" - - @property - def TEST(self) -> str: - return b"\xf0\x9f\xa7\xaa".decode() if NOT_WINDOWS else "TESTING:" - - @property - def COMMUNICATE(self) -> str: - return b"\xf0\x9f\x93\xa3".decode() if NOT_WINDOWS else "COMMUNICATING:" - - @property - def EXAMINE(self) -> str: - return b"\xf0\x9f\x94\x8d".decode() if NOT_WINDOWS else "VIEWING:" - - -msg_type = MsgType() - - -def git_init(c: Context, branch: str = "main"): - """Initialize a git repository if it does not exist yet.""" - # If no .git directory exits - if not Path(".git").exists(): - echo_header(f"{msg_type.DOING} Initializing Git repository") - c.run(f"git init -b {branch}") - c.run("git add .") - c.run("git commit -m 'Init'") - print(f"{msg_type.GOOD} Git repository initialized") - else: - print(f"{msg_type.GOOD} Git repository already initialized") - - -def setup_venv( - c: Context, - python_path: str, - venv_name: Optional[str] = None, -) -> str: - """Create a virtual environment if it does not exist yet. - - Args: - c: The invoke context. - python_path: The python executable to use. - venv_name: The name of the virtual environment. Defaults to ".venv". - """ - if venv_name is None: - venv_name = ".venv" - - if not Path(venv_name).exists(): - echo_header( - f"{msg_type.DOING} Creating virtual environment using {msg_type.PY}:{python_path}", - ) - c.run(f"{python_path} -m venv {venv_name}") - print(f"{msg_type.GOOD} Virtual environment created") - else: - print(f"{msg_type.GOOD} Virtual environment already exists") - return venv_name - - -def _add_commit(c: Context, msg: Optional[str] = None): - print(f"{msg_type.DOING} Adding and committing changes") - c.run("git add .") - - if msg is None: - msg = input("Commit message: ") - - c.run(f'git commit -m "{msg}"', pty=NOT_WINDOWS, hide=True) - print(f"{msg_type.GOOD} Changes added and committed") - - -def is_uncommitted_changes(c: Context) -> bool: - git_status_result: Result = c.run( - "git status --porcelain", - pty=NOT_WINDOWS, - hide=True, - ) - - uncommitted_changes = git_status_result.stdout != "" - return uncommitted_changes - - -def add_and_commit(c: Context, msg: Optional[str] = None): - """Add and commit all changes.""" - if is_uncommitted_changes(c): - uncommitted_changes_descr = c.run( - "git status --porcelain", - pty=NOT_WINDOWS, - hide=True, - ).stdout - - echo_header( - f"{msg_type.WARN} Uncommitted changes detected", - ) - - for line in uncommitted_changes_descr.splitlines(): - print(f" {line.strip()}") - print("\n") - _add_commit(c, msg=msg) - - -def branch_exists_on_remote(c: Context) -> bool: - branch_name = Path(".git/HEAD").read_text().split("/")[-1].strip() - - branch_exists_result: Result = c.run( - f"git ls-remote --heads origin {branch_name}", - hide=True, - ) - - return branch_name in branch_exists_result.stdout - - -def update_branch(c: Context): - echo_header(f"{msg_type.SYNC} Syncing branch with remote") - - if not branch_exists_on_remote(c): - c.run("git push --set-upstream origin HEAD") - else: - print("Pulling") - c.run("git pull") - print("Pushing") - c.run("git push") - - -def create_pr(c: Context): - c.run( - "gh pr create --web", - pty=NOT_WINDOWS, - ) - - -def update_pr(c: Context): - echo_header(f"{msg_type.COMMUNICATE} Syncing PR") - # Get current branch name - branch_name = Path(".git/HEAD").read_text().split("/")[-1].strip() - pr_result: Result = c.run( - "gh pr list --state OPEN", - pty=False, - hide=True, - ) - - if branch_name not in pr_result.stdout: - create_pr(c) - else: - open_web = input("Open in browser? [y/n] ") - if "y" in open_web.lower(): - c.run("gh pr view --web", pty=NOT_WINDOWS) - - -def exit_if_error_in_stdout(result: Result): - # Find N remaining using regex - - if "error" in result.stdout: - errors_remaining = re.findall(r"\d+(?=( remaining))", result.stdout)[ - 0 - ] # testing - if errors_remaining != "0": - exit(0) - - -def pre_commit(c: Context, auto_fix: bool): - """Run pre-commit checks.""" - - # Essential to have a clean working directory before pre-commit to avoid committing - # heterogenous files under a "style: linting" commit - if is_uncommitted_changes(c): - print( - f"{msg_type.WARN} Your git working directory is not clean. Stash or commit before running pre-commit.", - ) - exit(1) - - echo_header(f"{msg_type.CLEAN} Running pre-commit checks") - pre_commit_cmd = "pre-commit run --all-files" - result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True) - - exit_if_error_in_stdout(result) - - if ("fixed" in result.stdout or "reformatted" in result.stdout) and auto_fix: - _add_commit(c, msg="style: Auto-fixes from pre-commit") - - print(f"{msg_type.DOING} Fixed errors, re-running pre-commit checks") - second_result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True) - exit_if_error_in_stdout(second_result) - else: - if result.return_code != 0: - print(f"{msg_type.FAIL} Pre-commit checks failed") - exit(1) - - -@task -def static_type_checks(c: Context): - echo_header(f"{msg_type.CLEAN} Running static type checks") - c.run("tox -e type", pty=NOT_WINDOWS) - - -@task -def install( - c: Context, - pip_args: str = "", - msg: bool = True, - venv_path: Optional[str] = None, -): - """Install the project in editable mode using pip install""" - if msg: - echo_header(f"{msg_type.DOING} Installing project") - - extras = ".[dev,tests,docs]" if NOT_WINDOWS else ".[dev,tests,docs]" - install_cmd = f"pip install -e {extras} {pip_args}" - - if venv_path is not None and NOT_WINDOWS: - with c.prefix(f"source {venv_path}/bin/activate"): - c.run(install_cmd) - return - - c.run(install_cmd) - - -def get_python_path(preferred_version: str) -> Optional[str]: - """Get path to python executable.""" - preferred_version_path = shutil.which(f"python{preferred_version}") - - if preferred_version_path is not None: - return preferred_version_path - - print( - f"{msg_type.WARN}: python{preferred_version} not found, continuing with default python version", - ) - return shutil.which("python") - - -@task -def setup(c: Context, python_path: Optional[str] = None): - """Confirm that a git repo exists and setup a virtual environment. - - Args: - c: Invoke context - python_path: Path to the python executable to use for the virtual environment. Uses the return value of `which python` if not provided. - """ - git_init(c) - - if python_path is None: - # get path to python executable - python_path = get_python_path(preferred_version="3.9") - if not python_path: - print(f"{msg_type.FAIL} Python executable not found") - exit(1) - venv_name = setup_venv(c, python_path=python_path) - - install(c, pip_args="--upgrade", msg=False, venv_path=venv_name) - - if venv_name is not None: - print( - f"{msg_type.DOING} Activate your virtual environment by running: \n\n\t\t source {venv_name}/bin/activate \n", - ) - - -@task -def update(c: Context): - """Update dependencies.""" - echo_header(f"{msg_type.DOING} Updating project") - install(c, pip_args="--upgrade", msg=False) - - -@task(iterable="pytest_args") -def test( - c: Context, - python_versions: List[str] = (SUPPORTED_PYTHON_VERSIONS[0],), # type: ignore - pytest_args: List[str] = [], # noqa -): - """Run tests""" - # Invoke requires lists as type hints, but does not support lists as default arguments. - # Hence this super weird type hint and default argument for the python_versions arg. - echo_header(f"{msg_type.TEST} Running tests") - - python_version_strings = [f"py{v.replace('.', '')}" for v in python_versions] - python_version_arg_string = ",".join(python_version_strings) - - if not pytest_args: - pytest_args = [ - "tests", - "-rfE", - "--failed-first", - "-p no:cov", - "--disable-warnings", - "-q", - ] - - pytest_arg_str = " ".join(pytest_args) - - test_result: Result = c.run( - f"tox -e {python_version_arg_string} -- {pytest_arg_str}", - warn=True, - pty=NOT_WINDOWS, - ) - - # If "failed" in the pytest results - failed_tests = [line for line in test_result.stdout if line.startswith("FAILED")] - - if len(failed_tests) > 0: - print("\n\n\n") - echo_header("Failed tests") - print("\n\n\n") - echo_header("Failed tests") - - for line in failed_tests: - # Remove from start of line until /test_ - line_sans_prefix = line[line.find("test_") :] - - # Keep only that after :: - line_sans_suffix = line_sans_prefix[line_sans_prefix.find("::") + 2 :] - print(f"FAILED {msg_type.FAIL} #{line_sans_suffix} ") - - if test_result.return_code != 0: - exit(test_result.return_code) - - -def test_for_rej(): - # Get all paths in current directory or subdirectories that end in .rej - rej_files = list(Path(".").rglob("*.rej")) - - if len(rej_files) > 0: - print(f"\n{msg_type.FAIL} Found .rej files leftover from cruft update.\n") - for file in rej_files: - print(f" /{file}") - print("\nResolve the conflicts and try again. \n") - exit(1) - - -@task -def lint(c: Context, auto_fix: bool = False): - """Lint the project.""" - test_for_rej() - pre_commit(c=c, auto_fix=auto_fix) - static_type_checks(c) - - -@task -def pr(c: Context, auto_fix: bool = False): - """Run all checks and update the PR.""" - add_and_commit(c) - lint(c, auto_fix=auto_fix) - test(c, python_versions=SUPPORTED_PYTHON_VERSIONS) - update_branch(c) - update_pr(c) - - -@task -def docs(c: Context, view: bool = False, view_only: bool = False): - """ - Build and view docs. If neither build or view are specified, both are run. - """ - if not view_only: - echo_header(f"{msg_type.DOING}: Building docs") - c.run("tox -e docs") - - if view or view_only: - echo_header(f"{msg_type.EXAMINE}: Opening docs in browser") - # check the OS and open the docs in the browser - if platform.system() == "Windows": - c.run("start docs/_build/html/index.html") - else: - c.run("open docs/_build/html/index.html") diff --git a/training/main/requirements.txt b/training/main/requirements.txt index ae058fa2..a44d4e36 100644 --- a/training/main/requirements.txt +++ b/training/main/requirements.txt @@ -14,6 +14,3 @@ wandb >= 0.14.2 # for dataset handling conllu>=4.5.2 wikidata>=0.7.0 - -# style -black>=23.3.0 \ No newline at end of file