From 9da7e6cc0d4e45fd7abf1d2536ed59be76215fb6 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 20:25:06 +0200 Subject: [PATCH 01/17] ci: use nimble-python --- .cookiecutter.json | 5 +- .cruft.json | 9 +- .devcontainer/devcontainer.json | 68 ++++++++------- .github/workflows/dependabot_automerge.yml | 30 ------- .github/workflows/{validate.yml => tests.yml} | 25 +++--- .vscode/launch.json | 19 ---- .vscode/settings.json | 17 ++-- makefile | 55 +++++++----- pyproject.toml | 86 +++++++------------ readme.md | 64 ++++++++++++-- 10 files changed, 192 insertions(+), 186 deletions(-) delete mode 100644 .github/workflows/dependabot_automerge.yml rename .github/workflows/{validate.yml => tests.yml} (60%) delete mode 100644 .vscode/launch.json diff --git a/.cookiecutter.json b/.cookiecutter.json index e80c2a10..202857ac 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -2,11 +2,12 @@ "_copy_without_render": [ "*.github" ], - "_template": "https://github.com/MartinBernstorff/swift-python-cookiecutter", + "_template": "https://github.com/MartinBernstorff/nimble-python-cookiecutter", "author": "Martin Bernstorff", "copyright_year": "2023", - "email": "martinbernstorfff@gmail.com", + "email": "martinbernstorff@gmail.com", "friendly_name": "Personal Mnemonic Medium", + "github_repo": "personal-mnemonic-medium", "github_user": "MartinBernstorff", "license": "MIT", "package_name": "personal_mnemonic_medium", diff --git a/.cruft.json b/.cruft.json index 3f72bab4..3bcf6242 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { - "template": "https://github.com/MartinBernstorff/swift-python-cookiecutter", - "commit": "525b1f682621953d7ef9deb6ad6cf7359ebd43c8", + "template": "https://github.com/MartinBernstorff/nimble-python-cookiecutter", + "commit": "d65d50c7215714a4e6df85be23c5cd9066db9cc0", "checkout": null, "context": { "cookiecutter": { @@ -8,15 +8,16 @@ "package_name": "personal_mnemonic_medium", "friendly_name": "Personal Mnemonic Medium", "author": "Martin Bernstorff", - "email": "martinbernstorfff@gmail.com", + "email": "martinbernstorff@gmail.com", "github_user": "MartinBernstorff", + "github_repo": "personal-mnemonic-medium", "version": "0.0.0", "copyright_year": "2023", "license": "MIT", "_copy_without_render": [ "*.github" ], - "_template": "https://github.com/MartinBernstorff/swift-python-cookiecutter" + "_template": "https://github.com/MartinBernstorff/nimble-python-cookiecutter" } }, "directory": null diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 841db021..f049add4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,37 +1,39 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile { - "name": "Existing Dockerfile", - "build": { - // Sets the run context to one level up instead of the .devcontainer folder. - "context": "..", - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerfile": "../Dockerfile" - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "charliermarsh.ruff", - "ms-python.black-formatter", - "ms-azuretools.vscode-docker", - "ms-vscode.makefile-tools", - "github.vscode-github-actions" - ] - } - }, - "features": { - "ghcr.io/devcontainers/features/github-cli:1": {} - }, - "postStartCommand": "pip install -e ." - // Features to add to the dev container. More info: https://containers.dev/features. - // "features": {}, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - // Uncomment the next line to run commands after the container is created. - // "postCreateCommand": "cat /etc/os-release", - // Configure tool-specific properties. - // "customizations": {}, - // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. - // "remoteUser": "devcontainer" + "name": "Existing Dockerfile", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "../Dockerfile", + "cacheFrom": "ghcr.io/martinbernstorff/personal-mnemonic-medium:latest" + }, + // "features": {}, + "customizations": { + "vscode": { + "extensions": [ + "GitHub.copilot", + "charliermarsh.ruff", + "ms-python.python", + "ms-python.vscode-pylance", + "GitHub.vscode-pull-request-github", + "ms-vscode.makefile-tools", + "github.vscode-github-actions", + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + "postStartCommand": "make install" + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Uncomment the next line to run commands after the container is created. + // Configure tool-specific properties. + // "customizations": {}, + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" } \ No newline at end of file diff --git a/.github/workflows/dependabot_automerge.yml b/.github/workflows/dependabot_automerge.yml deleted file mode 100644 index 22d2ecd7..00000000 --- a/.github/workflows/dependabot_automerge.yml +++ /dev/null @@ -1,30 +0,0 @@ -# GitHub action to automerge dependabot PRs. Only merges if tests passes the -# branch protections in the repository settings. -# You can set branch protections in the repository under Settings > Branches > Add rule -name: automerge-bot-prs - -on: pull_request - -permissions: - contents: write - pull-requests: write - -jobs: - dependabot-automerge: - runs-on: ubuntu-latest - # if actor is dependabot or pre-commit-ci[bot] then run - if: ${{ github.actor == 'dependabot[bot]' }} - - steps: - # Checkout action is required for token to persist - - name: Enable auto-merge for Dependabot PRs - run: gh pr merge --auto --merge "$PR_URL" # Use Github CLI to merge automatically the PR - env: - PR_URL: ${{github.event.pull_request.html_url}} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Auto approve dependabot PRs - if: ${{ github.actor == 'dependabot[bot]' }} - uses: hmarr/auto-approve-action@v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/validate.yml b/.github/workflows/tests.yml similarity index 60% rename from .github/workflows/validate.yml rename to .github/workflows/tests.yml index 8064d438..ac2134b3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/tests.yml @@ -1,15 +1,17 @@ -# GitHub action to check if pre-commit has been run. Runs from .pre-commit-config.yaml, where the pre-commit actions are. - -name: validate +# This workflow will install Python dependencies, run pytests and run notebooks +# then it will in python 3.9 (ubuntu-latest) create a badge with the coverage +# and add it to the PR. This badge will be updated if the PR is updated. +name: Tests on: - pull_request: - branches: [main] push: branches: [main] + pull_request: + branches: [main] jobs: - build: + build-and-test: + permissions: write-all concurrency: group: "${{ github.workflow }} @ ${{ github.ref }}" cancel-in-progress: true @@ -22,14 +24,15 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: MartinBernstorff password: ${{ secrets.GITHUB_TOKEN }} - name: Pre-build dev container image uses: devcontainers/ci@v0.3 with: - imageName: ghcr.io/martinbernstorff/personal-mnemonic-medium - cacheFrom: ghcr.io/martinbernstorff/personal-mnemonic-medium - push: always + imageName: ghcr.io/martinbernstorff/personal-mnemonic-medium-devcontainer + cacheFrom: ghcr.io/martinbernstorff/personal-mnemonic-medium-devcontainer:latest + push: filter + refFilterForPush: refs/heads/main runCmd: - make validate \ No newline at end of file + make validate diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index b7594878..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Run main on Life Lessons", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "args": [ - "/input/", - "Life.apkg" - ], - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 190db2e1..2483f957 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,12 +1,19 @@ { + "python.analysis.typeCheckingMode": "strict", "python.testing.pytestArgs": [ - "tests" + "personal_mnemonic_medium" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, - "python.analysis.typeCheckingMode": "strict", - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "explorer.excludeGitIgnore": false, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/Thumbs.db": true, + "**/BUILD": true }, - "python.formatting.provider": "none" + "python.analysis.diagnosticMode": "workspace" } \ No newline at end of file diff --git a/makefile b/makefile index a547efa1..cf64fdf7 100644 --- a/makefile +++ b/makefile @@ -1,28 +1,43 @@ -lint: - @echo Running black - black . +SRC_PATH = personal_mnemonic_medium - @echo Running ruff +install-dev: + pip install -r dev-requirements.txt + +install-deps: + pip install -r requirements.txt + +install: + make install-deps + make install-dev + pip install -e . + +test: ## Run tests + pytest $(SRC_PATH) + +lint: ## Format code ruff check . --fix + ruff format . -test: - @echo ––– Testing ––– - pytest -n auto -rfE --failed-first --disable-warnings -q +type-check: ## Type-check code + pyright $(SRC_PATH) -type-check: - @echo ––– Running static type checks ––– - pyright . +validate: ## Run all checks + make lint + make type-check + make test -install: - pip install --upgrade -e .[dev,tests] +sync-pr: + git push --set-upstream origin HEAD + git push -validate: - @echo ––– Ensuring dependencies are up to date. This will take a few moments --- - @make install > /dev/null - @make lint && make type-check && make test +create-pr: + gh pr create -w || true -pr: - gh pr create -w +merge-pr: + gh pr merge --auto --merge --delete-branch + +pr: ## Run relevant tests before PR + make sync-pr + make create-pr make validate - git push - gh pr merge --auto --merge \ No newline at end of file + make merge-pr \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 66fb2f30..df59ba7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,22 @@ -# v1 -# v2 [build-system] requires = ["setuptools>=61.0.0", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [project] name = "personal-mnemonic-medium" -version = "0.2.0" -authors = [ - { name = "Martin Bernstorff", email = "martinbernstorfff@gmail.com" }, -] +version = "0.0.0" +authors = [{ name = "Martin Bernstorff", email = "martinbernstorff@gmail.com" }] description = "Personal Mnemonic Medium" -classifiers = [ - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Programming Language :: Python :: 3.10", -] -requires-python = ">=3.10" +classifiers = ["Programming Language :: Python :: 3.11"] +requires-python = ">=3.11" +[project.license] +file = "LICENSE" +name = "MIT" + +[project.readme] +file = "README.md" +content-type = "text/markdown" + dependencies = [ "misaka==2.1.1", "genanki==0.13.0", @@ -48,10 +47,8 @@ repository = "https://github.com/MartinBernstorff/personal-mnemonic-medium" documentation = "https://MartinBernstorff.github.io/personal-mnemonic-medium/" [tool.pyright] -exclude = [".*venv*", ".tox", "*.apkg"] +exclude = [".*venv*", ".tox"] pythonPlatform = "Darwin" -typeCheckingMode = "basic" - [tool.ruff] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. @@ -83,7 +80,23 @@ select = [ "SIM", "W", ] -ignore = ["ANN101", "ANN401", "E402", "E501", "F401", "F841", "UP006", "RET504"] +ignore = [ + "ANN101", + "ANN401", + "E402", + "E501", + "F401", + "F841", + "RET504", + "COM812", + "COM819", + "Q000", + "Q001", + "Q002", + "Q003", + "W191", +] +ignore-init-module-imports = true # Allow autofix for all enabled rules (when `--fix`) is provided. unfixable = ["ERA"] # Exclude a variety of commonly ignored directories. @@ -112,7 +125,7 @@ exclude = [ ] # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" -target-version = "py39" +target-version = "py311" [tool.ruff.flake8-annotations] mypy-init-return = true @@ -125,42 +138,5 @@ known-third-party = ["wandb"] # Unlike Flake8, default to a complexity level of 10. max-complexity = 10 -[tool.semantic_release] -branch = "main" -version_variable = ["pyproject.toml:version"] -upload_to_pypi = false -upload_to_release = false -build_command = "python -m pip install build; python -m build" - [tool.setuptools] include-package-data = true - - -[tool.tox] -legacy_tox_ini = """ -[tox] -envlist = py{39} - -[testenv] -description: run unit tests -extras = tests -use_develop = true -commands = - pytest -n auto {posargs:test} - -[testenv:type] -description: run type checks -extras = tests, dev -basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version -use_develop = true -commands = - pyright . - -[testenv:docs] -description: build docs -extras = docs -basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version -use_develop = true -commands = - sphinx-build -b html docs docs/_build/html -""" diff --git a/readme.md b/readme.md index 85108c0a..3eb868d9 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,18 @@ # Personal Mnemonic Medium -[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/MartinBernstorff/personal-mnemonic-medium/) + +[![PyPI](https://img.shields.io/pypi/v/personal-mnemonic-medium.svg)][pypi status] +[![Python Version](https://img.shields.io/pypi/pyversions/personal-mnemonic-medium)][pypi status] +[![documentation](https://github.com/MartinBernstorff/personal-mnemonic-medium/actions/workflows/documentation.yml/badge.svg)][documentation] +[![Tests](https://github.com/MartinBernstorff/personal-mnemonic-medium/actions/workflows/tests.yml/badge.svg)][tests] +[![Black](https://img.shields.io/badge/code%20style-black-000000.svg)][black] + +[pypi status]: https://pypi.org/project/personal-mnemonic-medium/ +[documentation]: https://MartinBernstorff.github.io/personal-mnemonic-medium/ +[tests]: https://github.com/MartinBernstorff/personal-mnemonic-medium/actions?workflow=Tests +[black]: https://github.com/psf/black + + + Extracting spaced repetition prompts (flashcards) from documents. @@ -11,7 +24,10 @@ A [Zettelkasten](https://medium.com/@martinbernstorf/why-you-need-an-idea-manage This thinking is largely inspired by Andy Matuschak's [Personal Mnemonic Medium](https://notes.andymatuschak.org/The_mnemonic_medium_can_be_extended_to_one%E2%80%99s_personal_notes), and the code is based on the unmaintained [Ankdown](https://github.com/benwr/ankdown). -FYI-style open source, maintenance is not guaranteed. + + +## Installation + ## Pipeline The left path describes the abstract pipeline, the right path the current instantiation in this repo. @@ -33,10 +49,44 @@ graph TD Prompts -- AnkiPackageGenerator --> Cards ``` -## Contributing -To get a full +### Setting up a dev environment +1. Install [Orbstack](https://orbstack.dev/) or Docker Desktop. Make sure to complete the full install process before continuing. +2. If not installed, install VSCode +3. Press this [link](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/Aarhus-Psychiatry-Research/psycop-common) +4. Complete the setup process + +## Usage + +TODO: Add minimal usage example + +To see more examples, see the [documentation]. + +# 📖 Documentation + +| Documentation | | +| --------------------- | -------------------------------------------------------- | +| 🔧 **[Installation]** | Installation instructions on how to install this package | +| 📖 **[Documentation]** | A minimal and developing documentation | +| 👩‍💻 **[Tutorials]** | Tutorials for using this package | +| 🎛️ **[API Reference]** | API reference for this package | +| 📚 **[FAQ]** | Frequently asked questions | + + +# 💬 Where to ask questions + +| Type | | +| ------------------------------ | ---------------------- | +| 📚 **FAQ** | [FAQ] | +| 🚨 **Bug Reports** | [GitHub Issue Tracker] | +| 🎁 **Feature Requests & Ideas** | [GitHub Issue Tracker] | +| 👩‍💻 **Usage Questions** | [GitHub Discussions] | +| 🗯 **General Discussion** | [GitHub Discussions] | -## Running through docker -To build and run the container, see `docker_cmd.sh`. +[Documentation]: https://MartinBernstorff.github.io/personal-mnemonic-medium/index.html +[Installation]: https://MartinBernstorff.github.io/personal-mnemonic-medium/installation.html +[Tutorials]: https://MartinBernstorff.github.io/personal-mnemonic-medium/tutorials.html +[API Reference]: https://MartinBernstorff.github.io/personal-mnemonic-medium/references.html +[FAQ]: https://MartinBernstorff.github.io/personal-mnemonic-medium/faq.html +[github issue tracker]: https://github.com/MartinBernstorff/personal-mnemonic-medium/issues +[github discussions]: https://github.com/MartinBernstorff/personal-mnemonic-medium/discussions - From bfbe55d2241c9ec260e0d2b9d762e30443f453bb Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 20:25:24 +0200 Subject: [PATCH 02/17] ci: remove pull-request template --- .github/pull_request_template.md | 8 -------- application/main.py | 4 ++-- src/personal_mnemonic_medium/card_pipeline.py | 4 ++-- .../exporters/anki/card_types/base.py | 11 ++++++----- .../exporters/anki/card_types/cloze.py | 5 +++-- .../exporters/anki/card_types/qa.py | 5 +++-- .../exporters/anki/globals.py | 2 +- .../exporters/anki/package_generator.py | 8 ++++---- src/personal_mnemonic_medium/exporters/anki/sync.py | 8 ++++---- src/personal_mnemonic_medium/note_factories/note.py | 2 +- .../prompt_extractors/cloze_extractor.py | 2 +- .../prompt_extractors/prompt.py | 2 +- .../prompt_extractors/qa_extractor.py | 2 +- tasks.py | 4 ++-- tests/exporters/anki/test_card_converter.py | 6 +++--- 15 files changed, 34 insertions(+), 39 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 56f513c2..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,8 +0,0 @@ -- [ ] I have considered whether this PR needs review, and requested a review if necessary. - -Fixes issue # - -# Notes for reviewers -Reviewers can skip X, but should pay attention to Y. - - \ No newline at end of file diff --git a/application/main.py b/application/main.py index 5a110106..367a8041 100644 --- a/application/main.py +++ b/application/main.py @@ -21,7 +21,7 @@ # helper for creating anki connect requests -def request(action: Any, **params: Any) -> Dict[str, Any]: +def request(action: Any, **params: Any) -> dict[str, Any]: return {"action": action, "params": params, "version": 6} @@ -83,6 +83,6 @@ def main( sleep(sleep_seconds) main(input_dir=input_dir, watch=watch, host_output_dir=host_output_dir) - + if __name__ == "__main__": typer.run(main) diff --git a/src/personal_mnemonic_medium/card_pipeline.py b/src/personal_mnemonic_medium/card_pipeline.py index af01007b..1bede07a 100644 --- a/src/personal_mnemonic_medium/card_pipeline.py +++ b/src/personal_mnemonic_medium/card_pipeline.py @@ -24,8 +24,8 @@ def __init__( def run( self, input_path: Path, - ) -> List[AnkiCard]: - notes: List[Document] = [] + ) -> list[AnkiCard]: + notes: list[Document] = [] if input_path.is_dir(): notes += list(self.document_factory.get_notes_from_dir(dir_path=input_path)) diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py index 109eb1d2..862822d5 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py @@ -2,8 +2,9 @@ import os import re from abc import ABC, abstractmethod +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable, List, Optional, Tuple +from typing import Any, List, Optional, Tuple import genanki from personal_mnemonic_medium.exporters.anki.globals import CONFIG @@ -22,7 +23,7 @@ class AnkiCard(ABC): def __init__( self, - fields: List[str], + fields: list[str], source_prompt: Prompt, url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, @@ -38,11 +39,11 @@ def source_markdown(self) -> str: return self.source_doc.content @property - def html_fields(self) -> List[str]: + def html_fields(self) -> list[str]: return list(map(self.html_compiler, self.markdown_fields)) @property - def tags(self) -> List[str]: + def tags(self) -> list[str]: return self.source_doc.tags @property @@ -132,7 +133,7 @@ def to_genanki_note(self) -> genanki.Note: tags=self.tags, ) - def make_ref_pair(self, filename: str) -> Tuple[Path, str]: + def make_ref_pair(self, filename: str) -> tuple[Path, str]: """Take a filename relative to the card, and make it absolute.""" newname = "%".join(filename.split(os.sep)) diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py index 3e5b854c..51df772a 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py @@ -1,6 +1,7 @@ import re +from collections.abc import Callable from pathlib import Path -from typing import Callable, List, Optional +from typing import List, Optional import genanki from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard @@ -18,7 +19,7 @@ class AnkiCloze(AnkiCard): def __init__( self, - fields: List[str], + fields: list[str], source_prompt: Prompt, url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py index 98310ec5..4ecccfa3 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py @@ -1,5 +1,6 @@ +from collections.abc import Callable from pathlib import Path -from typing import Callable, List, Optional +from typing import List, Optional import genanki from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard @@ -17,7 +18,7 @@ class AnkiQA(AnkiCard): def __init__( self, - fields: List[str], + fields: list[str], source_prompt: Prompt, url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, diff --git a/src/personal_mnemonic_medium/exporters/anki/globals.py b/src/personal_mnemonic_medium/exporters/anki/globals.py index 6f32482a..35f1d1bb 100644 --- a/src/personal_mnemonic_medium/exporters/anki/globals.py +++ b/src/personal_mnemonic_medium/exporters/anki/globals.py @@ -84,7 +84,7 @@ "card_model_template_cloze": CLOZE_MODEL_TEMPLATE, } -VERSION_LOG: Dict[Any, Any] = {} +VERSION_LOG: dict[Any, Any] = {} Q_TYPE_TAG = { "G": "med/type/1_GP", "A": "med/type/2_Acute_care", diff --git a/src/personal_mnemonic_medium/exporters/anki/package_generator.py b/src/personal_mnemonic_medium/exporters/anki/package_generator.py index 6fa31910..d99c28f7 100644 --- a/src/personal_mnemonic_medium/exporters/anki/package_generator.py +++ b/src/personal_mnemonic_medium/exporters/anki/package_generator.py @@ -43,7 +43,7 @@ def __getitem__(self, deckname: str) -> Any: @dataclass(frozen=True) class DeckBundle: deck: genanki.Deck - media: Set[str] + media: set[str] def get_package(self) -> genanki.Package: return genanki.Package(deck_or_decks=self.deck, media_files=list(self.media)) @@ -61,7 +61,7 @@ def __init__(self) -> None: pass @staticmethod - def cards_to_deck_bundle(cards: List[AnkiCard]) -> DeckBundle: + def cards_to_deck_bundle(cards: list[AnkiCard]) -> DeckBundle: """Take an iterable prompts, output an .apkg in a file called output_name. NOTE: We _must_ be in a temp directory. """ @@ -75,7 +75,7 @@ def cards_to_deck_bundle(cards: List[AnkiCard]) -> DeckBundle: @staticmethod def cards_to_deck( cards: Sequence[AnkiCard], - ) -> tuple[genanki.Deck, Set[str]]: + ) -> tuple[genanki.Deck, set[str]]: media = set() deck_name = cards[0].deckname @@ -103,7 +103,7 @@ def cards_to_deck( def prompts_to_cards( self, prompts: Sequence[Prompt], - ) -> List[AnkiCard]: + ) -> list[AnkiCard]: """Takes an iterable of prompts and turns them into AnkiCards""" cards: list[AnkiCard] = [] diff --git a/src/personal_mnemonic_medium/exporters/anki/sync.py b/src/personal_mnemonic_medium/exporters/anki/sync.py index 33cbc213..4996df2b 100644 --- a/src/personal_mnemonic_medium/exporters/anki/sync.py +++ b/src/personal_mnemonic_medium/exporters/anki/sync.py @@ -15,7 +15,7 @@ # helper for creating anki connect requests -def request(action: Any, **params: Any) -> Dict[str, Any]: +def request(action: Any, **params: Any) -> dict[str, Any]: return {"action": action, "params": params, "version": 6} @@ -139,19 +139,19 @@ def sync_deck( def get_md_note_infos(deck_bundle: DeckBundle) -> set[str]: - md_notes: List[Note] = deck_bundle.deck.notes + md_notes: list[Note] = deck_bundle.deck.notes md_note_guids = {str(n.guid) for n in md_notes} return md_note_guids def get_anki_note_infos(deck_bundle: DeckBundle) -> tuple[dict[str, Any], set[str]]: - anki_card_ids: List[int] = invoke( + anki_card_ids: list[int] = invoke( "findCards", query=f'"deck:{deck_bundle.deck.name}"', ) # get a list of anki notes in the deck - anki_note_ids: List[int] = invoke("cardsToNotes", cards=anki_card_ids) + anki_note_ids: list[int] = invoke("cardsToNotes", cards=anki_card_ids) # get the note info for the notes in the deck anki_notes_info = invoke("notesInfo", notes=anki_note_ids) diff --git a/src/personal_mnemonic_medium/note_factories/note.py b/src/personal_mnemonic_medium/note_factories/note.py index cb001725..67e5f4da 100644 --- a/src/personal_mnemonic_medium/note_factories/note.py +++ b/src/personal_mnemonic_medium/note_factories/note.py @@ -43,7 +43,7 @@ def _replace_alias_wiki_links(text: str) -> str: return text - def get_tags(self, input_str: str, import_time: str) -> List[str]: + def get_tags(self, input_str: str, import_time: str) -> list[str]: file_tags = [import_time] if self.has_supplementary_tags(input_str): diff --git a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py index 03f7d9bc..9e0a155f 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py @@ -19,7 +19,7 @@ def __init__(self) -> None: pass @staticmethod - def _break_string_by_two_or_more_newlines(string: str) -> List[str]: + def _break_string_by_two_or_more_newlines(string: str) -> list[str]: """Break string into a list by 2+ newlines in a row.""" return re.split(r"(\n\n)+", string) diff --git a/src/personal_mnemonic_medium/prompt_extractors/prompt.py b/src/personal_mnemonic_medium/prompt_extractors/prompt.py index b92c0f34..1aa05cf1 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/prompt.py +++ b/src/personal_mnemonic_medium/prompt_extractors/prompt.py @@ -8,7 +8,7 @@ def __init__( self, note_uuid: str, source_note: Document, - tags: Optional[List[str]] = None, + tags: Optional[list[str]] = None, line_nr: Optional[int] = None, ): self.tags = tags diff --git a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py index 50eb2b1a..39ed7608 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py @@ -50,7 +50,7 @@ def _get_first_answer(self, string: str) -> str: return answer[len(self.answer_prefix) + 2 :].rstrip() @staticmethod - def _break_string_by_two_or_more_newlines(string: str) -> List[str]: + def _break_string_by_two_or_more_newlines(string: str) -> list[str]: """Break string into a list by 2+ newlines in a row.""" return re.split(r"(\n\n)+", string) diff --git a/tasks.py b/tasks.py index 8de35b2c..f1caf01f 100644 --- a/tasks.py +++ b/tasks.py @@ -329,7 +329,7 @@ def update(c: Context): @task(iterable="pytest_args") def test( c: Context, - python_versions: List[str] = (SUPPORTED_PYTHON_VERSIONS[0],), # type: ignore + python_versions: list[str] = (SUPPORTED_PYTHON_VERSIONS[0],), # type: ignore pytest_args: List[str] = [], # noqa ): """Run tests""" @@ -382,7 +382,7 @@ def test( def test_for_rej(): # Get all paths in current directory or subdirectories that end in .rej - rej_files = list(Path(".").rglob("*.rej")) + rej_files = list(Path().rglob("*.rej")) if len(rej_files) > 0: print(f"\n{msg_type.FAIL} Found .rej files leftover from cruft update.\n") diff --git a/tests/exporters/anki/test_card_converter.py b/tests/exporters/anki/test_card_converter.py index 0e6c784e..0bb12180 100644 --- a/tests/exporters/anki/test_card_converter.py +++ b/tests/exporters/anki/test_card_converter.py @@ -28,8 +28,8 @@ def __init__( self, document_factory: DocumentFactory = MarkdownNoteFactory(), # noqa: B008 prompt_extractors: Sequence[PromptExtractor] = [ - QAPromptExtractor(), # noqa: B008 - ClozePromptExtractor(), # noqa: B008 + QAPromptExtractor(), + ClozePromptExtractor(), ], card_exporter: CardExporter = AnkiPackageGenerator(), # noqa: B008 ) -> None: @@ -42,7 +42,7 @@ def __init__( def test_card_pipeline( self, input_path: Path, - ) -> List[AnkiCard]: + ) -> list[AnkiCard]: return self.run( input_path=input_path, ) From 413edea11b6d2a04e02bb6401a34441a6988077c Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 18:43:29 +0000 Subject: [PATCH 03/17] cleanup --- Dockerfile | 8 +- makefile | 2 +- pyproject.toml | 1 + tasks.py | 428 ------------------------------------------------- 4 files changed, 6 insertions(+), 433 deletions(-) delete mode 100644 tasks.py diff --git a/Dockerfile b/Dockerfile index 2c144c25..a01c1081 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,16 +3,16 @@ FROM python:3.11-bookworm # Set the working directory to /app WORKDIR /app +RUN pip install pyright +RUN pyright . # Install deps COPY pyproject.toml ./ -RUN pip install .[dev] -RUN pip install .[tests] +RUN pip install --upgrade .[dev] +RUN pip install --upgrade .[tests] # Ensure pyright builds correctly. # If run in make validate, it is run in parallel, which breaks its installation. -RUN pyright . - # Install the entire app COPY . /app RUN pip install -e . diff --git a/makefile b/makefile index cf64fdf7..ec8ccf1e 100644 --- a/makefile +++ b/makefile @@ -15,7 +15,7 @@ test: ## Run tests pytest $(SRC_PATH) lint: ## Format code - ruff check . --fix + ruff . --fix ruff format . type-check: ## Type-check code diff --git a/pyproject.toml b/pyproject.toml index df59ba7a..ae4f4e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ documentation = "https://MartinBernstorff.github.io/personal-mnemonic-medium/" [tool.pyright] exclude = [".*venv*", ".tox"] pythonPlatform = "Darwin" +reportMissingTypeStubs = false [tool.ruff] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. diff --git a/tasks.py b/tasks.py deleted file mode 100644 index f1caf01f..00000000 --- a/tasks.py +++ /dev/null @@ -1,428 +0,0 @@ -""" -This project uses Invoke (pyinvoke.org) for task management. -Install it via: - -``` -pip install invoke -``` - -And then run: - -``` -inv --list -``` - -If you do not wish to use invoke you can simply delete this file. -""" - - -import platform -import re -import shutil -from pathlib import Path -from typing import List, Optional - -from invoke import Context, Result, task # type: ignore - -# Extract supported python versions from the pyproject.toml classifiers key -SUPPORTED_PYTHON_VERSIONS = [ - line.split("::")[-1].strip().replace('"', "").replace(",", "") - for line in Path("pyproject.toml").read_text().splitlines() - if "Programming Language :: Python ::" in line -] - -NOT_WINDOWS = platform.system() != "Windows" - - -def echo_header(msg: str): - print(f"\n--- {msg} ---") - - -class MsgType: - # Emojis have to be encoded as bytes to not break the terminal on Windows - @property - def DOING(self) -> str: - return b"\xf0\x9f\xa4\x96".decode() if NOT_WINDOWS else "DOING:" - - @property - def GOOD(self) -> str: - return b"\xe2\x9c\x85".decode() if NOT_WINDOWS else "DONE:" - - @property - def FAIL(self) -> str: - return b"\xf0\x9f\x9a\xa8".decode() if NOT_WINDOWS else "FAILED:" - - @property - def WARN(self) -> str: - return b"\xf0\x9f\x9a\xa7".decode() if NOT_WINDOWS else "WARNING:" - - @property - def SYNC(self) -> str: - return b"\xf0\x9f\x9a\x82".decode() if NOT_WINDOWS else "SYNCING:" - - @property - def PY(self) -> str: - return b"\xf0\x9f\x90\x8d".decode() if NOT_WINDOWS else "" - - @property - def CLEAN(self) -> str: - return b"\xf0\x9f\xa7\xb9".decode() if NOT_WINDOWS else "CLEANING:" - - @property - def TEST(self) -> str: - return b"\xf0\x9f\xa7\xaa".decode() if NOT_WINDOWS else "TESTING:" - - @property - def COMMUNICATE(self) -> str: - return b"\xf0\x9f\x93\xa3".decode() if NOT_WINDOWS else "COMMUNICATING:" - - @property - def EXAMINE(self) -> str: - return b"\xf0\x9f\x94\x8d".decode() if NOT_WINDOWS else "VIEWING:" - - -msg_type = MsgType() - - -def git_init(c: Context, branch: str = "main"): - """Initialize a git repository if it does not exist yet.""" - # If no .git directory exits - if not Path(".git").exists(): - echo_header(f"{msg_type.DOING} Initializing Git repository") - c.run(f"git init -b {branch}") - c.run("git add .") - c.run("git commit -m 'Init'") - print(f"{msg_type.GOOD} Git repository initialized") - else: - print(f"{msg_type.GOOD} Git repository already initialized") - - -def setup_venv( - c: Context, - python_path: str, - venv_name: Optional[str] = None, -) -> str: - """Create a virtual environment if it does not exist yet. - - Args: - c: The invoke context. - python_path: The python executable to use. - venv_name: The name of the virtual environment. Defaults to ".venv". - """ - if venv_name is None: - venv_name = ".venv" - - if not Path(venv_name).exists(): - echo_header( - f"{msg_type.DOING} Creating virtual environment using {msg_type.PY}:{python_path}", - ) - c.run(f"{python_path} -m venv {venv_name}") - print(f"{msg_type.GOOD} Virtual environment created") - else: - print(f"{msg_type.GOOD} Virtual environment already exists") - return venv_name - - -def _add_commit(c: Context, msg: Optional[str] = None): - print(f"{msg_type.DOING} Adding and committing changes") - c.run("git add .") - - if msg is None: - msg = input("Commit message: ") - - c.run(f'git commit -m "{msg}"', pty=NOT_WINDOWS, hide=True) - print(f"{msg_type.GOOD} Changes added and committed") - - -def is_uncommitted_changes(c: Context) -> bool: - git_status_result: Result = c.run( - "git status --porcelain", - pty=NOT_WINDOWS, - hide=True, - ) - - uncommitted_changes = git_status_result.stdout != "" - return uncommitted_changes - - -def add_and_commit(c: Context, msg: Optional[str] = None): - """Add and commit all changes.""" - if is_uncommitted_changes(c): - uncommitted_changes_descr = c.run( - "git status --porcelain", - pty=NOT_WINDOWS, - hide=True, - ).stdout - - echo_header( - f"{msg_type.WARN} Uncommitted changes detected", - ) - - for line in uncommitted_changes_descr.splitlines(): - print(f" {line.strip()}") - print("\n") - _add_commit(c, msg=msg) - - -def branch_exists_on_remote(c: Context) -> bool: - branch_name = Path(".git/HEAD").read_text().split("/")[-1].strip() - - branch_exists_result: Result = c.run( - f"git ls-remote --heads origin {branch_name}", - hide=True, - ) - - return branch_name in branch_exists_result.stdout - - -def update_branch(c: Context): - echo_header(f"{msg_type.SYNC} Syncing branch with remote") - - if not branch_exists_on_remote(c): - c.run("git push --set-upstream origin HEAD") - else: - print("Pulling") - c.run("git pull") - print("Pushing") - c.run("git push") - - -def create_pr(c: Context): - c.run( - "gh pr create --web", - pty=NOT_WINDOWS, - ) - - -def update_pr(c: Context): - echo_header(f"{msg_type.COMMUNICATE} Syncing PR") - # Get current branch name - branch_name = Path(".git/HEAD").read_text().split("/")[-1].strip() - pr_result: Result = c.run( - "gh pr list --state OPEN", - pty=False, - hide=True, - ) - - if branch_name not in pr_result.stdout: - create_pr(c) - else: - open_web = input("Open in browser? [y/n] ") - if "y" in open_web.lower(): - c.run("gh pr view --web", pty=NOT_WINDOWS) - - -def exit_if_error_in_stdout(result: Result): - # Find N remaining using regex - - if "error" in result.stdout: - errors_remaining = re.findall(r"\d+(?=( remaining))", result.stdout)[ - 0 - ] # testing - if errors_remaining != "0": - exit(0) - - -def pre_commit(c: Context, auto_fix: bool): - """Run pre-commit checks.""" - - # Essential to have a clean working directory before pre-commit to avoid committing - # heterogenous files under a "style: linting" commit - if is_uncommitted_changes(c): - print( - f"{msg_type.WARN} Your git working directory is not clean. Stash or commit before running pre-commit.", - ) - exit(1) - - echo_header(f"{msg_type.CLEAN} Running pre-commit checks") - pre_commit_cmd = "pre-commit run --all-files" - result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True) - - exit_if_error_in_stdout(result) - - if ("fixed" in result.stdout or "reformatted" in result.stdout) and auto_fix: - _add_commit(c, msg="style: Auto-fixes from pre-commit") - - print(f"{msg_type.DOING} Fixed errors, re-running pre-commit checks") - second_result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True) - exit_if_error_in_stdout(second_result) - else: - if result.return_code != 0: - print(f"{msg_type.FAIL} Pre-commit checks failed") - exit(1) - - -@task -def static_type_checks(c: Context): - echo_header(f"{msg_type.CLEAN} Running static type checks") - c.run("tox -e type", pty=NOT_WINDOWS) - - -@task -def install( - c: Context, - pip_args: str = "", - msg: bool = True, - venv_path: Optional[str] = None, -): - """Install the project in editable mode using pip install""" - if msg: - echo_header(f"{msg_type.DOING} Installing project") - - extras = ".[dev,tests,docs]" if NOT_WINDOWS else ".[dev,tests,docs]" - install_cmd = f"pip install -e {extras} {pip_args}" - - if venv_path is not None and NOT_WINDOWS: - with c.prefix(f"source {venv_path}/bin/activate"): - c.run(install_cmd) - return - - c.run(install_cmd) - - -def get_python_path(preferred_version: str) -> Optional[str]: - """Get path to python executable.""" - preferred_version_path = shutil.which(f"python{preferred_version}") - - if preferred_version_path is not None: - return preferred_version_path - - print( - f"{msg_type.WARN}: python{preferred_version} not found, continuing with default python version", - ) - return shutil.which("python") - - -@task -def setup(c: Context, python_path: Optional[str] = None): - """Confirm that a git repo exists and setup a virtual environment. - - Args: - c: Invoke context - python_path: Path to the python executable to use for the virtual environment. Uses the return value of `which python` if not provided. - """ - git_init(c) - - if python_path is None: - # get path to python executable - python_path = get_python_path(preferred_version="3.9") - if not python_path: - print(f"{msg_type.FAIL} Python executable not found") - exit(1) - venv_name = setup_venv(c, python_path=python_path) - - install(c, pip_args="--upgrade", msg=False, venv_path=venv_name) - - if venv_name is not None: - print( - f"{msg_type.DOING} Activate your virtual environment by running: \n\n\t\t source {venv_name}/bin/activate \n", - ) - - -@task -def update(c: Context): - """Update dependencies.""" - echo_header(f"{msg_type.DOING} Updating project") - install(c, pip_args="--upgrade", msg=False) - - -@task(iterable="pytest_args") -def test( - c: Context, - python_versions: list[str] = (SUPPORTED_PYTHON_VERSIONS[0],), # type: ignore - pytest_args: List[str] = [], # noqa -): - """Run tests""" - # Invoke requires lists as type hints, but does not support lists as default arguments. - # Hence this super weird type hint and default argument for the python_versions arg. - echo_header(f"{msg_type.TEST} Running tests") - - python_version_strings = [f"py{v.replace('.', '')}" for v in python_versions] - python_version_arg_string = ",".join(python_version_strings) - - if not pytest_args: - pytest_args = [ - "tests", - "-n auto", - "-rfE", - "--failed-first", - "-p no:cov", - "--disable-warnings", - "-q", - ] - - pytest_arg_str = " ".join(pytest_args) - - test_result: Result = c.run( - f"tox -e {python_version_arg_string} -- {pytest_arg_str}", - warn=True, - pty=NOT_WINDOWS, - ) - - # If "failed" in the pytest results - failed_tests = [line for line in test_result.stdout if line.startswith("FAILED")] - - if len(failed_tests) > 0: - print("\n\n\n") - echo_header("Failed tests") - print("\n\n\n") - echo_header("Failed tests") - - for line in failed_tests: - # Remove from start of line until /test_ - line_sans_prefix = line[line.find("test_") :] - - # Keep only that after :: - line_sans_suffix = line_sans_prefix[line_sans_prefix.find("::") + 2 :] - print(f"FAILED {msg_type.FAIL} #{line_sans_suffix} ") - - if test_result.return_code != 0: - exit(test_result.return_code) - - -def test_for_rej(): - # Get all paths in current directory or subdirectories that end in .rej - rej_files = list(Path().rglob("*.rej")) - - if len(rej_files) > 0: - print(f"\n{msg_type.FAIL} Found .rej files leftover from cruft update.\n") - for file in rej_files: - print(f" /{file}") - print("\nResolve the conflicts and try again. \n") - exit(1) - - -@task -def lint(c: Context, auto_fix: bool = False): - """Lint the project.""" - test_for_rej() - pre_commit(c=c, auto_fix=auto_fix) - static_type_checks(c) - - -@task -def pr(c: Context, auto_fix: bool = True): - """Run all checks and update the PR.""" - add_and_commit(c) - lint(c, auto_fix=auto_fix) - test(c, python_versions=SUPPORTED_PYTHON_VERSIONS) - update_branch(c) - update_pr(c) - - -@task -def docs(c: Context, view: bool = False, view_only: bool = False): - """ - Build and view docs. If neither build or view are specified, both are run. - """ - if not view_only: - echo_header(f"{msg_type.DOING}: Building docs") - c.run("tox -e docs") - - if view or view_only: - echo_header(f"{msg_type.EXAMINE}: Opening docs in browser") - # check the OS and open the docs in the browser - if platform.system() == "Windows": - c.run("start docs/_build/html/index.html") - else: - c.run("open docs/_build/html/index.html") From 72d106a5e9597798f329d502401c4d4848a9721d Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 18:43:52 +0000 Subject: [PATCH 04/17] feat: use functionalpy --- application/main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/application/main.py b/application/main.py index 367a8041..4c00df20 100644 --- a/application/main.py +++ b/application/main.py @@ -1,10 +1,11 @@ from collections import defaultdict from pathlib import Path from time import sleep -from typing import Annotated, Any, Dict +from typing import Annotated, Any import sentry_sdk import typer +from functionalpy import Seq from personal_mnemonic_medium.card_pipeline import CardPipeline from personal_mnemonic_medium.exporters.anki.package_generator import ( AnkiPackageGenerator, @@ -63,13 +64,11 @@ def main( input_path=input_dir, ) - decks = defaultdict(list) + grouped_cards = Seq(cards).group_by(lambda card: card.deckname).to_iter() - for card in cards: - decks[card.deckname] += [card] - - for deck in decks: - deck_bundle = AnkiPackageGenerator().cards_to_deck_bundle(cards=decks[deck]) + for group in grouped_cards: + cards = group.group_contents.to_list() + deck_bundle = AnkiPackageGenerator().cards_to_deck_bundle(cards=cards) sync_deck( deck_bundle=deck_bundle, sync_dir_path=host_output_dir, From b7352f1961177661ce7de8b5f4c5a754564140dd Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 18:43:52 +0000 Subject: [PATCH 05/17] feat: use functionalpy --- pyproject.toml | 1 + .../exporters/anki/card_types/base.py | 5 ++--- .../exporters/anki/card_types/cloze.py | 3 +-- .../exporters/anki/card_types/qa.py | 3 +-- .../exporters/anki/sync.py | 2 +- .../exporters/url_generators/obsidian_url.py | 2 +- .../note_factories/markdown.py | 2 +- .../note_factories/note.py | 4 ++-- .../prompt_extractors/cloze_extractor.py | 2 +- .../prompt_extractors/prompt.py | 6 ++---- tests/exporters/anki/test_card_converter.py | 14 +++++++------- tests/exporters/anki/test_package_generator.py | 4 ++-- 12 files changed, 22 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae4f4e1e..e1961c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ file = "README.md" content-type = "text/markdown" dependencies = [ + "functionalpy==0.6.0", "misaka==2.1.1", "genanki==0.13.0", "typer==0.9.0", diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py index 862822d5..e94c3c51 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py @@ -25,7 +25,7 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, + url_generator: Callable[[Path, int | None], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): self.markdown_fields = fields @@ -164,8 +164,7 @@ def process_match(m) -> str: # noqa current_stage = re.sub(regex, process_match, current_stage) - for r in results: - yield r + yield from results # Anki seems to hate alt tags :( self.html_fields[i] = re.sub(r'alt="[^"]*?"', "", current_stage) diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py index 51df772a..43216480 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py @@ -1,7 +1,6 @@ import re from collections.abc import Callable from pathlib import Path -from typing import List, Optional import genanki from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard @@ -21,7 +20,7 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, + url_generator: Callable[[Path, int | None], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): super().__init__( diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py index 4ecccfa3..67aa6093 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py @@ -1,6 +1,5 @@ from collections.abc import Callable from pathlib import Path -from typing import List, Optional import genanki from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard @@ -20,7 +19,7 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, Optional[int]], str] = get_obsidian_url, + url_generator: Callable[[Path, int | None], str] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): super().__init__( diff --git a/src/personal_mnemonic_medium/exporters/anki/sync.py b/src/personal_mnemonic_medium/exporters/anki/sync.py index 4996df2b..75096b57 100644 --- a/src/personal_mnemonic_medium/exporters/anki/sync.py +++ b/src/personal_mnemonic_medium/exporters/anki/sync.py @@ -3,7 +3,7 @@ import urllib.request from pathlib import Path from time import sleep -from typing import Any, Dict, List +from typing import Any from genanki import Model, Note from wasabi import Printer diff --git a/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py b/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py index 43f2f92c..96dd15a1 100644 --- a/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py +++ b/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py @@ -3,7 +3,7 @@ from typing import Optional -def get_obsidian_url(source_path: Path, line_nr: Optional[int] = None) -> str: +def get_obsidian_url(source_path: Path, line_nr: int | None = None) -> str: """Get the obsidian URI for the source document.""" vault: str = urllib.parse.quote(source_path.parent.name) # type: ignore file: str = urllib.parse.quote(source_path.name) # type: ignore diff --git a/src/personal_mnemonic_medium/note_factories/markdown.py b/src/personal_mnemonic_medium/note_factories/markdown.py index 8b548618..73a3ba4e 100644 --- a/src/personal_mnemonic_medium/note_factories/markdown.py +++ b/src/personal_mnemonic_medium/note_factories/markdown.py @@ -31,7 +31,7 @@ def get_and_append_new_uuid(self, file_path: Path) -> str: def get_note_id(self, file_string: str) -> str: return re.findall(r"" - expected_id = ( - r"" - ) + expected_id = r"" extracted_id = factory.get_note_id(note_str) diff --git a/tests/prompt_extractors/test_qa_prompt_extractor.py b/tests/prompt_extractors/test_qa_prompt_extractor.py index 233ecd49..66dca9b8 100644 --- a/tests/prompt_extractors/test_qa_prompt_extractor.py +++ b/tests/prompt_extractors/test_qa_prompt_extractor.py @@ -2,7 +2,9 @@ import pytest from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.qa_extractor import QAPromptExtractor +from personal_mnemonic_medium.prompt_extractors.qa_extractor import ( + QAPromptExtractor, +) @pytest.fixture() @@ -35,13 +37,19 @@ def test_has_qa_matches(qa_extractor: QAPromptExtractor): "QA. Testing something else, even with QA in it!", "\\Q. Testing newlines as well!", ] - matches = [string for string in example_strings if qa_extractor._has_qa(string)] + matches = [ + string for string in example_strings if qa_extractor._has_qa(string) + ] assert len(matches) == 3 def test_has_qa_does_not_match(qa_extractor: QAPromptExtractor): - example_strings = ["\nQ.E.D.", "> A question like this, or", "::Q. A comment!::"] + example_strings = [ + "\nQ.E.D.", + "> A question like this, or", + "::Q. A comment!::", + ] matches = 0 From 249592b84bf608fecbb22b8f8142ca41a4572b01 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 19:10:21 +0000 Subject: [PATCH 08/17] format: line length --- application/main.py | 30 +++++++++++---- pyproject.toml | 2 +- src/personal_mnemonic_medium/card_pipeline.py | 16 ++++++-- .../exporters/anki/card_types/base.py | 24 +++++++++--- .../exporters/anki/card_types/cloze.py | 8 +++- .../exporters/anki/card_types/qa.py | 8 +++- .../exporters/anki/globals.py | 12 ++++-- .../exporters/anki/package_generator.py | 20 +++++++--- .../exporters/anki/sync.py | 32 ++++++++++++---- .../exporters/base.py | 8 +++- .../markdown_to_html/html_compiler.py | 8 +++- .../exporters/url_generators/obsidian_url.py | 4 +- .../note_factories/base.py | 4 +- .../note_factories/markdown.py | 12 ++++-- .../note_factories/note.py | 12 ++++-- .../prompt_extractors/cloze_extractor.py | 37 +++++++++++++----- .../prompt_extractors/qa_extractor.py | 20 +++++++--- src/personal_mnemonic_medium/utils/hasher.py | 3 +- tests/exporters/anki/test_card_converter.py | 38 ++++++++++++++----- .../exporters/anki/test_package_generator.py | 16 ++++++-- .../note_factories/test_markdown_extractor.py | 20 ++++++++-- .../prompt_extractors/test_cloze_extractor.py | 4 +- .../test_qa_prompt_extractor.py | 4 +- 23 files changed, 257 insertions(+), 85 deletions(-) diff --git a/application/main.py b/application/main.py index 54b858d9..d457f889 100644 --- a/application/main.py +++ b/application/main.py @@ -10,7 +10,9 @@ AnkiPackageGenerator, ) from personal_mnemonic_medium.exporters.anki.sync import sync_deck -from personal_mnemonic_medium.note_factories.markdown import MarkdownNoteFactory +from personal_mnemonic_medium.note_factories.markdown import ( + MarkdownNoteFactory, +) from personal_mnemonic_medium.prompt_extractors.cloze_extractor import ( ClozePromptExtractor, ) @@ -32,12 +34,16 @@ def main( host_output_dir: Path, watch: Annotated[ bool, - typer.Option(help="Keep running, updating Anki deck every 15 seconds"), + typer.Option( + help="Keep running, updating Anki deck every 15 seconds" + ), ], ): """Run the thing.""" if not input_dir.exists(): - raise FileNotFoundError(f"Input directory {input_dir} does not exist") + raise FileNotFoundError( + f"Input directory {input_dir} does not exist" + ) if not host_output_dir.exists(): msg.info(f"Creating output directory {host_output_dir}") @@ -65,11 +71,15 @@ def main( input_path=input_dir, ) - grouped_cards = Seq(cards).group_by(lambda card: card.deckname).to_iter() + grouped_cards = ( + Seq(cards).group_by(lambda card: card.deckname).to_iter() + ) for group in grouped_cards: cards = group.group_contents.to_list() - deck_bundle = AnkiPackageGenerator().cards_to_deck_bundle(cards=cards) + deck_bundle = AnkiPackageGenerator().cards_to_deck_bundle( + cards=cards + ) sync_deck( deck_bundle=deck_bundle, sync_dir_path=host_output_dir, @@ -79,9 +89,15 @@ def main( if watch: sleep_seconds = 60 - msg.good(f"Sync complete, sleeping for {sleep_seconds} seconds") + msg.good( + f"Sync complete, sleeping for {sleep_seconds} seconds" + ) sleep(sleep_seconds) - main(input_dir=input_dir, watch=watch, host_output_dir=host_output_dir) + main( + input_dir=input_dir, + watch=watch, + host_output_dir=host_output_dir, + ) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index f426daaa..f3bcea6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ reportMissingTypeStubs = false [tool.ruff] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -line-length = 80 +line-length = 70 select = [ "A", "ANN", diff --git a/src/personal_mnemonic_medium/card_pipeline.py b/src/personal_mnemonic_medium/card_pipeline.py index f42e5111..cfb14ebd 100644 --- a/src/personal_mnemonic_medium/card_pipeline.py +++ b/src/personal_mnemonic_medium/card_pipeline.py @@ -1,11 +1,17 @@ from collections.abc import Sequence from pathlib import Path -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) from personal_mnemonic_medium.exporters.base import CardExporter -from personal_mnemonic_medium.note_factories.base import DocumentFactory +from personal_mnemonic_medium.note_factories.base import ( + DocumentFactory, +) from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.base import PromptExtractor +from personal_mnemonic_medium.prompt_extractors.base import ( + PromptExtractor, +) from personal_mnemonic_medium.prompt_extractors.prompt import Prompt @@ -27,7 +33,9 @@ def run( notes: list[Document] = [] if input_path.is_dir(): notes += list( - self.document_factory.get_notes_from_dir(dir_path=input_path) + self.document_factory.get_notes_from_dir( + dir_path=input_path + ) ) if not input_path.is_dir(): diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py index 25041df1..a0ab01ab 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py @@ -24,7 +24,9 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, int | None], str] = get_obsidian_url, + url_generator: Callable[ + [Path, int | None], str + ] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): self.markdown_fields = fields @@ -112,7 +114,9 @@ def to_genanki_note(self) -> genanki.Note: ) if len(self.html_fields) < len(self.genanki_model.fields): # type: ignore - while len(self.html_fields) < len(self.genanki_model.fields): # type: ignore + while len(self.html_fields) < len( + self.genanki_model.fields + ): # type: ignore before_extras_field = len(self.html_fields) == 2 if before_extras_field: self.add_field(self.get_source_button()) @@ -146,7 +150,9 @@ def get_deck_dir(self) -> Path: # This is all it takes return Path(self.source_doc.source_path).parent - def determine_media_references(self) -> Iterator[tuple[Path, Path]]: + def determine_media_references( + self + ) -> Iterator[tuple[Path, Path]]: """Find all media references in a card""" for i, field in enumerate(self.html_fields): current_stage = field @@ -157,13 +163,19 @@ def determine_media_references(self) -> Iterator[tuple[Path, Path]]: def process_match(m) -> str: # noqa # type: ignore initial_contents = m.group(1) # type: ignore - abspath, newpath = self.make_ref_pair(initial_contents) # type: ignore + abspath, newpath = self.make_ref_pair( + initial_contents + ) # type: ignore results.append((abspath, newpath)) # noqa # type: ignore return r'src="' + newpath + '"' - current_stage = re.sub(regex, process_match, current_stage) # type: ignore + current_stage = re.sub( + regex, process_match, current_stage + ) # type: ignore yield from results # Anki seems to hate alt tags :( - self.html_fields[i] = re.sub(r'alt="[^"]*?"', "", current_stage) + self.html_fields[i] = re.sub( + r'alt="[^"]*?"', "", current_stage + ) diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py index 65001871..a8736b30 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/cloze.py @@ -3,7 +3,9 @@ from pathlib import Path import genanki -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) from personal_mnemonic_medium.exporters.markdown_to_html.html_compiler import ( compile_field, ) @@ -19,7 +21,9 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, int | None], str] = get_obsidian_url, + url_generator: Callable[ + [Path, int | None], str + ] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): super().__init__( diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py index 45f4b024..1ef7ab1d 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/qa.py @@ -2,7 +2,9 @@ from pathlib import Path import genanki -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) from personal_mnemonic_medium.exporters.markdown_to_html.html_compiler import ( compile_field, ) @@ -18,7 +20,9 @@ def __init__( self, fields: list[str], source_prompt: Prompt, - url_generator: Callable[[Path, int | None], str] = get_obsidian_url, + url_generator: Callable[ + [Path, int | None], str + ] = get_obsidian_url, html_compiler: Callable[[str], str] = compile_field, ): super().__init__( diff --git a/src/personal_mnemonic_medium/exporters/anki/globals.py b/src/personal_mnemonic_medium/exporters/anki/globals.py index 037b3bd6..c633ea8d 100644 --- a/src/personal_mnemonic_medium/exporters/anki/globals.py +++ b/src/personal_mnemonic_medium/exporters/anki/globals.py @@ -2,7 +2,9 @@ import textwrap from typing import Any -from personal_mnemonic_medium.exporters.anki.anki_css import CARD_MODEL_CSS +from personal_mnemonic_medium.exporters.anki.anki_css import ( + CARD_MODEL_CSS, +) ANKICONNECT_URL = ( "http://host.docker.internal:8765" @@ -19,8 +21,12 @@ QUESTION_STR = r"{{ Question }}" ANSWER_STR = r"{{ Answer }}" EXTRA_STR = r"{{ Extra }}" -TTS_QUESTION_STR = r"{{ tts en_US voices=Apple_Samantha speed=1.05:Question }}" -TTS_ANSWER_STR = r"{{ tts en_US voices=Apple_Samantha speed=1.05:Answer }}" +TTS_QUESTION_STR = ( + r"{{ tts en_US voices=Apple_Samantha speed=1.05:Question }}" +) +TTS_ANSWER_STR = ( + r"{{ tts en_US voices=Apple_Samantha speed=1.05:Answer }}" +) QA_MODEL_TEMPLATE = [ { diff --git a/src/personal_mnemonic_medium/exporters/anki/package_generator.py b/src/personal_mnemonic_medium/exporters/anki/package_generator.py index ea110507..ac29f79b 100644 --- a/src/personal_mnemonic_medium/exporters/anki/package_generator.py +++ b/src/personal_mnemonic_medium/exporters/anki/package_generator.py @@ -11,15 +11,23 @@ import genanki -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard -from personal_mnemonic_medium.exporters.anki.card_types.cloze import AnkiCloze -from personal_mnemonic_medium.exporters.anki.card_types.qa import AnkiQA +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) +from personal_mnemonic_medium.exporters.anki.card_types.cloze import ( + AnkiCloze, +) +from personal_mnemonic_medium.exporters.anki.card_types.qa import ( + AnkiQA, +) from personal_mnemonic_medium.exporters.base import CardExporter from personal_mnemonic_medium.prompt_extractors.cloze_extractor import ( ClozePrompt, ) from personal_mnemonic_medium.prompt_extractors.prompt import Prompt -from personal_mnemonic_medium.prompt_extractors.qa_extractor import QAPrompt +from personal_mnemonic_medium.prompt_extractors.qa_extractor import ( + QAPrompt, +) from personal_mnemonic_medium.utils.hasher import simple_hash log = logging.getLogger(__name__) @@ -84,7 +92,9 @@ def cards_to_deck( ) # This is inefficient but definitely works on all platforms. media.add(newpath) except FileNotFoundError as e: - log.debug(f"Could not find file {abspath} for media, {e}.") + log.debug( + f"Could not find file {abspath} for media, {e}." + ) try: deck.add_note(card.to_genanki_note()) diff --git a/src/personal_mnemonic_medium/exporters/anki/sync.py b/src/personal_mnemonic_medium/exporters/anki/sync.py index e457746f..a2bf6e3b 100644 --- a/src/personal_mnemonic_medium/exporters/anki/sync.py +++ b/src/personal_mnemonic_medium/exporters/anki/sync.py @@ -8,8 +8,12 @@ from genanki import Model, Note from wasabi import Printer -from personal_mnemonic_medium.exporters.anki.globals import ANKICONNECT_URL -from personal_mnemonic_medium.exporters.anki.package_generator import DeckBundle +from personal_mnemonic_medium.exporters.anki.globals import ( + ANKICONNECT_URL, +) +from personal_mnemonic_medium.exporters.anki.package_generator import ( + DeckBundle, +) msg = Printer(timestamp=True) @@ -29,7 +33,9 @@ def invoke(action: Any, **params: Any) -> Any: Returns: Any: the response from anki connect """ - requestJson = json.dumps(request(action, **params)).encode("utf-8") + requestJson = json.dumps(request(action, **params)).encode( + "utf-8" + ) response = json.load( urllib.request.urlopen( urllib.request.Request(ANKICONNECT_URL, requestJson) @@ -84,7 +90,9 @@ def sync_deck( return # get a list of anki cards in the deck - anki_note_info_by_guid, anki_note_guids = get_anki_note_infos(deck_bundle) + anki_note_info_by_guid, anki_note_guids = get_anki_note_infos( + deck_bundle + ) # get the unique guids of the md notes md_note_guids = get_md_note_infos(deck_bundle) @@ -125,7 +133,9 @@ def sync_deck( "deleteNotes", notes=note_ids, ) - msg.good(f"Deleted {len(guids_to_delete)} notes") + msg.good( + f"Deleted {len(guids_to_delete)} notes" + ) except Exception: msg.fail( @@ -159,7 +169,9 @@ def get_anki_note_infos( ) # get a list of anki notes in the deck - anki_note_ids: list[int] = invoke("cardsToNotes", cards=anki_card_ids) + anki_note_ids: list[int] = invoke( + "cardsToNotes", cards=anki_card_ids + ) # get the note info for the notes in the deck anki_notes_info = invoke("notesInfo", notes=anki_note_ids) @@ -186,7 +198,9 @@ def sync_model(model: Model): if model.name not in model_names_to_ids: return except Exception as e: - msg.good("\tUnable to fetch existing model names and ids from anki") + msg.good( + "\tUnable to fetch existing model names and ids from anki" + ) msg.good(f"\t\t{e}") if anki_connect_is_live(): @@ -206,7 +220,9 @@ def sync_model(model: Model): ) msg.good(f"\tUpdated model {model.name} template") except Exception as e: - msg.good(f"\tUnable to update model {model.name} template") + msg.good( + f"\tUnable to update model {model.name} template" + ) msg.good(f"\t\t{e}") try: diff --git a/src/personal_mnemonic_medium/exporters/base.py b/src/personal_mnemonic_medium/exporters/base.py index f489121c..3e13b75f 100644 --- a/src/personal_mnemonic_medium/exporters/base.py +++ b/src/personal_mnemonic_medium/exporters/base.py @@ -1,11 +1,15 @@ from abc import ABC, abstractmethod from collections.abc import Sequence -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) from personal_mnemonic_medium.prompt_extractors.prompt import Prompt class CardExporter(ABC): @abstractmethod - def prompts_to_cards(self, prompts: Sequence[Prompt]) -> list[AnkiCard]: + def prompts_to_cards( + self, prompts: Sequence[Prompt] + ) -> list[AnkiCard]: pass diff --git a/src/personal_mnemonic_medium/exporters/markdown_to_html/html_compiler.py b/src/personal_mnemonic_medium/exporters/markdown_to_html/html_compiler.py index b88f3268..294b33bd 100644 --- a/src/personal_mnemonic_medium/exporters/markdown_to_html/html_compiler.py +++ b/src/personal_mnemonic_medium/exporters/markdown_to_html/html_compiler.py @@ -50,7 +50,11 @@ def field_to_html(field: Any) -> str: def compile_field(fieldtext: str) -> str: """Turn source markdown into an HTML field suitable for Anki.""" - fieldtext_sans_wiki = fieldtext.replace("[[", "").replace("]]", "") - fieldtext_sans_comments = re.sub(r"", "", fieldtext_sans_wiki) + fieldtext_sans_wiki = fieldtext.replace("[[", "").replace( + "]]", "" + ) + fieldtext_sans_comments = re.sub( + r"", "", fieldtext_sans_wiki + ) return field_to_html(fieldtext_sans_comments) diff --git a/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py b/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py index a53ea404..75d4d036 100644 --- a/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py +++ b/src/personal_mnemonic_medium/exporters/url_generators/obsidian_url.py @@ -2,7 +2,9 @@ from pathlib import Path -def get_obsidian_url(source_path: Path, line_nr: int | None = None) -> str: +def get_obsidian_url( + source_path: Path, line_nr: int | None = None +) -> str: """Get the obsidian URI for the source document.""" vault: str = urllib.parse.quote(source_path.parent.name) # type: ignore file: str = urllib.parse.quote(source_path.name) # type: ignore diff --git a/src/personal_mnemonic_medium/note_factories/base.py b/src/personal_mnemonic_medium/note_factories/base.py index cd64ce8e..75f91c78 100644 --- a/src/personal_mnemonic_medium/note_factories/base.py +++ b/src/personal_mnemonic_medium/note_factories/base.py @@ -7,7 +7,9 @@ class DocumentFactory(ABC): @abstractmethod - def get_notes_from_dir(self, dir_path: Path) -> Sequence[Document]: + def get_notes_from_dir( + self, dir_path: Path + ) -> Sequence[Document]: pass @abstractmethod diff --git a/src/personal_mnemonic_medium/note_factories/markdown.py b/src/personal_mnemonic_medium/note_factories/markdown.py index da978188..089d77c8 100644 --- a/src/personal_mnemonic_medium/note_factories/markdown.py +++ b/src/personal_mnemonic_medium/note_factories/markdown.py @@ -5,7 +5,9 @@ from tqdm import tqdm -from personal_mnemonic_medium.note_factories.base import DocumentFactory +from personal_mnemonic_medium.note_factories.base import ( + DocumentFactory, +) from personal_mnemonic_medium.note_factories.note import Document @@ -42,7 +44,9 @@ def get_note_from_file(self, file_path: Path) -> Document | None: note_title = file_path.stem if self.cut_note_after in file_contents: - file_contents = file_contents.split(self.cut_note_after)[0] + file_contents = file_contents.split( + self.cut_note_after + )[0] return Document( title=note_title, @@ -51,7 +55,9 @@ def get_note_from_file(self, file_path: Path) -> Document | None: source_path=file_path, ) - def get_notes_from_dir(self, dir_path: Path) -> Sequence[Document]: + def get_notes_from_dir( + self, dir_path: Path + ) -> Sequence[Document]: notes: list[Document] = [] for parent_dir, _, files in os.walk(dir_path): diff --git a/src/personal_mnemonic_medium/note_factories/note.py b/src/personal_mnemonic_medium/note_factories/note.py index 4b35706d..e8bd73c5 100644 --- a/src/personal_mnemonic_medium/note_factories/note.py +++ b/src/personal_mnemonic_medium/note_factories/note.py @@ -16,7 +16,9 @@ def __init__( self.content = self.replace_alias_wiki_links(content) self.source_path = source_path - import_time_formatted = datetime.datetime.now().strftime("%Y-%m-%d") + import_time_formatted = datetime.datetime.now().strftime( + "%Y-%m-%d" + ) self.tags = self.get_tags( self.content, import_time=import_time_formatted @@ -25,7 +27,9 @@ def __init__( @staticmethod def replace_alias_wiki_links(text: str) -> str: tokens_in_link = r"[\w|\s|\d|\(|\)\-]" - regex_pattern = rf"\[\[{tokens_in_link}+\|{tokens_in_link}+\]\]" + regex_pattern = ( + rf"\[\[{tokens_in_link}+\|{tokens_in_link}+\]\]" + ) pattern_matches = re.findall( pattern=regex_pattern, string=text, @@ -34,7 +38,9 @@ def replace_alias_wiki_links(text: str) -> str: for match in pattern_matches: link_name = ( - re.findall(pattern=rf"\|{tokens_in_link}+\]\]", string=match)[0] + re.findall( + pattern=rf"\|{tokens_in_link}+\]\]", string=match + )[0] .replace("|", "") .replace("]", "") ) diff --git a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py index d1fb4054..ba12e993 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py @@ -4,7 +4,9 @@ from typing import Any from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.base import PromptExtractor +from personal_mnemonic_medium.prompt_extractors.base import ( + PromptExtractor, +) from personal_mnemonic_medium.prompt_extractors.prompt import Prompt @@ -19,7 +21,9 @@ def __init__(self) -> None: pass @staticmethod - def _break_string_by_two_or_more_newlines(string: str) -> list[str]: + def _break_string_by_two_or_more_newlines( + string: str + ) -> list[str]: """Break string into a list by 2+ newlines in a row.""" return re.split(r"(\n\n)+", string) @@ -52,11 +56,16 @@ def _replace_cloze_id_with_unique( if selected_cloze is not None: selected_clozes = [selected_cloze] else: - selected_clozes = re.findall(r"{(?!BearID).[^}]*}", string) + selected_clozes = re.findall( + r"{(?!BearID).[^}]*}", string + ) for cloze in selected_clozes: output_hash = ( - int(hashlib.sha256(cloze.encode("utf-8")).hexdigest(), 16) + int( + hashlib.sha256(cloze.encode("utf-8")).hexdigest(), + 16, + ) % 10**3 ) @@ -66,19 +75,27 @@ def _replace_cloze_id_with_unique( return string - def extract_prompts(self, note: Document) -> Sequence[ClozePrompt]: + def extract_prompts( + self, note: Document + ) -> Sequence[ClozePrompt]: prompts = [] - blocks = self._break_string_by_two_or_more_newlines(note.content) + blocks = self._break_string_by_two_or_more_newlines( + note.content + ) for block_string in blocks: if self._has_cloze(block_string): - clozes = re.findall(r"{(?!BearID).[^}]*}", block_string) + clozes = re.findall( + r"{(?!BearID).[^}]*}", block_string + ) for selected_cloze in clozes: - prompt_content = self._replace_cloze_id_with_unique( - block_string, - selected_cloze=selected_cloze, + prompt_content = ( + self._replace_cloze_id_with_unique( + block_string, + selected_cloze=selected_cloze, + ) ) prompts.append( diff --git a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py index 701527de..655538ae 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py @@ -4,7 +4,9 @@ from typing import Any from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.base import PromptExtractor +from personal_mnemonic_medium.prompt_extractors.base import ( + PromptExtractor, +) from personal_mnemonic_medium.prompt_extractors.prompt import Prompt log = logging.getLogger(__name__) @@ -54,7 +56,9 @@ def _get_first_answer(self, string: str) -> str: return answer[len(self.answer_prefix) + 2 :].rstrip() @staticmethod - def _break_string_by_two_or_more_newlines(string: str) -> list[str]: + def _break_string_by_two_or_more_newlines( + string: str + ) -> list[str]: """Break string into a list by 2+ newlines in a row.""" return re.split(r"(\n\n)+", string) @@ -63,7 +67,9 @@ def _has_qa(self, string: str) -> bool: if ( len( re.findall( - r"^(?![:>]).*" + self.question_prefix + r"{0,1}\. ", + r"^(?![:>]).*" + + self.question_prefix + + r"{0,1}\. ", string, flags=re.DOTALL, ), @@ -76,7 +82,9 @@ def _has_qa(self, string: str) -> bool: def extract_prompts(self, note: Document) -> Sequence[QAPrompt]: prompts = [] - blocks = self._break_string_by_two_or_more_newlines(note.content) + blocks = self._break_string_by_two_or_more_newlines( + note.content + ) block_starting_line_nr = 1 for block_string in blocks: @@ -101,7 +109,9 @@ def extract_prompts(self, note: Document) -> Sequence[QAPrompt]: ), ) - block_lines = len(re.findall(r"\n", block_string, flags=re.DOTALL)) + block_lines = len( + re.findall(r"\n", block_string, flags=re.DOTALL) + ) block_starting_line_nr += block_lines return prompts diff --git a/src/personal_mnemonic_medium/utils/hasher.py b/src/personal_mnemonic_medium/utils/hasher.py index d36db008..631538df 100644 --- a/src/personal_mnemonic_medium/utils/hasher.py +++ b/src/personal_mnemonic_medium/utils/hasher.py @@ -4,7 +4,8 @@ def simple_hash(text: str) -> int: """MD5 of text, mod 2^63. Probably not a great hash function.""" comp_hash = ( - int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) % 10**10 + int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) + % 10**10 ) return comp_hash diff --git a/tests/exporters/anki/test_card_converter.py b/tests/exporters/anki/test_card_converter.py index 628e68e4..5766db75 100644 --- a/tests/exporters/anki/test_card_converter.py +++ b/tests/exporters/anki/test_card_converter.py @@ -3,16 +3,26 @@ import genanki from personal_mnemonic_medium.card_pipeline import CardPipeline -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard -from personal_mnemonic_medium.exporters.anki.card_types.qa import AnkiQA +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) +from personal_mnemonic_medium.exporters.anki.card_types.qa import ( + AnkiQA, +) from personal_mnemonic_medium.exporters.anki.package_generator import ( AnkiPackageGenerator, ) from personal_mnemonic_medium.exporters.base import CardExporter -from personal_mnemonic_medium.note_factories.base import DocumentFactory -from personal_mnemonic_medium.note_factories.markdown import MarkdownNoteFactory +from personal_mnemonic_medium.note_factories.base import ( + DocumentFactory, +) +from personal_mnemonic_medium.note_factories.markdown import ( + MarkdownNoteFactory, +) from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.base import PromptExtractor +from personal_mnemonic_medium.prompt_extractors.base import ( + PromptExtractor, +) from personal_mnemonic_medium.prompt_extractors.cloze_extractor import ( ClozePromptExtractor, ) @@ -94,7 +104,9 @@ def test_qa_uuid_generation(): / "test_md_files" / "test_card_guid.md" ) - cards = TestCardPipeline(prompt_extractors=[QAPromptExtractor()]).run( + cards = TestCardPipeline( + prompt_extractors=[QAPromptExtractor()] + ).run( input_path=file_path, ) notes = [c.to_genanki_note() for c in cards] @@ -137,15 +149,23 @@ def test_get_bear_id(): def test_alias_wiki_link_substitution(): alias = "Here I am [[alias|wiki link]], and another [[alias2|wiki link2]]" output = Document.replace_alias_wiki_links(alias) - assert output == "Here I am [[wiki link]], and another [[wiki link2]]" + assert ( + output + == "Here I am [[wiki link]], and another [[wiki link2]]" + ) no_alias = "Here I am [[wiki link]] and another [[wiki link2]]" output = Document.replace_alias_wiki_links(no_alias) - assert output == "Here I am [[wiki link]] and another [[wiki link2]]" + assert ( + output == "Here I am [[wiki link]] and another [[wiki link2]]" + ) test_3 = "How was ice climbing [[Franz Josef]] with [[Vibeke Christiansen|Vibeke]]?" output = Document.replace_alias_wiki_links(test_3) - assert output == "How was ice climbing [[Franz Josef]] with [[Vibeke]]?" + assert ( + output + == "How was ice climbing [[Franz Josef]] with [[Vibeke]]?" + ) alias = "[[Isolation (database design)|Isolation]]" output = Document.replace_alias_wiki_links(alias) diff --git a/tests/exporters/anki/test_package_generator.py b/tests/exporters/anki/test_package_generator.py index 47f68733..0676951d 100644 --- a/tests/exporters/anki/test_package_generator.py +++ b/tests/exporters/anki/test_package_generator.py @@ -1,13 +1,19 @@ from pathlib import Path import genanki -from personal_mnemonic_medium.exporters.anki.card_types.base import AnkiCard -from personal_mnemonic_medium.exporters.anki.card_types.qa import AnkiQA +from personal_mnemonic_medium.exporters.anki.card_types.base import ( + AnkiCard, +) +from personal_mnemonic_medium.exporters.anki.card_types.qa import ( + AnkiQA, +) from personal_mnemonic_medium.exporters.anki.package_generator import ( AnkiPackageGenerator, ) from personal_mnemonic_medium.note_factories.note import Document -from personal_mnemonic_medium.prompt_extractors.qa_extractor import QAPrompt +from personal_mnemonic_medium.prompt_extractors.qa_extractor import ( + QAPrompt, +) def test_cards_to_decks(): @@ -31,7 +37,9 @@ def test_cards_to_decks(): for _ in range(4) ] - deck, media = AnkiPackageGenerator().cards_to_deck(cards=genanki_notes) + deck, media = AnkiPackageGenerator().cards_to_deck( + cards=genanki_notes + ) assert isinstance(deck, genanki.Deck) assert isinstance(media, set) diff --git a/tests/note_factories/test_markdown_extractor.py b/tests/note_factories/test_markdown_extractor.py index ddd8636c..cb091064 100644 --- a/tests/note_factories/test_markdown_extractor.py +++ b/tests/note_factories/test_markdown_extractor.py @@ -2,7 +2,9 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent -from personal_mnemonic_medium.note_factories.markdown import MarkdownNoteFactory +from personal_mnemonic_medium.note_factories.markdown import ( + MarkdownNoteFactory, +) def test_get_notes_from_dir(): @@ -11,6 +13,16 @@ def test_get_notes_from_dir(): ) assert len(notes) == 4 - assert len([note for note in notes if note.title == "test_card_guid"]) == 1 - assert len([note for note in notes if "7696CDCD" in note.content]) == 1 - assert len([note for note in notes if "7696CDCD" in note.uuid]) == 1 + assert ( + len( + [note for note in notes if note.title == "test_card_guid"] + ) + == 1 + ) + assert ( + len([note for note in notes if "7696CDCD" in note.content]) + == 1 + ) + assert ( + len([note for note in notes if "7696CDCD" in note.uuid]) == 1 + ) diff --git a/tests/prompt_extractors/test_cloze_extractor.py b/tests/prompt_extractors/test_cloze_extractor.py index 1d5f5678..8267a4bb 100644 --- a/tests/prompt_extractors/test_cloze_extractor.py +++ b/tests/prompt_extractors/test_cloze_extractor.py @@ -32,6 +32,8 @@ def test_cloze_no_hits(): source_path=Path(__file__), ) - prompts = ClozePromptExtractor().extract_prompts(note_without_cloze) + prompts = ClozePromptExtractor().extract_prompts( + note_without_cloze + ) assert len(prompts) == 0 diff --git a/tests/prompt_extractors/test_qa_prompt_extractor.py b/tests/prompt_extractors/test_qa_prompt_extractor.py index 66dca9b8..9a96dc0c 100644 --- a/tests/prompt_extractors/test_qa_prompt_extractor.py +++ b/tests/prompt_extractors/test_qa_prompt_extractor.py @@ -38,7 +38,9 @@ def test_has_qa_matches(qa_extractor: QAPromptExtractor): "\\Q. Testing newlines as well!", ] matches = [ - string for string in example_strings if qa_extractor._has_qa(string) + string + for string in example_strings + if qa_extractor._has_qa(string) ] assert len(matches) == 3 From abae734e498eedc95537e2579101f56e32d75a26 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 19:17:23 +0000 Subject: [PATCH 09/17] ci: simplify --- makefile | 4 +++- pyproject.toml | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/makefile b/makefile index 0f59df4b..d6b7b627 100644 --- a/makefile +++ b/makefile @@ -16,7 +16,9 @@ test: ## Run tests lint: ## Format code ruff format . - ruff . --fix --extend-select F401 + ruff . --fix \ + --extend-select F401 \ + --extend-select F841 type-check: ## Type-check code pyright $(SRC_PATH) diff --git a/pyproject.toml b/pyproject.toml index f3bcea6b..fbd7e780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,6 @@ select = [ "PLW", "PT", "UP", - "Q", "PTH", "RSE", "RET", @@ -93,10 +92,6 @@ ignore = [ "RET504", "COM812", "COM819", - "Q000", - "Q001", - "Q002", - "Q003", "W191", ] ignore-init-module-imports = true From 01ec6484cce41b46b382fb0bd544a0e6f27b018e Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 19:23:17 +0000 Subject: [PATCH 10/17] misc. --- application/main.py | 4 +--- pyproject.toml | 4 ++++ src/personal_mnemonic_medium/card_pipeline.py | 9 +++----- .../exporters/anki/card_types/base.py | 9 ++++---- .../exporters/anki/globals.py | 10 ++++----- .../exporters/anki/package_generator.py | 21 ++++++------------- .../exporters/anki/sync.py | 17 +++++---------- .../note_factories/note.py | 10 ++------- .../prompt_extractors/cloze_extractor.py | 5 ++--- .../prompt_extractors/qa_extractor.py | 10 ++++----- tests/exporters/anki/test_card_converter.py | 17 ++++----------- .../exporters/anki/test_package_generator.py | 4 +--- .../note_factories/test_markdown_extractor.py | 2 +- 13 files changed, 42 insertions(+), 80 deletions(-) diff --git a/application/main.py b/application/main.py index d457f889..22d55ec4 100644 --- a/application/main.py +++ b/application/main.py @@ -67,9 +67,7 @@ def main( ClozePromptExtractor(), ], card_exporter=AnkiPackageGenerator(), # Step 3, get the cards from the prompts - ).run( - input_path=input_dir, - ) + ).run(input_path=input_dir) grouped_cards = ( Seq(cards).group_by(lambda card: card.deckname).to_iter() diff --git a/pyproject.toml b/pyproject.toml index fbd7e780..06645150 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,12 +125,16 @@ exclude = [ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" target-version = "py311" +[tool.ruff.format] +skip-magic-trailing-comma = true + [tool.ruff.flake8-annotations] mypy-init-return = true suppress-none-returning = true [tool.ruff.isort] known-third-party = ["wandb"] +split-on-trailing-comma = false [tool.ruff.mccabe] # Unlike Flake8, default to a complexity level of 10. diff --git a/src/personal_mnemonic_medium/card_pipeline.py b/src/personal_mnemonic_medium/card_pipeline.py index cfb14ebd..c11becad 100644 --- a/src/personal_mnemonic_medium/card_pipeline.py +++ b/src/personal_mnemonic_medium/card_pipeline.py @@ -26,10 +26,7 @@ def __init__( self.prompt_extractors = prompt_extractors self.card_exporter = card_exporter - def run( - self, - input_path: Path, - ) -> list[AnkiCard]: + def run(self, input_path: Path) -> list[AnkiCard]: notes: list[Document] = [] if input_path.is_dir(): notes += list( @@ -40,7 +37,7 @@ def run( if not input_path.is_dir(): note_from_file = self.document_factory.get_note_from_file( - file_path=input_path, + file_path=input_path ) notes.append(note_from_file) @@ -51,6 +48,6 @@ def run( collected_prompts += extractor.extract_prompts(note) cards: list[AnkiCard] = self.card_exporter.prompts_to_cards( - prompts=collected_prompts, + prompts=collected_prompts ) return cards diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py index a0ab01ab..4b66a982 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py @@ -84,7 +84,7 @@ def deckname(self) -> str: + self.subdeck ) raise ValueError( - "Subdeck length is 0", + "Subdeck length is 0" ) # This is purposefully non-valid code except: # noqa return "0. Don't click me::1. Active::Personal Mnemonic Medium" @@ -100,8 +100,7 @@ def add_field(self, field: Any): def get_source_button(self) -> str: """Get the button to open the source document.""" url = self.url_generator( - self.source_doc.source_path, - self.source_prompt.line_nr, + self.source_doc.source_path, self.source_prompt.line_nr ) html = f'

Open

' return html @@ -110,7 +109,7 @@ def to_genanki_note(self) -> genanki.Note: """Produce a genanki. Note with the specified guid.""" if len(self.html_fields) > len(self.genanki_model.fields): # type: ignore raise ValueError( - f"Too many fields for model {self.genanki_model.name}: {self.html_fields}", # type: ignore + f"Too many fields for model {self.genanki_model.name}: {self.html_fields}" # type: ignore ) if len(self.html_fields) < len(self.genanki_model.fields): # type: ignore @@ -157,7 +156,7 @@ def determine_media_references( for i, field in enumerate(self.html_fields): current_stage = field for regex in [ - r'src="([^"]*?)"', + r'src="([^"]*?)"' ]: # TODO not sure how this should work:, r'\[sound:(.*?)\]']: results = [] diff --git a/src/personal_mnemonic_medium/exporters/anki/globals.py b/src/personal_mnemonic_medium/exporters/anki/globals.py index c633ea8d..b0adbbd1 100644 --- a/src/personal_mnemonic_medium/exporters/anki/globals.py +++ b/src/personal_mnemonic_medium/exporters/anki/globals.py @@ -13,7 +13,7 @@ CARD_MATHJAX_CONTENT = textwrap.dedent( """\ -""", +""" ) VERSION = "0.1" @@ -52,19 +52,19 @@ """, - }, + } ] CLOZE_MODEL_TEMPLATE = [ { "name": "Ankdown Cloze Card with UUID", "qfmt": r"{{{{cloze:Text}}}}\n
{{{{Extra}}}}
\n{}".format( - CARD_MATHJAX_CONTENT, + CARD_MATHJAX_CONTENT ), "afmt": r"{{{{cloze:Text}}}}\n
{{{{Extra}}}}
\n{}".format( - CARD_MATHJAX_CONTENT, + CARD_MATHJAX_CONTENT ), - }, + } ] CONFIG = { diff --git a/src/personal_mnemonic_medium/exporters/anki/package_generator.py b/src/personal_mnemonic_medium/exporters/anki/package_generator.py index ac29f79b..ae8d2b93 100644 --- a/src/personal_mnemonic_medium/exporters/anki/package_generator.py +++ b/src/personal_mnemonic_medium/exporters/anki/package_generator.py @@ -68,14 +68,11 @@ def cards_to_deck_bundle(cards: list[AnkiCard]) -> DeckBundle: """ deck, media = AnkiPackageGenerator.cards_to_deck(cards=cards) - return DeckBundle( - deck=deck, - media=media, - ) + return DeckBundle(deck=deck, media=media) @staticmethod def cards_to_deck( - cards: Sequence[AnkiCard], + cards: Sequence[AnkiCard] ) -> tuple[genanki.Deck, set[str]]: media = set() @@ -87,8 +84,7 @@ def cards_to_deck( for abspath, newpath in card.determine_media_references(): try: copyfile( - abspath, - newpath, + abspath, newpath ) # This is inefficient but definitely works on all platforms. media.add(newpath) except FileNotFoundError as e: @@ -106,8 +102,7 @@ def cards_to_deck( return deck, media def prompts_to_cards( - self, - prompts: Sequence[Prompt], + self, prompts: Sequence[Prompt] ) -> list[AnkiCard]: """Takes an iterable of prompts and turns them into AnkiCards""" @@ -116,16 +111,12 @@ def prompts_to_cards( for prompt in prompts: if isinstance(prompt, QAPrompt): card = AnkiQA( - fields=[ - prompt.question, - prompt.answer, - ], + fields=[prompt.question, prompt.answer], source_prompt=prompt, ) elif isinstance(prompt, ClozePrompt): card = AnkiCloze( - fields=[prompt.content], - source_prompt=prompt, + fields=[prompt.content], source_prompt=prompt ) else: raise NotImplementedError( diff --git a/src/personal_mnemonic_medium/exporters/anki/sync.py b/src/personal_mnemonic_medium/exporters/anki/sync.py index a2bf6e3b..4c2e8d39 100644 --- a/src/personal_mnemonic_medium/exporters/anki/sync.py +++ b/src/personal_mnemonic_medium/exporters/anki/sync.py @@ -39,7 +39,7 @@ def invoke(action: Any, **params: Any) -> Any: response = json.load( urllib.request.urlopen( urllib.request.Request(ANKICONNECT_URL, requestJson) - ), + ) ) if len(response) != 2: raise Exception("response has an unexpected number of fields") @@ -60,7 +60,7 @@ def anki_connect_is_live() -> bool: except Exception as err: msg.info(f"Attempted connection on {ANKICONNECT_URL}") msg.info( - "Unable to reach anki connect. Make sure anki is running and the Anki Connect addon is installed.", + "Unable to reach anki connect. Make sure anki is running and the Anki Connect addon is installed." ) msg.fail(f"Error was {err}") @@ -129,10 +129,7 @@ def sync_deck( for guid in guids_to_delete ] - invoke( - "deleteNotes", - notes=note_ids, - ) + invoke("deleteNotes", notes=note_ids) msg.good( f"Deleted {len(guids_to_delete)} notes" ) @@ -164,8 +161,7 @@ def get_anki_note_infos( deck_bundle: DeckBundle ) -> tuple[dict[str, Any], set[str]]: anki_card_ids: list[int] = invoke( - "findCards", - query=f'"deck:{deck_bundle.deck.name}"', + "findCards", query=f'"deck:{deck_bundle.deck.name}"' ) # get a list of anki notes in the deck @@ -228,10 +224,7 @@ def sync_model(model: Model): try: invoke( "updateModelStyling", - model={ - "name": model.name, - "css": model.css, - }, + model={"name": model.name, "css": model.css}, ) msg.good(f"\tUpdated model {model.name} css") except Exception as e: diff --git a/src/personal_mnemonic_medium/note_factories/note.py b/src/personal_mnemonic_medium/note_factories/note.py index e8bd73c5..57180361 100644 --- a/src/personal_mnemonic_medium/note_factories/note.py +++ b/src/personal_mnemonic_medium/note_factories/note.py @@ -5,11 +5,7 @@ class Document: def __init__( - self, - title: str, - content: str, - uuid: str, - source_path: Path, + self, title: str, content: str, uuid: str, source_path: Path ): self.title = title self.uuid = uuid @@ -31,9 +27,7 @@ def replace_alias_wiki_links(text: str) -> str: rf"\[\[{tokens_in_link}+\|{tokens_in_link}+\]\]" ) pattern_matches = re.findall( - pattern=regex_pattern, - string=text, - flags=re.DOTALL, + pattern=regex_pattern, string=text, flags=re.DOTALL ) for match in pattern_matches: diff --git a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py index ba12e993..46caab1f 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/cloze_extractor.py @@ -42,8 +42,7 @@ def _has_cloze(string: str) -> bool: @staticmethod def _replace_cloze_id_with_unique( - string: str, - selected_cloze: str | None = None, + string: str, selected_cloze: str | None = None ) -> str: """Each cloze deletion in a note is numbered sequentially. @@ -104,7 +103,7 @@ def extract_prompts( tags=note.tags, note_uuid=note.uuid, source_note=note, - ), + ) ) return prompts diff --git a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py index 655538ae..cc2bdd39 100644 --- a/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py +++ b/src/personal_mnemonic_medium/prompt_extractors/qa_extractor.py @@ -12,9 +12,7 @@ log = logging.getLogger(__name__) # Log to disk, not to console. logging.basicConfig( - filename="qa_extractor.log", - filemode="w", - level=logging.DEBUG, + filename="qa_extractor.log", filemode="w", level=logging.DEBUG ) @@ -72,7 +70,7 @@ def _has_qa(self, string: str) -> bool: + r"{0,1}\. ", string, flags=re.DOTALL, - ), + ) ) != 0 ): @@ -94,7 +92,7 @@ def extract_prompts(self, note: Document) -> Sequence[QAPrompt]: answer = self._get_first_answer(block_string) except IndexError: logging.warn( - f"Could not find answer in {note.title} for {question}", + f"Could not find answer in {note.title} for {question}" ) continue @@ -106,7 +104,7 @@ def extract_prompts(self, note: Document) -> Sequence[QAPrompt]: note_uuid=note.uuid, source_note=note, line_nr=block_starting_line_nr, - ), + ) ) block_lines = len( diff --git a/tests/exporters/anki/test_card_converter.py b/tests/exporters/anki/test_card_converter.py index 5766db75..fd2ab764 100644 --- a/tests/exporters/anki/test_card_converter.py +++ b/tests/exporters/anki/test_card_converter.py @@ -48,13 +48,8 @@ def __init__( card_exporter=card_exporter, ) - def test_card_pipeline( - self, - input_path: Path, - ) -> list[AnkiCard]: - return self.run( - input_path=input_path, - ) + def test_card_pipeline(self, input_path: Path) -> list[AnkiCard]: + return self.run(input_path=input_path) def test_custom_card_to_genanki_card(): @@ -106,9 +101,7 @@ def test_qa_uuid_generation(): ) cards = TestCardPipeline( prompt_extractors=[QAPromptExtractor()] - ).run( - input_path=file_path, - ) + ).run(input_path=file_path) notes = [c.to_genanki_note() for c in cards] field_guids = {note.guid for note in notes} @@ -126,9 +119,7 @@ def test_cloze_uuid_generation(): ) cloze_cards = TestCardPipeline( prompt_extractors=[ClozePromptExtractor()] - ).run( - input_path=file_path, - ) + ).run(input_path=file_path) cloze_generated_guids = {card.card_uuid for card in cloze_cards} cloze_reference_guids = {3001245253, 952903559} diff --git a/tests/exporters/anki/test_package_generator.py b/tests/exporters/anki/test_package_generator.py index 0676951d..95e549c7 100644 --- a/tests/exporters/anki/test_package_generator.py +++ b/tests/exporters/anki/test_package_generator.py @@ -66,6 +66,4 @@ def test_package_generators(): for _ in range(4) ] - AnkiPackageGenerator().cards_to_deck_bundle( - cards=genanki_notes, - ) + AnkiPackageGenerator().cards_to_deck_bundle(cards=genanki_notes) diff --git a/tests/note_factories/test_markdown_extractor.py b/tests/note_factories/test_markdown_extractor.py index cb091064..df8a0459 100644 --- a/tests/note_factories/test_markdown_extractor.py +++ b/tests/note_factories/test_markdown_extractor.py @@ -9,7 +9,7 @@ def test_get_notes_from_dir(): notes = MarkdownNoteFactory().get_notes_from_dir( - PROJECT_ROOT / "tests" / "test_md_files", + PROJECT_ROOT / "tests" / "test_md_files" ) assert len(notes) == 4 From 40cc8e69d74327a2ce50acc657a361a653ae34d3 Mon Sep 17 00:00:00 2001 From: Martin Bernstorff Date: Fri, 27 Oct 2023 19:38:40 +0000 Subject: [PATCH 11/17] types: ignore unknown types --- pyproject.toml | 1 + .../exporters/anki/card_types/base.py | 8 +- .../exporters/anki/package_generator.py | 6 +- .../exporters/anki/sync.py | 128 ++++++++++-------- .../note_factories/markdown.py | 14 +- .../prompt_extractors/cloze_extractor.py | 4 +- .../prompt_extractors/qa_extractor.py | 4 +- .../test_qa_prompt_extractor.py | 4 +- 8 files changed, 91 insertions(+), 78 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 06645150..29031246 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ select = [ "ARG", "B", "C4", + "C90", "COM", "D417", "E", diff --git a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py index 4b66a982..80ad1eca 100644 --- a/src/personal_mnemonic_medium/exporters/anki/card_types/base.py +++ b/src/personal_mnemonic_medium/exporters/anki/card_types/base.py @@ -114,7 +114,7 @@ def to_genanki_note(self) -> genanki.Note: if len(self.html_fields) < len(self.genanki_model.fields): # type: ignore while len(self.html_fields) < len( - self.genanki_model.fields + self.genanki_model.fields # type: ignore ): # type: ignore before_extras_field = len(self.html_fields) == 2 if before_extras_field: @@ -137,7 +137,7 @@ def to_genanki_note(self) -> genanki.Note: def make_ref_pair(self, filename: str) -> tuple[Path, str]: """Take a filename relative to the card, and make it absolute.""" - newname = "%".join(filename.split(os.sep)) + newname = "%".join(filename.split(os.sep)) # type: ignore # noqa: PTH206 if os.path.isabs(filename): # noqa abspath = Path(filename) @@ -163,13 +163,13 @@ def determine_media_references( def process_match(m) -> str: # noqa # type: ignore initial_contents = m.group(1) # type: ignore abspath, newpath = self.make_ref_pair( - initial_contents + initial_contents # type: ignore ) # type: ignore results.append((abspath, newpath)) # noqa # type: ignore return r'src="' + newpath + '"' current_stage = re.sub( - regex, process_match, current_stage + regex, process_match, current_stage # type: ignore ) # type: ignore yield from results diff --git a/src/personal_mnemonic_medium/exporters/anki/package_generator.py b/src/personal_mnemonic_medium/exporters/anki/package_generator.py index ae8d2b93..07a822cf 100644 --- a/src/personal_mnemonic_medium/exporters/anki/package_generator.py +++ b/src/personal_mnemonic_medium/exporters/anki/package_generator.py @@ -74,7 +74,7 @@ def cards_to_deck_bundle(cards: list[AnkiCard]) -> DeckBundle: def cards_to_deck( cards: Sequence[AnkiCard] ) -> tuple[genanki.Deck, set[str]]: - media = set() + media = set() # type: ignore deck_name = cards[0].deckname deck_id = simple_hash(deck_name) @@ -86,7 +86,7 @@ def cards_to_deck( copyfile( abspath, newpath ) # This is inefficient but definitely works on all platforms. - media.add(newpath) + media.add(newpath) # type: ignore except FileNotFoundError as e: log.debug( f"Could not find file {abspath} for media, {e}." @@ -99,7 +99,7 @@ def cards_to_deck( f"Could not add card {card} to deck {deck_name}, {e}." ) - return deck, media + return deck, media # type: ignore def prompts_to_cards( self, prompts: Sequence[Prompt] diff --git a/src/personal_mnemonic_medium/exporters/anki/sync.py b/src/personal_mnemonic_medium/exporters/anki/sync.py index 4c2e8d39..a9bbc6b1 100644 --- a/src/personal_mnemonic_medium/exporters/anki/sync.py +++ b/src/personal_mnemonic_medium/exporters/anki/sync.py @@ -99,60 +99,78 @@ def sync_deck( note_diff = md_note_guids.symmetric_difference(anki_note_guids) if note_diff: - msg.info(" Syncing deck: ") - msg.info(f"\t{deck_bundle.deck.name}") - - added_note_guids = md_note_guids - anki_note_guids - if added_note_guids: - msg.info("\tNotes added: ") - msg.info(f"\t\t{added_note_guids}") - - removed_note_guids = anki_note_guids - md_note_guids - if removed_note_guids: - msg.info("\tNotes removed: ") - msg.info(f"\t\t{removed_note_guids}") - - package_path = deck_bundle.save_deck_to_file( - save_dir_path / "deck.apkg" + _sync_deck( + deck_bundle=deck_bundle, + save_dir_path=save_dir_path, + sync_dir_path=sync_dir_path, + delete_cards=delete_cards, + anki_note_info_by_guid=anki_note_info_by_guid, + anki_note_guids=anki_note_guids, + md_note_guids=md_note_guids, ) - try: - sync_path = str(sync_dir_path / "deck.apkg") - invoke("importPackage", path=sync_path) - print(f"Imported {deck_bundle.deck.name}!") - - if delete_cards: - try: - guids_to_delete = anki_note_guids - md_note_guids - if guids_to_delete: - note_ids = [ - anki_note_info_by_guid[guid]["noteId"] - for guid in guids_to_delete - ] - - invoke("deleteNotes", notes=note_ids) - msg.good( - f"Deleted {len(guids_to_delete)} notes" - ) - - except Exception: - msg.fail( - f"Unable to delete cards in {deck_bundle.deck.name}" - ) - # Print full stack trace - traceback.print_exc() - except Exception as e: - print(f"Unable to sync {package_path} to anki") - print(f"{e}") - traceback.print_exc() else: msg.info("Skipped") - msg.info(f"{deck_bundle.deck.name}") + msg.info(f"{deck_bundle.deck.name}") # type: ignore msg.info("\tNo notes added or removed") print("\n") +def _sync_deck( + deck_bundle: DeckBundle, + save_dir_path: Path, + sync_dir_path: Path, + delete_cards: bool, + anki_note_info_by_guid: dict[str, Any], + anki_note_guids: set[str], + md_note_guids: set[str], +): + msg.info(" Syncing deck: ") + msg.info(f"\t{deck_bundle.deck.name}") # type: ignore + + added_note_guids = md_note_guids - anki_note_guids + if added_note_guids: + msg.info("\tNotes added: ") + msg.info(f"\t\t{added_note_guids}") + + removed_note_guids = anki_note_guids - md_note_guids + if removed_note_guids: + msg.info("\tNotes removed: ") + msg.info(f"\t\t{removed_note_guids}") + + package_path = deck_bundle.save_deck_to_file( + save_dir_path / "deck.apkg" + ) + try: + sync_path = str(sync_dir_path / "deck.apkg") + invoke("importPackage", path=sync_path) + print(f"Imported {deck_bundle.deck.name}!") # type: ignore + + if delete_cards: + try: + guids_to_delete = anki_note_guids - md_note_guids + if guids_to_delete: + note_ids = [ + anki_note_info_by_guid[guid]["noteId"] + for guid in guids_to_delete + ] + + invoke("deleteNotes", notes=note_ids) + msg.good(f"Deleted {len(guids_to_delete)} notes") + + except Exception: + msg.fail( + f"Unable to delete cards in {deck_bundle.deck.name}" # type: ignore + ) + # Print full stack trace + traceback.print_exc() + except Exception as e: + print(f"Unable to sync {package_path} to anki") + print(f"{e}") + traceback.print_exc() + + def get_md_note_infos(deck_bundle: DeckBundle) -> set[str]: - md_notes: list[Note] = deck_bundle.deck.notes + md_notes: list[Note] = deck_bundle.deck.notes # type: ignore md_note_guids = {str(n.guid) for n in md_notes} return md_note_guids @@ -161,7 +179,7 @@ def get_anki_note_infos( deck_bundle: DeckBundle ) -> tuple[dict[str, Any], set[str]]: anki_card_ids: list[int] = invoke( - "findCards", query=f'"deck:{deck_bundle.deck.name}"' + "findCards", query=f'"deck:{deck_bundle.deck.name}"' # type: ignore ) # get a list of anki notes in the deck @@ -191,7 +209,7 @@ def sync_model(model: Model): model_names_to_ids = {} try: model_names_to_ids = invoke("modelNamesAndIds") - if model.name not in model_names_to_ids: + if model.name not in model_names_to_ids: # type: ignore return except Exception as e: msg.good( @@ -204,29 +222,29 @@ def sync_model(model: Model): invoke( "updateModelTemplates", model={ - "name": model.name, + "name": model.name, # type: ignore "templates": { t["name"]: { "qfmt": t["qfmt"], "afmt": t["afmt"], } - for t in model.templates + for t in model.templates # type: ignore }, }, ) - msg.good(f"\tUpdated model {model.name} template") + msg.good(f"\tUpdated model {model.name} template") # type: ignore except Exception as e: msg.good( - f"\tUnable to update model {model.name} template" + f"\tUnable to update model {model.name} template" # type: ignore ) msg.good(f"\t\t{e}") try: invoke( "updateModelStyling", - model={"name": model.name, "css": model.css}, + model={"name": model.name, "css": model.css}, # type: ignore ) - msg.good(f"\tUpdated model {model.name} css") + msg.good(f"\tUpdated model {model.name} css") # type: ignore except Exception as e: - msg.good(f"\tUnable to update model {model.name} css") + msg.good(f"\tUnable to update model {model.name} css") # type: ignore msg.good(f"\t\t{e}") diff --git a/src/personal_mnemonic_medium/note_factories/markdown.py b/src/personal_mnemonic_medium/note_factories/markdown.py index 089d77c8..b082da2a 100644 --- a/src/personal_mnemonic_medium/note_factories/markdown.py +++ b/src/personal_mnemonic_medium/note_factories/markdown.py @@ -32,7 +32,7 @@ def get_and_append_new_uuid(self, file_path: Path) -> str: def get_note_id(self, file_string: str) -> str: return re.findall(r"