diff --git a/.gitignore b/.gitignore index 4fc3d4b..142a665 100644 --- a/.gitignore +++ b/.gitignore @@ -16,10 +16,12 @@ build/ dist/ # build dir -setup/build/ -setup/*dist/ -setup/wiki_music.spec -setup/unnecessary_dependencies.txt +freeze/build/ +freeze/*dist/ +freeze/upx/ +freeze/wiki_music.spec +freeze/unnecessary_dependencies.txt +freeze/numpy-1.16.5+vanilla-cp37-cp37m-win_amd64.whl # dir for file output, profiling, logs wiki_music/logs/ @@ -42,4 +44,7 @@ deprecated/ # built version of documentation docs/_build/ docs/_static/ -docs/_templates/ \ No newline at end of file +docs/_templates/ + +# PyPi tokens +.pypirc \ No newline at end of file diff --git a/README.md b/README.md index 61eb361..a18d91e 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ For more detailes refer to Documentation which can be found at: Anyone is welcome to use it or contribute. All of the dependencies are fairly common so you shouldn't encounter any problems. Curentlly supported versions of -python are **3.6** and **3.7**. +python are **3.6** - **3.8**. ## Bugs & Features diff --git a/code_changelog.md b/code_changelog.md index 7727180..1f5ec03 100644 --- a/code_changelog.md +++ b/code_changelog.md @@ -1,16 +1,14 @@ # To-Do ### Main problems ordered by targeted release -- 0.4a0 - - make docs 'about' page and open right url from GUI - - test frozen gui app - - create a github release - 0.5a0 + - fix extraction for endless forms most beautiful - fix as many bugs as possible + - make parallel freezing, and package for release + - fix gui startup and show file speed, too slow, mybe big cover art? - preload needs a complete rewrite the logic is horribly complex, too many classes manipulate preload related variables - 0.xb0 - - use underscores to mark private things - fix gui scaling and elements moving around - convert constants to re patterns for better matching and use more re for better extraction - try to setup some CI system @@ -22,7 +20,8 @@ - support more music formats ### Freezing problems -- try compression options - upx +- upx probably messes some dll, PIXmap does not work, pictures are blank +- pyinstaller now does not include wiki_music in frozen app ### Ideas - parser probably should have its own lock? - access to its mutable variables should be guarded see 13.1.2019 entry in changelog @@ -36,8 +35,8 @@ - research PIL interface to PyQt, and what about Pyside? - cover art search could anounce new downloaded images by signals if we were using QThreads - use custom widgets to simplify GUI https://www.learnpyqt.com/courses/qt-creator/embed-pyqtgraph-custom-widgets-qt-app/ e.g. tableWiew -- implement main GUI progressbar - parser locks could be implemented easilly by getattr and set attr only for public attributes. +- cells with dropdowns for subtracks ### Individual problem cases - load guests as in https://en.wikipedia.org/wiki/Emerald_Forest_and_the_Blackbird @@ -47,6 +46,25 @@ # Change Log +### 16.10.2019 0.4a0 +- selenium dependency was caused by high download limit for + google_images_download, limit is now set to 100 +- fixed premature preload start +- experiment with UPX +- cleanup in parser.in_out +- some dlls must be excluded from UPX compression otherwise they are messed up + and the executable is not working +- we got ~37% reduction in size of GUI app and 25% for CLI app + +### 15.10.2019 - 0.3a4 +- fixed some gui scaling problems +- parser get methods now return so they can be used directly +- fixed marking of private methods in parser +- added GUI progressbar +- implemeted theadpool progressbar +- added selenium dependency for google_images_download +- added pypiwin32 dependency for building frozen app + ### 14.10.2019 - 0.3a4 - completelly reworked logging - finally readthedocs build is passing at long last, had to install with pip diff --git a/docs/api.rst b/docs/api.rst index 412d2e5..3722f2a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,16 +1,20 @@ API reference ============= +.. warning:: + Documentation is stil under construction some things might not be up to + date. This file describes wiki_music API reference. Beware, some things might not be up to date and all is subject to change since we are still in early development phase. -We often use throughout the documentation notation same as python -`typing `_. -module to mark variable types as it is richer and preserves more information. -e.g. List[str] obviously means list of strings. More on the matter can be -read in the typing module documentation. +.. note:: + We often use throughout the documentation notation same as python + `typing `_. + module to mark variable types as it is richer and preserves more + information. e.g. List[str] obviously means list of strings. More on the + matter can be read in the typing module documentation. .. toctree:: :maxdepth: 2 diff --git a/docs/api_constants.rst b/docs/api_constants.rst index c85e0c5..2745f76 100644 --- a/docs/api_constants.rst +++ b/docs/api_constants.rst @@ -2,9 +2,8 @@ wiki_music.constants module =========================== .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. constants --------- diff --git a/docs/api_external_libraries.rst b/docs/api_external_libraries.rst index 0d3fe18..4ba2d27 100644 --- a/docs/api_external_libraries.rst +++ b/docs/api_external_libraries.rst @@ -2,9 +2,8 @@ wiki_music.external_libraries module ==================================== .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. Here the quality of the documentation is not guaranted since we rely only on the docstrings provided by the autors of respectable packages, which are diff --git a/docs/api_gui.rst b/docs/api_gui.rst index fa42306..226f27e 100644 --- a/docs/api_gui.rst +++ b/docs/api_gui.rst @@ -2,9 +2,8 @@ wiki_music.gui_lib module ========================= .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. gui_lib.qt_importer ------------------- diff --git a/docs/api_library_lyrics.rst b/docs/api_library_lyrics.rst index 9af1171..b2ccaec 100644 --- a/docs/api_library_lyrics.rst +++ b/docs/api_library_lyrics.rst @@ -2,9 +2,8 @@ wiki_music.library.lyrics module ================================ .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. library.lyrics -------------- diff --git a/docs/api_library_parser.rst b/docs/api_library_parser.rst index 6a5be26..bee2349 100644 --- a/docs/api_library_parser.rst +++ b/docs/api_library_parser.rst @@ -2,9 +2,8 @@ wiki_music.library.parser module ================================ .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. library.parser.WikipediaRunner ------------------------------ diff --git a/docs/api_library_tags.rst b/docs/api_library_tags.rst index db24ee4..500d8ea 100644 --- a/docs/api_library_tags.rst +++ b/docs/api_library_tags.rst @@ -2,9 +2,8 @@ wiki_music.library.tags_handler and tags_io modules =================================================== .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. library.tags_io --------------- diff --git a/docs/api_utilities.rst b/docs/api_utilities.rst index 8c573fb..296476f 100644 --- a/docs/api_utilities.rst +++ b/docs/api_utilities.rst @@ -2,9 +2,8 @@ wiki_music.utilities module =========================== .. warning:: - This file describes wiki_music API reference. Beware, some things might not - be up to date and all is subject to change since we are still in early - development stage. + Documentation is stil under construction some things might not be up to + date. utilities.gui_utils ------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index ffa1d82..ca2f5f1 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1,6 +1,10 @@ Contributing Guide ================== +.. warning:: + Documentation is stil under construction some things might not be up to + date. + Running the tests ----------------- diff --git a/docs/index.rst b/docs/index.rst index f8a5be5..8e47f3b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,9 @@ Welcome to Wiki music's documentation! ====================================== -Documentation is still under construction !!! but most parts are commplete. +.. warning:: + Documentation is stil under construction some things might not be up to + date. Projects home directory is: `wikipedia-music-tags `_. diff --git a/docs/instalation.rst b/docs/instalation.rst index fb5f742..913fa65 100644 --- a/docs/instalation.rst +++ b/docs/instalation.rst @@ -17,8 +17,15 @@ for GUI and one for CLI version. .. code-block:: bash - wiki_music_gui(.exe) - wiki_music_cli(.exe) + wiki-music-gui(.exe) + wiki-music-cli(.exe) + +There are also binary releases available on +`Github Releases `_. + +.. warning:: + Binary releases are still in very early development stage and may not work + properly. For Developers -------------- @@ -36,7 +43,7 @@ With only some minor modifications it should be able to run on Linux and Os X too. Problems concern mainly default paths and interaction with clipboard in GUI. -Multiple Qt backends are supported with the help of QtPy. So you can substitute +Multiple Qt backends are supported thanks to QtPy. So you can substitute PyQt5 for: PyQt4, PySide2 or PySide. However, for now only compatibility with PyQt5 is tested, so naturally it is also recomended. @@ -103,8 +110,8 @@ Building frozen app you can use option ``--exclude-module=`` in freeze.py to exclude unwanted libraries. See section `Creating virtual environment`_ -There is one optional optimization which you can do before building frozen app. -You can use 'vanilla' numpy to further reduce size of freezed app. Vanilla +There are few optional optimization which you can do before building frozen +app. You can use 'vanilla' numpy to further reduce size of freezed app. Vanilla numpy build can be downloaded from here: `numpy vanilla `_. The problem with regular numpy is building against OPENBLAS (pip version ~40MB) @@ -115,6 +122,15 @@ for your python version, install it by: pip install .whl +Other than that you can use `UPX `_ to compress the app +to a smaller size. It prooves to be quite effective reducing app size.. +If you want to use it go to the provided link and download apropriate +version for your system. Then unpack it in upx folder under wiki_music/freeze. + +.. warning:: + This is not recomended in debugging stage as it adds another layer of + complexity. + Now you are ready to go: .. code-block:: bash @@ -127,8 +143,9 @@ To build the CLI app: python freeze.py cli -When building in virtual env the frozen app should have ~ 75MB. -Without vanilla numpy ~105MB. +When building in virtual env the frozen app should have ~75MB. +With UPX compression and vanilla Numpy ~56MB +With OPENBLAS numpy and UPX compression ~105MB. To build the GUI app: @@ -136,8 +153,9 @@ To build the GUI app: python freeze.py gui -When building in virtual env the frozen app should have ~ 120MB. -Without vanilla numpy ~150MB. +When building in virtual env the frozen app should have ~120MB. +With UPX compression and vanilla Numpy ~75MB +With OPENBLAS numpy and UPX compression ~150MB. This will generate list three directories under setup/ folder: gdist/ cdist/ and build/. Build contains just pyinstaller help files and **(g/c)dist/wiki_music** diff --git a/docs/usage.rst b/docs/usage.rst index 20cf526..06bf7b2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,6 +1,10 @@ Usage instructions ================== +.. warning:: + Documentation is stil under construction some things might not be up to + date. + As an application ----------------- @@ -9,13 +13,13 @@ use two scripts provided by the package: .. code-block:: bash - wiki_music_cli(.exe) + wiki-music-cli(.exe) which runs the CLI app, and .. code-block:: bash - wiki_music_gui(.exe) + wiki-music-gui(.exe) which runs the GUI app. Both can be run with -h or --help to list addinional command line parameters. @@ -84,16 +88,10 @@ You can initialize the parser and call its methods: 'https://en.wikipedia.org/wiki/When_a_Shadow_Is_Forced_into_the_Light' >>> parser.cook_soup() >>> parser.get_contents() - >>> parser.contents - ['Track listing', 'Personnel', 'Charts', 'References'] - >>> parser.get_contents() - >>> parser.contents ['Track listing', 'Personnel', 'Charts', 'References'] >>> parser.get_genres() - >>> parser.genres ['Post-metal', 'gothic metal', 'black metal', 'doom metal'] >>> parser.get_personnel() - >>> parser.personnel ['Mikko Kotamäki', 'Juho Räihä', 'Juha Raivio', 'Jaani Peuhu', 'Matti Honkonen', 'Juuso Raatikainen'] diff --git a/setup/custom_hooks/__init__.py b/freeze/custom_hooks/__init__.py similarity index 100% rename from setup/custom_hooks/__init__.py rename to freeze/custom_hooks/__init__.py diff --git a/setup/custom_hooks/hook-PyQt5.py b/freeze/custom_hooks/hook-PyQt5.py similarity index 100% rename from setup/custom_hooks/hook-PyQt5.py rename to freeze/custom_hooks/hook-PyQt5.py diff --git a/setup/custom_hooks/hook-nltk.py b/freeze/custom_hooks/hook-nltk.py similarity index 100% rename from setup/custom_hooks/hook-nltk.py rename to freeze/custom_hooks/hook-nltk.py diff --git a/setup/freeze.py b/freeze/freeze.py similarity index 83% rename from setup/freeze.py rename to freeze/freeze.py index 20c5922..158b56e 100644 --- a/setup/freeze.py +++ b/freeze/freeze.py @@ -1,8 +1,11 @@ +"""Freezes wiki_music into standalone executable.""" + import PyInstaller import argparse +from typing import Set if float(PyInstaller.__version__[0]) >= 4: - from PyInstaller.depend.imphook import ModuleHook + from PyInstaller.depend.imphook import ModuleHook # pylint: disable=no-name-in-module, import-error else: from PyInstaller.building.imphook import ModuleHook @@ -19,10 +22,11 @@ PATCH_HOOKS = [path.join(HOOKS_DIR, f.name) for f in scandir(HOOKS_DIR) if f.name != "__init__.py"] -patched = set() +patched: Set[str] = set() + -# monkey patch hooks def patched_load_hook_module(self): + """Monkey pytch PyInstaller hooks, with ones from custom_hooks dir.""" global patched for ph in PATCH_HOOKS: @@ -39,8 +43,7 @@ def patched_load_hook_module(self): def input_parser(): - """ Parse command line input parameters. """ - + """Parse command line input parameters.""" parser = argparse.ArgumentParser(description="script to build freezed app") parser.add_argument("mode", type=str, help="choose CLI/GUI build mode", choices=["gui", "cli"]) @@ -48,6 +51,7 @@ def input_parser(): return args.mode.upper() + # monkey patch hooks original_load_hook_module = ModuleHook._load_hook_module ModuleHook._load_hook_module = patched_load_hook_module @@ -83,14 +87,20 @@ def input_parser(): # constnts build options "--clean", "--noconfirm", - #"--noupx", - #"--upx-dir=", - #"--version-file=", + # "--version-file=", # debbugging options - #"--debug=bootloader", - #"--debug=all", - #"--debug=noarchive", + # "--debug=bootloader", + # "--debug=all", + # "--debug=noarchive", + + # upx options + "--noupx", + "--upx-exclude=vcruntime140.dll", + "--upx-exclude=msvcp140.dll", + "--upx-exclude=qwindows.dll", + "--upx-exclude=qwindowsvistastyle.dll", + f"--upx-dir={path.join(WORK_DIR, 'upx')}", # pyinstaller data paths and hooks f"--paths={PACKAGE_PATH}", @@ -102,7 +112,7 @@ def input_parser(): # what to build "--onedir", - #"--onefile", + # "--onefile", "--name=wiki_music" ] @@ -137,4 +147,3 @@ def input_parser(): for f in files: if f.endswith(".pyo"): remove(path.join(root, f)) - diff --git a/setup/hooks/hook-lazy_import.py b/freeze/hooks/hook-lazy_import.py similarity index 100% rename from setup/hooks/hook-lazy_import.py rename to freeze/hooks/hook-lazy_import.py diff --git a/setup/hooks/hook-wiki_music.py b/freeze/hooks/hook-wiki_music.py similarity index 67% rename from setup/hooks/hook-wiki_music.py rename to freeze/hooks/hook-wiki_music.py index e1365e0..bfc7914 100644 --- a/setup/hooks/hook-wiki_music.py +++ b/freeze/hooks/hook-wiki_music.py @@ -1,4 +1,10 @@ -hiddenimports =[ +"""Hook for wiki_music.""" + +from PyInstaller.utils.hooks import collect_data_files +from wiki_music.constants import ROOT_DIR +import os + +hiddenimports = [ "wiki_music.version", # all the following are lazy loaded so pyinstaller cannot find them "wiki_music.library.tags_handler.tag_base", @@ -12,4 +18,12 @@ "wiki_music.external_libraries.lyricsfinder.extractors.lyrical_nonsense", "wiki_music.external_libraries.lyricsfinder.extractors.lyricsmode", "wiki_music.external_libraries.lyricsfinder.extractors.musixmatch", -] \ No newline at end of file +] + +# collect the ui files +UI_PATH = os.path.join(ROOT_DIR, "ui") + +datas = [] +for f in os.scandir(UI_PATH): + if f.is_file() and f.name.endswith(".ui"): + datas.append((os.path.join(UI_PATH, f.name), "ui")) diff --git a/setup/notes_on_build.txt b/freeze/notes_on_build.txt similarity index 100% rename from setup/notes_on_build.txt rename to freeze/notes_on_build.txt diff --git a/setup/requirements.txt b/freeze/requirements.txt similarity index 75% rename from setup/requirements.txt rename to freeze/requirements.txt index ad9f1ea..87ab533 100644 --- a/setup/requirements.txt +++ b/freeze/requirements.txt @@ -1,3 +1,4 @@ # optional - for freezing # installs: altgraph, future, pefile, pywin32-ctypes -pyinstaller>=3.5 \ No newline at end of file +pyinstaller>=3.5 +pypiwin32 \ No newline at end of file diff --git a/setup/rhooks/pyi_rth_nltk.py b/freeze/rhooks/pyi_rth_nltk.py similarity index 100% rename from setup/rhooks/pyi_rth_nltk.py rename to freeze/rhooks/pyi_rth_nltk.py diff --git a/requirements.txt b/requirements.txt index 01421dd..4d6657e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ PyQt5>=5.11.3 python-Levenshtein>=0.12.0 QtPy>=1.7.0 requests>=2.18.4 -wikipedia>=1.4.0 \ No newline at end of file +wikipedia>=1.4.0 diff --git a/setup.cfg b/setup.cfg index 0d244b6..42f571d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,4 @@ ignore = E402, E701 [pydocstyle] -ignore = D413, D416, D203, D107, D405, D401 \ No newline at end of file +ignore = D413, D416, D203, D107, D405, D401, D212, D213 \ No newline at end of file diff --git a/setup.py b/setup.py index 61bf15b..e4ee422 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Multimedia :: Sound/Audio :: Analysis", "Typing :: Typed" ], @@ -53,8 +54,8 @@ python_requires=">=3.6", entry_points={ "console_scripts": [ - "wiki_music_gui=wiki_music.app_gui:main", - "wiki_music_cli=wiki_music.app_cli:main", + "wiki-music-gui=wiki_music.app_gui:main", + "wiki-music-cli=wiki_music.app_cli:main", ] }, ) diff --git a/setup/pywin32-225.win-amd64-py3.7 .exe b/setup/pywin32-225.win-amd64-py3.7 .exe deleted file mode 100644 index f702dd4..0000000 Binary files a/setup/pywin32-225.win-amd64-py3.7 .exe and /dev/null differ diff --git a/tests/setup_tests.py b/tests/setup_tests.py index ff5f934..2ce18af 100644 --- a/tests/setup_tests.py +++ b/tests/setup_tests.py @@ -1,4 +1,4 @@ -""" Manipulate path so tests can import wiki_music_package. """ +"""Manipulate path so tests can import wiki_music_package""" import os import sys diff --git a/tests/test_tags.py b/tests/test_tags.py index 7473120..7a00473 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -9,7 +9,7 @@ class TestTagConsistency(unittest.TestCase): - """ Test that tags in constnts and tag handlers are consistent """ + """Test that tags in constnts and tag handlers are consistent """ def test_flac_consistency(self): self.assertEqual(sorted(TagFlac._map_keys.values()), sorted(TAGS + ("COMMENT",))) diff --git a/wiki_music/app_cli.py b/wiki_music/app_cli.py index d3328b2..ebd9b7c 100644 --- a/wiki_music/app_cli.py +++ b/wiki_music/app_cli.py @@ -37,17 +37,18 @@ def main(): parser = WikipediaRunner(GUI=False) - parser.album = album - parser.band = band + parser.ALBUM = album + parser.ALBUMARTIST = band parser.work_dir = work_dir parser.with_log = with_log - parser.list_files() if only_lyrics: parser.run_lyrics() else: parser.run_wiki() + input("\nPress ENTER to continue") + if __name__ == "__main__": main() diff --git a/wiki_music/constants/__init__.py b/wiki_music/constants/__init__.py index 8ba0fbb..326b788 100644 --- a/wiki_music/constants/__init__.py +++ b/wiki_music/constants/__init__.py @@ -1,4 +1,4 @@ -""" Module which holds all the important constants used throughout the package. +"""Module which holds all the important constants used throughout the package. """ import logging diff --git a/wiki_music/constants/gui_const.py b/wiki_music/constants/gui_const.py index 1d9a85f..d3f8c5f 100644 --- a/wiki_music/constants/gui_const.py +++ b/wiki_music/constants/gui_const.py @@ -1,4 +1,4 @@ -""" Constants used by :mod:`wiki_music.gui_lib` """ +"""Constants used by :mod:`wiki_music.gui_lib` """ from os import path from typing import Tuple diff --git a/wiki_music/constants/parser_const.py b/wiki_music/constants/parser_const.py index f3f5dff..a768c36 100644 --- a/wiki_music/constants/parser_const.py +++ b/wiki_music/constants/parser_const.py @@ -1,4 +1,4 @@ -""" Holds constants that are used in parser. """ +"""Holds constants that are used in parser""" import re # lazy loaded from typing import Tuple, Pattern diff --git a/wiki_music/constants/paths.py b/wiki_music/constants/paths.py index f6165db..a9a36f4 100644 --- a/wiki_music/constants/paths.py +++ b/wiki_music/constants/paths.py @@ -51,9 +51,9 @@ def module_path() -> str: """ if hasattr(sys, "frozen"): - return path.dirname(sys.executable) + return path.abspath(path.dirname(sys.executable)) else: - return path.join(path.dirname(__file__), "..") + return path.abspath(path.join(path.dirname(__file__), "..")) #: toplevel directory for file saving diff --git a/wiki_music/constants/tags.py b/wiki_music/constants/tags.py index 914239f..c4d6e96 100644 --- a/wiki_music/constants/tags.py +++ b/wiki_music/constants/tags.py @@ -1,4 +1,4 @@ -""" Constants used by whole :mod:`wiki_music` module.""" +"""Constants used by whole :mod:`wiki_music` module.""" from typing import Tuple __all__ = ["TAGS", "EXTENDED_TAGS", "STR_TAGS"] diff --git a/wiki_music/external_libraries/google_images_download/google_images_download.py b/wiki_music/external_libraries/google_images_download/google_images_download.py index c582e64..336044b 100644 --- a/wiki_music/external_libraries/google_images_download/google_images_download.py +++ b/wiki_music/external_libraries/google_images_download/google_images_download.py @@ -4,10 +4,11 @@ ###### Searching and Downloading Google Images to the local disk ###### -# libraries for wiki_music +# ! added for wiki_music ################################################## from wiki_music.utilities.gui_utils import get_sizes import logging import queue +# ! added for wiki_music ################################################## # Import Libraries import sys diff --git a/wiki_music/external_libraries/google_images_download/google_images_download_offline.py b/wiki_music/external_libraries/google_images_download/google_images_download_offline.py index 1795192..e27dfe5 100644 --- a/wiki_music/external_libraries/google_images_download/google_images_download_offline.py +++ b/wiki_music/external_libraries/google_images_download/google_images_download_offline.py @@ -14,7 +14,7 @@ class googleimagesdownload: - """ Offline version imitating google images download. Main puprose is + """Offline version imitating google images download. Main puprose is offline testing. Attributes @@ -35,7 +35,7 @@ def __init__(self) -> None: self._files: List[str] = [] def download(self, arguments: dict): - """ Start reding images from files. + """Start reding images from files. Parameters ---------- @@ -77,12 +77,12 @@ def download(self, arguments: dict): print(f"\nErrors: {errorCount}\n") def close(self): - """ Stop downloading images. """ + """Stop downloading images""" self._exit = True @property def max(self) -> int: - """ Returns maximum number of loadable images. Needed to set progresbar + """Returns maximum number of loadable images. Needed to set progresbar in GUI. The value is cached for later use. See also @@ -100,7 +100,7 @@ def max(self) -> int: @property def files(self) -> List[str]: - """ List of image files to load in direstory. + """List of image files to load in direstory. See also -------- diff --git a/wiki_music/gui_lib/__init__.py b/wiki_music/gui_lib/__init__.py index c741b99..bb2075b 100644 --- a/wiki_music/gui_lib/__init__.py +++ b/wiki_music/gui_lib/__init__.py @@ -1,4 +1,4 @@ -""" wiki_music submodule which provides all the GUI functionallity built with +"""wiki_music submodule which provides all the GUI functionallity built with support of all major python Qt bindings. """ diff --git a/wiki_music/gui_lib/base.py b/wiki_music/gui_lib/base.py index 574b966..712f17a 100644 --- a/wiki_music/gui_lib/base.py +++ b/wiki_music/gui_lib/base.py @@ -1,4 +1,4 @@ -""" The base module for Qt frontend. """ +"""The base module for Qt frontend.""" import ctypes import logging @@ -17,9 +17,10 @@ # inherit base from QMainWindow and lyaout from Ui_MainWindow class BaseGui(QMainWindow): - """Base class for all GUI classes, initializes UI from Qt Designer - generated files. then sets up needed variables. Connects buttons and input - fields signals to methods. All GUI classes should subclass this class. + """Base for all GUI classes, initializes UI from Qt Designer ui files. + + Then sets up needed variables. Connects buttons and input fields signals to + methods. All GUI classes should subclass this class. Warnings -------- @@ -39,23 +40,21 @@ def __init__(self) -> None: # call QMainWindow __init__ method super().__init__() + + # misc + self.work_dir: str = "" + self._log: MultiLog = MultiLog(log) + # call Ui_MainWindow user interface setup method uic.loadUi(MAIN_WINDOW_UI, self) # initialize - self.__initUI__() - - # misc - self.work_dir: str = "" - self.log: MultiLog = MultiLog(log) + self._initUI() log.debug("init base done") - def __initUI__(self): - """ Has three responsibilities: load and set window and tray icon and - Set application name. - """ - + def _initUI(self): + """Load and set window tray icon and set application name.""" self.setWindowTitle("Wiki Music") myappid = "WikiMusic" ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid) @@ -65,16 +64,15 @@ def __initUI__(self): tray_icon.show() def _do_nothing(self): - """ Developement convenience function, shows messagebox with a warning - about functionality not being implemented yet. - """ + """Show warning about functionality not being implemented yet. + Developement convenience function. + """ log.warning("Not implemented yet") QMessageBox(QMessageBox.Warning, "Info", "Not implemented yet!").exec_() @abstractmethod def _display_image(self, image=None): - """ Will be reimpemented in :mod:`wiki_music.gui_lib.data_model` module - """ + """Will be reimpemented in :mod:`wiki_music.gui_lib.data_model`.""" abstract_warning() diff --git a/wiki_music/gui_lib/cover_art.py b/wiki_music/gui_lib/cover_art.py index d1c3462..bdbf462 100644 --- a/wiki_music/gui_lib/cover_art.py +++ b/wiki_music/gui_lib/cover_art.py @@ -1,4 +1,4 @@ -""" This module houses classes that are responsible for search and retrieval +"""This module houses classes that are responsible for search and retrieval of cover art picture. """ @@ -84,11 +84,11 @@ def __init__(self, bytes_image: bytearray, log: MultiLog) -> None: self.bytes_image_resz = None self.resize(800, 800) - self.log = log + self._log = log @property def current_image_c(self) -> bytearray: - """ Get image to compress. + """Get image to compress. Returns ------- @@ -108,7 +108,7 @@ def current_image_c(self) -> bytearray: @property def current_image_r(self) -> bytearray: - """ get image to resize. + """get image to resize. Returns ------- @@ -123,7 +123,7 @@ def current_image_r(self) -> bytearray: return self.bytes_image_orig def compress_image(self, quality: int): - """ Apply defined compresion to image and show result. + """Apply defined compresion to image and show result. Parameters ---------- @@ -142,10 +142,10 @@ def compress_image(self, quality: int): size = get_image_size(self.bytes_image_edit) self.disksizeChanged.emit(size) - self.log.info(f"Compressed cover art to: {size}Kb") + self._log.info(f"Compressed cover art to: {size}Kb") def resize_image(self, x: int, y: int, quality: int): - """ Resize image to set dimensions, then apply defined compresion and + """Resize image to set dimensions, then apply defined compresion and show result. Parameters @@ -167,11 +167,11 @@ def resize_image(self, x: int, y: int, quality: int): self.bytes_image_resz = self.bytes_image_edit self.update_pixmap(self.bytes_image_resz) - self.log.info(f"Cover art resized to: {x}x{y}") + self._log.info(f"Cover art resized to: {x}x{y}") self.disksizeChanged.emit(get_image_size(self.bytes_image_edit)) def crop_image(self, quality: int): - """ Crops image to part selected by rubberband, then applies set level + """Crops image to part selected by rubberband, then applies set level of compresin and shows image in GUI. Parameters @@ -194,10 +194,10 @@ def crop_image(self, quality: int): self.dimensionsChanged.emit(*self.image_dims) - self.log.info("Cover art cropped to: {}x{}".format(*self.image_dims)) + self._log.info("Cover art cropped to: {}x{}".format(*self.image_dims)) def save_image(self, path: str): - """ Saves the most current version of the image to file with name + """Saves the most current version of the image to file with name Folder.jpg """ @@ -206,12 +206,12 @@ def save_image(self, path: str): # TODO currently not used # def send2clipboard(self): - # """ Copies the image to clipboard """ + # """Copies the image to clipboard """ # send_to_clipboard(self.bytes_image_edit) class SearchDialog(QDialog): - """ Manages the search dialog window, displays found picture thumbnails and + """Manages the search dialog window, displays found picture thumbnails and houses some basic search controls. Parameters @@ -232,7 +232,7 @@ class SearchDialog(QDialog): :class:`wiki_music.ui.cover_art_base.Ui_cover_art_search` this is the template class for this class layout :class:`wiki_music.gui_lib.custom_classes.ImageTable` - this is the class ttaht manages the table itself + this is the class that manages the table itself """ table: ImageTable @@ -265,7 +265,7 @@ def __init__(self, query: str) -> None: self.cancel_button.clicked.connect(self.done) def table_cell_clicked(self, row: int, col: int): - """ Emits signal when cell in table is clicked. From the row and + """Emits signal when cell in table is clicked. From the row and column position calculates the picture position in the list. Parameters @@ -279,7 +279,7 @@ def table_cell_clicked(self, row: int, col: int): @property def max_columns(self) -> int: - """ Get fixed predefined number of row columns. + """Get fixed predefined number of row columns. Returns ------- @@ -289,7 +289,7 @@ def max_columns(self) -> int: return self.table.MAX_COLUMNS def add_pic(self, dimension: str, thumbnail: bytearray): - """ Add new picture to the table. + """Add new picture to the table. Parameters ---------- @@ -302,7 +302,7 @@ def add_pic(self, dimension: str, thumbnail: bytearray): class PictureEdit(QDialog): - """ Manages picture editor window, with cropping, resizing and compressing + """Manages picture editor window, with cropping, resizing and compressing abilities. Parameters @@ -341,12 +341,12 @@ def __init__(self, dimensions: Tuple[int, int], clicked_image: bytearray, # setup ui from template uic.loadUi(COVER_ART_EDIT_UI, self) - self.log = log + self._log = log self.save_dir = save_dir self.setWindowTitle("Edit cover art") self.setWindowIcon(QIcon(get_icon())) - self.picture = PictureContainer(clicked_image, self.log) + self.picture = PictureContainer(clicked_image, self._log) # variables self.original_ratio = dimensions[0] / dimensions[1] @@ -362,10 +362,10 @@ def __init__(self, dimensions: Tuple[int, int], clicked_image: bytearray, self.size_box.setText(get_image_size(clicked_image)) self.picture_frame.addWidget(self.picture) - self.__setup_overlay__() + self._setup_overlay() - def __setup_overlay__(self): - """ Connect all the dialog elements to coresponding signals. """ + def _setup_overlay(self): + """Connect all the dialog elements to coresponding signals""" # editor signals self.picture.selectionActive.connect(self.cancel_crop.setEnabled) @@ -400,14 +400,14 @@ def __setup_overlay__(self): @property def image_dims(self) -> Tuple[int, int]: - """ Edited image dimensions. + """Edited image dimensions. :type: Tuple[int, int] """ return self.size_spinbox_X.value(), self.size_spinbox_Y.value() def set_image_dims(self, x: int, y: int): - """ passes current image dimension to spinboxes. + """passes current image dimension to spinboxes. Parameters ---------- @@ -422,7 +422,7 @@ def set_image_dims(self, x: int, y: int): @property def clipboard(self) -> bool: - """ Clipboard checkbox state. + """Clipboard checkbox state. :type: bool """ @@ -430,7 +430,7 @@ def clipboard(self) -> bool: @property def save_file(self) -> bool: - """ Save to disk checkbox state. + """Save to disk checkbox state. :type: bool """ @@ -438,7 +438,7 @@ def save_file(self) -> bool: @property def preserve_ratio(self) -> bool: - """ Preserve ration checkbox state. + """Preserve ration checkbox state. :type: bool """ @@ -446,7 +446,7 @@ def preserve_ratio(self) -> bool: @property def compresion(self) -> int: - """ Current compresion slider and spinbox value. + """Current compresion slider and spinbox value. :type: int """ @@ -454,7 +454,7 @@ def compresion(self) -> int: @exception(log) def _get_crop_ratio(self, value: str): - """ Sets the desired aspect ratio for cropping + """Sets the desired aspect ratio for cropping See also -------- @@ -488,7 +488,7 @@ def _get_crop_ratio(self, value: str): self.picture.set_aspect_ratio(crop_ratio) def _aspect_ratio_sync(self, dim: int, activated: str): - """ Synchronize the aspect ratio spinboxes. Employs complex logic to + """Synchronize the aspect ratio spinboxes. Employs complex logic to avoid recursion. Parameters @@ -518,7 +518,7 @@ def _aspect_ratio_sync(self, dim: int, activated: str): self.original_ratio = x / y def _get_compressed(self, comp_value: int, activated: str): - """ Compresses the image to desired value. + """Compresses the image to desired value. Parameters ---------- @@ -541,7 +541,7 @@ def _get_compressed(self, comp_value: int, activated: str): self.picture.compress_image(comp_value) def picture_save(self): - """ Saves picture to Folder.jpg in directory with + """Saves picture to Folder.jpg in directory with currently edited files. Warnings @@ -557,16 +557,16 @@ def picture_save(self): """ if self.save_file: - self.log.info("Cover art saved to file") + self._log.info("Cover art saved to file") self.picture.save_image(path.join(self.save_dir, "Folder.jpg")) if self.clipboard: - self.log.info("Cover art copied to clipboard") + self._log.info("Cover art copied to clipboard") # TODO this is not used for now see discussion in gui_utils module # self.picture.send2clipboard() class CoverArtSearch(BaseGui): - """ Main class that handles cover art search, calls all the apropriate + """Main class that handles cover art search, calls all the apropriate methods. First initialize google_images_download in a separate thread, then initializes Search dialog window. After that it reads downloaded image thumbnails and displays them inn the dialog. @@ -599,7 +599,7 @@ class that shows dialog downloaded cover art images and some basic @exception(log) def cover_art_search(self): - """ Method that takes care of cover art search, download and edit. + """Method that takes care of cover art search, download and edit. See also -------- @@ -622,9 +622,12 @@ def cover_art_search(self): self._max_count = 20 query = f"{self.ALBUMARTIST} {self.ALBUM}" + + # limit can be no more than 100, otherwise chromedriver and selenium + # are needed, because other download method is used arguments = { "keywords": query, - "limit": 500, + "limit": 100, "size": "large", "no_download": True, "no_download_thumbs": True, @@ -644,6 +647,9 @@ def cover_art_search(self): self.search_dialog.load_button.clicked.connect(self._load_more) # stop image download on dialog close self.search_dialog.finished.connect(self.gimd.close) + # TODO unreliable + # on search dialog exit remove message from progressbar + self.search_dialog.destroyed.connect(lambda: self._log.info("")) # TODO connect self.search_dialog.search_button.clicked.connect(self._do_nothing) self.search_dialog.browser_button.clicked.connect(self._do_nothing) @@ -652,10 +658,10 @@ def cover_art_search(self): QTimer.singleShot(0, self._async_loader) - self.log.info("Searching for Cover Art") + self._log.info("Searching for Cover Art") def _load_more(self): - """ Method that raises the maximum number of images to download and + """Method that raises the maximum number of images to download and then continues the search. """ @@ -667,7 +673,7 @@ def _load_more(self): if len(self.images) >= self.gimd.max: QMessageBox(QMessageBox.Information, "Message", "No more images to load").exec_() - self.log.debug("No more images to load") + self._log.debug("No more images to load") return if self._max_count > self.gimd.max: @@ -678,7 +684,7 @@ def _load_more(self): @exception(log) def _async_loader(self): - """ Periodically checks background thread (every 50 ms) for new + """Periodically checks background thread (every 50 ms) for new downloaded images. When a new image is found it is loaded and passed to GUI dialog to display. @@ -690,7 +696,7 @@ def _async_loader(self): dim: str def continue_load(flip: bool = False) -> Tuple[bool, bool]: - """ Decides if more images should be loaded from background thread. + """Decides if more images should be loaded from background thread. Returns ------- @@ -719,7 +725,7 @@ def continue_load(flip: bool = False) -> Tuple[bool, bool]: f"{(image['dim'][0] / 1024):.2f}Kb") except TypeError as e: # if we couldnat load image parameters, don't show it - self.log.debug(e) + self._log.debug(e) pass else: # show and store image @@ -743,7 +749,7 @@ def continue_load(flip: bool = False) -> Tuple[bool, bool]: @exception(log) def _select_picture(self, index: int): - """ Method that initializes necessary classes for selected picture + """Method that initializes necessary classes for selected picture editing. Parameters @@ -756,11 +762,11 @@ def _select_picture(self, index: int): dimensions: Tuple[int, int] = self.images[index]["dim"][1] url: str = self.images[index]["url"] - self.log.info(f"Downloading full size cover art from: {url}") + self._log.info(f"Downloading full size cover art from: {url}") # create dialog to handle image editing - self.picture_editor = PictureEdit(dimensions, get_image(url), self.log, - self.work_dir) + self.picture_editor = PictureEdit(dimensions, get_image(url), + self._log, self.work_dir) # if choice was accepted close download, show selected image in gui # close search dialog, and save picture self.picture_editor.accepted.connect(self.gimd.close) diff --git a/wiki_music/gui_lib/custom_classes.py b/wiki_music/gui_lib/custom_classes.py index fbb2d7b..4093286 100644 --- a/wiki_music/gui_lib/custom_classes.py +++ b/wiki_music/gui_lib/custom_classes.py @@ -105,7 +105,7 @@ def data(self, role: Qt.ItemDataRole) -> QVariant: return self._filtered def setData(self, value: QVariant, role: Qt.UserRole): - """ Reimplemented, sets new data for QStandardItem. + """Reimplemented, sets new data for QStandardItem. also sets the :attr:`_filtered` to None so the path for the new item can be filtered again. @@ -121,7 +121,7 @@ def setData(self, value: QVariant, role: Qt.UserRole): super().setData(value, role) def real_data(self, role: Qt.ItemDataRole) -> str: - """ Workaround to show real contained data because the + """Workaround to show real contained data because the :meth:`QStandardItem.data` method is overridden. Parameters @@ -159,15 +159,17 @@ def text(self, split=False) -> Union[str, list]: if split and "," in data: return [x.strip() for x in data.split(",")] + else: + return data class CustomQStandardItemModel(QStandardItemModel): - """ Overrides the default impementation adds `__getitem__` method so the + """Overrides the default impementation adds `__getitem__` method so the table columns ca be indexed by its names. """ def __getitem__(self, name: str) -> int: - """ Column index from column header name. + """Column index from column header name. Parameters ---------- @@ -221,7 +223,7 @@ def __init__(self, text: str, img: bytearray, self.initUi() def initUi(self): - """ Sets the text and image to be visible. """ + """Sets the text and image to be visible""" image = QImage() image.loadFromData(self._img) @@ -395,9 +397,14 @@ class ResizablePixmap(QLabel): bytes_image_edit: bytearray - def __init__(self, bytes_image: bytearray) -> None: + def __init__(self, bytes_image: bytearray, stretch: bool = True) -> None: QLabel.__init__(self) - self.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored) + + if stretch: + self.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored) + else: + self.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Ignored) + self.setAlignment(Qt.AlignCenter) self.setStyleSheet("background-color: #ffffff;") @@ -464,7 +471,7 @@ def _bytes2pixmap(raw_image: bytearray) -> QPixmap: @staticmethod def _pixmap2bytes(pixmap: QPixmap) -> bytearray: - """ Convert `QPixmap` to bytes image. + """Convert `QPixmap` to bytes image. Parameters ---------- @@ -593,7 +600,7 @@ def cancel_selection(self): self.selectionActive.emit(False) def scale(self, fromResize: bool = False): - """ Handle picture and selection scaling caused by window resize. + """Handle picture and selection scaling caused by window resize. Parameters ---------- @@ -671,7 +678,7 @@ def eventFilter(self, source, event: QEvent): return super(SelectablePixmap, self).eventFilter(source, event) def mousePressEvent(self, event: QEvent): - """ Handles left mouse button cliks. + """Handles left mouse button cliks. If the clicked position is inside the current selection than that selection is moved. If it is outside tahn a new selection is created. @@ -824,19 +831,26 @@ def __init__(self, window_instance: object) -> None: # ensure directory for storing file exists makedirs(path.dirname(DIR_FILE), exist_ok=True) - def get_dir(self) -> str: + def get_dir(self) -> Tuple[str, bool]: """Shows a folder selection dialog with the last visited dir as root. After the user has selected directory it is remenbered, returned to user and upon context exit saved to file. + + Returns + ------- + str + string with directory name + bool + True if some directory was selected, False if dialog was canceled """ self._start_dir = QFileDialog.getExistingDirectory( self.window_instance, "Open Folder", self._start_dir) - return self._start_dir + return self._start_dir, bool(self._start_dir) def __enter__(self): - """ Load last visited directory from file. + """Load last visited directory from file. If the file could not be read, try to get music path on local PC """ @@ -853,7 +867,7 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_traceback): - """ On context exit try to save opened directory to file. """ + """On context exit try to save opened directory to file""" with open(DIR_FILE, "w") as f: f.write(self._start_dir) diff --git a/wiki_music/gui_lib/data_model.py b/wiki_music/gui_lib/data_model.py index 0514578..c5624e4 100644 --- a/wiki_music/gui_lib/data_model.py +++ b/wiki_music/gui_lib/data_model.py @@ -4,7 +4,7 @@ -------- All parser interaciton should be hanhled by this class. Parser methods and attributes should not be accesed directly in GUI. -""" +""" import logging from typing import Iterable, List, Optional, Union @@ -15,7 +15,7 @@ ResizablePixmap) from wiki_music.gui_lib.qt_importer import (QImage, QLabel, QModelIndex, QPixmap, QStandardItemModel, - QTimer) + QTimer, QMessageBox) from wiki_music.library.parser import WikipediaRunner from wiki_music.utilities import SharedVars, exception @@ -62,11 +62,11 @@ def GENRE(self) -> str: :type: str """ - return self._parser.selected_genre + return self._parser.GENRE @GENRE.setter def GENRE(self, value: str): - self._parser.selected_genre = value + self._parser.GENRE = value self.genre_entry.setText(value) @property @@ -82,11 +82,11 @@ def DATE(self) -> str: :type: str """ - return self._parser.release_date + return self._parser.DATE @DATE.setter def DATE(self, value: str): - self._parser.release_date = value + self._parser.DATE = value self.year_entry.setText(value) @property @@ -104,11 +104,11 @@ def ALBUM(self) -> str: :type: str """ - return self._parser.album + return self._parser.ALBUM @ALBUM.setter def ALBUM(self, value: str): - self._parser.album = value + self._parser.ALBUM = value self.album_entry.setText(value) self.album_entry_input.setText(value) @@ -127,11 +127,11 @@ def ALBUMARTIST(self) -> str: :type: str """ - return self._parser.band + return self._parser.ALBUMARTIST @ALBUMARTIST.setter def ALBUMARTIST(self, value: str): - self._parser.band = value + self._parser.ALBUMARTIST = value self.band_entry.setText(value) self.band_entry_input.setText(value) @@ -169,8 +169,6 @@ def work_dir(self) -> str: # type: ignore -------- :attr:`wiki_music.library.parser.base.ParserBase.work_dir` parser attribute tied to this property - :class:`BaseGui.display_dir` - Qt entry field tied to this property :type: str """ @@ -178,8 +176,9 @@ def work_dir(self) -> str: # type: ignore @work_dir.setter def work_dir(self, value: str): - self.display_dir.setText(value) self._parser.work_dir = value + if value: + self.setWindowTitle(f"Wiki Music - {value}") @property def COVERART(self): @@ -250,7 +249,7 @@ def number_of_tracks(self): Number of tracks detected by parser on wikipedia or load from disk. - :type: int + :type: int """ return len(self._parser) @@ -268,10 +267,8 @@ def reinit_parser(self): Reinitializes parser and SharedVars attributes so a new search can be performed. """ - # TODO non-atomic self._parser.__init__(protected_vars=False) - self._parser.list_files() SharedVars.re_init() @@ -317,6 +314,47 @@ def __init__(self) -> None: log.debug("init data model done") + def _input_is_present(self, with_warn=False) -> bool: + """Check if apropriate input to conduct search is present. + + Returns + ------- + bool + true if all requested inputs are present + """ + group = [self.ALBUMARTIST, self.ALBUM, self.work_dir] + if all(group): + return True + else: + alb, bnd, wkd, inp, com, _and = [""] * 6 + if not self.ALBUM: + alb = " album" + if not self.ALBUMARTIST: + bnd = " band" + if not self.work_dir: + wkd = " select working directory" + + if any([alb, bnd]): + inp = "input" + if all([alb, bnd, wkd]): + com = "," + elif all([alb, bnd]): + com = " and" + if all([alb, wkd]) or all([wkd, bnd]): + _and = " and" + + msg = f"You must {inp}{alb}{com}{bnd}{_and}{wkd}" + + if with_warn: + message = QMessageBox() + message.setIcon(QMessageBox.Information) + message.setWindowTitle("Message") + message.setText(msg) + message.exec_() + self._log.warning(msg) + + return False + def _gui_to_parser(self): """Transfers data from GUI to parser. @@ -326,7 +364,6 @@ def _gui_to_parser(self): locking mechanism. We rely on the fact that only GUI or the parser are trying to access data at a given time. """ - col: int # TODO non-atomic @@ -346,21 +383,20 @@ def _display_image(self, image: Optional[bytearray] = None): See also -------- :class:`wiki_music.gui_lib.custom_classes.ResizablePixmap` - + Parameters ---------- image: bytearray force displaying of the input image instead of the one contained in parser. Also the input image is saved to parser. """ - if image: self.COVERART = image if self.cover_art: self.cover_art.update_pixmap(self.COVERART) else: - self.cover_art = ResizablePixmap(self.COVERART) + self.cover_art = ResizablePixmap(self.COVERART, stretch=False) self.picture_layout.addWidget(self.cover_art) @exception(log) @@ -414,7 +450,6 @@ def _detail(self, proxy_index: QModelIndex): index of the proxy model cell, needs to be translated to real table cell index """ - # proxy_index is QModelIndex proxyIndex Class type # it is index of data mapping as shown in gui # real indeces are different @@ -445,11 +480,11 @@ def _detail(self, proxy_index: QModelIndex): # None is the default argument for text since QTextEdit # signal doesn´t retutn text ent.textChanged.connect( - lambda text=None, c=col: self.__text_check__(row, c, text)) + lambda text=None, c=col: self._text_check(row, c, text)) - def __text_check__(self, row: int, col: int, text: Optional[str]): + def _text_check(self, row: int, col: int, text: Optional[str]): """Writes changed text in detail tab to table. - + Parameters ---------- row: int @@ -459,7 +494,6 @@ def __text_check__(self, row: int, col: int, text: Optional[str]): text: Optional[str] text put in cell, if None read it from :attr:`lyrics_detail` """ - if text is None: text = self.lyrics_detail.toPlainText() diff --git a/wiki_music/gui_lib/main_window.py b/wiki_music/gui_lib/main_window.py index f8528a7..1a7f1b5 100644 --- a/wiki_music/gui_lib/main_window.py +++ b/wiki_music/gui_lib/main_window.py @@ -9,10 +9,13 @@ from threading import Thread from typing import Optional, Union -from wiki_music.constants import API_KEY_MESSAGE, MP3_TAG, ROOT_DIR +from wiki_music import __version__ +from wiki_music.constants import API_KEY_MESSAGE, LOG_DIR, MP3_TAG, ROOT_DIR from wiki_music.gui_lib import BaseGui, CoverArtSearch, DataModel, RememberDir from wiki_music.gui_lib.qt_importer import (QAbstractItemView, QFileDialog, - QInputDialog, QMessageBox, QTimer) + QInputDialog, QMessageBox, + QProgressBar, QProgressDialog, Qt, + QTimer) from wiki_music.utilities import (Mp3tagNotFoundException, SharedVars, exception, synchronized, warning, we_are_frozen) @@ -33,7 +36,7 @@ def __init__(self) -> None: super().__init__() - self._remember: Optional[str] = None + self.progressShow: Optional[QProgressDialog] = None def _init_checkers(self): """Initializes timers for periodically repeating methods. @@ -47,6 +50,8 @@ def _init_checkers(self): with errors :meth:`Checkers._conditions_check` checks for parser progres descritions + :meth:`Checkers._threadpool_check` + checks for progress of function running in threadpool :meth:`Checkers._start_checkers` starts the initialized timers """ @@ -62,6 +67,10 @@ def _init_checkers(self): self.conditions_timer.timeout.connect(self._conditions_check) self.conditions_timer.setSingleShot(True) + self.threadpool_timer = QTimer() + self.threadpool_timer.timeout.connect(self._threadpool_check) + self.threadpool_timer.setSingleShot(True) + def _start_checkers(self): """Initializes timers of periodically repeating methods. @@ -70,7 +79,6 @@ def _start_checkers(self): :meth:`Checkers._init_checkers` initializes timers """ - self.description_timer.start(200) self.exception_timer.start(500) self.conditions_timer.start(400) @@ -83,7 +91,6 @@ def _conditions_check(self): Periodically checks if parser requires user input, if so displays dialog with appropriate description and input options. """ - def msg_process(out: QMessageBox.StandardButton) -> Union[str, bool]: if out == QMessageBox.Yes: return True @@ -133,11 +140,20 @@ def msg_process(out: QMessageBox.StandardButton) -> Union[str, bool]: SharedVars.write_lyrics = msg_process(msg.exec_()) + if SharedVars.write_lyrics: + # show download progress + self.progressShow = QProgressDialog("Downloading lyrics", + "", 0, + self.number_of_tracks, + self) + self.progressShow.setCancelButton(None) + self._threadpool_check() + if SharedVars.switch == "api_key": log.debug("ask to get google api key") msg = QMessageBox(QMessageBox.Information, "Warning", API_KEY_MESSAGE.replace("\n", ""), - QMessageBox.Yes | QMessageBox.No | + QMessageBox.Yes | QMessageBox.No | QMessageBox.Ignore) # change button text dont_bother_button = msg.button(QMessageBox.Ignore) @@ -151,7 +167,7 @@ def msg_process(out: QMessageBox.StandardButton) -> Union[str, bool]: dialog.setInputMode(QInputDialog.TextInput) dialog.setLabelText("Key:") dialog.setWindowTitle("Paste goole API key") - dialog.resize(350, 70) + dialog.resize(350, 70) if dialog.exec_(): SharedVars.get_api_key = dialog.textValue() @@ -180,12 +196,12 @@ def _exception_check(self): displays message with information and if applicable option to exit the app """ - if SharedVars.has_exception: msg = QMessageBox(QMessageBox.Critical, "Exception", SharedVars.has_exception) # TODO set right logging file - msg.setDetailedText(open("logger","r").read()) + path = os.path.join(LOG_DIR, "wiki_music_library.log") + msg.setDetailedText(open(path, "r").read()) msg.exec_() SharedVars.has_exception = "" @@ -221,19 +237,21 @@ def _description_check(self): Runs periodically. """ + for item in ("Done", "Preload"): + if item in SharedVars.describe: + self.progressBar.setValue(self.progressBar.maximum()) + break + else: + self.progressBar.setValue(SharedVars.progress) - if " . . ." in SharedVars.describe: - SharedVars.describe = SharedVars.describe.replace(" . . .", "") + self.progressBar.setFormat(SharedVars.describe) - self._remember = SharedVars.describe + def _threadpool_check(self): - if SharedVars.describe.strip(): - if (self._remember == SharedVars.describe and - "Done" not in SharedVars.describe): # noqa E129 - SharedVars.describe += " ." - self.statusbar.showMessage(SharedVars.describe) - else: - self.statusbar.showMessage("") + self.progressShow.setValue(SharedVars.threadpool_prog) + + if self.progressShow.maximum() != SharedVars.threadpool_prog: + self.threadpool_timer.start(50) class Buttons(BaseGui): @@ -245,11 +263,19 @@ class Buttons(BaseGui): """ @exception(log) - def __open_dir__(self): - """Opens the current search folder.""" - - if self.work_dir: - self.log.info("Opening folder...") + def _open_dir(self, folder: str = ""): + """Opens the current search folder, or the one from input. + + Parameters + ---------- + folder: str + force opening this folder instead of working directory + """ + if folder: + self._log.info("Opening folder...") + os.startfile(folder) + elif self.work_dir: + self._log.info("Opening folder...") os.startfile(self.work_dir) else: QMessageBox(QMessageBox.Information, "Message", @@ -258,8 +284,8 @@ def __open_dir__(self): def _select_file(self, description: str = "Select song file", file_types: str = "Audio files (*.mp3 *.flac *.m4a)" ) -> str: - """Handle file selection - + """Handle file selection. + Parameters ---------- description: str @@ -278,7 +304,6 @@ def _select_file(self, description: str = "Select song file", self.file_detail.setText(_file[0]) return _file[0] - @exception(log) def _open_browser(self, url: Optional[str] = None): """Opens the found wikipedia page in default web browser. @@ -288,14 +313,13 @@ def _open_browser(self, url: Optional[str] = None): url: Optional[str] force opening of input url instead of parser wikipedia url """ - if url: webbrowser.open_new_tab(url) else: if self.url and not SharedVars.offline_debbug: webbrowser.open_new_tab(self.url) elif SharedVars.offline_debbug: - self.log.warning("You are in offline debugging mode") + self._log.warning("You are in offline debugging mode") else: QMessageBox(QMessageBox.Information, "Message", "You must run the search first!").exec_() @@ -310,7 +334,6 @@ def _run_Mp3tag(self): This action works only in Windods if Mp3tag is not installed in the right directory a dialog to set the path is displayed. """ - global MP3_TAG path_file = os.path.join(ROOT_DIR, "files", "MP3_TAG_PATH.txt") @@ -320,7 +343,8 @@ def _run_Mp3tag(self): if not MP3_TAG: msg = QMessageBox(QMessageBox.Warning, "Unable to locate Mp3tag", - "Do you want to set the location?", + "Mp3tag is a complementary app to this one. " + "Do you want to set its location?", QMessageBox.Yes | QMessageBox.No) if msg.exec_() == QMessageBox.No: return @@ -350,40 +374,41 @@ def _run_Mp3tag(self): def _show_help(self, help_type): - if help_type == "idx": + if help_type == "index": self._open_browser("https://wikipedia-music-tags.readthedocs.io" "/en/latest/index.html") - elif help_type == "about": - self._open_browser("https://github.com/marian-code/" - "wikipedia-music-tags") elif help_type == "git": self._open_browser("https://github.com/marian-code/" "wikipedia-music-tags") + elif help_type == "version": + QMessageBox(QMessageBox.Information, "Message", + f"You are using version: {__version__}").exec_() + elif help_type == "logs": + self._open_dir(LOG_DIR) + - def __entry_band__(self): + def _entry_band(self): """Connect to albumartist entry field.""" self.ALBUMARTIST = self.band_entry_input.text() - self.start_preload() - def __entry_album__(self): + def _entry_album(self): """Connect to album entry field.""" self.ALBUM = self.album_entry_input.text() - self.start_preload() - def __select_json__(self): + def _select_json(self): """Connect to json checkbox.""" SharedVars.write_json = self.json_write_sw.isChecked() - def __select_offline_debbug__(self): + def _select_offline_debbug(self): """Connect to offline debug checkbox. - + Note ---- - If the checkbox is unselected preload is stopped. + Restarts the preload with right settings. """ SharedVars.offline_debbug = self.offline_debbug_sw.isChecked() - if SharedVars.offline_debbug: - self.stop_preload() + if SharedVars.offline_debbug and self._input_is_present(): + self.start_preload() class Window(DataModel, Checkers, Buttons, CoverArtSearch): @@ -399,7 +424,7 @@ def __init__(self, debug): log.debug("setup overlay") # set overlay functions - self.__setup_overlay__() + self._setup_overlay() log.debug("start checkers") # start checkers @@ -407,21 +432,19 @@ def __init__(self, debug): self._start_checkers() # setup methods - def __setup_overlay__(self): + def _setup_overlay(self): """Sets up GUI input elements siganl connections.""" - # map buttons to functions # must use lambda otherwise wrapper doesnt work correctly - self.browse_button.clicked.connect(lambda: self.__select_dir__()) - self.wiki_search_button.clicked.connect(lambda: self.__run_search__()) + self.browse_button.clicked.connect(lambda: self._select_dir()) + self.wiki_search_button.clicked.connect(lambda: self._run_search()) self.coverArt.clicked.connect(lambda: self.cover_art_search()) - self.lyrics_button.clicked.connect( - lambda: self.__run_lyrics_search__()) + self.lyrics_button.clicked.connect(lambda: self._run_lyrics_search()) self.toolButton.clicked.connect(lambda: self._select_file()) # connect the album and band entry field to preload function - self.band_entry_input.editingFinished.connect(self.__entry_band__) - self.album_entry_input.editingFinished.connect(self.__entry_album__) + self.band_entry_input.editingFinished.connect(self._entry_band) + self.album_entry_input.editingFinished.connect(self._entry_album) # create table-proxy mapping for sorting self.tableView.setModel(self.proxy) @@ -441,41 +464,55 @@ def __setup_overlay__(self): # connect menubar actions to functions # must use lambda otherwise wrapper doesnt work correctly - self.actionDirectory.triggered.connect(lambda: self.__open_dir__()) + self.actionDirectory.triggered.connect(lambda: self._open_dir()) self.actionWikipedia.triggered.connect(lambda: self._open_browser()) self.actionMp3_Tag.triggered.connect(lambda: self._run_Mp3tag()) - self.actionAll_Tags.triggered.connect(lambda: self.__save_all__()) - # TODO get rid of save only lyrics button, we now save inteligently - # TODO only changed tags - self.actionOnly_Lyrics.triggered.connect(lambda: self.__save_all__()) + + # save action + self.actionAll_Tags.triggered.connect(lambda: self._save_all()) # show help - self.actionHelp_Index.triggered.connect(lambda: self._show_help("idx")) - self.actionAbout.triggered.connect(lambda: self._show_help("about")) + self.actionHelp.triggered.connect(lambda: self._show_help("index")) self.actionGit.triggered.connect(lambda: self._show_help("git")) + self.actionLogs.triggered.connect(lambda: self._show_help("logs")) + self.actionVersion.triggered.connect( + lambda: self._show_help("version")) + + # search actons + self.actionWikipedia.triggered.connect(lambda: self._run_search()) + self.actionLyrics.triggered.connect(lambda: self.cover_art_search()) + self.actionCoverArt.triggered.connect( + lambda: self._run_lyrics_search()) + + # main actions + self.actionOpen.triggered.connect(lambda: self._select_dir()) # TODO menubar buttons taht are not implemented self.actionNew.triggered.connect(self._do_nothing) - self.actionOpen.triggered.connect(self._do_nothing) self.actionExit.triggered.connect(self._do_nothing) self.actionSave.triggered.connect(self._do_nothing) - # TODO this needs to be hidden alfo for pip + # add starusbar to progress bar + self.progressBar = QProgressBar() + self.progressBar.setTextVisible(True) + self.progressBar.setAlignment(Qt.AlignCenter) + self.progressBar.setFormat("") + + self.statusbar.addPermanentWidget(self.progressBar) + if not self._DEBUG: - # show switches only if not frozen self.offline_debbug_sw.hide() self.json_write_sw.hide() else: # connect switches to functions self.offline_debbug_sw.stateChanged.connect( - self.__select_offline_debbug__) - self.json_write_sw.stateChanged.connect(self.__select_json__) + self._select_offline_debbug) + self.json_write_sw.stateChanged.connect(self._select_json) # methods that bind to gui elements @exception(log) - def __save_all__(self): + def _save_all(self): """Save changes to file tags, after saving reload files from disk.""" - # first stop any running preload, as it is not needed any more self.stop_preload() @@ -489,11 +526,17 @@ def __save_all__(self): "You must specify directory with files!").exec_() return + # show save progress + self.progressShow = QProgressDialog("Writing tags", "", 0, + self.number_of_tracks, self) + self.progressShow.setCancelButton(None) + self._threadpool_check() + if not self.write_tags(): msg = ("Cannot write tags because there are no " "coresponding files") QMessageBox(QMessageBox.Information, "Info", msg) - self.log.info(msg) + self._log.info(msg) # reload files from disc after save self.reinit_parser() @@ -501,11 +544,16 @@ def __save_all__(self): self._parser_to_gui() @exception(log) - def __select_dir__(self): + def _select_dir(self): """Select working directory, read found files and start preload.""" - with RememberDir(self) as rd: - self.work_dir = rd.get_dir() + self.work_dir, load_ok = rd.get_dir() + + # if dialog was canceled, return imediatelly + if not load_ok: + return + + self._init_progress_bar(0, 2) # TODO non-atomic # read files and start preload @@ -514,54 +562,28 @@ def __select_dir__(self): self._parser_to_gui() - def __check_input_is_present__(self) -> bool: - """Check if apropriate input to conduct search is present. + def _init_progress_bar(self, minimum: int, maximum: int): + """Resets main progresbar to 0 and sets range. - Returns - ------- - bool - true if all requested inputs are present + Parameters + ---------- + minimum: int + minimal progressbar value + maximum: int + maximum progressbar value """ - - group = [self.ALBUMARTIST, self.ALBUM, self.work_dir] - if all(group): - return True - else: - alb, bnd, wkd, inp, com, _and = [""] * 6 - if not self.ALBUM: - alb = " album" - if not self.ALBUMARTIST: - bnd = " band" - if not self.work_dir: - wkd = " select working directory" - - if any([alb, bnd]): - inp = "input" - if all([alb, bnd, wkd]): - com = "," - elif all([alb, bnd]): - com = " and" - if all([alb, wkd]) or all([wkd, bnd]): - _and = " and" - - msg = f"You must {inp}{alb}{com}{bnd}{_and}{wkd}" - - message = QMessageBox() - message.setIcon(QMessageBox.Information) - message.setWindowTitle("Message") - message.setText(msg) - message.exec_() - self.log.warning(msg) - - return False + SharedVars.progress = minimum + self.progressBar.setRange(minimum, maximum) + self.progressBar.setValue(minimum) @exception(log) - def __run_search__(self): + def _run_search(self): """Start wikipedia search in background thread.""" - - if not self.__check_input_is_present__(): + if not self._input_is_present(with_warn=True): return + self._init_progress_bar(0, 16) + log.info("starting wikipedia search") # TODO non-atomic @@ -573,14 +595,21 @@ def __run_search__(self): main_app.start() @exception(log) - def __run_lyrics_search__(self): + def _run_lyrics_search(self): """Start only lyric search in background thread.""" - self.stop_preload() - if not self.__check_input_is_present__(): + if not self._input_is_present(with_warn=True): return + self._init_progress_bar(0, 2) + + # show download progress + self.progressShow = QProgressDialog("Downloading lyrics", "", 0, + self.number_of_tracks, self) + self.progressShow.setCancelButton(None) + self._threadpool_check() + log.info("starting lyrics search") SharedVars.write_lyrics = True diff --git a/wiki_music/gui_lib/qt_importer.py b/wiki_music/gui_lib/qt_importer.py index 18b99f4..1cdb907 100644 --- a/wiki_music/gui_lib/qt_importer.py +++ b/wiki_music/gui_lib/qt_importer.py @@ -1,4 +1,4 @@ -""" Module for importing all Qt classes used by GUI. +"""Module for importing all Qt classes used by GUI. Note ---- @@ -6,29 +6,46 @@ Pyside2 and PySide. """ -# debug readthedocs build +import logging + +log = logging.getLogger(__name__) +# This exception is recurring quite often so here are imports to debug it +# normally these are hiddnen because they are caught in qtpy +""" try: - from PyQt5.QtCore import PYQT_VERSION_STR + from PyQt5.QtCore import PYQT_VERSION_STR as PYQT_VERSION except ImportError as e: - print(e) + log.exception(e) try: - from PyQt5.QtCore import QT_VERSION_STR + from PyQt5.QtCore import QT_VERSION_STR as QT_VERSION except ImportError as e: - print(e) + log.exception(e) try: - from PySide2 import __version__ + from PySide2 import __version__ as PYSIDE_VERSION except ImportError as e: - print(e) + log.exception(e) try: - from PySide2.QtCore import __version__ + from PySide2.QtCore import __version__ as QT_VERSION except ImportError as e: - print(e) -# end of reradthedocs debug - -import logging - -log = logging.getLogger(__name__) + log.exception(e) +try: + from PyQt4.Qt import PYQT_VERSION_STR as PYQT_VERSION +except ImportError as e: + log.exception(e) +try: + from PyQt4.Qt import QT_VERSION_STR as QT_VERSION +except ImportError as e: + log.exception(e) +try: + from PySide import __version__ as PYSIDE_VERSION +except ImportError as e: + log.exception(e) +try: + from PySide.QtCore import __version__ as QT_VERSION +except ImportError as e: + log.exception(e) +""" try: from qtpy.QtWidgets import (QMainWindow, QFileDialog, QApplication, @@ -36,7 +53,8 @@ QAbstractItemView, QInputDialog, QLabel, QVBoxLayout, QTableWidget, QWidget, QDialog, QStatusBar, QSystemTrayIcon, QSizePolicy, - QRubberBand, QStyleFactory, QSizeGrip) + QRubberBand, QStyleFactory, QSizeGrip, + QProgressBar, QProgressDialog) from qtpy.QtGui import (QStandardItemModel, QStandardItem, QImage, QPixmap, QIcon, QPainter, QResizeEvent) from qtpy.QtCore import (Qt, QSortFilterProxyModel, QTimer, QObject, @@ -45,7 +63,7 @@ QModelIndex, QVariant) from qtpy import uic except ImportError as e: - log.critical("None of the Qt backends is available! Aborting") - raise ImportError("None of the Qt backends is available! Aborting") + log.critical(f"None of the Qt backends is available: {e}") + raise ImportError(f"None of the Qt backends is available: {e}") else: log.debug("Qt imports done") diff --git a/wiki_music/library/lyrics.py b/wiki_music/library/lyrics.py index 5198150..9aeef24 100644 --- a/wiki_music/library/lyrics.py +++ b/wiki_music/library/lyrics.py @@ -27,7 +27,7 @@ def save_lyrics(tracks: List[str], types: List[str], band: str, album: str, GUI: bool) -> List[str]: - """ This function does some preprocessing before it starts the lyricsfinder + """This function does some preprocessing before it starts the lyricsfinder module and downloads the lyrics. In preproces, tracks which will have same lyrics are identified so the same lyrics are not downloaded twice. The lyrics are then downloaded asynchronously each in separate thread for @@ -130,7 +130,7 @@ def save_lyrics(tracks: List[str], types: List[str], band: str, album: str, def _get_lyrics(manager: 'LyricsManager', artist: str, album: str, song: str, GOOGLE_API_KEY: str ) -> Dict[str, Optional[Union[str, Dict[str, str]]]]: - """ Function which calls lyricsfinder.LyricsManager.search_lyrics method + """Function which calls lyricsfinder.LyricsManager.search_lyrics method to find and download lyrics for specified song. See also diff --git a/wiki_music/library/parser/__init__.py b/wiki_music/library/parser/__init__.py index 4890330..7d3bfa5 100644 --- a/wiki_music/library/parser/__init__.py +++ b/wiki_music/library/parser/__init__.py @@ -13,7 +13,7 @@ class WikipediaRunner(WikipediaParser): - """ Toplevel Wikipedia Parser class which inherits all other parser + """Toplevel Wikipedia Parser class which inherits all other parser subclasses. This is the class that is intended for user interaction. Its methods know how to run the parser in order to produce meaningfull results. @@ -35,22 +35,22 @@ def __init__(self, GUI: bool = True, protected_vars: bool = True) -> None: log.debug("init parser runner") super().__init__(protected_vars=protected_vars) - self.GUI = GUI + self._GUI = GUI self.with_log = False log.debug("init parser runner done") @exception(log) def run_wiki(self): - """ Runs the whole wikipedia search, together with lyrics finding. """ + """Runs the whole wikipedia search, together with lyrics finding""" - if self.GUI: + if self._GUI: self._run_wiki_gui() else: self._run_wiki_nogui() def _run_wiki_gui(self): - """ Runs wikipedia search with specifics of the GUI mode. """ + """Runs wikipedia search with specifics of the GUI mode""" def wait_select(switch: str): SharedVars.switch = switch SharedVars.wait = True @@ -59,39 +59,36 @@ def wait_select(switch: str): # download wikipedia page if not SharedVars.offline_debbug: - self.log.info(f"Searching for: {self.album} by {self.band}") + self._log.info(f"Searching for: {self.ALBUM} by " + f"{self.ALBUMARTIST}") else: - self.log.info("Using offline cached page insted of web page") + self._log.info("Using offline cached page insted of web page") error_msg = self.get_wiki() if error_msg: - self.log.exception(error_msg) + self._log.exception(error_msg) return else: - self.log.info(f"Found at: {self.url}") - self.log.info("Cooking Soup") + self._log.info(f"Found at: {self.url}") + self._log.info("Cooking Soup") error_msg = self.cook_soup() if error_msg: - self.log.exception(error_msg) + self._log.exception(error_msg) return else: - self.log.info("Soup ready") - - # get page contents - self.get_contents() + self._log.info("Soup ready") # find release date - self.get_release_date() - self.log.info(f"Found release date {self.release_date}") + self._log.info(f"Found release date: {self.get_release_date()}") # find list of genres self.get_genres() - self.log.info(f"Found genre(s): {', '.join(self.genres)}") + self._log.info(f"Found genre(s): {', '.join(self.genres)}") # download cover art from wikipedia - self.log.info("Downloading cover art") + self._log.info("Downloading cover art") self.get_cover_art() if not we_are_frozen(): @@ -99,47 +96,32 @@ def wait_select(switch: str): self.basic_out() # print out page contents - self.log.info(f"Found page contents: {', '.join(self.contents)}") + self._log.info(f"Found page contents: {', '.join(self.get_contents())}") # extract track list - self.log.info("Extracting tracks") - self._get_tracks() + self._log.info("Extracting tracks") + self.get_tracks() # extract personel names - self.log.info("Extracting additional personnel") + self._log.info("Extracting additional personnel") self.get_personnel() - # complete artists names - self.complete() - - # look for aditional artists in brackets behind track names and - # complete artists names again with new info - self.log.info("Getting tracks info") - self.info_tracks() - # extract writers, composers - self.log.info("Extracting composers") + self._log.info("Extracting composers") self.get_composers() - # complete artists names - self.complete() - if not we_are_frozen(): # save to files - self.log.info("Writing to disc") + self._log.info("Writing to disc") self.disk_write() - # merge artists and personel which have some apperences - self.log.info("merge artists and personnel") - self.merge_artist_personnel() - # select genre - self.log.info("Select genre") - if not self.selected_genre: + self._log.info("Select genre") + if not self.GENRE: wait_select("genres") # decide what to do with artists - self.log.info("Assign artists to composers") + self._log.info("Assign artists to composers") # first load already known values to GUI # wait must be set before load, @@ -154,7 +136,7 @@ def wait_select(switch: str): self.merge_artist_composers() # decide if you want to find lyrics - self.log.info("Searching for Lyrics") + self._log.info("Searching for Lyrics") wait_select("lyrics") self.save_lyrics() @@ -162,18 +144,18 @@ def wait_select(switch: str): # announce that main app thread has reached the barrier SharedVars.barrier.wait() - self.log.info("Done") + self._log.info("Done") def _run_wiki_nogui(self): - """ Runs wikipedia search with specifics of the CLI mode. """ + """Runs wikipedia search with specifics of the CLI mode""" # download wikipedia page if not SharedVars.offline_debbug: self._log_print(msg_WHITE="Accessing Wikipedia...") - print("Searching for: " + GREEN + self.album + RESET + " by " + - GREEN + self.band) + print("Searching for: " + GREEN + self.ALBUM + RESET + " by " + + GREEN + self.ALBUMARTIST) else: self._log_print(msg_GREEN="Using offline cached page insted " @@ -197,13 +179,9 @@ def _run_wiki_nogui(self): else: self._log_print(msg_WHITE="Soup ready") - # get page contents - self.get_contents() - # find release date - self.get_release_date() - self._log_print(msg_GREEN="Found release date", - msg_WHITE=self.release_date) + self._log_print(msg_GREEN="Found release date:", + msg_WHITE=self.get_release_date()) # find list of genres self.get_genres() @@ -214,12 +192,12 @@ def _run_wiki_nogui(self): # basic html textout for debug self.basic_out() - # print out page contents + # get and print out page contents self._log_print(msg_GREEN="Found page contents", - msg_WHITE="\n".join(self.contents)) + msg_WHITE="\n".join(self.get_contents())) # extract track list - self._get_tracks() + self.get_tracks() # extract personel names self._log_print(msg_GREEN="Extracting additional personnel") @@ -230,22 +208,11 @@ def _run_wiki_nogui(self): if not we_are_frozen(): print(self.personnel_2_str()) - # complete artists names - self.complete() - - # look for aditional artists in brackets behind track names and - # complete artists names again with new info - self.info_tracks() - # extract writers, composers self._log_print(msg_GREEN="Extracting composers") - self.get_composers() - - # complete artists names - self.complete() self._log_print(msg_GREEN="Found composers", - msg_WHITE="\n".join(flatten_set(self.composers))) + msg_WHITE="\n".join(flatten_set(self.get_composers()))) if not we_are_frozen(): # save to files @@ -256,11 +223,8 @@ def _run_wiki_nogui(self): self._log_print(msg_GREEN="Found Track list(s)") self.print_tracklist() - # merge artists and personel which have some apperences - self.merge_artist_personnel() - # select genre - if not self.selected_genre: + if not self.GENRE: if not self.genres: print(CYAN + "Input genre:", end="") self.genre = input() @@ -276,7 +240,7 @@ def _run_wiki_nogui(self): except ValueError: index = 0 - self.selected_genre = self.genres[index] + self.GENRE = self.genres[index] # decide what to do with artists print(CYAN + "Do you want to assign artists to composers? ([y]/n)", @@ -305,29 +269,29 @@ def _run_wiki_nogui(self): @exception(log) def run_lyrics(self): - """ Runs only the lyrics search. """ + """Runs only the lyrics search""" - if self.GUI: + if self._GUI: self._run_lyrics_gui() else: self._run_lyrics_nogui() def _run_lyrics_gui(self): - """ Runs only lyrics search with specifics of the GUI mode. """ + """Runs only lyrics search with specifics of the GUI mode""" - self.log.info("Searching for lyrics") + self._log.info("Searching for lyrics") self.save_lyrics() - self.log.info("Done") + self._log.info("Done") SharedVars.done = True - self.log.info("wait barrier") + self._log.debug("wait barrier") # announce that main app thread has reached the barrier SharedVars.barrier.wait() def _run_lyrics_nogui(self): - """ Runs only lyrics search with specifics of the CLI mode. """ + """Runs only lyrics search with specifics of the CLI mode""" self.read_files() @@ -346,7 +310,7 @@ def _log_print(self, msg_GREEN: str = "", msg_WHITE: str = "", level: str = "INFO"): - """ Redirects the input to sandard print function and to logger. + """Redirects the input to sandard print function and to logger. Parameters ---------- diff --git a/wiki_music/library/parser/base.py b/wiki_music/library/parser/base.py index 7c359f2..9886137 100644 --- a/wiki_music/library/parser/base.py +++ b/wiki_music/library/parser/base.py @@ -1,4 +1,4 @@ -""" Base module for all parser classes from which they import ParserBase +"""Base module for all parser classes from which they import ParserBase class that sets all the default attributes. """ @@ -41,30 +41,30 @@ class ParserBase: Attributes ---------- - contents: List[str] + _contents: List[str] stores the wikipedia page contents - disk_sep: List[int] + _disk_sep: List[int] list of tracks separating disks e.g. if CD 1 = (1, 13) and - CD 2 = (14, 20), disk_sep = [0, 12, 19] the offset by one if because of + CD 2 = (14, 20), _disk_sep = [0, 12, 19] the offset by one if because of zero first index - disks: List[list] + _disks: List[list] holds album disks titles genres: List[str] list of genres found in wikipedia page - header: List[str] + _header: List[str] tracklist table headers NLTK_names: List[str] list of Person Named Entities extracted from wikipedia page by nltk. - See :meth:`wiki_music.library.parser.process_page.WikipediaParser.extract_names` + See :meth:`wiki_music.library.parser.process_page.WikipediaParser._extract_names` for details on how and from which parts of test the names are extracted - personnel: List[str] + _personnel: List[str] list holding adittional personnel participating on album - appearences: List[List[int]] + _appearences: List[List[int]] list coresponding to personnel holding for each person list of tracks that the said person has appeared on - subtracks: List[List[str]] + _subtracks: List[List[str]] each entry holds list of subtracks for one track - sub_types: List[List[str]] + _subtypes: List[List[str]] each entry holds list of types for each subtrack work_dir: str string with path to directory with music files, this variable can be @@ -72,7 +72,7 @@ class ParserBase: log: :class:`wiki_music.utilities.utils.MultiLog` instance of MUltiLog which sends messages to logger and :class:`wiki_music.utilities.sync.SharedVars` - sections: Dict[str, List[Bs4Soup]] + _sections: Dict[str, List[Bs4Soup]] dictionary of lists of BeautifulSoup objects each entry in the dict contains one whole section of the page and is indexed by that section title @@ -80,114 +80,114 @@ class ParserBase: files: SList bracketed_types: SList - sections: Dict[str, List["Bs4Soup"]] - page: "WikiPage" - soup: "Bs4Soup" + _sections: Dict[str, List["Bs4Soup"]] + _page: "WikiPage" + _soup: "Bs4Soup" def __init__(self, protected_vars: bool) -> None: log.debug("parser base") # lists 1D - self.contents: SList = [] - self.tracks: SList = [] - self.types: SList = [] - self.disc_num: IList = [] - self.disk_sep: IList = [] - self.disks: List[list] = [] + self._contents: SList = [] + self._tracks: SList = [] + self._types: SList = [] + self._disc_num: IList = [] + self._disk_sep: IList = [] + self._disks: List[list] = [] self.genres: SList = [] - self.header: SList = [] - self.lyrics: SList = [] - self.NLTK_names: SList = [] - self.numbers: SList = [] - self.personnel: SList = [] + self._header: SList = [] + self._lyrics: SList = [] + self._NLTK_names: SList = [] + self._numbers: SList = [] + self._personnel: SList = [] self._bracketed_types: SList = [] self._files: SList = [] # lists 2D - self.appearences: NIList = [] - self.artists: NSList = [] - self.composers: NSList = [] - self.sub_types: NSList = [] - self.subtracks: NSList = [] + self._appearences: NIList = [] + self._artists: NSList = [] + self._composers: NSList = [] + self._subtypes: NSList = [] + self._subtracks: NSList = [] # bytearray self.cover_art: bytearray = bytearray() # strings - self.release_date: str = "" - self.selected_genre: str = "" + self._release_date: str = "" + self._selected_genre: str = "" # atributes protected from GUIs reinit method # when new search is started if protected_vars: - self.album: str = "" - self.band: str = "" + self._album: str = "" + self._band: str = "" self.work_dir: str = "" - self.log: MultiLog = MultiLog(log) - self.GUI = False + self._log: MultiLog = MultiLog(log) + self._GUI = False - self.log.debug("parser base done") + self._log.debug("parser base done") def __len__(self): - return len(self.numbers) + return len(self._numbers) def __bool__(self): return bool(self.__len__()) @property def ALBUM(self) -> str: - """ string with album name, this variable can be protected from + """string with album name, this variable can be protected from reseting in __init__ method. :type: str """ - return self.album + return self._album @ALBUM.setter def ALBUM(self, value: str): - self.album = value + self._album = value @property def ALBUMARTIST(self) -> str: - """ string with band name, this variable can be protected from reseting + """string with band name, this variable can be protected from reseting in __init__ method. :type: str """ - return self.band + return self._band @ALBUMARTIST.setter def ALBUMARTIST(self, value: str): - self.band = value + self._band = value @property def ARTIST(self) -> NSList: - """ Each entry in list holds list of artists for one track. + """Each entry in list holds list of artists for one track. :type: List[List[str]] """ - return self.artists + return self._artists @ARTIST.setter def ARTIST(self, value: NSList): - self.artists = value + self._artists = value @property def COMPOSER(self) -> NSList: - """ Each entry in list holds list of composers for one track. + """Each entry in list holds list of composers for one track. :type: List[List[str]] """ - return self.composers + return self._composers @COMPOSER.setter def COMPOSER(self, value: NSList): - self.composers = value + self._composers = value @property def COVERART(self) -> bytearray: - """ Holds coverart read into memory as a bytes object. + """Holds coverart read into memory as a bytes object. :type: bytearray """ @@ -199,15 +199,15 @@ def COVERART(self, value: bytearray): @property def DATE(self) -> str: - """ Album release date. + """Album release date. :type: str """ - return self.release_date + return self._release_date @DATE.setter def DATE(self, value: str): - self.release_date = value + self._release_date = value @property def DISCNUMBER(self) -> IList: @@ -215,11 +215,11 @@ def DISCNUMBER(self) -> IList: :type: List[int] """ - return self.disc_num + return self._disc_num @DISCNUMBER.setter def DISCNUMBER(self, value: IList): - self.disc_num = value + self._disc_num = value @property def GENRE(self) -> str: @@ -228,11 +228,11 @@ def GENRE(self) -> str: :type: str """ - return self.selected_genre + return self._selected_genre @GENRE.setter def GENRE(self, value: str): - self.selected_genre = value + self._selected_genre = value @property def LYRICS(self) -> SList: @@ -240,39 +240,39 @@ def LYRICS(self) -> SList: :type: List[str] """ - return self.lyrics + return self._lyrics @LYRICS.setter def LYRICS(self, value: SList): - self.lyrics = value + self._lyrics = value @property def TITLE(self) -> SList: - """ List of track names. + """List of track names. :type: List[str] """ - return self.tracks + return self._tracks @TITLE.setter def TITLE(self, value: SList): - self.tracks = value + self._tracks = value @property def TRACKNUMBER(self) -> SList: - """ List of numbers for each track. + """List of numbers for each track. :type: List[str] """ - return self.numbers + return self._numbers @TRACKNUMBER.setter def TRACKNUMBER(self, value: SList): - self.numbers = value + self._numbers = value @property def FILE(self) -> SList: - """ List of files on local disk corresponding to each track. + """List of files on local disk corresponding to each track. :type: List[str] """ @@ -284,7 +284,7 @@ def FILE(self, value: SList): @property def TYPE(self) -> SList: - """ List of track types. + """List of track types. :type: List[str] """ diff --git a/wiki_music/library/parser/in_out.py b/wiki_music/library/parser/in_out.py index ecf5300..637c008 100644 --- a/wiki_music/library/parser/in_out.py +++ b/wiki_music/library/parser/in_out.py @@ -1,3 +1,5 @@ +"""Module with parser inpu-output methods.""" + import logging import os import pickle # lazy loaded @@ -23,10 +25,13 @@ class ParserInOut(ParserBase): - """ Class that is inherited by + """Encapsulates parser input and output methods. + + Class is inherited by :class:`wiki_music.library.parser.process_page.WikipediaParser`. Takes care of outputing and loading information. """ + def __init__(self, protected_vars): super().__init__(protected_vars=protected_vars) @@ -34,12 +39,12 @@ def __init__(self, protected_vars): self._debug_folder: str = "" @abstractmethod - def info_tracks(self): + def _info_tracks(self): raise NotImplementedError("Call to abstract method") @property def bracketed_types(self) -> List[str]: # type: ignore - """ takes `_bracketed_types` list, populates and returns it. + """Takes `_bracketed_types` list, populates and returns it. See also -------- @@ -48,27 +53,24 @@ def bracketed_types(self) -> List[str]: # type: ignore :type: List[str] """ - if not self._bracketed_types: - self._bracketed_types = bracket(self.types) - return self._bracketed_types - else: - return self._bracketed_types + self._bracketed_types = bracket(self._types) + + return self._bracketed_types @property # type: ignore def files(self) -> List[str]: # type: ignore - """ Gets list of music files in currently set working directory. + """Gets list of music files in currently set working directory. :type: List[str] See also -------- - :meth:`reassign_files` + :meth:`_reassign_files` """ - - if len(self._files) < len(self.tracks): - self.reassign_files() + if len(self._files) < len(self._tracks) or not self._files: + self._reassign_files() return self._files @@ -82,92 +84,84 @@ def debug_folder(self) -> str: :type: str """ - if not self._debug_folder: - _win_name = win_naming_convetion(self.album, dir_name=True) + _win_name = win_naming_convetion(self._album, dir_name=True) self._debug_folder = os.path.join(OUTPUT_FOLDER, _win_name) os.makedirs(self._debug_folder, exist_ok=True) return self._debug_folder - def list_files(self): - """Lists files in current working directory. + def _reassign_files(self): + """Search current working directory and assign files to tracks.""" + wnc = win_naming_convetion - See also - -------- - :func:`wiki_music.utilities.utils.list_files` - function used for file searching - :attr:`work_dir` - """ - self.files = list_files(self.work_dir) + # write data to ID3 tags + disk_files = list_files(self.work_dir) - def reassign_files(self): - """Searched the current working directory and tries to assign files to - the tracks. - """ + # max() argument must have len >= 1 + if self._tracks: + max_length = len(max(self._tracks, key=len)) - wnc = win_naming_convetion - max_length = len(max(self.tracks, key=len)) + files = [] - # write data to ID3 tags - disk_files = (self.work_dir) - files = [] + print(GREEN + "\nFound files:") + print(*disk_files, sep="\n") + print(GREEN + "\nAssigning files to tracks:") - print(GREEN + "\nFound files:") - print(*disk_files, sep="\n") - print(GREEN + "\nAssigning files to tracks:") + for i, tr in enumerate(self._tracks): + self._tracks[i] = tr.strip() - for i, tr in enumerate(self.tracks): - self.tracks[i] = tr.strip() + for path in disk_files: + f = os.path.split(path)[1] + if (nc(wnc(tr)) in nc(f) and nc(self._types[i]) in nc(f)): # noqa E129 - for path in disk_files: - f = os.path.split(path)[1] - if (nc(wnc(tr)) in nc(f) - and nc(self.types[i]) in nc(f)): # noqa E129 + print(LBLUE + tr + RESET, + "-" * (1 + max_length - len(tr)) + ">", path) + files.append(path) + break + else: + print(YELLOW + tr + RESET, + "." * (2 + max_length - len(tr)), + "Does not have a matching file!") - print(LBLUE + tr + RESET, - "-" * (1 + max_length - len(tr)) + ">", path) - files.append(path) - break - else: - print(YELLOW + tr + RESET, "." * (2 + max_length - len(tr)), - "Does not have a matching file!") + files.append(None) - files.append(None) + self.files = files - self.files = files + else: + self.files = disk_files def basic_out(self): - """ Outputs files in three basic formats:\n + """Outputs files in three basic formats. + 1. pickled version of the downloaded wikipedia page 2. nicely formated html version of the wikipedia page 3. plain text version of the wikipedia page """ - + # ensure directory for results storing exists os.makedirs(self.debug_folder, exist_ok=True) # save page object for offline debbug fname = os.path.join(self.debug_folder, 'page.pkl') if not os.path.isfile(fname): with open(fname, 'wb') as f: - pickle.dump(self.page, f) + pickle.dump(self._page, f) # save formated html to file fname = os.path.join(self.debug_folder, 'page.html') if not os.path.isfile(fname): with open(fname, 'w', encoding='utf8') as f: - f.write(self.soup.prettify()) + f.write(self._soup.prettify()) # save html converted to text fname = os.path.join(self.debug_folder, 'page.txt') if not os.path.isfile(fname): with open(fname, 'w', encoding='utf8') as f: - f.write(self.soup.get_text()) + f.write(self._soup.get_text()) def disk_write(self): - """ Saves tracklist and album personnel to disk in plain text formats. - """ - + """Save tracklist and personnel to disk in plain text format.""" + # save tracklist to file for i, tracklist in enumerate(self.tracklist_2_str(to_file=True), 1): fname = os.path.join(self.debug_folder, f"tracklist_{i}.txt") with open(fname, "w", encoding="utf-8") as f: @@ -181,7 +175,7 @@ def disk_write(self): f.write(self.personnel_2_str()) def tracklist_2_str(self, to_file=True) -> list: - """ Convert tracklist to string to print out or write to disk. + """Convert tracklist to string to print out or write to disk. Parameters ---------- @@ -201,14 +195,14 @@ def _set_color(): return GREEN, LGREEN, RESET # compute number of spaces, wrong mypy detection - spaces, length = count_spaces(self.tracks, + spaces, length = count_spaces(self._tracks, self.bracketed_types) # type: ignore G, LG, R = _set_color() # convert to string tracklists = [] - for j, (ds, hd) in enumerate(zip(self.disks, self.header)): + for j, (ds, hd) in enumerate(zip(self._disks, self._header)): s = "" s += f"{G}\n{j + 1}. Disk:{R} {ds}" if len(hd) >= 2: @@ -219,13 +213,13 @@ def _set_color(): s += f", {hd[3]}" s += f"{R}\n" - for i in range(self.disk_sep[j], self.disk_sep[j + 1]): - s += (f"{self.numbers[i]:>2}. {self.tracks[i]} " + for i in range(self._disk_sep[j], self._disk_sep[j + 1]): + s += (f"{self._numbers[i]:>2}. {self._tracks[i]} " f"{self.bracketed_types[i]}{spaces[i]} " - f"{', '.join(self.artists[i] + self.composers[i])}\n") + f"{', '.join(self._artists[i] + self._composers[i])}\n") for k, (sbtr, sbtp) in enumerate( - zip(self.subtracks[i], self.sub_types[i])): + zip(self._subtracks[i], self._subtypes[i])): s += (f" {write_roman(k + 1)}. {sbtr} {sbtp}\n") tracklists.append(s) @@ -233,45 +227,43 @@ def _set_color(): return tracklists def print_tracklist(self): - """ Prints tracklist to console. + """Prints tracklist to console. See also -------- :func:`tracklist_2_str` """ - print("\n".join(self.tracklist_2_str(to_file=False))) def personnel_2_str(self): - """ Convert album personnel to string to print out or write to disk. + """Convert album personnel to string to print out or write to disk. Returns ------- str nicely formated string representation of personnel """ - s = "" - if not self.personnel: + if not self._personnel: s += "---\n" - for pers, app in zip(self.personnel, self.appearences): + for pers, app in zip(self._personnel, self._appearences): if app: s += pers + " - " temp = 1000 for k, a in enumerate(sorted(app)): - for j, _ in enumerate(self.disk_sep[:-1]): + for j, _ in enumerate(self._disk_sep[:-1]): - if (a >= self.disk_sep[j] - and a < self.disk_sep[j + 1]): # noqa E129 + if (a >= self._disk_sep[j] + and a < self._disk_sep[j + 1]): # noqa E129 if j != temp: - s += f"{self.disks[j]}: {self.numbers[a]}" + s += f"{self._disks[j]}: {self._numbers[a]}" temp = j else: - s += self.numbers[a] + s += self._numbers[a] if k != len(app) - 1: s += ", " @@ -287,9 +279,10 @@ def personnel_2_str(self): def data_to_dict(self ) -> List[Dict[str, Union[str, int, bytearray, list]]]: - """ Converts parser data to list of dictionaries. If yaml_dump is - enabled list is written to - + """Converts parser data to list of dictionaries. + + If yaml_dump is enabled list is written to file. + Warnings -------- This class is not ment to be instantiated, only inherited. @@ -305,9 +298,8 @@ def data_to_dict(self List[Dict[str, Union[str, int, bytearray, list]]] each dictionary in list represents tags of one song """ - dict_data = [] - for i, _ in enumerate(self.tracks): + for i, _ in enumerate(self._tracks): tags = dict() @@ -327,8 +319,7 @@ def data_to_dict(self return dict_data def write_tags(self) -> bool: - """ Write tags to coresponding files. Writing is done in a parallel - maner to speed things up. + """Write tags to coresponding files. Writing is done in a parallel. See also -------- @@ -344,7 +335,6 @@ class that handles paralelism bool If writing was successfull return true value """ - if not any(self.files): return False else: @@ -354,8 +344,7 @@ class that handles paralelism return True def save_lyrics(self): - """Function that calls lyricsfinder to search for and save lyrics for - all tracks. + """Calls lyricsfinder to search for and save lyrics for all tracks. See also -------- @@ -367,29 +356,23 @@ def save_lyrics(self): lyrics search is controled by :attr:`wiki_music.utilities.sync.SharedVars.write_lyrics` """ - if SharedVars.write_lyrics: - self.lyrics = save_lyrics(self.tracks, self.types, self.band, - self.album, self.GUI) + self._lyrics = save_lyrics(self._tracks, self._types, self._band, + self._album, self._GUI) else: - self.lyrics = [""] * len(self) + self._lyrics = [""] * len(self) def read_files(self): - """ Read tags from files in working directory. + """Read tags from files in working directory. See also -------- - :meth:`list_files` - method that takes care of file searching :func:`wiki_music.library.tags_io.read_tags` function that thandles tag reading """ - # initialize variables self.__init__(protected_vars=False) - self.list_files() - for fl in self.files: for key, value in read_tags(fl).items(): @@ -407,13 +390,13 @@ def read_files(self): if self.files: # look for aditional artists in brackets behind track names and # complete artists names again with new info - self.info_tracks() + self._info_tracks() - self.log.info("Files loaded sucesfully") + self._log.info("Files loaded sucesfully") else: - self.album = "" - self.band = "" - self.selected_genre = "" - self.release_date = "" + self._album = "" + self._band = "" + self._selected_genre = "" + self._release_date = "" - self.log.info("No music files to Load") + self._log.info("No music files to Load") diff --git a/wiki_music/library/parser/preload.py b/wiki_music/library/parser/preload.py index 5e1274f..1cf8cf9 100644 --- a/wiki_music/library/parser/preload.py +++ b/wiki_music/library/parser/preload.py @@ -54,25 +54,26 @@ class WikiCooker(ParserBase): ---------- page: wikipedia.WikipediaPage downloaded page to be parsed by BeautifulSoup - soup: bs4.BeautifulSoup + _soup: bs4.BeautifulSoup BeautibulSoup object representing the whole page - sections: Dict[str, bs4.BeautifulSoup] - the :attr:`soup` split to page sections indexed by their titles + _sections: Dict[str, bs4.BeautifulSoup] + the :attr:`_soup` split to page sections indexed by their titles """ + def __init__(self, protected_vars: bool) -> None: super().__init__(protected_vars=protected_vars) - self.log.debug("cooker imports") + self._log.debug("cooker imports") if protected_vars: - self.sections = {} - self.page = None - self.soup = None + self._sections = {} + self._page = None + self._soup = None # control download and cook status are indexed by na of album - self.wiki_downloaded: queue.Queue = queue.Queue(maxsize=1) - self.soup_ready: queue.Queue = queue.Queue(maxsize=1) + self._wiki_downloaded: queue.Queue = queue.Queue(maxsize=1) + self._soup_ready: queue.Queue = queue.Queue(maxsize=1) # control self.preload_running: bool = False @@ -84,7 +85,7 @@ def __init__(self, protected_vars: bool) -> None: # pass reference of current class instance to subclass self.Preload.outer_instance = self - self.log.debug("cooker imports done") + self._log.debug("cooker imports done") # TODO maybe we can move needed methods from outer class to preload so we # can have more instances of preload running at once, the results of each @@ -95,6 +96,7 @@ class Preload: """Contolling the preload of wikipedia page. It is totally self-contained exposes only start and stop methods. + Aborts automatically when no album or band is specified. Attributes ---------- @@ -116,23 +118,25 @@ def start(cls): ---- Currently running preload is stopped before new one is started """ - cls.stop() # first empty records of previous download try: - cls.outer_instance.wiki_downloaded.get(block=False) + cls.outer_instance._wiki_downloaded.get(block=False) except queue.Empty: pass try: - cls.outer_instance.wiki_downloaded.get(block=False) + cls.outer_instance._wiki_downloaded.get(block=False) except queue.Empty: pass cls.outer_instance.error_msg = None + if not all([cls.outer_instance._album, cls.outer_instance._band]): + return + log.debug(f"Starting wikipedia preload for: " - f"{cls.outer_instance.album} by " - f"{cls.outer_instance.band}") + f"{cls.outer_instance._album} by " + f"{cls.outer_instance._band}") cls._preload_thread = ThreadWithTrace( target=cls.outer_instance._preload_run, name="WikiPreload") @@ -144,8 +148,8 @@ def stop(cls): if cls.outer_instance.preload_running: log.debug(f"Stoping wikipedia preload for: " - f"{cls.outer_instance.album} by " - f"{cls.outer_instance.band}") + f"{cls.outer_instance._album} by " + f"{cls.outer_instance._band}") cls._preload_thread.kill() cls._preload_thread.join() @@ -169,16 +173,17 @@ def url(self) -> str: """ if not self._url: if SharedVars.offline_debbug: - self._url = path.join(OUTPUT_FOLDER, self.album, "page.pkl") + self._url = path.abspath(path.join(OUTPUT_FOLDER, self._album, + "page.pkl")) else: - self._url = str(self.page.url) # type: ignore + self._url = str(self._page.url) # type: ignore return self._url # TODO doesn't work without GUI def terminate(self, message: str): """Send message to GUI to ask user if he wishes to terminate the app. - + If the answer if yes than parser is destroyed and GUI terminated. See also @@ -208,8 +213,9 @@ def terminate(self, message: str): # TODO if the propper page cannot be found we could show a dialog with a # list of possible matches for user to choose from def _check_band(self) -> bool: - """Check if the artist that was input in search is the same as the one - found on wikipedia page. If not issues warning about mismatch and asks + """Check if artist from input is the same as the one on wikipedia page. + + If the artist is not the same issues warning about mismatch and asks user if he wants to continue. See also @@ -217,19 +223,19 @@ def _check_band(self) -> bool: :meth:`terminate` method that takes care of ending the app execution """ - album_artist = self.sections["infobox"].find( + album_artist = self._sections["infobox"].find( href="/wiki/Album") # type: ignore if album_artist: album_artist = album_artist.parent.get_text() - if fuzz.token_set_ratio(nc(self.band), nc(album_artist)) > 90: + if fuzz.token_set_ratio(nc(self._band), nc(album_artist)) > 90: return True else: b = re.sub(r"[Bb]y|[Ss]tudio album", "", album_artist).strip() - m = (f"The Wikipedia entry for album: {self.album} belongs to " - f"band: {b}\nThis probably means that entry for: " - f"{self.album} by {self.band} does not exist.") - self.log.exception(m) + m = (f"The Wikipedia entry for album: {self._album} belongs to" + f" band: {b}\nThis probably means that entry for: " + f"{self._album} by {self._band} does not exist.") + self._log.exception(m) self.terminate(m) return False @@ -237,8 +243,9 @@ def _check_band(self) -> bool: return False def _preload_run(self): - """ The main method which runs in the preload thread and calls other - methods based on input to load and parse the wikipedia page. + """Organizes the preload thread and calls other methods. + + Based on input decides how to load and parse the wikipedia page. See also -------- @@ -247,7 +254,6 @@ def _preload_run(self): :meth:`cook_soup` method to parse the page """ - self.preload_running = True self.get_wiki(preload=True) @@ -256,16 +262,20 @@ def _preload_run(self): self.cook_soup() if not self.error_msg: - self.log.info(f"Preload finished successfully, " - f"found: {self.url}") + # file path can be too long to show in GUI + if "pkl" in self.url: + url = "..." + self.url.rsplit("wiki_music", 1)[1] + else: + url = self.url + self._log.info(f"Found: {url}") if self.error_msg: - self.log.info(f"Preload unsucessfull: {self.error_msg}") + self._log.info(f"Preload unsucessfull: {self.error_msg}") self.preload_running = False def get_wiki(self, preload=False) -> Optional[str]: - """ Gets wikipedia page uses offline or online version. + """Gets wikipedia page uses offline or online version. Parameters ---------- @@ -281,7 +291,6 @@ def get_wiki(self, preload=False) -> Optional[str]: :meth:`_from_web` fetches the online version of page """ - # when function is called from application, # wait until all preloads are finished and then continue if not preload: @@ -289,14 +298,14 @@ def get_wiki(self, preload=False) -> Optional[str]: time.sleep(0.05) try: - downloaded = self.wiki_downloaded.get(block=False) == self.ALBUM + downloaded = self._wiki_downloaded.get(block=False) == self.ALBUM except queue.Empty: downloaded = False finally: if downloaded: return self.error_msg - self.log.debug("getting wiki") + self._log.debug("getting wiki") if SharedVars.offline_debbug: return self._from_disk() @@ -304,28 +313,26 @@ def get_wiki(self, preload=False) -> Optional[str]: return self._from_web() def _from_web(self) -> Optional[str]: - """ Guesses the right wikipedia page from innput artist and album name - and downloads it. + """Guesses the right wikipedia page from input and downloads it. Returns ------- Optional[str] if some error occured return string with its description """ - - self.log.debug("from web") + self._log.debug("from web") searches = [ - f"{self.album} ({self.band} album)", f"{self.album} (album)", - self.album + f"{self._album} ({self._band} album)", f"{self._album} (album)", + self._album ] try: for query in searches: - self.log.debug(f"trying query: {query}") - self.page = wiki.page(title=query, auto_suggest=True) - summ = nc(self.page.summary) - if nc(self.band) in summ and nc(self.album) in summ: + self._log.debug(f"trying query: {query}") + self._page = wiki.page(title=query, auto_suggest=True) + summ = nc(self._page.summary) + if nc(self._band) in summ and nc(self._album) in summ: break else: self.error_msg = "Could not get wikipedia page." @@ -333,9 +340,9 @@ def _from_web(self) -> Optional[str]: except wiki.exceptions.DisambiguationError as e: print("Found entries: {}\n...".format("\n".join(e.options[:3]))) for option in e.options: - if self.band in option: + if self._band in option: print(f"\nSelecting: {option}\n") - self.page = wiki.page(option) + self._page = wiki.page(option) break else: self.error_msg = ("Couldn't select best album entry " @@ -343,46 +350,47 @@ def _from_web(self) -> Optional[str]: except wiki.exceptions.PageError: try: - self.page = wiki.page(f"{self.album} {self.band}") + self._page = wiki.page(f"{self._album} {self._band}") except wiki.exceptions.PageError as e: self.error_msg = "Album was not found on wikipedia" - # TODO this is dangerous, can hide other types of exceptions - except (wiki.exceptions.HTTPTimeoutError, Exception) as e: + except wiki.exceptions.HTTPTimeoutError as e: self.error_msg = ("Search failed probably due to " - "poor internet connetion.") + "poor internet connetion:") + except Exception as e: + self.error_msg = (f"Search failed with unspecified exception: {e}") else: self.error_msg = None - self.wiki_downloaded.put(self.ALBUM) + self._wiki_downloaded.put(self.ALBUM) return self.error_msg def _from_disk(self) -> Optional[str]: - """Loads wikipedia page from pickle file on disk. + """Load wikipedia page from pickle file on disk. Returns ------- Optional[str] if some error occured return string with its description """ - # TODO pickle probably cannot handle some complex pages - self.log.debug(f"loading pickle file {self.url}") + self._log.debug(f"loading pickle file {self.url}") if path.isfile(self.url): with open(self.url, 'rb') as f: - self.log.debug("loading ...") - self.page = pickle.load(f) - self.log.debug("done") + self._log.debug("loading ...") + self._page = pickle.load(f) + self._log.debug("done") self.error_msg = None - self.wiki_downloaded.put(self.ALBUM) + self._wiki_downloaded.put(self.ALBUM) else: self.error_msg = "Cannot find cached offline version of page." return self.error_msg def cook_soup(self) -> Optional[str]: - """ Takes page downloaded from wikipedia and and parses it with use - of bs4. Then splits the page to dictionary of sections each section + """Parse downloaded wikipedia page with bs4 to BeautifulSoup object. + + Then splits the page to dictionary of sections, where each section is indexed by its name. Returns @@ -390,9 +398,8 @@ def cook_soup(self) -> Optional[str]: Optional[str] if some error occured return string with its description """ - try: - cooked = self.soup_ready.get(block=False) == self.ALBUM + cooked = self._soup_ready.get(block=False) == self.ALBUM except queue.Empty: cooked = False finally: @@ -400,14 +407,14 @@ def cook_soup(self) -> Optional[str]: return self.error_msg # make BeautifulSoup black magic - self.soup = bs4.BeautifulSoup(self.page.html(), - features="lxml") # type: ignore + self._soup = bs4.BeautifulSoup(self._page.html(), + features="lxml") # type: ignore # split page to parts - self.sections = collections.OrderedDict() + self._sections = collections.OrderedDict() # h2 mark the main haedings in document - for h2 in self.soup.find_all("h2"): + for h2 in self._soup.find_all("h2"): # heading should have a name marked by css class mw-headline # if not skip it @@ -428,15 +435,15 @@ def cook_soup(self) -> Optional[str]: else: value.append(s) - self.sections[name] = value + self._sections[name] = value # add infobox to the sections - self.sections["infobox"] = self.soup.find( + self._sections["infobox"] = self._soup.find( "table", class_="infobox vevent haudio") # check if the album belongs to band that was requested if self._check_band(): - self.soup_ready.put(self.ALBUM) + self._soup_ready.put(self.ALBUM) self.error_msg = None else: self.error_msg = "Album doesnt't belong to the requested band" diff --git a/wiki_music/library/parser/process_page.py b/wiki_music/library/parser/process_page.py index 89686f2..d4611b5 100644 --- a/wiki_music/library/parser/process_page.py +++ b/wiki_music/library/parser/process_page.py @@ -1,4 +1,4 @@ -""" This module containns the whole parser with all the inherited subclasses. +"""This module containns the whole parser with all the inherited subclasses. Class :class:`WikipediaParser` has complete functionallity but its methods need to be called in the correst order to give sensible results. """ @@ -7,7 +7,7 @@ import re # lazy loaded from os import path from threading import Thread -from typing import List +from typing import List, Tuple import datefinder # lazy loaded import fuzzywuzzy.fuzz as fuzz # lazy loaded @@ -39,7 +39,7 @@ @for_all_methods(time_methods) class WikipediaParser(DataExtractors, WikiCooker, ParserInOut): - """ Class for parsing the wikipedia page and extracting tags data from it. + """Class for parsing the wikipedia page and extracting tags data from it. Warnings -------- @@ -58,7 +58,7 @@ class WikipediaParser(DataExtractors, WikiCooker, ParserInOut): protected_vars: bool defines if certain variables should be initialized by __init__ method or not - """ + """ def __init__(self, protected_vars: bool = True) -> None: @@ -70,29 +70,36 @@ def __init__(self, protected_vars: bool = True) -> None: log.debug("init parser done") @warning(log) - def get_release_date(self): - """ Gets album release date from information box in the + def get_release_date(self) -> str: + """Gets album release date from information box in the top right corner of wikipedia page. Populates:attr:`wiki_music.DATE` Raises ------ :exc:`utilities.exceptions.NoReleaseDateException` raised if no release date was extracted + + Returns + ------- + str + release year as a string """ - dates = self.sections["infobox"].find(class_="published") + dates = self._sections["infobox"].find(class_="published") if dates: dates = datefinder.find_dates(dates.get_text()) date_year = [d.strftime('%Y') for d in dates] - self.release_date = list(set(date_year))[0] + self._release_date = list(set(date_year))[0] else: - self.release_date = "" + self._release_date = "" raise NoReleaseDateException + return self._release_date + @warning(log) - def get_genres(self): - """ Gets list of album genres from information box in the + def get_genres(self) -> List[str]: + """Gets list of album genres from information box in the top right corner of wikipedia page. If found genre if only one then assigns is value to :attr:`GENRE` @@ -100,9 +107,14 @@ def get_genres(self): ------ :exc:`wiki_music.utilities.exceptions.NoGenreException` if no genres could be extracted from page + + Returns + ------- + List[str] + list of found genres """ - genres = self.sections["infobox"].find(class_="category hlist") + genres = self._sections["infobox"].find(class_="category hlist") if genres: self.genres = [g.string for g in genres.find_all(href=WIKI_GENRES, @@ -113,11 +125,13 @@ def get_genres(self): # auto select genre if only one was found if len(self.genres) == 1: - self.selected_genre = self.genres[0] + self._selected_genre = self.genres[0] + + return self.genres @warning(log) def get_cover_art(self): - """ Gets album cover art information box in the top right corner + """Gets album cover art information box in the top right corner of wikipedia page. Runs in a separate thread because the cover art data is not used by parser in any way, so it can be downloaded in the background. Populates :attr:`COVERART` @@ -133,7 +147,7 @@ def get_cover_art(self): def cover_art_getter(): """ - for child in self.sections["infobox"].children: + for child in self._sections["infobox"].children: if child.find("img") is not None: image = child.find("img") image_url = f"https:{image['src']}" @@ -141,9 +155,9 @@ def cover_art_getter(): self.cover_art = get_image(image_url) break """ - for img in self.sections["infobox"].find_all("img", src=True, - alt=True): - if fuzz.token_set_ratio(img["alt"], self.album) > 90: + for img in self._sections["infobox"].find_all("img", src=True, + alt=True): + if fuzz.token_set_ratio(img["alt"], self._album) > 90: break if img: @@ -156,8 +170,8 @@ def cover_art_getter(): Thread(target=cover_art_getter, name="CoverArtGetter").start() - def get_composers(self): - """ Extracts composers from wikipedia page. Employs complex logic. + def get_composers(self) -> List[List[str]]: + """Extracts composers from wikipedia page. Employs complex logic. First Person named entities are extracted by nltk. Then merges them with composers. After that uses this list of names to try to guess composers and coresponding tracks from short text above the table. @@ -166,16 +180,21 @@ def get_composers(self): -------- :meth:`get_personnel` this method should run first because it populates the - :attr:`personnel` used by this method + :attr:`_personnel` used by this method Warnings -------- This method is not as robust as it should be. It fails for many types of formating. + + Returns + ------- + List[List[str]] + list of composers for every track """ def check_name_complete(name, text): - """ Checks if name retrieved fromm text is complete or only part by + """Checks if name retrieved fromm text is complete or only part by checking the following words in text. Warnings @@ -205,15 +224,12 @@ def check_name_complete(name, text): else: return name - # extract names from text using NLTK - self.extract_names() - - self.NLTK_names = set(self.NLTK_names + self.personnel) + NLTK_names = set(self.NLTK_names + self._personnel) # get the short comment above the table which is marked as html # paragraph with

...

- if self.sections["track_listing"][0].name == "p": - html = self.sections["track_listing"][0].get_text() + if self._sections["track_listing"][0].name == "p": + html = self._sections["track_listing"][0].get_text() else: html = "" @@ -233,47 +249,58 @@ def check_name_complete(name, text): # find which track are affected by except index = [] if len(parts) == 2: - for i, tr in enumerate(self.tracks): + for i, tr in enumerate(self._tracks): if caseless_contains(tr, parts[1]): index.append(i) # assign composers to tracks - for name in self.NLTK_names: + for name in NLTK_names: if len(parts) == 2: if caseless_contains(name, parts[1]): n = check_name_complete(name, parts[1]) for ind in index: - self.composers[ind].append(n) + self._composers[ind].append(n) if caseless_contains(name, parts[0]): n = check_name_complete(name, parts[0]) - for i, comp in enumerate(self.composers): + for i, comp in enumerate(self._composers): if i not in index: if n not in comp: - self.composers[i].append(n) + self._composers[i].append(n) + + self._complete() + + return self._composers @warning(log) - def get_contents(self): - """ Extract page contets from keys in :attr:`sections` dictionary. - + def get_contents(self) -> List[str]: + """Extract page contets from keys in :attr:`_sections` dictionary. + Raises ------ :exc:`wiki_music.utilities.exceptions.NoContentsException` if no contents were retrieved + + Returns + ------- + List[str] + page contents as a list """ # the last element is infobox which was added manually - self.contents = [s.replace("_", " ").capitalize() - for s in self.sections.keys()][:-1] + self._contents = [s.replace("_", " ").capitalize() + for s in self._sections.keys()][:-1] - if len(self.contents) == 0: + if len(self._contents) == 0: raise NoContentsException + return self._contents + @warning(log) - def get_personnel(self): - """ Extract personnel from wikipedia page sections defined by: + def get_personnel(self) -> Tuple[List[str], List[List[int]]]: + """Extract personnel from wikipedia page sections defined by: :const:`wiki_music.constants.parser_const.PERSONNEL_SECTIONS` then parse these entries for additional data like apperences on tracks. @@ -281,28 +308,31 @@ def get_personnel(self): ------ :exc:`wiki_music.utilities.exceptions.NoPersonnelException` if no table or list with personnel was found + + Returns + ------- + Tuple[List[str], List[List[int]]] + two lists, first contains found personnel and the second has for + each person list of tracks on which the person appeared """ personnel = [] appearences = [] - + for s in PERSONNEL_SECTIONS: - if s not in self.sections: + if s not in self._sections: continue - for html in self.sections[s]: + for html in self._sections[s]: # if the toplevel tag is the list itself if html.name in ("ul", "ol"): personnel.extend(self._html2python_list(html)) - + # if the list is nested inside some other tags for h in html.find_all(["ul", "ol"]): personnel.extend(self._html2python_list(h)) - if len(personnel) == 0: - raise NoPersonnelException - for i, person in enumerate(personnel): # make space in appearences @@ -310,7 +340,7 @@ def get_personnel(self): # remove reference person = re.sub(r"\[ *\d+ *\]", "", person) - + # split to person and the rest try: person, appear = re.split(r" \W | as ", person, 1, flags=re.I) @@ -330,7 +360,7 @@ def get_personnel(self): appearences[i].append(int(app)) # find references to song names - for j, t in enumerate(self.tracks): + for j, t in enumerate(self._tracks): if re.match(t, appear, re.I): appearences[i].append(j) @@ -339,15 +369,21 @@ def get_personnel(self): appearences[i] = [a - 1 for a in appearences[i]] personnel[i] = person - #print(person, "|", appear, "|", appearences[i]) + self._personnel = personnel + self._appearences = appearences - self.personnel = personnel - self.appearences = appearences + self._complete() + self._info_tracks() + self._merge_artist_personnel() + + if len(personnel) == 0: + raise NoPersonnelException + + return self._personnel, self._appearences @warning(log) - def _get_tracks(self): - """ Method that attempts to extract tracklist from table or list - format on the wikipedia page. + def get_tracks(self) -> Tuple[List[str], List[List[str]]]: + """Attempt to extract tracklist from html table or list on wikipedia. See also -------- @@ -355,7 +391,7 @@ def _get_tracks(self): used to parse tracklist in htlm table :meth:`_from_list` used to parse tracklist in html list - :meth:`process_tracks` + :meth:`_process_tracks` method called to parse raw extracted table and get song numbers, artists, composers ... @@ -363,23 +399,38 @@ def _get_tracks(self): ------ :exc:`wiki_music.utilities.exceptions.NoTracklistException` raised if no tracklist in any format was found + + Returns + ------- + Tuple[List[str], List[List[str]]] + list of tracks and for each track list of atrists """ - tables = self.soup.find_all("table", class_="tracklist") + tables = [] + for html in self._sections["track_listing"]: + # if the toplevel tag is the table itself + if html.name == "table": + if "tracklist" in html["class"]: + tables.extend(html) + + # if the list is nested inside some other tags + for h in html.find_all("table", class_="tracklist"): + tables.extend(h) if tables: data = self._from_table(tables) else: try: tables = [] - for s in self.sections["track_listing"]: + for s in self._sections["track_listing"]: if s.name in ("ul", "ol"): tables.append(s) else: tables.extend(s.find_all(["ul", "ol"])) - except AttributeError as e: + except AttributeError: msg = (f"No tracklist found!\nURL: {self.url}\nprobably " - f"doesn´t belong to album: {self.album} by {self.band}") + f"doesn´t belong to album: {self._album} by " + f"{self._band}") SharedVars.warning(msg) raise NoTracklistException(msg) else: @@ -387,36 +438,43 @@ def _get_tracks(self): for t in tables: data.extend(self._from_list(t)) - self.process_tracks(data) + return self._process_tracks(data) - def process_tracks(self, data: List[List[str]]): - """ Process raw extracted list of tables with trackist for - track details. """ + def _process_tracks(self, data: List[List[str]] + ) -> Tuple[List[str], List[List[str]]]: + """Process raw extracted list of tables with trackists for + track details. + + Returns + ------- + Tuple[List[str], List[List[str]]] + list of tracks and for each track list of atrists + """ - self.disk_sep.append(0) + self._disk_sep.append(0) index = 1 for CD in data: - self.disks.append([f"{self.album} CD {index}", len(self)]) + self._disks.append([f"{self._album} CD {index}", len(self)]) index += 1 for song in CD: if "no" in nc(song[0]): # table header - possibly use in future # posladny stlpec je length a ten väčšinou netreba if "length" in nc(song[-1]): - self.header.append(song[:-1]) + self._header.append(song[:-1]) else: - self.header.append(song) + self._header.append(song) elif re.match(ORDER_NUMBER, song[0]): - self.numbers.append(song[0].replace(".", "")) + self._numbers.append(song[0].replace(".", "")) tmp1, tmp2 = self._get_track(song[1]) - self.tracks.append(tmp1) - self.subtracks.append(tmp2) + self._tracks.append(tmp1) + self._subtracks.append(tmp2) if len(song) > 3: - self.artists.append([]) - self.composers.append([]) + self._artists.append([]) + self._composers.append([]) # all columns between track and track length belong to # artists we assign them to artists or composers @@ -425,63 +483,66 @@ def process_tracks(self, data: List[List[str]]): a = self._get_artist(song[i]) try: is_composer = process.extractOne( - self.header[-1][i], COMPOSER_HEADER, + self._header[-1][i], COMPOSER_HEADER, score_cutoff=90, scorer=fuzz.ratio) except: - self.artists[-1].extend(a) + self._artists[-1].extend(a) else: if is_composer: - self.composers[-1].extend(a) + self._composers[-1].extend(a) else: - self.artists[-1].extend(a) + self._artists[-1].extend(a) else: - self.artists.append([]) - self.composers.append([]) + self._artists.append([]) + self._composers.append([]) elif "total" in nc(song[0]): # total length summary pass else: # if all else passes than line must be disc title - self.disks[-1] = [song[0], len(self)] + self._disks[-1] = [song[0], len(self)] # bonus track are sometimes marked as another disc - disks_filtered = [d for d in self.disks if "bonus" not in nc(d[0])] + disks_filtered = [d for d in self._disks if "bonus" not in nc(d[0])] - self.disk_sep = [i[1] for i in disks_filtered] - self.disk_sep.append(len(self)) - self.disks = [i[0] for i in disks_filtered] + self._disk_sep = [i[1] for i in disks_filtered] + self._disk_sep.append(len(self)) + self._disks = [i[0] for i in disks_filtered] # assign disc number to tracks - for i, _ in enumerate(self.tracks): - for j, _ in enumerate(self.disk_sep[:-1]): - if self.disk_sep[j] <= i and i < self.disk_sep[j + 1]: - self.disc_num.append(j + 1) - - def info_tracks(self): - """ Parse track names for aditional information like - artist, composer, type... . Also get rid of useless strings like + for i, _ in enumerate(self._tracks): + for j, _ in enumerate(self._disk_sep[:-1]): + if self._disk_sep[j] <= i and i < self._disk_sep[j + 1]: + self._disc_num.append(j + 1) + + return self._tracks, self._artists + + def _info_tracks(self): + """Parse track names for aditional information. + + Like artist, composer, type... . Also get rid of useless strings like bonus track, featuring... . These informations are assumed to be enclosed in brackets behind the track name. """ - self.types = [] - self.sub_types = [] + self._types = [] + self._subtypes = [] - comp_flat = flatten_set(self.composers) + comp_flat = flatten_set(self._composers) # hladanie umelcov ked su v zatvorke za skladbou # + zbavovanie sa bonus track a pod. - for i, tr in enumerate(self.tracks): + for i, tr in enumerate(self._tracks): - self.types.append("") - self.sub_types.append([]) + self._types.append("") + self._subtypes.append([]) start_list = [m.start() for m in re.finditer(r'\(', tr)] end_list = [m.start() for m in re.finditer(r'\)', tr)] for start, end in zip(start_list, end_list): - self.tracks[i] = re.sub(TO_DELETE, "", tr, re.I) + self._tracks[i] = re.sub(TO_DELETE, "", tr, re.I) artist = re.sub("[,:]", "", tr[start + 1:end]) artist = re.split(r",|\/|\\", artist) @@ -492,28 +553,28 @@ def info_tracks(self): # TODO all these maybe should be in the UNWANTED loop?? # check against additional personnel - for person in self.personnel: + for person in self._personnel: if fuzz.token_set_ratio(art, person) > 90: - self.artists[i].append(person) - self.tracks[i] = self._cut_out(tr, start, end) + self._artists[i].append(person) + self._tracks[i] = self._cut_out(tr, start, end) # check agains composers for comp in comp_flat: if fuzz.token_set_ratio(art, comp) > 90: - self.artists[i].append(comp) - self.tracks[i] = self._cut_out(tr, start, end) + self._artists[i].append(comp) + self._tracks[i] = self._cut_out(tr, start, end) # check if instrumental, ... _type, score = process.extractOne( art, DEF_TYPES, scorer=fuzz.token_sort_ratio) if score > 90: - self.types[i] = art # _type - self.tracks[i] = self._cut_out(tr, start, end) + self._types[i] = art # _type + self._tracks[i] = self._cut_out(tr, start, end) - if self.subtracks: - for j, sbtr in enumerate(self.subtracks[i]): + if self._subtracks: + for j, sbtr in enumerate(self._subtracks[i]): - self.sub_types[i] = [""] * len(sbtr) + self._subtypes[i] = [""] * len(sbtr) start_list = [m.start() for m in re.finditer(r'\(', sbtr)] end_list = [m.start() for m in re.finditer(r'\)', sbtr)] @@ -524,27 +585,27 @@ def info_tracks(self): for a in art: # check against additional personnel - for person in self.personnel: + for person in self._personnel: if fuzz.token_set_ratio(a, person) > 90: - self.artists[i].append(person) + self._artists[i].append(person) sbtr = self._cut_out(sbtr, start, end) - self.subtracks[i][j] = sbtr + self._subtracks[i][j] = sbtr # check if instrumental, ... _type, score = process.extractOne( a, DEF_TYPES, scorer=fuzz.token_sort_ratio) if score > 90: - self.sub_types[i][j] = _type + self._subtypes[i][j] = _type sbtr = self._cut_out(sbtr, start, end) - self.subtracks[i][j] = sbtr + self._subtracks[i][j] = sbtr - def complete(self): - """ Recursively traverses: :attr:`composers`, :attr:`artists` and - :attr:`personnel` and checks each name with each if some is found to be - incomplete then it is replaced by longer version from other list. - """ + def _complete(self): + """Recursively traverses: :attr:`_composers`, :attr:`artists` and + :attr:`_personnel` and checks each name with each if some is found to + be incomplete then it is replaced by longer version from other list. + """ - to_complete = (self.composers, self.artists, self.personnel) + to_complete = (self._composers, self._artists, self._personnel) delete: list = ["", " "] # complete everything with everything @@ -553,8 +614,8 @@ def complete(self): complete_N_dim(to_replace, to_find) # sort artist alphabeticaly - if self.artists: - for a in self.artists: + if self._artists: + for a in self._artists: a.sort() for tc in to_complete: @@ -571,27 +632,29 @@ def complete(self): if isinstance(t, list): tc[i] = sorted(list(set(t))) - def merge_artist_personnel(self): - """ Assigns personnel which have appearences specified to coresponding + def _merge_artist_personnel(self): + """Assigns personnel which have appearences specified to coresponding list of track artists. """ + self._log.debug("merge artists and personnel") - for person, appear in zip(self.personnel, self.appearences): + for person, appear in zip(self._personnel, self._appearences): for a in appear: - self.artists[a].append(person) + self._artists[a].append(person) def merge_artist_composers(self): - """ Move all artists to composers list. This is done or left out based - on user input. """ + """Move all artists to composers list. This is done or left out based + on user input""" - for i, (c, a) in enumerate(zip(self.composers, self.artists)): - self.composers[i] = sorted(list(filter(None, set(c + a)))) + for i, (c, a) in enumerate(zip(self._composers, self._artists)): + self._composers[i] = sorted(list(filter(None, set(c + a)))) - self.artists = [""] * len(self.composers) + self._artists = [""] * len(self._composers) + @property @warning(log, show_GUI=False) - def extract_names(self): - """ Used nltk to estract person names from supplied sections of the + def NLTK_names(self): + """Use nltk to extract person names from supplied sections of the wikipedia page. See also @@ -609,52 +672,58 @@ def extract_names(self): cluttered by hardly classifiable information. """ - document = "" - for key, value in self.sections.items(): - - if key in (PERSONNEL_SECTIONS + ("track_listing", )): - for val in value: - document += val.get_text(" ") + if not self._NLTK_names: - # if none of the sections is present exit method - if not document: - raise NoNames2ExtractException + document = "" + for key, value in self._sections.items(): - stop = NLTK.nltk.corpus.stopwords.words('english') + if key in (PERSONNEL_SECTIONS + ("track_listing", )): + for val in value: + document += val.get_text(" ") - document = ' '.join([i for i in document.split() if i not in stop]) - sentences = NLTK.nltk.tokenize.sent_tokenize(document) - sentences = [NLTK.nltk.word_tokenize(sent) for sent in sentences] - sentences = [NLTK.nltk.pos_tag(sent) for sent in sentences] + # if none of the sections is present exit method + if not document: + raise NoNames2ExtractException - names = [] - for tagged_sentence in sentences: - for chunk in NLTK.nltk.ne_chunk(tagged_sentence): - if type(chunk) == NLTK.nltk.tree.Tree: - if chunk.label() == 'PERSON': - names.append(' '.join([c[0] for c in chunk])) - - names = sorted(list(set(names))) - - # TODO smetimes when two names are separated only by "and", "by".. - # or such short word the two names are found together as one - # and the sunsequent completion messes the right names that were found - """ - # filter incomplete names - selected_names = [] - for n1 in names: - name = n1 - for n2 in names: - if name in n2: - name = n2 - else: - pass + try: + stop = NLTK.nltk.corpus.stopwords.words('english') + except AttributeError: + raise NltkUnavailableException("NLTK not available!") + + document = ' '.join([i for i in document.split() if i not in stop]) + sentences = NLTK.nltk.tokenize.sent_tokenize(document) + sentences = [NLTK.nltk.word_tokenize(sent) for sent in sentences] + sentences = [NLTK.nltk.pos_tag(sent) for sent in sentences] + + names = [] + for tagged_sentence in sentences: + for chunk in NLTK.nltk.ne_chunk(tagged_sentence): + if type(chunk) == NLTK.nltk.tree.Tree: + if chunk.label() == 'PERSON': + names.append(' '.join([c[0] for c in chunk])) + + names = sorted(list(set(names))) + + # TODO smetimes when two names are separated only by "and", "by".. + # or such short word the two names are found together as one + # and the sunsequent completion messes the right names + # that were found + """ + # filter incomplete names + selected_names = [] + for n1 in names: + name = n1 + for n2 in names: + if name in n2: + name = n2 + else: + pass - selected_names.append(name) - """ + selected_names.append(name) + """ - # filter out already found tracks - filtered_names = [n for n in names - if fuzz.token_set_ratio(n, self.tracks) < 90] + # filter out already found tracks + filtered_names = [n for n in names + if fuzz.token_set_ratio(n, self._tracks) < 90] - self.NLTK_names = filtered_names + return self._NLTK_names diff --git a/wiki_music/library/tags_handler/flac.py b/wiki_music/library/tags_handler/flac.py index 082275f..dc1cc85 100644 --- a/wiki_music/library/tags_handler/flac.py +++ b/wiki_music/library/tags_handler/flac.py @@ -14,7 +14,7 @@ class TagFlac(TagBase): - """A low level implementation of tag handling for flac files. """ + """A low level implementation of tag handling for flac files""" __doc__ += TagBase.__doc__ # type: ignore _map_keys = OrderedDict([ @@ -33,7 +33,7 @@ class TagFlac(TagBase): ) def _open(self, filename: str): - """Function reading flac file to mutagen.flac.FLAC class. """ + """Function reading flac file to mutagen.flac.FLAC class""" try: self._song = FLAC(filename=filename) diff --git a/wiki_music/library/tags_handler/m4a.py b/wiki_music/library/tags_handler/m4a.py index 4602949..281b2d6 100644 --- a/wiki_music/library/tags_handler/m4a.py +++ b/wiki_music/library/tags_handler/m4a.py @@ -13,7 +13,7 @@ class TagM4a(TagBase): - """A low level implementation of tag handling for m4a files. """ + """A low level implementation of tag handling for m4a files""" __doc__ += TagBase.__doc__ # type: ignore _map_keys = OrderedDict([ @@ -32,7 +32,7 @@ class TagM4a(TagBase): ) def _open(self, filename: str): - """Function reading m4a file to mutagen.mp4.MP4 class. """ + """Function reading m4a file to mutagen.mp4.MP4 class""" try: self._song = MP4(filename=filename) diff --git a/wiki_music/library/tags_handler/mp3.py b/wiki_music/library/tags_handler/mp3.py index 6e756a7..228e839 100644 --- a/wiki_music/library/tags_handler/mp3.py +++ b/wiki_music/library/tags_handler/mp3.py @@ -15,7 +15,7 @@ class TagMp3(TagBase): - """A low level implementation of tag handling for mp3 files. """ + """A low level implementation of tag handling for mp3 files""" __doc__ += TagBase.__doc__ # type: ignore _map_keys = OrderedDict([ @@ -34,7 +34,7 @@ class TagMp3(TagBase): ) def _open(self, filename: str): - """Function reading mp3 file to mutagen.id3.ID3 class. """ + """Function reading mp3 file to mutagen.id3.ID3 class""" try: self._song = ID3(filename=filename) diff --git a/wiki_music/library/tags_handler/tag_base.py b/wiki_music/library/tags_handler/tag_base.py index 80190a1..22b390e 100644 --- a/wiki_music/library/tags_handler/tag_base.py +++ b/wiki_music/library/tags_handler/tag_base.py @@ -1,4 +1,4 @@ -""" Base module for all tag handlers. """ +"""Base module for all tag handlers""" import collections import logging diff --git a/wiki_music/library/tags_io.py b/wiki_music/library/tags_io.py index 98f1e9d..ae42e29 100644 --- a/wiki_music/library/tags_io.py +++ b/wiki_music/library/tags_io.py @@ -101,7 +101,7 @@ def write(tag, value=None): @exception(log) def read_tags(song_file: str) -> Dict[str, Union[str, list, bytearray]]: - """ Convenience function which takes care of reading tags from file. + """Convenience function which takes care of reading tags from file. Abstracts away from low level mutagen API. If no tags are read, function can guess track title from file name, assumming some decent formating. diff --git a/wiki_music/ui/MainWindow.ui b/wiki_music/ui/MainWindow.ui index 5cad800..136db66 100644 --- a/wiki_music/ui/MainWindow.ui +++ b/wiki_music/ui/MainWindow.ui @@ -14,112 +14,122 @@ MainWindow - - - - - - 600 - 16777215 - - - - Input + + + 5 + + + 10 + + + 5 + + + 5 + + + + + QLayout::SetMinimumSize - - - - - - - - - - - - - - - - - - - - - Input Artist - - - - - - - - - - Input Album - - - - - - - - + - - - - - 600 - 16777215 - - - - Common Tags + + + + QLayout::SetMaximumSize - - - - - Album Artist - - - - - - - Album - - - - - - - Year - - - - - - - Genre - - - - - - - - - - - - - - - - - + + + + + 600 + 16777215 + + + + Input + + + + + + + + + Input Artist + + + + + + + Input Album + + + + + + + + + + + + + + 600 + 16777215 + + + + Common Tags + + + + + + Album Artist + + + + + + + Album + + + + + + + Year + + + + + + + Genre + + + + + + + + + + + + + + + + + + + @@ -138,48 +148,48 @@ Detail - - + + Song title - + - + Number - + - + Artists - + - + Composers - + - + @@ -197,25 +207,22 @@ - + - + Lyrics - + - - - @@ -223,6 +230,59 @@ + + + + QLayout::SetMinimumSize + + + + + + 93 + 16777215 + + + + browse + + + + + + + + 93 + 16777215 + + + + Lyrics search + + + + + + + Cover Art + + + + + + + + 93 + 16777215 + + + + Wiki Search + + + + + @@ -262,6 +322,9 @@ + + QLayout::SetMinimumSize + @@ -290,63 +353,6 @@ - - - - - - - 93 - 16777215 - - - - browse - - - - - - - - 93 - 16777215 - - - - Lyrics search - - - - - - - - 93 - 16777215 - - - - Wiki Search - - - - - - - Cover Art - - - - - - - Reload - - - - - @@ -385,20 +391,29 @@ Save - Help - + - + + + + + + Search + + + + + @@ -451,7 +466,7 @@ Mp3Tag - + Help Index @@ -466,6 +481,31 @@ About + + + Version + + + + + Wikipedia + + + + + Lyrics + + + + + Cover Art + + + + + Logs + + diff --git a/wiki_music/utilities/exceptions.py b/wiki_music/utilities/exceptions.py index 462ca98..7763b1e 100644 --- a/wiki_music/utilities/exceptions.py +++ b/wiki_music/utilities/exceptions.py @@ -7,7 +7,8 @@ __all__ = ["NoTracklistException", "NoReleaseDateException", "NoGenreException", "NoCoverArtException", "NoNames2ExtractException", "NoContentsException", - "NoPersonnelException", "Mp3tagNotFoundException"] + "NoPersonnelException", "Mp3tagNotFoundException", + "NltkUnavailableException"] class NoTracklistException(Exception): @@ -56,3 +57,9 @@ class Mp3tagNotFoundException(Exception): """Exception raised when Mp3tag could not be run.""" pass + + +class NltkUnavailableException(Exception): + """Exception raised when Mp3tag could not be run.""" + + pass diff --git a/wiki_music/utilities/gui_utils.py b/wiki_music/utilities/gui_utils.py index 3031965..7d78317 100644 --- a/wiki_music/utilities/gui_utils.py +++ b/wiki_music/utilities/gui_utils.py @@ -24,7 +24,7 @@ def abstract_warning(): - """ Raises error when abstract method is called directly. + """Raises error when abstract method is called directly. Raises ------ NotImplementedError @@ -106,7 +106,7 @@ def get_music_path() -> str: def get_image(address: str) -> Optional[bytes]: - """ Based on addres decides if the image is online or local. If address + """Based on addres decides if the image is online or local. If address has http prefix, image is downloaded from internet. If not then it is read from disk. @@ -136,7 +136,7 @@ def get_image(address: str) -> Optional[bytes]: def comp_res(image: bytearray, quality: int, x: int = 0, y: int = 0) -> bytes: - """ Compress and/or change image resolution. If x and y dimension are + """Compress and/or change image resolution. If x and y dimension are both specified than image is resized to these dimension otherwise it is only compressed @@ -176,7 +176,7 @@ def comp_res(image: bytearray, quality: int, x: int = 0, y: int = 0) -> bytes: def get_image_size(image: bytearray) -> str: - """ get size of image in memory + """get size of image in memory Parameters ---------- @@ -193,7 +193,7 @@ def get_image_size(image: bytearray) -> str: def get_icon() -> str: - """ returns application icon path + """returns application icon path Raises ------ @@ -214,7 +214,7 @@ def get_icon() -> str: def get_sizes(uri: str) -> Tuple[Optional[int], Optional[Tuple[int, int]]]: - """ Get file size and image size (None if not known) of picture on the + """Get file size and image size (None if not known) of picture on the internet without downloading it. Parameters diff --git a/wiki_music/utilities/loggers.py b/wiki_music/utilities/loggers.py index 7441cb6..0b6925e 100644 --- a/wiki_music/utilities/loggers.py +++ b/wiki_music/utilities/loggers.py @@ -52,6 +52,7 @@ def set_log_handles(level: int): already_set: Set[str] = set() for name in logging.root.manager.loggerDict: if "wiki_music" in name: + log = logging.getLogger(name) # assumed format is wiki_music...<...> try: log_name = name.split(".")[1] @@ -67,10 +68,10 @@ def set_log_handles(level: int): fh.setLevel(level) fh.setFormatter(FORMATTER) - log = logging.getLogger(name) - log.setLevel(level) log.addHandler(fh) already_set.add(log_name) + finally: + log.setLevel(level) return log diff --git a/wiki_music/utilities/parser_utils.py b/wiki_music/utilities/parser_utils.py index 583ab84..327006d 100644 --- a/wiki_music/utilities/parser_utils.py +++ b/wiki_music/utilities/parser_utils.py @@ -15,8 +15,9 @@ from wiki_music.constants.colors import GREEN, RESET from .utils import normalize +from .sync import SharedVars -logging.getLogger(__name__) +log = logging.getLogger(__name__) if TYPE_CHECKING: from logging import Logger @@ -94,25 +95,26 @@ def nltk(cls): class NLTK(metaclass=_NltkMeta): - """A thread safe nltk importer. Will make other threads wait if they want - to access nltk until it is imported. + """A thread safe nltk importer. + + Will make other threads wait if they want to access nltk + until it is imported. """ _import_running: bool = False # nltk class attribute is provided by metaclass _nltk = None _lock: Lock = Lock() - + @classmethod def run_import(cls, logger: "Logger"): """Import nltk in separate thread and assign it to class attribute. - + Parameters ---------- logger: logging.Logger instance of a logger to log import messages """ - def imp(): with cls._lock: logger.debug("import nltk") @@ -134,7 +136,7 @@ def imp(): class ThreadWithReturn(Thread): - """ Subclass of python threading.Thread which can return result of + """Subclass of python threading.Thread which can return result of running function. The result is return by calling the Thread.join() method. @@ -160,15 +162,15 @@ def __init__(self, *args, **kwargs) -> None: self._return: Any = None def run(self): - """ Override standard threading.Thread.run() method to store + """Override standard threading.Thread.run() method to store running function return value. """ if self._target is not None: self._return = self._target(*self._args, **self._kwargs) - def join(self, timeout:Optional[float] = None) -> Any: - """ Override standard threading.Thread.join() method to return + def join(self, timeout: Optional[float] = None) -> Any: + """Override standard threading.Thread.join() method to return running function return value. """ @@ -178,7 +180,7 @@ def join(self, timeout:Optional[float] = None) -> Any: class ThreadPool: - """ Spawns pool of threads to excecute function. If the list of arguments + """Spawns pool of threads to excecute function. If the list of arguments contains only one tuple, run the function in the calling thread to avoid unnecessary overhead as a result of spawning a new thread. @@ -188,22 +190,27 @@ class ThreadPool: callable that each thread should run args: List[tuple] each tuple in list contains args for one thread running target - + See Also -------- :class:`ThreadWithReturn` """ - def __init__(self, target:Callable[..., list] = lambda *args: [], - args:List[tuple] = [tuple()]) -> None: + def __init__(self, target: Callable[..., list] = lambda *args: [], + args: List[tuple] = [tuple()]) -> None: self._args = args self._target = target def run(self, timeout: Optional[float] = 60) -> list: - """ Starts the execution of threads in pool. returns after all threads + """Starts the execution of threads in pool. returns after all threads join() metod has returned. + See also + -------- + :meth:`wiki_music.utilities.sync.SharedVars.set_threadpool_prog` + inform GUI of threadpool progress + Parameters ---------- timeout: Optional[float] @@ -225,15 +232,24 @@ def run(self, timeout: Optional[float] = 60) -> list: name=f"ThreadPoolWorker-{i}")) threads[-1].daemon = True threads[-1].start() - + + # report progress to gui + while True: + count = [t.is_alive() for t in threads].count(False) + SharedVars.set_threadpool_prog(count) + if count == len(threads): + break + + time.sleep(0.05) + for i, l in enumerate(threads): threads[i] = l.join(timeout=timeout) return threads - + def bracket(data: List[str]) -> List[str]: - """ Puts elements of the list in brackets.\n + """Puts elements of the list in brackets. Parameters ---------- @@ -257,7 +273,8 @@ def bracket(data: List[str]) -> List[str]: def write_roman(num: Union[int, str]): - """ Convert integer to roman number + """Convert integer to roman number. + Parameters ---------- num: int @@ -273,39 +290,34 @@ def write_roman(num: Union[int, str]): roman number converted from integer """ - roman_numerals = [ - ('M', 1000), - ('CM', 900), - ('D', 500), - ('CD', 400), - ('C', 100), - ('XC', 90), - ('L', 50), - ('XL', 40), - ('X', 10), - ('IX', 9), - ('V', 5), - ('IV', 4), - ('I', 1) - ] - - num = str(num) - - ix = 0 - result = 0 - while ix < len(num): - for k, v in roman_numerals: - if num.startswith(k, ix): - result += v - ix += len(k) + roman = collections.OrderedDict() + roman[1000] = "M" + roman[900] = "CM" + roman[500] = "D" + roman[400] = "CD" + roman[100] = "C" + roman[90] = "XC" + roman[50] = "L" + roman[40] = "XL" + roman[10] = "X" + roman[9] = "IX" + roman[5] = "V" + roman[4] = "IV" + roman[1] = "I" + + def roman_num(num): + for r in roman.keys(): + x, y = divmod(num, r) + yield roman[r] * x + num -= (r * x) + if num <= 0: break - else: - raise ValueError('Invalid Roman number.') - return result + + return "".join([a for a in roman_num(num)]) def normalize_caseless(text: str) -> str: - """ NFKD casefold string normalization + """NFKD casefold string normalization. Parameters ---------- @@ -321,7 +333,7 @@ def normalize_caseless(text: str) -> str: def caseless_equal(left: str, right: str) -> bool: - """ Check for normalized string equality + """Check for normalized string equality. Parameters ---------- @@ -343,7 +355,7 @@ def caseless_equal(left: str, right: str) -> bool: def caseless_contains(string: str, in_text: str) -> bool: - """ Check if string is contained in text.\n + """Check if string is contained in text. Parameters ---------- @@ -361,7 +373,6 @@ def caseless_contains(string: str, in_text: str) -> bool: -------- :func:`normalize_caseless` """ - if normalize_caseless(string) in normalize_caseless(in_text): return True else: @@ -369,7 +380,7 @@ def caseless_contains(string: str, in_text: str) -> bool: def count_spaces(*lists: Tuple[List[str], ...]) -> Tuple[List[str], int]: - """ Counts max length of elements in list and croesponding spaces for + """Counts max length of elements in list and coresponding spaces for each item to fit that length. Parameters @@ -384,7 +395,6 @@ def count_spaces(*lists: Tuple[List[str], ...]) -> Tuple[List[str], int]: list of number os apces to append to list elements to make them span max length """ - transposed: List[List[str]] = list(map(list, zip(*lists))) max_length: int = 0 spaces: List[str] = [] @@ -401,25 +411,24 @@ def count_spaces(*lists: Tuple[List[str], ...]) -> Tuple[List[str], int]: def yaml_dump(dict_data: List[Dict[str, str]], save_dir: str): - """ Save yaml file to disk. Each dictionary in list contains tags - of one album track + """Save yaml tracklist file to disk. Parameters ---------- dict_data: List[Dict[str, str]] - list of dictionarie to save to disk + list of dictionarie to save to disk, each dictionary in list contains + tags of one album track save_dir: str directory to save to """ - - _path = os.path.join(save_dir, "database.yaml") - print(GREEN + "\nSaving YAML file: " + RESET + _path + "\n") - with open(_path, "w") as outfile: - yaml.dump(dict_data, outfile, default_flow_style=False) + path = os.path.join(save_dir, "database.yaml") + print(GREEN + "\nSaving YAML file: " + RESET + path + "\n") + with open(path, "w") as f: + yaml.dump(dict_data, f, default_flow_style=False) def yaml_load(yml_file: str) -> List[dict]: - """ Loads yaml format file to dictionary. + """Loads yaml format file to dictionary. Parameters ---------- @@ -431,14 +440,13 @@ def yaml_load(yml_file: str) -> List[dict]: List[dict] list of loaded dictionaries """ - with open(yml_file, "r") as infile: return yaml.full_load(infile) def _find_N_dim(array: Union[list, str], template: str - ) -> Optional[Union[list, str]]: - """ Recursive helper function with two nested list as input. array is + ) -> Optional[Union[list, str]]: + """Recursive helper function with two nested list as input. array is traversed and its elements are fuzzy tested if they match expresion in template @@ -455,7 +463,6 @@ def _find_N_dim(array: Union[list, str], template: str :func:`complete_N_dim` :func:`replace_N_dim` """ - if isinstance(array, list): for a in array: ret = _find_N_dim(a, template) @@ -469,9 +476,9 @@ def _find_N_dim(array: Union[list, str], template: str def complete_N_dim(to_complete: list, to_find: list): - """ Recursive function with two list as input, one list contains incomplete + """Recursive function with two list as input, one list contains incomplete versions of strings and the other has full versions. Lists can be nested. - both are then traversed and the strings in the first list are completed + both are then traversed and the strings in the first list are completed with strings from the second list. Changes are made in place. Parameters @@ -488,7 +495,6 @@ def complete_N_dim(to_complete: list, to_find: list): -------- :func:`_find_N_dim` """ - if isinstance(to_complete, list): for i, _ in enumerate(to_complete): ret = complete_N_dim(to_complete[i], to_find) @@ -499,7 +505,7 @@ def complete_N_dim(to_complete: list, to_find: list): def replace_N_dim(to_replace: list, to_find: str): - """ Recursive function with nested list as input. The nested list elements + """Recursive function with nested list as input. The nested list elements are traversed and defined expresion is replaced by empty string in each element. Changes are made in place. @@ -514,7 +520,6 @@ def replace_N_dim(to_replace: list, to_find: str): -------- :func:`_find_N_dim` """ - if isinstance(to_replace, list): for i, _ in enumerate(to_replace): ret = replace_N_dim(to_replace[i], to_find) @@ -525,7 +530,7 @@ def replace_N_dim(to_replace: list, to_find: str): def delete_N_dim(to_delete: list, to_find: list) -> list: # type: ignore - """ Recursive function with nested list as input. The nested list elements + """Recursive function with nested list as input. The nested list elements are traversed and each that is equal to one of the elements in to_find list is deleted. Changes are made in place. @@ -536,7 +541,6 @@ def delete_N_dim(to_delete: list, to_find: list) -> list: # type: ignore to_find: str list of unwanted elements """ - if to_delete: if isinstance(to_delete[0], list): for i, td in enumerate(to_delete): diff --git a/wiki_music/utilities/sync.py b/wiki_music/utilities/sync.py index f92d8f8..b52813b 100644 --- a/wiki_music/utilities/sync.py +++ b/wiki_music/utilities/sync.py @@ -1,4 +1,4 @@ -""" Module for variable synchronization between application and GUI """ +"""Module for variable synchronization between application and GUI """ import logging from threading import Barrier, Lock @@ -8,14 +8,11 @@ __all__ = ["SharedVars"] -ClsStr = ClassVar[str] -ClsBool = ClassVar[bool] - class SharedVars: - """ Class for synchronizing info between threads SharedVars class provides - means to synchronize some variables between gui and application, - it serves to pass the questions asked in úarser to to PyQt GIU. + """Class for synchronizing variables between parser and GUI thread. + + Serves to pass the questions asked in parser to to PyQt GIU. Should not be instantiated, all methods and attributes belog to the class. The class implements API similar to logging.Logger with info(), warning() and exception() methods. These are prefered to directly @@ -31,7 +28,7 @@ class SharedVars: decides which dialog should be displayed in GUI - lyrics, genres, or lyrics write_json : bool - write yaml fiel contining tags for all tracks + write yaml file contining tags for all tracks offline_debbug : bool switch to offline debugging mode. Instead of online page a local pickle version is used of wikipedia.Page object and cover art @@ -62,32 +59,35 @@ class SharedVars: """ # action description - describe: ClsStr = "" - ask_exit: ClsStr = "" - switch: ClsStr = "" + progress: ClassVar[int] = 0 + threadpool_prog: ClassVar[int] = 0 + describe: ClassVar[str] = "" + ask_exit: ClassVar[str] = "" + switch: ClassVar[str] = "" # switches - write_json: ClsBool = False - offline_debbug: ClsBool = False - write_lyrics: ClsBool = False - assign_artists: ClsBool = False + write_json: ClassVar[bool] = False + offline_debbug: ClassVar[bool] = False + write_lyrics: ClassVar[bool] = False + assign_artists: ClassVar[bool] = False get_api_key: ClassVar[Union[str, bool]] = True # exceptions - has_warning: ClsStr = "" - has_exception: ClsStr = "" + has_warning: ClassVar[str] = "" + has_exception: ClassVar[str] = "" # control - wait_exit: ClsBool = False - terminate_app: ClsBool = False - wait: ClsBool = False - done: ClsBool = False - load: ClsBool = False + wait_exit: ClassVar[bool] = False + terminate_app: ClassVar[bool] = False + wait: ClassVar[bool] = False + done: ClassVar[bool] = False + load: ClassVar[bool] = False lock: ClassVar[Lock] = Lock() barrier: ClassVar[Barrier] = Barrier(2) @classmethod def re_init(cls): + """Initializes class variables to default values.""" cls.write_lyrics = None cls.select_genre = None @@ -98,6 +98,10 @@ def re_init(cls): cls.done = False cls.load = False + # progressbars + cls.progress = 0 + cls.threadpool_prog = 0 + # action description cls.describe = "" @@ -116,16 +120,67 @@ def re_init(cls): cls.has_warning = None cls.has_exception = None + @classmethod + def increment_progress(cls): + """Increments progress read by GUI main progressbar. + + See also + -------- + :meth:`wiki_music.gui_lib.main_window.Checkers._description_check` + periodically running method that displays progress in GUI. + """ + cls.lock.acquire() + cls.progress += 1 + cls.lock.release() + + @classmethod + def set_threadpool_prog(cls, count): + """Increments progress read by GUI threadpool progressbar dialog. + + See also + -------- + :meth:`wiki_music.gui_lib.main_window.Checkers._threadpool_check` + periodically running method that displays progress in GUI. + """ + cls.threadpool_prog = count + @classmethod def info(cls, msg: str): + """Messages to be displayed in GUI progressbar. + + Parameters + ---------- + msg: str + message text + """ + if len(msg) > 100 and ":" in msg: + msg = msg.split(": ", 1)[1] cls.lock.acquire() - cls.describe = str(msg) + cls.describe = str(msg).strip() cls.lock.release() @classmethod def warning(cls, msg: Union[Exception, str]): + """Warnings to be displayed in GUI. + + Warnings, only inform user but do not interupt the program. + + Parameters + ---------- + msg: str + message text + """ cls.has_warning = str(msg) @classmethod def exception(cls, msg: Union[Exception, str]): + """Exceptions to be displayed in GUI. + + Exceptions can interupt running program, based on their severity. + + Parameters + ---------- + msg: str + message text + """ cls.has_exception = str(msg) diff --git a/wiki_music/utilities/utils.py b/wiki_music/utilities/utils.py index 7cf3a71..fd92e44 100644 --- a/wiki_music/utilities/utils.py +++ b/wiki_music/utilities/utils.py @@ -1,4 +1,4 @@ -""" Basic utilities used by the whole package. """ +"""Basic utilities used by the whole package""" import argparse # lazy loaded import logging @@ -26,7 +26,7 @@ class MultiLog: - """ Passes the messages to logger instance and to SharedVars sychronization + """Passes the messages to logger instance and to SharedVars sychronization class where applicable as SharedVars does not implement whole Logger API See also @@ -40,39 +40,40 @@ class passing the messages to GUI Logger instance """ def __init__(self, logger): - self.logger = logger + self._logger = logger def debug(self, message: Any): - """ Issue a debug message. """ - self.logger.debug(message) + """Issue a debug message""" + self._logger.debug(message) def info(self, message: Any): - """ Issue a info message. """ - self.logger.info(message) + """Issue a info message""" + self._logger.info(message) SharedVars.info(message) + SharedVars.increment_progress() def warning(self, message: Any): - """ Issue a warning message. """ - self.logger.warning(message) + """Issue a warning message""" + self._logger.warning(message) SharedVars.has_warning = message def error(self, message: Any): - """ Issue a error message. """ - self.logger.error(message) + """Issue a error message""" + self._logger.error(message) def critical(self, message: Any): - """ Issue a critical message. """ - self.logger.critical(message) + """Issue a critical message""" + self._logger.critical(message) def exception(self, message: Any): - """ Issue a exception message. """ - self.logger.exception(message) + """Issue a exception message""" + self._logger.exception(message) SharedVars.has_exception = message def list_files(work_dir: str, file_type: str = "music", recurse: bool = True) -> List[str]: - """ List music files in directory. + """List music files in directory. Parameters ---------- @@ -123,7 +124,7 @@ def list_files(work_dir: str, file_type: str = "music", def to_bool(string: Union[str, bool]) -> bool: - """ Coverts string (yes, no, y, n adn capitalized versions) to bool.\n + """Coverts string (yes, no, y, n adn capitalized versions) to bool.\n Parameters ---------- @@ -147,7 +148,7 @@ def to_bool(string: Union[str, bool]) -> bool: def normalize(text: str) -> str: - """ NFKD string normalization + """NFKD string normalization Parameters ---------- @@ -163,7 +164,7 @@ def normalize(text: str) -> str: def we_are_frozen() -> bool: - """ Checks if the running code is frozen (e.g by cx-Freeze, pyinstaller). + """Checks if the running code is frozen (e.g by cx-Freeze, pyinstaller). Returns ------- @@ -176,7 +177,7 @@ def we_are_frozen() -> bool: def read_google_api_key(GUI) -> Optional[str]: - """ Reads google api key needed by lyricsfinder in external libraries from + """Reads google api key needed by lyricsfinder in external libraries from file. Returns @@ -258,7 +259,7 @@ def _get_google_api_key(GUI: bool) -> Optional[str]: def win_naming_convetion(string: str, dir_name=False) -> str: - """ Returns Windows normalized path name with removed forbiden + """Returns Windows normalized path name with removed forbiden characters. If platworm is not windows string is returned without changes. Parameters @@ -283,7 +284,7 @@ def win_naming_convetion(string: str, dir_name=False) -> str: def flatten_set(array: List[list]) -> set: - """ Converst 2D list to 1D set. + """Converst 2D list to 1D set. Parameters ---------- @@ -386,7 +387,7 @@ def input_parser() -> Tuple[bool, bool, bool, str, str, str, bool]: def loading(): - """ CLI loading marker + """CLI loading marker Warnings -------- diff --git a/wiki_music/utilities/wrappers.py b/wiki_music/utilities/wrappers.py index 2c54db3..66f952c 100644 --- a/wiki_music/utilities/wrappers.py +++ b/wiki_music/utilities/wrappers.py @@ -1,4 +1,4 @@ -""" Fancy wrapper function used in whole package. """ +"""Fancy wrapper functions used in whole package.""" import logging import os @@ -20,9 +20,10 @@ def exception(logger: "Logger", show_GUI: bool = True) -> Callable: - """ - A decorator that wraps the passed in function and logs exceptions should - one. Messages are sent to gui through SharedVars and to logger. + """Wraps the passed in function and logs exceptions should one occure. + + Messages are sent to gui through SharedVars and to logger. Application + is not interupted. Warnings -------- @@ -56,15 +57,16 @@ def wrapper(*args, **kwargs): def synchronized(lock: "Lock") -> Callable: - """ Synchronization decorator. Syncs all callables wrapped by this - decorator which are passed the same lock + """Synchronization decorator. + + Syncs all decorated callables which are passed the same lock. Warnings -------- Do not use on computationally heavy functions, as this could cause GUI freezing. When dealing with such functions always lock only single variables - + Parameters ---------- lock: threading.Lock @@ -75,7 +77,6 @@ def synchronized(lock: "Lock") -> Callable: Callable callable synchronized with passed lock """ - def real_wrapper(function: Callable) -> Callable: @wraps(function) def wrapper(*args, **kwargs): @@ -89,10 +90,10 @@ def wrapper(*args, **kwargs): def warning(logger: "Logger", show_GUI: bool = True) -> Callable: - """ - A decorator that wraps the passed in function and logs AttributeErrors - to logger warning and Exceptions to logger exceptions. Messages are sent to - gui through SharedVars and to logger. + """Catch and inform user of module defined exceptions. + + A decorator that wraps the passed in function and logs wiki_music defined + errors to logger warning. Messages are sent to gui through SharedVars. Warnings -------- @@ -111,7 +112,6 @@ def warning(logger: "Logger", show_GUI: bool = True) -> Callable: Callable wrapped callable which will not crash app when it raises error """ - def real_wrapper(function: Callable) -> Callable: @wraps(function) def wrapper(*args, **kwargs): @@ -120,7 +120,8 @@ def wrapper(*args, **kwargs): except (NoTracklistException, NoReleaseDateException, NoGenreException, NoCoverArtException, NoNames2ExtractException, NoContentsException, - NoPersonnelException, Mp3tagNotFoundException) as e: + NoPersonnelException, Mp3tagNotFoundException, + NltkUnavailableException) as e: logger.warning(e) if show_GUI: SharedVars.warning(e) @@ -130,7 +131,7 @@ def wrapper(*args, **kwargs): class Timer: - """ Timing context manager. measures execution time of a function. + """Timing context manager. measures execution time of a function. Parameters ---------- @@ -174,8 +175,7 @@ def __exit__(self, *args): def time_methods(function: Callable) -> Callable: - """ A decorator that wraps the passed in function and measures execution - time. + """Wraps the passed in function and measures execution time. Parameters ---------- @@ -188,10 +188,11 @@ def wrapper(*args, **kwargs) -> Callable: return wrapper + # TODO why exceptions are thrown for static methods?? def for_all_methods(decorator: Callable[[Any], Callable], exclude: List[str] = []) -> Callable: - """ Decorates class methods, except the ones in excluded list. + """Decorates class methods, except the ones in excluded list. Warnings -------- @@ -201,7 +202,7 @@ def for_all_methods(decorator: Callable[[Any], Callable], References ---------- https://stackoverflow.com/questions/6307761/how-to-decorate-all-functions-of-a-class-without-typing-it-over-and-over-for-eac - + Parameters ---------- decorator: Callable @@ -209,7 +210,6 @@ def for_all_methods(decorator: Callable[[Any], Callable], exclude: List[str] list of method names to exclude from decorating """ - @wraps(decorator) def decorate(cls: Callable) -> Callable: for attr in cls.__dict__: diff --git a/wiki_music/version.py b/wiki_music/version.py index 18cb5f1..893ea34 100644 --- a/wiki_music/version.py +++ b/wiki_music/version.py @@ -1 +1 @@ -__version__ = "0.3a4" \ No newline at end of file +__version__ = "0.4a0" \ No newline at end of file