diff --git a/nb/download_tablefile.ipynb b/nb/download_tablefile.ipynb index 8bb7743..1702ee9 100644 --- a/nb/download_tablefile.ipynb +++ b/nb/download_tablefile.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -747,7 +747,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.9.12 ('pygenesis')", "language": "python", "name": "python3" }, @@ -761,11 +761,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "ee0113c470e2ab03fd08da308faad6cb3acef2959a5a1adc44423161e2606732" + "hash": "c50015765afe066708d859da3faaa0505e12b679b95f6727e524b172064c6917" } } }, diff --git a/nb/parse_cube.ipynb b/nb/parse_cube.ipynb index 6c84c2e..b03dddb 100644 --- a/nb/parse_cube.ipynb +++ b/nb/parse_cube.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "scrolled": true }, @@ -641,21 +641,9 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['2015', '2016', '2017', '2018', '2019', '2020', '2021'],\n", - " dtype=object)" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rename_axes(cube)[\"QEI\"][\"JAHR\"].unique()" ] @@ -669,11 +657,8 @@ } ], "metadata": { - "interpreter": { - "hash": "02e23b522f8c3795158421909d41ced4ef90521258d58d1c53bee449d96f71e3" - }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.9.12 ('pygenesis')", "language": "python", "name": "python3" }, @@ -687,7 +672,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.9.12" + }, + "vscode": { + "interpreter": { + "hash": "c50015765afe066708d859da3faaa0505e12b679b95f6727e524b172064c6917" + } } }, "nbformat": 4, diff --git a/src/pygenesis/cache.py b/src/pygenesis/cache.py index ca5372e..bf0fd70 100644 --- a/src/pygenesis/cache.py +++ b/src/pygenesis/cache.py @@ -1,9 +1,10 @@ """Module provides functions/decorators to cache downloaded data.""" import logging +import shutil from datetime import date from functools import wraps from pathlib import Path -from typing import Callable +from typing import Callable, Optional import pandas as pd @@ -50,3 +51,38 @@ def wrapper_func(**kwargs): return data return wrapper_func + + +# TODO: Write test, use ID instead of file +def clean_cache(file: Optional[Path]) -> None: + """Clean the data cache by overall or specific file removal. + + Args: + file (Path, optional): Path to the file which should be removed from cache directory. + """ + config = load_config() + + # check for cache_dir in DATA section of the config.ini + # TODO: What happens if this key is not defined? is that error understandable? + cache_dir = Path(config["DATA"]["cache_dir"]) + + if not cache_dir.is_dir() or not cache_dir.exists(): + logger.critical( + "Cache dir does not exist! Please make sure init_config() was run properly. Path: %s", + cache_dir, + ) + + # remove (previously specified) file(s) from the data cache + files = [cache_dir / file] if file is not None else cache_dir.glob(file) + + # TODO: remove complete tree according to ID file tree structure + for filename in files: + file_path = cache_dir / filename + try: + if file_path.is_file() or file_path.is_symlink(): + file_path.unlink() + elif file_path.is_dir(): + shutil.rmtree(file_path) + # TODO: narrow down this exception + except Exception as e: + print(f"Failed to delete {file_path}. Reason: {e}") diff --git a/src/pygenesis/config.py b/src/pygenesis/config.py index 84ab33f..8f48d2b 100644 --- a/src/pygenesis/config.py +++ b/src/pygenesis/config.py @@ -8,11 +8,8 @@ If there is no config.ini in the given config_dir, a default config will be created with empty credentials. """ import logging -import os -import shutil from configparser import ConfigParser from pathlib import Path -from typing import Optional PKG_NAME = __name__.split(".", maxsplit=1)[0] @@ -151,52 +148,4 @@ def _create_default_config() -> ConfigParser: return config -# TODO: Decide where this function should go... Maybe a feature of the new cache.py? -def clean_cache(file: Optional[Path]) -> None: - """Clean the data cache by overall or specific file removal. - - Args: - file (Path, optional): Path to the file which should be removed from cache directory. - """ - config_file = get_config_path_from_settings() - config = _load_config(config_file) - - # check for cache_dir in DATA section of the config.ini - if config.has_section("DATA"): - logger.info("Cache config %s was loaded successfully.", config_file) - - if not config.get("DATA", "cache_dir") or not os.path.isdir( - config.get("DATA", "cache_dir") - ): - logger.critical( - "Cache directory not set and/or corrupted! " - "Please make sure to run init_config() and set up the data cache appropriately. " - ) - raise KeyError( - "Issue with 'cache_dir' in the config.ini. Please rerun init_config()." - ) - - # load the folder path - cache_dir = config["DATA"]["cache_dir"] - - # remove (previously specified) file(s) from the data cache - files = ( - [os.path.join(cache_dir, file)] - if file is not None - else os.listdir(cache_dir) - ) - - for filename in files: - file_path = os.path.join(cache_dir, filename) - try: - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - print("Failed to delete %s. Reason: %s" % (file_path, e)) - - return None - - create_settings() diff --git a/src/pygenesis/http_helper.py b/src/pygenesis/http_helper.py index 2427385..c66839c 100644 --- a/src/pygenesis/http_helper.py +++ b/src/pygenesis/http_helper.py @@ -48,12 +48,12 @@ def _check_invalid_status_code(status_code: int) -> None: status_code (int): Status code from the response object Raises: - Exception: Generic exception if status 4xx or 5xx is returned + AssertionError: Assert that status is not 4xx or 5xx """ - if (status_code // 100) in [4, 5]: - raise Exception( - f"Error {status_code}: The server returned a {status_code} status code" - ) + assert status_code // 100 not in [ + 4, + 5, + ], f"Error {status_code}: The server returned a {status_code} status code" return None @@ -73,13 +73,11 @@ def _check_invalid_destatis_status_code(response: requests.Response) -> None: return None _check_destatis_status(response_dict.get("Status", {})) - return None - def _check_destatis_status(destatis_status: dict) -> None: """ Helper method which checks the status message from Destatis. - If the status message is erroneous an exception will be raised. + If the status message is erroneous an error will be raised. Possible Codes (2.1.2 Grundstruktur der Responses): - 0: "erfolgreich" (Type: "Information") @@ -90,10 +88,10 @@ def _check_destatis_status(destatis_status: dict) -> None: destatis_status (dict): Status response dict from Destatis Raises: - Exception: Generic exception if the status code displays an error + # TODO: Is this a Value or KeyError? + ValueError: If the status code or type displays an error (caused by the user inputs) """ - # -1 is a status code that according to the documentation should not occur - # and thus only is found if the status response dict is empty + # -1 status code for unexpected errors and if no status code is given (faulty response) destatis_status_code = destatis_status.get("Code", -1) destatis_status_type = destatis_status.get("Type") destatis_status_content = destatis_status.get("Content") @@ -103,14 +101,14 @@ def _check_destatis_status(destatis_status: dict) -> None: # check for generic/ system error if destatis_status_code == -1: - raise Exception( + raise ValueError( "Error: There is a system error.\ Please check your query parameters." ) # check for destatis/ query errors elif (destatis_status_code == 104) or (destatis_status_type in error_en_de): - raise Exception(destatis_status_content) + raise ValueError(destatis_status_content) # print warnings to user elif (destatis_status_code == 22) or ( @@ -118,4 +116,5 @@ def _check_destatis_status(destatis_status: dict) -> None: ): warnings.warn(destatis_status_content, UserWarning, stacklevel=2) - return None + # TODO: pass response information to user, however logger.info might be overlooked + # as standard only shows beyond warning -> HowTo? diff --git a/tests/unit_tests/test_http_helper.py b/tests/unit_tests/test_http_helper.py index f7a03a6..9607574 100644 --- a/tests/unit_tests/test_http_helper.py +++ b/tests/unit_tests/test_http_helper.py @@ -18,7 +18,7 @@ def test__check_invalid_status_code_with_error(): for _handle_status_code method. """ for status_code in [400, 500]: - with pytest.raises(Exception) as e: + with pytest.raises(AssertionError) as e: _check_invalid_status_code(status_code) assert ( str(e.value) @@ -96,7 +96,7 @@ def test__check_invalid_destatis_status_code_with_error(): # extract status content which is raised status_content = status.json().get("Status").get("Content") - with pytest.raises(Exception) as e: + with pytest.raises(ValueError) as e: _check_invalid_destatis_status_code(status) assert str(e.value) == status_content