Merged changes from draft branch, including pylint fixes, Exception a…

…nd Error clarification. Also moved clean_cache function. Some ToDos still left, #45.

nb/download_tablefile.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -747,7 +747,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.9.12 ('pygenesis')",
    "language": "python",
    "name": "python3"
   },
@@ -761,11 +761,11 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.7"
+   "version": "3.9.12"
   },
   "vscode": {
    "interpreter": {
-    "hash": "ee0113c470e2ab03fd08da308faad6cb3acef2959a5a1adc44423161e2606732"
+    "hash": "c50015765afe066708d859da3faaa0505e12b679b95f6727e524b172064c6917"
    }
   }
  },

nb/parse_cube.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -12,7 +12,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -49,7 +49,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "scrolled": true
    },
@@ -641,21 +641,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "array(['2015', '2016', '2017', '2018', '2019', '2020', '2021'],\n",
-       "      dtype=object)"
-      ]
-     },
-     "execution_count": 21,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "rename_axes(cube)[\"QEI\"][\"JAHR\"].unique()"
    ]
@@ -669,11 +657,8 @@
   }
  ],
  "metadata": {
-  "interpreter": {
-   "hash": "02e23b522f8c3795158421909d41ced4ef90521258d58d1c53bee449d96f71e3"
-  },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.9.12 ('pygenesis')",
    "language": "python",
    "name": "python3"
   },
@@ -687,7 +672,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.7"
+   "version": "3.9.12"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "c50015765afe066708d859da3faaa0505e12b679b95f6727e524b172064c6917"
+   }
   }
  },
  "nbformat": 4,

src/pygenesis/cache.py

@@ -1,9 +1,10 @@
 """Module provides functions/decorators to cache downloaded data."""
 import logging
+import shutil
 from datetime import date
 from functools import wraps
 from pathlib import Path
-from typing import Callable
+from typing import Callable, Optional
 
 import pandas as pd
 
@@ -50,3 +51,38 @@ def wrapper_func(**kwargs):
         return data
 
     return wrapper_func
+
+
+# TODO: Write test, use ID instead of file
+def clean_cache(file: Optional[Path]) -> None:
+    """Clean the data cache by overall or specific file removal.
+
+    Args:
+        file (Path, optional): Path to the file which should be removed from cache directory.
+    """
+    config = load_config()
+
+    # check for cache_dir in DATA section of the config.ini
+    # TODO: What happens if this key is not defined? is that error understandable?
+    cache_dir = Path(config["DATA"]["cache_dir"])
+
+    if not cache_dir.is_dir() or not cache_dir.exists():
+        logger.critical(
+            "Cache dir does not exist! Please make sure init_config() was run properly. Path: %s",
+            cache_dir,
+        )
+
+    # remove (previously specified) file(s) from the data cache
+    files = [cache_dir / file] if file is not None else cache_dir.glob(file)
+
+    # TODO: remove complete tree according to ID file tree structure
+    for filename in files:
+        file_path = cache_dir / filename
+        try:
+            if file_path.is_file() or file_path.is_symlink():
+                file_path.unlink()
+            elif file_path.is_dir():
+                shutil.rmtree(file_path)
+        # TODO: narrow down this exception
+        except Exception as e:
+            print(f"Failed to delete {file_path}. Reason: {e}")

src/pygenesis/config.py

@@ -8,11 +8,8 @@
 If there is no config.ini in the given config_dir, a default config will be created with empty credentials.
 """
 import logging
-import os
-import shutil
 from configparser import ConfigParser
 from pathlib import Path
-from typing import Optional
 
 PKG_NAME = __name__.split(".", maxsplit=1)[0]
 
@@ -151,52 +148,4 @@ def _create_default_config() -> ConfigParser:
     return config
 
 
-# TODO: Decide where this function should go... Maybe a feature of the new cache.py?
-def clean_cache(file: Optional[Path]) -> None:
-    """Clean the data cache by overall or specific file removal.
-
-    Args:
-        file (Path, optional): Path to the file which should be removed from cache directory.
-    """
-    config_file = get_config_path_from_settings()
-    config = _load_config(config_file)
-
-    # check for cache_dir in DATA section of the config.ini
-    if config.has_section("DATA"):
-        logger.info("Cache config %s was loaded successfully.", config_file)
-
-        if not config.get("DATA", "cache_dir") or not os.path.isdir(
-            config.get("DATA", "cache_dir")
-        ):
-            logger.critical(
-                "Cache directory not set and/or corrupted! "
-                "Please make sure to run init_config() and set up the data cache appropriately. "
-            )
-            raise KeyError(
-                "Issue with 'cache_dir' in the config.ini. Please rerun init_config()."
-            )
-
-    # load the folder path
-    cache_dir = config["DATA"]["cache_dir"]
-
-    # remove (previously specified) file(s) from the data cache
-    files = (
-        [os.path.join(cache_dir, file)]
-        if file is not None
-        else os.listdir(cache_dir)
-    )
-
-    for filename in files:
-        file_path = os.path.join(cache_dir, filename)
-        try:
-            if os.path.isfile(file_path) or os.path.islink(file_path):
-                os.unlink(file_path)
-            elif os.path.isdir(file_path):
-                shutil.rmtree(file_path)
-        except Exception as e:
-            print("Failed to delete %s. Reason: %s" % (file_path, e))
-
-    return None
-
-
 create_settings()

src/pygenesis/http_helper.py

@@ -48,12 +48,12 @@ def _check_invalid_status_code(status_code: int) -> None:
         status_code (int): Status code from the response object
 
     Raises:
-        Exception: Generic exception if status 4xx or 5xx is returned
+        AssertionError: Assert that status is not 4xx or 5xx
     """
-    if (status_code // 100) in [4, 5]:
-        raise Exception(
-            f"Error {status_code}: The server returned a {status_code} status code"
-        )
+    assert status_code // 100 not in [
+        4,
+        5,
+    ], f"Error {status_code}: The server returned a {status_code} status code"
 
     return None
 
@@ -73,13 +73,11 @@ def _check_invalid_destatis_status_code(response: requests.Response) -> None:
         return None
     _check_destatis_status(response_dict.get("Status", {}))
 
-    return None
-
 
 def _check_destatis_status(destatis_status: dict) -> None:
     """
     Helper method which checks the status message from Destatis.
-    If the status message is erroneous an exception will be raised.
+    If the status message is erroneous an error will be raised.
 
     Possible Codes (2.1.2 Grundstruktur der Responses):
     - 0: "erfolgreich" (Type: "Information")
@@ -90,10 +88,10 @@ def _check_destatis_status(destatis_status: dict) -> None:
         destatis_status (dict): Status response dict from Destatis
 
     Raises:
-        Exception: Generic exception if the status code displays an error
+        # TODO: Is this a Value or KeyError?
+        ValueError: If the status code or type displays an error (caused by the user inputs)
     """
-    # -1 is a status code that according to the documentation should not occur
-    # and thus only is found if the status response dict is empty
+    # -1 status code for unexpected errors and if no status code is given (faulty response)
     destatis_status_code = destatis_status.get("Code", -1)
     destatis_status_type = destatis_status.get("Type")
     destatis_status_content = destatis_status.get("Content")
@@ -103,19 +101,20 @@ def _check_destatis_status(destatis_status: dict) -> None:
 
     # check for generic/ system error
     if destatis_status_code == -1:
-        raise Exception(
+        raise ValueError(
             "Error: There is a system error.\
                 Please check your query parameters."
         )
 
     # check for destatis/ query errors
     elif (destatis_status_code == 104) or (destatis_status_type in error_en_de):
-        raise Exception(destatis_status_content)
+        raise ValueError(destatis_status_content)
 
     # print warnings to user
     elif (destatis_status_code == 22) or (
         destatis_status_type in warning_en_de
     ):
         warnings.warn(destatis_status_content, UserWarning, stacklevel=2)
 
-    return None
+    # TODO: pass response information to user, however logger.info might be overlooked
+    # as standard only shows beyond warning -> HowTo?

tests/unit_tests/test_http_helper.py

@@ -18,7 +18,7 @@ def test__check_invalid_status_code_with_error():
     for _handle_status_code method.
     """
     for status_code in [400, 500]:
-        with pytest.raises(Exception) as e:
+        with pytest.raises(AssertionError) as e:
             _check_invalid_status_code(status_code)
         assert (
             str(e.value)
@@ -96,7 +96,7 @@ def test__check_invalid_destatis_status_code_with_error():
         # extract status content which is raised
         status_content = status.json().get("Status").get("Content")
 
-        with pytest.raises(Exception) as e:
+        with pytest.raises(ValueError) as e:
             _check_invalid_destatis_status_code(status)
         assert str(e.value) == status_content