feat: refactor progress reporting (#82)

* feat: refactore verbosity levels and add new mode * ci: change macos conda installation * ci: change windows gdal installation * ci: restore paths * feat: finish progress bar refactor * ci: fix macos tests * ci: change manual tests matrix * feat: change default osm extract source to geofabrik * chore: add link to the osm tags filters example * chore: add no cover pragma * chore: add badges to readme * chore: add changelog entry * feat: add new tests and refactor osm_fr iteration logic * chore: change test parameters * chore: change test parameters
kraina-ai · Apr 24, 2024 · 087d021 · 087d021
1 parent b94055f
commit 087d021
Show file tree

Hide file tree

Showing 16 changed files with 669 additions and 259 deletions.
diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml
@@ -11,7 +11,7 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
         include:
-          - os: macos-latest
+          - os: macos-13
             python-version: "3.12"
           - os: windows-latest
             python-version: "3.12"
@@ -24,11 +24,19 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install GDAL (unix)
-        if: matrix.os != 'windows-latest'
+      - name: Install GDAL (linux)
+        if: matrix.os == 'ubuntu-latest'
         run: |
           $CONDA/bin/conda install -c conda-forge gdal
           $CONDA/bin/ogr2ogr --version
+      - name: Install GDAL (macos)
+        if: matrix.os == 'macos-13'
+        run: |
+          CONDA=$HOME/miniconda
+          wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          bash Miniconda3-latest-MacOSX-x86_64.sh -b -p $CONDA
+          $CONDA/bin/conda install -c conda-forge gdal
+          $CONDA/bin/ogr2ogr --version
       - name: Install GDAL (windows)
         if: matrix.os == 'windows-latest'
         run: |
@@ -56,9 +64,15 @@ jobs:
           key: tox-cache-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pdm.lock') }}
           restore-keys: |
             tox-cache-${{ matrix.os }}-${{ matrix.python-version }}-
-      - name: Run tests with tox (unix)
-        if: matrix.os != 'windows-latest'
+      - name: Run tests with tox (linux)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          PATH=$CONDA/bin:$PATH
+          pdm run tox -e python${{ matrix.python-version }}
+      - name: Run tests with tox (macos)
+        if: matrix.os == 'macos-13'
         run: |
+          CONDA=$HOME/miniconda
           PATH=$CONDA/bin:$PATH
           pdm run tox -e python${{ matrix.python-version }}
       - name: Run tests with tox (windows)

diff --git a/.github/workflows/manual_tests.yml b/.github/workflows/manual_tests.yml
@@ -13,6 +13,8 @@ jobs:
           include:
             - os: macos-latest
               python-version: "3.12"
+            - os: macos-13
+              python-version: "3.12"
             - os: windows-latest
               python-version: "3.12"
       env:
@@ -24,11 +26,19 @@ jobs:
           uses: actions/setup-python@v4
           with:
             python-version: ${{ matrix.python-version }}
-        - name: Install GDAL (unix)
-          if: matrix.os != 'windows-latest'
+        - name: Install GDAL (linux)
+          if: matrix.os == 'ubuntu-latest'
           run: |
             $CONDA/bin/conda install -c conda-forge gdal
             $CONDA/bin/ogr2ogr --version
+        - name: Install GDAL (macos)
+          if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
+          run: |
+            CONDA=$HOME/miniconda
+            wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh
+            bash Miniconda3-latest-MacOSX-arm64.sh -b -p $CONDA
+            $CONDA/bin/conda install -c conda-forge gdal
+            $CONDA/bin/ogr2ogr --version
         - name: Install GDAL (windows)
           if: matrix.os == 'windows-latest'
           run: |
@@ -40,9 +50,18 @@ jobs:
           run: pdm lock --lockfile pdm.newest.lock --strategy no_cross_platform -dG:all
         - name: Install quackosm and tests dependencies
           run: pdm install --lockfile pdm.newest.lock -dG:all
-        - name: Run tests with pytest (unix)
-          if: matrix.os != 'windows-latest'
+        - name: Run tests with pytest (linux)
+          if: matrix.os == 'ubuntu-latest'
+          run: |
+            PATH=$CONDA/bin:$PATH
+            pdm run pytest -v -s --durations=20 --doctest-modules --doctest-continue-on-failure quackosm
+            pdm run pytest -v -s --durations=20 tests/base
+            pdm run pytest -v -s --durations=20 tests/optional_imports
+            pdm run pytest -v -s --durations=20 tests/benchmark
+        - name: Run tests with pytest (macos)
+          if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
           run: |
+            CONDA=$HOME/miniconda
             PATH=$CONDA/bin:$PATH
             pdm run pytest -v -s --durations=20 --doctest-modules --doctest-continue-on-failure quackosm
             pdm run pytest -v -s --durations=20 tests/base
@@ -52,7 +71,6 @@ jobs:
           if: matrix.os == 'windows-latest'
           run: |
             $env:Path = "$env:CONDA\Library\bin;" + $env:Path
-            $env:PYTHONIOENCODING = "utf-8"
             pdm run pytest -v -s --durations=20 --doctest-modules --doctest-continue-on-failure quackosm
             pdm run pytest -v -s --durations=20 tests/base
             pdm run pytest -v -s --durations=20 tests/optional_imports

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Transient mode of reporting progress with output being removed after operation [#77](https://github.com/kraina-ai/quackosm/issues/77)
+- Tracking for multiple files within single operation
+- New tests for all 3 methods of combining result files together with duplicated features removal
+
+### Changed
+
+- Refactored internal Rich progress reporting process
+- Replaced `silent_mode` parameter with `verbosity_mode` argument
+- Changed default `OSMExtractSource` value from `any` to `Geofabrik`
+- Modified OpenStreetMap_fr scraping process with better progress bar UI
+
+### Removed
+
+- `silent_mode` parameter from the Python API
+
+### Fixed
+
+- Replaced slash characters in Geofabrik index IDs with underscore to prevent nested directories creation
+- Added additional check on number of points in a LineString when trying to represent them as a polygon
+
 ## [0.6.1] - 2024-04-17
 
 ### Changed

diff --git a/README.md b/README.md
@@ -3,6 +3,31 @@
   <small>Generated using DALL·E 3 model with this prompt: A logo for a python library with White background, high quality, 8k. Cute duck and globe with cartography elements. Library for reading OpenStreetMap data using DuckDB.</small>
 </p>
 
+<p align="center">
+    <img alt="GitHub" src="https://img.shields.io/github/license/kraina-ai/quackosm?logo=apache&logoColor=%23fff">
+    <img src="https://img.shields.io/github/checks-status/kraina-ai/quackosm/main?logo=GitHubActions&logoColor=%23fff" alt="Checks">
+    <a href="https://github.com/kraina-ai/quackosm/actions/workflows/ci-dev.yml" target="_blank">
+        <img alt="GitHub Workflow Status - DEV" src="https://img.shields.io/github/actions/workflow/status/kraina-ai/quackosm/ci-dev.yml?label=build-dev&logo=GitHubActions&logoColor=%23fff">
+    </a>
+    <a href="https://github.com/kraina-ai/quackosm/actions/workflows/ci-prod.yml" target="_blank">
+        <img alt="GitHub Workflow Status - PROD" src="https://img.shields.io/github/actions/workflow/status/kraina-ai/quackosm/ci-prod.yml?label=build-prod&logo=GitHubActions&logoColor=%23fff">
+    </a>
+    <a href="https://results.pre-commit.ci/latest/github/kraina-ai/quackosm/main" target="_blank">
+        <img src="https://results.pre-commit.ci/badge/github/kraina-ai/quackosm/main.svg" alt="pre-commit.ci status">
+    </a>
+    <a href="https://www.codefactor.io/repository/github/kraina-ai/quackosm"><img alt="CodeFactor Grade" src="https://img.shields.io/codefactor/grade/github/kraina-ai/quackosm?logo=codefactor&logoColor=%23fff"></a>
+    <a href="https://app.codecov.io/gh/kraina-ai/quackosm/tree/main"><img alt="Codecov" src="https://img.shields.io/codecov/c/github/kraina-ai/quackosm?logo=codecov&token=PRS4E02ZX0&logoColor=%23fff"></a>
+    <a href="https://pypi.org/project/quackosm" target="_blank">
+        <img src="https://img.shields.io/pypi/v/quackosm?color=%2334D058&label=pypi%20package&logo=pypi&logoColor=%23fff" alt="Package version">
+    </a>
+    <a href="https://pypi.org/project/quackosm" target="_blank">
+        <img src="https://img.shields.io/pypi/pyversions/quackosm.svg?color=%2334D058&logo=python&logoColor=%23fff" alt="Supported Python versions">
+    </a>
+    <a href="https://pypi.org/project/quackosm" target="_blank">
+        <img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/quackosm">
+    </a>
+</p>
+
 # QuackOSM
 
 An open-source tool for reading OpenStreetMap PBF files using DuckDB.

diff --git a/dev/generate_resources_usage_plot.ipynb b/dev/generate_resources_usage_plot.ipynb
@@ -112,7 +112,7 @@
     "        pbf_path=pbf_path,\n",
     "        working_directory=working_directory,\n",
     "        ignore_cache=True,\n",
-    "        silent_mode=True,\n",
+    "        verbosity_mode=\"silent\",\n",
     "    )\n",
     "\n",
     "\n",

diff --git a/examples/advanced_examples/osm_tags_filter.ipynb b/examples/advanced_examples/osm_tags_filter.ipynb
@@ -161,7 +161,7 @@
    "outputs": [],
    "source": [
     "tags_filter = {\"amenity\": \"bench\"}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True)"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\")"
    ]
   },
   {
@@ -178,7 +178,7 @@
    "outputs": [],
    "source": [
     "tags_filter = {\"amenity\": [\"cafe\", \"restaurant\", \"bar\"]}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True)"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\")"
    ]
   },
   {
@@ -195,7 +195,7 @@
    "outputs": [],
    "source": [
     "tags_filter = {\"amenity\": True, \"leisure\": True}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True)"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\")"
    ]
   },
   {
@@ -215,7 +215,7 @@
     "    \"shopping\": {\"shop\": True, \"landuse\": \"retail\"},\n",
     "    \"tourism\": {\"tourism\": True, \"historic\": True},\n",
     "}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=grouped_tags_filter, silent_mode=True)"
+    "get_features_gdf(monaco_pbf_file, tags_filter=grouped_tags_filter, verbosity_mode=\"silent\")"
    ]
   },
   {
@@ -362,7 +362,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()"
    ]
   },
   {
@@ -381,7 +381,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "get_features_gdf(monaco_pbf_file, tags_filter=None, silent_mode=True).sort_index()"
+    "get_features_gdf(monaco_pbf_file, tags_filter=None, verbosity_mode=\"silent\").sort_index()"
    ]
   },
   {
@@ -404,7 +404,7 @@
     "    tags_filter=tags_filter,\n",
     "    explode_tags=True,\n",
     "    keep_all_tags=False,\n",
-    "    silent_mode=True,\n",
+    "    verbosity_mode=\"silent\",\n",
     ").sort_index()"
    ]
   },
@@ -428,7 +428,7 @@
     "    tags_filter=tags_filter,\n",
     "    explode_tags=True,\n",
     "    keep_all_tags=True,\n",
-    "    silent_mode=True,\n",
+    "    verbosity_mode=\"silent\",\n",
     ").sort_index()"
    ]
   },
@@ -452,7 +452,7 @@
     "    tags_filter=tags_filter,\n",
     "    explode_tags=False,\n",
     "    keep_all_tags=False,\n",
-    "    silent_mode=True,\n",
+    "    verbosity_mode=\"silent\",\n",
     ").sort_index()"
    ]
   },
@@ -476,7 +476,7 @@
     "    tags_filter=tags_filter,\n",
     "    explode_tags=False,\n",
     "    keep_all_tags=True,\n",
-    "    silent_mode=True,\n",
+    "    verbosity_mode=\"silent\",\n",
     ").sort_index()"
    ]
   },
@@ -531,7 +531,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()"
    ]
   },
   {
@@ -572,7 +572,7 @@
     "get_features_gdf(\n",
     "    monaco_pbf_file,\n",
     "    tags_filter=tags_filter,\n",
-    "    silent_mode=True,\n",
+    "    verbosity_mode=\"silent\",\n",
     "    keep_all_tags=True,\n",
     "    explode_tags=True,\n",
     ").sort_index()"
@@ -617,7 +617,7 @@
    "outputs": [],
    "source": [
     "get_features_gdf(\n",
-    "    monaco_pbf_file, tags_filter=tags_filter, keep_all_tags=True, silent_mode=True\n",
+    "    monaco_pbf_file, tags_filter=tags_filter, keep_all_tags=True, verbosity_mode=\"silent\"\n",
     ").sort_index()"
    ]
   },
@@ -650,7 +650,7 @@
    "outputs": [],
    "source": [
     "tags_filter = {\"addr:*\": True}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()"
    ]
   },
   {
@@ -672,7 +672,7 @@
    "source": [
     "tags_filter = {\"building\": True, \"addr:*\": False}\n",
     "get_features_gdf(\n",
-    "    monaco_pbf_file, tags_filter=tags_filter, keep_all_tags=True, silent_mode=True\n",
+    "    monaco_pbf_file, tags_filter=tags_filter, keep_all_tags=True, verbosity_mode=\"silent\"\n",
     ").sort_index()"
    ]
   },
@@ -692,7 +692,7 @@
    "outputs": [],
    "source": [
     "tags_filter = {\"highway\": \"*ary\"}\n",
-    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()"
+    "get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()"
    ]
   },
   {
@@ -712,7 +712,7 @@
    "source": [
     "tags_filter = {\"name:*\": \"*Grand Prix*\"}\n",
     "get_features_gdf(\n",
-    "    monaco_pbf_file, tags_filter=tags_filter, silent_mode=True\n",
+    "    monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\"\n",
     ").sort_index()"
    ]
   },
@@ -737,7 +737,7 @@
    "source": [
     "try:\n",
     "    tags_filter = {\"group_1\": {\"amenity\": \"bench\"}, \"group_2\": {\"amenity\": False}}\n",
-    "    get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()\n",
+    "    get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()\n",
     "except ValueError as ex:\n",
     "    print(ex)"
    ]
@@ -750,7 +750,7 @@
    "source": [
     "try:\n",
     "    tags_filter = {\"name:en\": True, \"name:*\": False}\n",
-    "    get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, silent_mode=True).sort_index()\n",
+    "    get_features_gdf(monaco_pbf_file, tags_filter=tags_filter, verbosity_mode=\"silent\").sort_index()\n",
     "except ValueError as ex:\n",
     "    print(ex)"
    ]

diff --git a/examples/command_line_interface.ipynb b/examples/command_line_interface.ipynb
@@ -401,7 +401,7 @@
     "\n",
     "OSM tags filter logic is based on the filter from the [`OSMnx`](https://github.com/gboeing/osmnx) library.\n",
     "\n",
-    "Filter is expected to be in the form of dictionary with `keys` as string and `values` as one of the types: string, list of strings, bool `True` value.\n",
+    "Filter is expected to be in the form of dictionary with `keys` as string and `values` as one of the types: string, list of strings or bool value. Full tutorial for OSM tags filters can be accessed [here](../advanced_examples/osm_tags_filter).\n",
     "\n",
     "Example filters:\n",
     "\n",