From 0f24183db7125a710a30de66ffd29e12b4a204d7 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Sun, 27 Aug 2023 15:05:51 +0200 Subject: [PATCH 1/8] use pydantic_settings to handle settings with env variables --- mapchete/io/raster.py | 6 +++--- mapchete/io/settings.py | 14 +++++++++----- mapchete/io/vector.py | 4 ++-- pyproject.toml | 1 + requirements.txt | 1 + 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mapchete/io/raster.py b/mapchete/io/raster.py index 30fde218..89a8562a 100644 --- a/mapchete/io/raster.py +++ b/mapchete/io/raster.py @@ -26,7 +26,7 @@ from mapchete._timer import Timer from mapchete.errors import MapcheteIOError from mapchete.io import copy -from mapchete.io.settings import MAPCHETE_IO_RETRY_SETTINGS +from mapchete.io.settings import IORetrySettings from mapchete.path import MPath, fs_from_path from mapchete.tile import BufferedTile from mapchete.validate import validate_write_window_params @@ -516,7 +516,7 @@ def _get_warped_array( raise -@retry(logger=logger, exceptions=RasterioIOError, **MAPCHETE_IO_RETRY_SETTINGS) +@retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) def _rasterio_read( input_file=None, indexes=None, @@ -621,7 +621,7 @@ def read_raster_no_crs(input_file, indexes=None, gdal_opts=None): FileNotFoundError if file cannot be found. """ - @retry(logger=logger, exceptions=RasterioIOError, **MAPCHETE_IO_RETRY_SETTINGS) + @retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) def _read(): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/mapchete/io/settings.py b/mapchete/io/settings.py index 4c175b2d..a9aa083e 100644 --- a/mapchete/io/settings.py +++ b/mapchete/io/settings.py @@ -1,6 +1,8 @@ import logging import os +from pydantic_settings import BaseSettings, SettingsConfigDict + logger = logging.getLogger(__name__) @@ -33,8 +35,10 @@ def _merge_gdal_defaults_with_env(): VSI_CACHE_SIZE=5_000_000, ) GDAL_HTTP_OPTS = _merge_gdal_defaults_with_env() -MAPCHETE_IO_RETRY_SETTINGS = { - "tries": int(os.environ.get("MAPCHETE_IO_RETRY_TRIES", "3")), - "delay": float(os.environ.get("MAPCHETE_IO_RETRY_DELAY", "1")), - "backoff": float(os.environ.get("MAPCHETE_IO_RETRY_BACKOFF", "1")), -} + + +class IORetrySettings(BaseSettings): + tries: int = 3 + delay: float = 1.0 + backoff: float = 1.0 + model_config = SettingsConfigDict(env_prefix="MAPCHETE_IO_RETRY_") diff --git a/mapchete/io/vector.py b/mapchete/io/vector.py index 7c420eae..568326eb 100644 --- a/mapchete/io/vector.py +++ b/mapchete/io/vector.py @@ -25,7 +25,7 @@ segmentize_geometry, to_shape, ) -from mapchete.io.settings import MAPCHETE_IO_RETRY_SETTINGS +from mapchete.io.settings import IORetrySettings from mapchete.path import MPath, fs_from_path, path_exists from mapchete.types import Bounds from mapchete.validate import validate_bounds @@ -323,7 +323,7 @@ def __exit__(self, *args): @retry( logger=logger, exceptions=(DriverError, FionaError, FionaValueError), - **MAPCHETE_IO_RETRY_SETTINGS, + **IORetrySettings(), ) def _get_reprojected_features( inp=None, diff --git a/pyproject.toml b/pyproject.toml index 8d8dc68c..33f0c6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "numpy>=1.16", "oyaml", "pydantic<2.0.0", + "pydantic_settings", "pyproj", "python-dateutil", "rasterio>1.2.10", diff --git a/requirements.txt b/requirements.txt index 2f2b00c1..2dd34c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ numpy>=1.16 oyaml>=0.9 pyproj pydantic<2.0.0 +pydantic_settings retry>=0.9.2 rasterio>1.2.10 Shapely>=2.0.0 From 6e3919278473bd0ddd1333d695032609a7ac827f Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 28 Aug 2023 08:43:50 +0200 Subject: [PATCH 2/8] use pydantic 2.0 --- mapchete/config.py | 22 +++++++++++----------- mapchete/io/raster.py | 4 ++-- mapchete/io/vector.py | 2 +- pyproject.toml | 4 ++-- requirements.txt | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mapchete/config.py b/mapchete/config.py index ebcd6277..e74c3d81 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -81,16 +81,16 @@ def _metatiling(cls, value: int) -> int: # pragma: no cover class ProcessConfig(BaseModel, arbitrary_types_allowed=True): pyramid: PyramidConfig output: dict - zoom_levels: Union[dict, int, list] - process: Union[str, MPath, List[str], None] - baselevels: Union[dict, None] - input: Union[dict, None] - config_dir: Union[str, MPath, None] - area: Union[str, MPath, BaseGeometry, None] - area_crs: Union[dict, str, None] - bounds: Union[Tuple[float, float, float, float], None] - bounds_crs: Union[dict, str, None] - process_parameters: Union[dict, None] + zoom_levels: Union[dict, int, list, ZoomLevels] + process: Union[str, MPath, List[str], None] = None + baselevels: Union[dict, None] = None + input: Union[dict, None] = None + config_dir: Union[str, MPath, None] = None + area: Union[str, MPath, BaseGeometry, None] = None + area_crs: Union[dict, str, None] = None + bounds: Union[Tuple[float, float, float, float], Bounds, None] = None + bounds_crs: Union[dict, str, None] = None + process_parameters: Union[dict, None] = None _RESERVED_PARAMETERS = tuple(ProcessConfig.__fields__.keys()) @@ -286,7 +286,7 @@ def __init__( logger.debug(f"parsing {input_config}") try: self.parsed_config = parse_config(input_config, strict=stric_parsing) - self.parsed_config.dict() + self.parsed_config.model_dump() except Exception as exc: raise MapcheteConfigError(exc) self._init_zoom_levels = zoom diff --git a/mapchete/io/raster.py b/mapchete/io/raster.py index 89a8562a..bbb715ff 100644 --- a/mapchete/io/raster.py +++ b/mapchete/io/raster.py @@ -516,7 +516,7 @@ def _get_warped_array( raise -@retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) +@retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) def _rasterio_read( input_file=None, indexes=None, @@ -621,7 +621,7 @@ def read_raster_no_crs(input_file, indexes=None, gdal_opts=None): FileNotFoundError if file cannot be found. """ - @retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) + @retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) def _read(): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/mapchete/io/vector.py b/mapchete/io/vector.py index 568326eb..00ab336c 100644 --- a/mapchete/io/vector.py +++ b/mapchete/io/vector.py @@ -323,7 +323,7 @@ def __exit__(self, *args): @retry( logger=logger, exceptions=(DriverError, FionaError, FionaValueError), - **IORetrySettings(), + **dict(IORetrySettings()), ) def _get_reprojected_features( inp=None, diff --git a/pyproject.toml b/pyproject.toml index 33f0c6d0..af59617c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,8 @@ dependencies = [ "importlib-resources", "numpy>=1.16", "oyaml", - "pydantic<2.0.0", - "pydantic_settings", + "pydantic>=2.0.0", + "pydantic_settings>=2.0.0", "pyproj", "python-dateutil", "rasterio>1.2.10", diff --git a/requirements.txt b/requirements.txt index 2dd34c47..fa986b8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,8 +19,8 @@ matplotlib>=3.2.1 numpy>=1.16 oyaml>=0.9 pyproj -pydantic<2.0.0 -pydantic_settings +pydantic>=2.0.0 +pydantic_settings>=2.0.0 retry>=0.9.2 rasterio>1.2.10 Shapely>=2.0.0 From 7a946c967fe593a79c62f30c5debb0fb55e8f1d6 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Sun, 27 Aug 2023 15:05:51 +0200 Subject: [PATCH 3/8] use pydantic_settings to handle settings with env variables --- mapchete/io/raster.py | 4 ++-- mapchete/io/vector.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapchete/io/raster.py b/mapchete/io/raster.py index bbb715ff..89a8562a 100644 --- a/mapchete/io/raster.py +++ b/mapchete/io/raster.py @@ -516,7 +516,7 @@ def _get_warped_array( raise -@retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) +@retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) def _rasterio_read( input_file=None, indexes=None, @@ -621,7 +621,7 @@ def read_raster_no_crs(input_file, indexes=None, gdal_opts=None): FileNotFoundError if file cannot be found. """ - @retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) + @retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) def _read(): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/mapchete/io/vector.py b/mapchete/io/vector.py index e4885d1a..54a975ee 100644 --- a/mapchete/io/vector.py +++ b/mapchete/io/vector.py @@ -324,7 +324,7 @@ def __exit__(self, *args): @retry( logger=logger, exceptions=(DriverError, FionaError, FionaValueError), - **dict(IORetrySettings()), + **IORetrySettings(), ) def _get_reprojected_features( inp=None, From b14ee955244a3af96f1c72b472e061221e40618d Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 28 Aug 2023 08:43:50 +0200 Subject: [PATCH 4/8] use pydantic 2.0 --- .ipynb_checkpoints/Untitled-checkpoint.ipynb | 136 +++++++++++++++++++ Untitled.ipynb | 135 ++++++++++++++++++ herbert.tif | Bin 0 -> 4155 bytes mapchete/config.py | 3 +- mapchete/io/raster.py | 4 +- mapchete/io/vector.py | 2 +- mapchete/job.py | 63 +++++++++ mapchete/protocols.py | 29 ++++ 8 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 .ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 Untitled.ipynb create mode 100644 herbert.tif create mode 100644 mapchete/job.py create mode 100644 mapchete/protocols.py diff --git a/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 00000000..7d2f9bbf --- /dev/null +++ b/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2acea21f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{}" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from enum import Enum\n", + "import os\n", + "\n", + "class IOSettings(Enum):\n", + " tries: int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", + " delay: float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", + " backoff: float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", + "\n", + "dict(IOSettings)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "80563ece", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tries': 3, 'delay': 1.0, 'backoff': 5.0}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pydantic import BaseModel\n", + "\n", + "class IOSettings(BaseModel):\n", + " tries: int = int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", + " delay: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", + " backoff: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", + "\n", + "dict(IOSettings())" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "15e4b6fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3\n" + ] + }, + { + "data": { + "text/plain": [ + "{'tries': 3, 'delay': 1.0, 'backoff': 1.0}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pydantic_settings import BaseSettings, SettingsConfigDict\n", + "\n", + "\n", + "class IOSettings(BaseSettings):\n", + " tries: int = 3\n", + " delay: float = 1.0\n", + " backoff: float = 1.0\n", + " model_config = SettingsConfigDict(env_prefix='MAPCHETE_IO_RETRY_')\n", + "\n", + "print(os.environ[\"MAPCHETE_IO_RETRY_BACKOFF\"])\n", + "dict(IOSettings())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33b7f6d4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ed1875b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 00000000..c0d66829 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,135 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "id": "2acea21f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from enum import Enum\n", + "import os\n", + "\n", + "class IOSettings(Enum):\n", + " tries: int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", + " delay: float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", + " backoff: float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", + "\n", + "dict(IOSettings)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "80563ece", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tries': 3, 'delay': 1.0, 'backoff': 3.0}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pydantic import BaseModel\n", + "\n", + "class IOSettings(BaseModel):\n", + " tries: int = int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", + " delay: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", + " backoff: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", + "\n", + "dict(IOSettings())" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "15e4b6fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3\n" + ] + }, + { + "data": { + "text/plain": [ + "{'tries': 3, 'delay': 1.0, 'backoff': 3.0}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pydantic_settings import BaseSettings, SettingsConfigDict\n", + "\n", + "\n", + "class IOSettings(BaseSettings):\n", + " tries: int = 3\n", + " delay: float = 1.0\n", + " backoff: float = 1.0\n", + " model_config = SettingsConfigDict(env_prefix='MAPCHETE_IO_RETRY_')\n", + "\n", + "dict(IOSettings())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33b7f6d4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa3e6925", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/herbert.tif b/herbert.tif new file mode 100644 index 0000000000000000000000000000000000000000..fd358a44106588c8a24f8f16005a81a76ec145c8 GIT binary patch literal 4155 zcmd^BX;c$g8ZE#PP#RouV{2&!K@?D0*^&?-VptMP0*EXnC=I~|6j@{qiW*u3nikz~R^E z0h<6oK?zWRGmzaB{(u!>tOT3uVY||bT?Kw#y@C~Cym?6}_hK1Bq)#UgK%)M|?4Ok7^*T9R_R3$(U#?~;tPgRpi z1Mx7fp(+Bh7_Y!Mr9=@hg*DpYaURApA2WE!E*1Gt0p4E$u$8sWEhsABWI$8^a*dvz zEjB7BOfNhjEC{g^?@54bNkqH3yJB%pAllVwClZH2!zPlgsm$+R2@ZHyJOSa85i)Qh;>U78aGOT@oqNvnwMVMSD3B^(-wC0ofTkS1>c=Q)%#IGcan^R4KIV z*}6Z(;#Own!S1X1=;w~wPuFHX6{Q|_R7=xKQ^-zxOrZ^V)K$dmCaLiqC?kV|!Qe@j zgP$k;STtt?tpDe`?9Rq;OiIblU0^4J9akj0T@d+-97pocF1Kn3?#dz*6_!Se3c=K}=|YMOD|w4DtZ@pJoj zmGmu1`7~O+^Z{T&E$x^yzp=g6d#A17Erlh85;?=798?y7pMgY?#V_I8=(4EU_Ud&< z;e4lVgR~JaQKxp0*xcyPU_9r6mZcQgpDZ~kMeUiV72H~)g+|@U*whIIO&{(`7#RK8 zhYco6xE&fkloP<-LhV<*#-k&K3wth)+_H2z%wE^w5I5hP--GWQ)6;pgg>7w7TXQ>1 zx%S&p5rx87$Xork0skL3N%pjOM^qW)qVE13zgC4Z)lW15$_@j^ie5(Z8cYl( z-O=4u>Q4d# zwykOvOx~a~tlO&SwgU91g{EM^pPKWm(7LnRl^YHLwbd3PDs2i6J`7*gSs%r#lN z%waD%zemDC3_j~MGE5YWcE_CaGHgB8R-U=Jj#^z)+_7QedC-je7PYCqwgoDfv`|fF zAlh(+g@tuL<8p#Rnri!Mkz)zdFMSeGe@EByyNwE&drBFX`J(e9zU_9EgEy^XK0aH0 zOe*-&W!ex9q!d;lc(!FwP(G3%9LOF^XSeetY0$kdkW=1}gD^T1Ea#6m&)uBmPD+Pkj^5MIJYrHM<_Xxe zh@sU$6Hy;ksU=+4u!J6GKJ}{Q01di9*Wh}9>y1K~cJtWL}tvj@(cfqu< zAvHi0856TiCpJc}cR!GLsuB16{T#LamKOh#kY=uxvz1WIZLZKZpM**QN0fCgQo`?} z`zRg4Xc6OSZxmDWR#8R}IQeuSFbZ7C4HYI*KMQ!y=+tPWQ*VCyR zHnaf{5fr5-^bauEWUHbZr^=E+_5S3R5~T&K_<4r`WN2*L@X5VE`BbNZuwxOGFM9N0 zd|U&$e0(Ywc$?Q1>!)KRDqk;cblWj6I?VCD6g7E=J2>2AZSCH$!E?XZs4CUlQ-ioi zxyD#olC?BE&}aZGPRusVh1byDLVlBwp#p8uw~Je?Q2gae%-9Lv`PcelLseAJvSmrV z^{%93$;WF#D0X)6ot&t#G`!oWmy#wP=(-jw#H6yB+<)08H50)wwcRzdg!eDeGBDvsw#`ds2EEZabHmhSY zCG8{D7hV{3PNIjLGc9`W<|^%>6?A>`OW(^Ilo5G-vYm9d1;q)qTM?tEim0j&(D;4s z;Nnj%X6$w~ivQ;HO69iqQBX;@qo~=-KRHOgVWn5~p*bA^>WrI96}3JRrTwPDVF{ZQ zP84QC`;o?l{+ecAC?Afa{X!ly!v`PM$g5LlSacM_XQ4#}@z) zotn;9#=gaA@)L`Fr*MyZE=-yFeogAv;39Gh1xGL8sX$rV!?%PkvyiReaxb!&rf!+J zok~VB&G(f=MM7SFHYTqpKTpohQ<*48{AQ2BeN!f4$yar@|3Ue3ATEd9kFT5P-P-T({eHkicAWfuy*QsrIV}6depz8 z6?bh>)UnAR=|iHIi6+6g^wv~5doU)q+#=9v<|Q5>;mdiZ&CB+jX`<0*+PHJ|0er+n zN||}7sXjV=px^~}VCLtV=yl~v!Y?;9(`QxD=H0a@R+Zn^H{)prLwYC&xv2ECv}NI~ z=OEYe3g(`ooq=~PsEqQv15w5-+l*sF1kcB_z#)0dC-Oif_C47vzAiJaN`Rt&*B#ms z`(muz5*koM_8Hsk(pgjjdxkoHK!0PcO+hWbz3tPms;zZUJn+(!F=Kzd=uAlLi+OhB zPokTe7ex!d)gARkIj?JYB>Otuw?q1QyK11sTzOj6t-?@7jsd;GqPCLR*>-BnfPKq1NYsl+^lGwcT-De2OTt!Upr4{fnX`O#KW1{c$xWx$GEG#oS{k) z9O=(ImDhZ=qy30>6{G%AYJAv3Tf}Kd>1%wgNlomJkF{E(d&J^8@RugeX2{4u&rM*>mKvI z-5~883Rt+`HW+@vXp5q+rA_y&dfMy9NlQZw@}p@pH^{-n5t_KCPQtn95*VW`o5ikY zkjH0ZEpxrDc3mhJ9$}N!w8LiHaT+E5CXLl5vjPc19%!>@HsS8(z-Qm@)%6w-!#TzJ zZtOq_Ggr<31M+|MbNr0PoA$ z5m4A$+E5~~E{#ZNA%-_7Wj$%<1kGoA_iwagcDeU5e;JBSZj}d2)lKI6IK_JjP?EpJ zNE@hJr*?`BcYr8#krewo$A>m`%nl2~Ok196$1;ch2BJR>}3&@A7`se!}`u~=(G(xJ}c!;VX8~*^= C%00gT literal 0 HcmV?d00001 diff --git a/mapchete/config.py b/mapchete/config.py index 68e4ecb7..eb81209c 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -88,10 +88,9 @@ class ProcessConfig(BaseModel, arbitrary_types_allowed=True): config_dir: Union[str, MPath, None] = None area: Union[str, MPath, BaseGeometry, None] = None area_crs: Union[dict, str, None] = None - bounds: Union[Bounds, Tuple[float, float, float, float], None] = None + bounds: Union[Tuple[float, float, float, float], Bounds, None] = None bounds_crs: Union[dict, str, None] = None process_parameters: Union[dict, None] = None - mapchete_file: Union[str, MPath, None] = None _RESERVED_PARAMETERS = tuple(ProcessConfig.model_fields.keys()) diff --git a/mapchete/io/raster.py b/mapchete/io/raster.py index 89a8562a..bbb715ff 100644 --- a/mapchete/io/raster.py +++ b/mapchete/io/raster.py @@ -516,7 +516,7 @@ def _get_warped_array( raise -@retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) +@retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) def _rasterio_read( input_file=None, indexes=None, @@ -621,7 +621,7 @@ def read_raster_no_crs(input_file, indexes=None, gdal_opts=None): FileNotFoundError if file cannot be found. """ - @retry(logger=logger, exceptions=RasterioIOError, **IORetrySettings()) + @retry(logger=logger, exceptions=RasterioIOError, **dict(IORetrySettings())) def _read(): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/mapchete/io/vector.py b/mapchete/io/vector.py index 54a975ee..e4885d1a 100644 --- a/mapchete/io/vector.py +++ b/mapchete/io/vector.py @@ -324,7 +324,7 @@ def __exit__(self, *args): @retry( logger=logger, exceptions=(DriverError, FionaError, FionaValueError), - **IORetrySettings(), + **dict(IORetrySettings()), ) def _get_reprojected_features( inp=None, diff --git a/mapchete/job.py b/mapchete/job.py new file mode 100644 index 00000000..22a2ee08 --- /dev/null +++ b/mapchete/job.py @@ -0,0 +1,63 @@ +""" + + # Config object --> define & validate with pydantic + + # relative output paths are not useful, so raise exception + if not config.out_path.is_remote(out_path) and not config.out_path.is_absolute(): + raise ValueError(f"process output path must be absolute: {out_path}") + + # Mapchete now will initialize the process and prepare all the tasks required. + # this means the full task graph will be built during this time + job = Job( + config, + **config.params.items() + concurrency="dask", + ) + + with dask_cluster(**dask_cluster_setup, dask_specs=dask_specs) as cluster: + with dask_client( + dask_cluster_setup=dask_cluster_setup, cluster=cluster + ) as client: + + job.set_executor_kwargs(dict(dask_client=client)) + + with Timer() as timer_job: + + adapt_options = dask_specs.get("adapt_options") + cluster_adapt( + cluster, + flavor=dask_cluster_setup.get("flavor"), + adapt_options=adapt_options, + ) + + for i, _ in enumerate(job, 1): + state = backend_db.job(job_id)["properties"]["state"] + if state == "aborting": # pragma: no cover + logger.info( + "job %s abort state caught: %s", job_id, state + ) + # By calling the job's cancel method, all pending futures will be cancelled. + try: + job.cancel() + except Exception: + # catching possible Exceptions (due to losing scheduler before all futures are + # cancelled, etc.) makes sure, the job gets the correct cancelled state + pass + break + else: + # job finished successfully + backend_db.set( + job_id, + state="done", + results={ + "imagesOutput": { + "href": str(job.stac_item_path), + "type": "application/json", + } + }, + ) +""" + + +class DaskJob: + pass \ No newline at end of file diff --git a/mapchete/protocols.py b/mapchete/protocols.py new file mode 100644 index 00000000..6e25193f --- /dev/null +++ b/mapchete/protocols.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass +from typing import Protocol + +from affine import Affine +from rasterio.crs import CRS + + +class GeoGridProtocol(Protocol): + transform: Affine + crs: CRS + width: int + height: int + + +@dataclass +class GeoGrid: + transform: Affine + crs: CRS + width: int + height: int + + + +""" +* rasterio Dataset +* BufferedTile +* ReferencedRaster +* Grid +""" \ No newline at end of file From 2a67258cc2269642a743d506a7457c972887ef1e Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 19 Sep 2023 15:17:02 +0200 Subject: [PATCH 5/8] remove accidentally added files --- .ipynb_checkpoints/Untitled-checkpoint.ipynb | 136 ------------------- Untitled.ipynb | 135 ------------------ herbert.tif | Bin 4155 -> 0 bytes 3 files changed, 271 deletions(-) delete mode 100644 .ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 Untitled.ipynb delete mode 100644 herbert.tif diff --git a/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 7d2f9bbf..00000000 --- a/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "2acea21f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{}" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from enum import Enum\n", - "import os\n", - "\n", - "class IOSettings(Enum):\n", - " tries: int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", - " delay: float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", - " backoff: float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", - "\n", - "dict(IOSettings)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "80563ece", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tries': 3, 'delay': 1.0, 'backoff': 5.0}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pydantic import BaseModel\n", - "\n", - "class IOSettings(BaseModel):\n", - " tries: int = int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", - " delay: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", - " backoff: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", - "\n", - "dict(IOSettings())" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "15e4b6fe", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3\n" - ] - }, - { - "data": { - "text/plain": [ - "{'tries': 3, 'delay': 1.0, 'backoff': 1.0}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pydantic_settings import BaseSettings, SettingsConfigDict\n", - "\n", - "\n", - "class IOSettings(BaseSettings):\n", - " tries: int = 3\n", - " delay: float = 1.0\n", - " backoff: float = 1.0\n", - " model_config = SettingsConfigDict(env_prefix='MAPCHETE_IO_RETRY_')\n", - "\n", - "print(os.environ[\"MAPCHETE_IO_RETRY_BACKOFF\"])\n", - "dict(IOSettings())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33b7f6d4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4ed1875b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/Untitled.ipynb b/Untitled.ipynb deleted file mode 100644 index c0d66829..00000000 --- a/Untitled.ipynb +++ /dev/null @@ -1,135 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 9, - "id": "2acea21f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{}" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from enum import Enum\n", - "import os\n", - "\n", - "class IOSettings(Enum):\n", - " tries: int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", - " delay: float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", - " backoff: float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", - "\n", - "dict(IOSettings)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "80563ece", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tries': 3, 'delay': 1.0, 'backoff': 3.0}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pydantic import BaseModel\n", - "\n", - "class IOSettings(BaseModel):\n", - " tries: int = int(os.environ.get(\"MAPCHETE_IO_RETRY_TRIES\", 3))\n", - " delay: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_DELAY\", 1))\n", - " backoff: float = float(os.environ.get(\"MAPCHETE_IO_RETRY_BACKOFF\", 1))\n", - "\n", - "dict(IOSettings())" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "15e4b6fe", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3\n" - ] - }, - { - "data": { - "text/plain": [ - "{'tries': 3, 'delay': 1.0, 'backoff': 3.0}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pydantic_settings import BaseSettings, SettingsConfigDict\n", - "\n", - "\n", - "class IOSettings(BaseSettings):\n", - " tries: int = 3\n", - " delay: float = 1.0\n", - " backoff: float = 1.0\n", - " model_config = SettingsConfigDict(env_prefix='MAPCHETE_IO_RETRY_')\n", - "\n", - "dict(IOSettings())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33b7f6d4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa3e6925", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/herbert.tif b/herbert.tif deleted file mode 100644 index fd358a44106588c8a24f8f16005a81a76ec145c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4155 zcmd^BX;c$g8ZE#PP#RouV{2&!K@?D0*^&?-VptMP0*EXnC=I~|6j@{qiW*u3nikz~R^E z0h<6oK?zWRGmzaB{(u!>tOT3uVY||bT?Kw#y@C~Cym?6}_hK1Bq)#UgK%)M|?4Ok7^*T9R_R3$(U#?~;tPgRpi z1Mx7fp(+Bh7_Y!Mr9=@hg*DpYaURApA2WE!E*1Gt0p4E$u$8sWEhsABWI$8^a*dvz zEjB7BOfNhjEC{g^?@54bNkqH3yJB%pAllVwClZH2!zPlgsm$+R2@ZHyJOSa85i)Qh;>U78aGOT@oqNvnwMVMSD3B^(-wC0ofTkS1>c=Q)%#IGcan^R4KIV z*}6Z(;#Own!S1X1=;w~wPuFHX6{Q|_R7=xKQ^-zxOrZ^V)K$dmCaLiqC?kV|!Qe@j zgP$k;STtt?tpDe`?9Rq;OiIblU0^4J9akj0T@d+-97pocF1Kn3?#dz*6_!Se3c=K}=|YMOD|w4DtZ@pJoj zmGmu1`7~O+^Z{T&E$x^yzp=g6d#A17Erlh85;?=798?y7pMgY?#V_I8=(4EU_Ud&< z;e4lVgR~JaQKxp0*xcyPU_9r6mZcQgpDZ~kMeUiV72H~)g+|@U*whIIO&{(`7#RK8 zhYco6xE&fkloP<-LhV<*#-k&K3wth)+_H2z%wE^w5I5hP--GWQ)6;pgg>7w7TXQ>1 zx%S&p5rx87$Xork0skL3N%pjOM^qW)qVE13zgC4Z)lW15$_@j^ie5(Z8cYl( z-O=4u>Q4d# zwykOvOx~a~tlO&SwgU91g{EM^pPKWm(7LnRl^YHLwbd3PDs2i6J`7*gSs%r#lN z%waD%zemDC3_j~MGE5YWcE_CaGHgB8R-U=Jj#^z)+_7QedC-je7PYCqwgoDfv`|fF zAlh(+g@tuL<8p#Rnri!Mkz)zdFMSeGe@EByyNwE&drBFX`J(e9zU_9EgEy^XK0aH0 zOe*-&W!ex9q!d;lc(!FwP(G3%9LOF^XSeetY0$kdkW=1}gD^T1Ea#6m&)uBmPD+Pkj^5MIJYrHM<_Xxe zh@sU$6Hy;ksU=+4u!J6GKJ}{Q01di9*Wh}9>y1K~cJtWL}tvj@(cfqu< zAvHi0856TiCpJc}cR!GLsuB16{T#LamKOh#kY=uxvz1WIZLZKZpM**QN0fCgQo`?} z`zRg4Xc6OSZxmDWR#8R}IQeuSFbZ7C4HYI*KMQ!y=+tPWQ*VCyR zHnaf{5fr5-^bauEWUHbZr^=E+_5S3R5~T&K_<4r`WN2*L@X5VE`BbNZuwxOGFM9N0 zd|U&$e0(Ywc$?Q1>!)KRDqk;cblWj6I?VCD6g7E=J2>2AZSCH$!E?XZs4CUlQ-ioi zxyD#olC?BE&}aZGPRusVh1byDLVlBwp#p8uw~Je?Q2gae%-9Lv`PcelLseAJvSmrV z^{%93$;WF#D0X)6ot&t#G`!oWmy#wP=(-jw#H6yB+<)08H50)wwcRzdg!eDeGBDvsw#`ds2EEZabHmhSY zCG8{D7hV{3PNIjLGc9`W<|^%>6?A>`OW(^Ilo5G-vYm9d1;q)qTM?tEim0j&(D;4s z;Nnj%X6$w~ivQ;HO69iqQBX;@qo~=-KRHOgVWn5~p*bA^>WrI96}3JRrTwPDVF{ZQ zP84QC`;o?l{+ecAC?Afa{X!ly!v`PM$g5LlSacM_XQ4#}@z) zotn;9#=gaA@)L`Fr*MyZE=-yFeogAv;39Gh1xGL8sX$rV!?%PkvyiReaxb!&rf!+J zok~VB&G(f=MM7SFHYTqpKTpohQ<*48{AQ2BeN!f4$yar@|3Ue3ATEd9kFT5P-P-T({eHkicAWfuy*QsrIV}6depz8 z6?bh>)UnAR=|iHIi6+6g^wv~5doU)q+#=9v<|Q5>;mdiZ&CB+jX`<0*+PHJ|0er+n zN||}7sXjV=px^~}VCLtV=yl~v!Y?;9(`QxD=H0a@R+Zn^H{)prLwYC&xv2ECv}NI~ z=OEYe3g(`ooq=~PsEqQv15w5-+l*sF1kcB_z#)0dC-Oif_C47vzAiJaN`Rt&*B#ms z`(muz5*koM_8Hsk(pgjjdxkoHK!0PcO+hWbz3tPms;zZUJn+(!F=Kzd=uAlLi+OhB zPokTe7ex!d)gARkIj?JYB>Otuw?q1QyK11sTzOj6t-?@7jsd;GqPCLR*>-BnfPKq1NYsl+^lGwcT-De2OTt!Upr4{fnX`O#KW1{c$xWx$GEG#oS{k) z9O=(ImDhZ=qy30>6{G%AYJAv3Tf}Kd>1%wgNlomJkF{E(d&J^8@RugeX2{4u&rM*>mKvI z-5~883Rt+`HW+@vXp5q+rA_y&dfMy9NlQZw@}p@pH^{-n5t_KCPQtn95*VW`o5ikY zkjH0ZEpxrDc3mhJ9$}N!w8LiHaT+E5CXLl5vjPc19%!>@HsS8(z-Qm@)%6w-!#TzJ zZtOq_Ggr<31M+|MbNr0PoA$ z5m4A$+E5~~E{#ZNA%-_7Wj$%<1kGoA_iwagcDeU5e;JBSZj}d2)lKI6IK_JjP?EpJ zNE@hJr*?`BcYr8#krewo$A>m`%nl2~Ok196$1;ch2BJR>}3&@A7`se!}`u~=(G(xJ}c!;VX8~*^= C%00gT From a7c6603707870ed9eddd360eab1d10b19d53769f Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 19 Sep 2023 15:18:37 +0200 Subject: [PATCH 6/8] remove accidentally added files --- mapchete/job.py | 63 ------------------------------------------- mapchete/protocols.py | 29 -------------------- 2 files changed, 92 deletions(-) delete mode 100644 mapchete/job.py delete mode 100644 mapchete/protocols.py diff --git a/mapchete/job.py b/mapchete/job.py deleted file mode 100644 index 22a2ee08..00000000 --- a/mapchete/job.py +++ /dev/null @@ -1,63 +0,0 @@ -""" - - # Config object --> define & validate with pydantic - - # relative output paths are not useful, so raise exception - if not config.out_path.is_remote(out_path) and not config.out_path.is_absolute(): - raise ValueError(f"process output path must be absolute: {out_path}") - - # Mapchete now will initialize the process and prepare all the tasks required. - # this means the full task graph will be built during this time - job = Job( - config, - **config.params.items() - concurrency="dask", - ) - - with dask_cluster(**dask_cluster_setup, dask_specs=dask_specs) as cluster: - with dask_client( - dask_cluster_setup=dask_cluster_setup, cluster=cluster - ) as client: - - job.set_executor_kwargs(dict(dask_client=client)) - - with Timer() as timer_job: - - adapt_options = dask_specs.get("adapt_options") - cluster_adapt( - cluster, - flavor=dask_cluster_setup.get("flavor"), - adapt_options=adapt_options, - ) - - for i, _ in enumerate(job, 1): - state = backend_db.job(job_id)["properties"]["state"] - if state == "aborting": # pragma: no cover - logger.info( - "job %s abort state caught: %s", job_id, state - ) - # By calling the job's cancel method, all pending futures will be cancelled. - try: - job.cancel() - except Exception: - # catching possible Exceptions (due to losing scheduler before all futures are - # cancelled, etc.) makes sure, the job gets the correct cancelled state - pass - break - else: - # job finished successfully - backend_db.set( - job_id, - state="done", - results={ - "imagesOutput": { - "href": str(job.stac_item_path), - "type": "application/json", - } - }, - ) -""" - - -class DaskJob: - pass \ No newline at end of file diff --git a/mapchete/protocols.py b/mapchete/protocols.py deleted file mode 100644 index 6e25193f..00000000 --- a/mapchete/protocols.py +++ /dev/null @@ -1,29 +0,0 @@ -from dataclasses import dataclass -from typing import Protocol - -from affine import Affine -from rasterio.crs import CRS - - -class GeoGridProtocol(Protocol): - transform: Affine - crs: CRS - width: int - height: int - - -@dataclass -class GeoGrid: - transform: Affine - crs: CRS - width: int - height: int - - - -""" -* rasterio Dataset -* BufferedTile -* ReferencedRaster -* Grid -""" \ No newline at end of file From d56a0efa4b2665224c164f6f740f62ad64f6e077 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 19 Sep 2023 16:17:53 +0200 Subject: [PATCH 7/8] convert default GDAL HTTP options into a basesettings model --- mapchete/io/__init__.py | 4 ++-- mapchete/io/settings.py | 40 +++++++++++++++++----------------------- mapchete/path.py | 4 ++-- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/mapchete/io/__init__.py b/mapchete/io/__init__.py index 769d3e97..7f5cea93 100644 --- a/mapchete/io/__init__.py +++ b/mapchete/io/__init__.py @@ -9,7 +9,7 @@ tile_to_zoom_level, ) from mapchete.io.raster import rasterio_open -from mapchete.io.settings import GDAL_HTTP_OPTS +from mapchete.io.settings import GDALHTTPOptions from mapchete.io.vector import fiona_open from mapchete.path import ( MPath, @@ -25,7 +25,7 @@ __all__ = [ "copy", "fs_from_path", - "GDAL_HTTP_OPTS", + "GDALHTTPOptions", "get_best_zoom_level", "get_segmentize_value", "tile_to_zoom_level", diff --git a/mapchete/io/settings.py b/mapchete/io/settings.py index a9aa083e..9825ca51 100644 --- a/mapchete/io/settings.py +++ b/mapchete/io/settings.py @@ -1,40 +1,34 @@ -import logging -import os +""" +Combine default values with environment variable values. +""" from pydantic_settings import BaseSettings, SettingsConfigDict -logger = logging.getLogger(__name__) - - -def _merge_gdal_defaults_with_env(): - return {k: os.environ.get(k, v) for k, v in GDAL_HTTP_DEFAULTS.items()} - # defaults sets according to the recommendations given at # https://developmentseed.org/titiler/advanced/performance_tuning/ -GDAL_HTTP_DEFAULTS = dict( +class GDALHTTPOptions(BaseSettings): # this will be set later on depending on the opened file - CPL_VSIL_CURL_ALLOWED_EXTENSIONS="", + CPL_VSIL_CURL_ALLOWED_EXTENSIONS: str = "" # 200MB - CPL_VSIL_CURL_CACHE_SIZE=200_000_000, + CPL_VSIL_CURL_CACHE_SIZE: int = 200_000_000 # alternative: ARRAY - GDAL_BAND_BLOCK_CACHE="HASHSET", + GDAL_BAND_BLOCK_CACHE: str = "HASHSET" # # 200MB # GDAL_CACHEMAX=200, --> activating this seems to let the tests stall at some point # don't make LIST request - GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", - GDAL_HTTP_TIMEOUT=30, - GDAL_HTTP_MAX_RETRY=3, - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=True, - GDAL_HTTP_MULTIPLEX=True, - GDAL_HTTP_RETRY_DELAY=5, - GDAL_HTTP_VERSION=2, + GDAL_DISABLE_READDIR_ON_OPEN: str = "EMPTY_DIR" + GDAL_HTTP_TIMEOUT: int = 30 + GDAL_HTTP_MAX_RETRY: int = 3 + GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: bool = True + GDAL_HTTP_MULTIPLEX: bool = True + GDAL_HTTP_RETRY_DELAY: int = 5 + GDAL_HTTP_VERSION: int = 2 # let GDAL cache internally - VSI_CACHE=True, + VSI_CACHE: bool = True # 5MB cache per file - VSI_CACHE_SIZE=5_000_000, -) -GDAL_HTTP_OPTS = _merge_gdal_defaults_with_env() + VSI_CACHE_SIZE: int = 5_000_000 + model_config = SettingsConfigDict() class IORetrySettings(BaseSettings): diff --git a/mapchete/path.py b/mapchete/path.py index a8b41373..3a074bfe 100644 --- a/mapchete/path.py +++ b/mapchete/path.py @@ -15,7 +15,7 @@ from rasterio.session import Session as RioSession from mapchete._executor import Executor -from mapchete.io.settings import GDAL_HTTP_OPTS +from mapchete.io.settings import GDALHTTPOptions from mapchete.tile import BufferedTile logger = logging.getLogger(__name__) @@ -380,7 +380,7 @@ def gdal_env_params( # for remote paths, we need some special settings if self.is_remote(): - gdal_opts = GDAL_HTTP_OPTS.copy() + gdal_opts = dict(GDALHTTPOptions()) # we cannot know at this point which file types the VRT or STACTA JSON # is pointing to, so in order to play safe, we remove the extensions constraint here From 0e37dc30436fe3c85418c4fd05714327c214e8e0 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 19 Sep 2023 16:38:16 +0200 Subject: [PATCH 8/8] change type ordering --- mapchete/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapchete/config.py b/mapchete/config.py index eb81209c..0fc6992c 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -57,7 +57,7 @@ class OutputConfigBase(BaseModel): metatiling: Union[int, None] = 1 pixelbuffer: Union[NonNegativeInt, None] = 0 - @field_validator("metatiling", mode='before') + @field_validator("metatiling", mode="before") def _metatiling(cls, value: int) -> int: # pragma: no cover _metatiling_opts = [2**x for x in range(10)] if value not in _metatiling_opts: @@ -70,7 +70,7 @@ class PyramidConfig(BaseModel): metatiling: Union[int, None] = 1 pixelbuffer: Union[NonNegativeInt, None] = 0 - @field_validator("metatiling", mode='before') + @field_validator("metatiling", mode="before") def _metatiling(cls, value: int) -> int: # pragma: no cover _metatiling_opts = [2**x for x in range(10)] if value not in _metatiling_opts: @@ -88,7 +88,7 @@ class ProcessConfig(BaseModel, arbitrary_types_allowed=True): config_dir: Union[str, MPath, None] = None area: Union[str, MPath, BaseGeometry, None] = None area_crs: Union[dict, str, None] = None - bounds: Union[Tuple[float, float, float, float], Bounds, None] = None + bounds: Union[Bounds, Tuple[float, float, float, float], None] = None bounds_crs: Union[dict, str, None] = None process_parameters: Union[dict, None] = None