diff --git a/CHANGELOG.md b/CHANGELOG.md index d5160f37..9d7effdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Add `step fill-metadata` CLI command that populates transform meta-table with all indices to process * Add helm chart for running regular loops in k8s as `CronJob` +* Switch from vanilla `tqdm` to `tqdm_loggable` for better display in logs * Add `step run-idx` CLI command * Executors: `datapipe.executor.SingleThreadExecutor`, diff --git a/datapipe/compute.py b/datapipe/compute.py index 7cee6979..aa9d44dd 100644 --- a/datapipe/compute.py +++ b/datapipe/compute.py @@ -6,7 +6,7 @@ from typing import Dict, Iterable, List, Optional, Tuple from opentelemetry import trace -from tqdm import tqdm +from tqdm_loggable.auto import tqdm from datapipe.datatable import DataStore, DataTable from datapipe.executor import Executor, ExecutorConfig, SingleThreadExecutor diff --git a/datapipe/core_steps.py b/datapipe/core_steps.py index ddb5d15f..2846538d 100644 --- a/datapipe/core_steps.py +++ b/datapipe/core_steps.py @@ -19,7 +19,7 @@ import pandas as pd from opentelemetry import trace from sqlalchemy import alias, and_, column, func, literal, or_, select, tuple_ -from tqdm import tqdm +from tqdm_loggable.auto import tqdm from datapipe.compute import Catalog, ComputeStep, PipelineStep from datapipe.datatable import DataStore, DataTable diff --git a/datapipe/executor/__init__.py b/datapipe/executor/__init__.py index 06019ac1..9b4cf301 100644 --- a/datapipe/executor/__init__.py +++ b/datapipe/executor/__init__.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Iterable, Optional, Protocol -from tqdm import tqdm +from tqdm_loggable.auto import tqdm from datapipe.datatable import DataStore from datapipe.run_config import RunConfig diff --git a/datapipe/executor/ray.py b/datapipe/executor/ray.py index 413a561b..cbef98c0 100644 --- a/datapipe/executor/ray.py +++ b/datapipe/executor/ray.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Iterable, Optional import ray -from tqdm import tqdm +from tqdm_loggable.auto import tqdm from datapipe.datatable import DataStore from datapipe.executor import Executor, ExecutorConfig, ProcessFn diff --git a/pyproject.toml b/pyproject.toml index 6d5fab2d..834dd4b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datapipe-core" -version = "0.13.0-alpha.2" +version = "0.13.0-alpha.3" description = "`datapipe` is a realtime incremental ETL library for Python application" readme = "README.md" repository = "https://github.com/epoch8/datapipe" @@ -26,6 +26,7 @@ Pillow = "^9.0.0" iteration-utilities = ">=0.11.0" cityhash = "^0.4.2" +epoch8-tqdm-loggable = "^0.1.4" traceback-with-variables = "^2.0.4" pymilvus = {version="^2.0.2", optional=true}