pyproject.toml

[project]
name = "etl"
version = "0.1.0"
description = ""
authors = [
    {name = "Our World in Data", email = "tech@ourworldindata.org"},
]
requires-python = ">=3.10, <3.12"
dependencies = [
    "click>=8.0.1",
    "PyYAML>=6.0.1",
    "papermill>=2.3.3",
    "jupyterlab>=3.1.13",
    # we had to pin this because of this bug https://github.com/owid/etl/issues/2320, versions newer than 2.2.4 might be working again
    "SPARQLWrapper>=1.8.5",
    "openpyxl>=3.0.9",
    "sh==1.14.3",
    "Unidecode>=1.3.2",
    "python-dotenv>=0.19.0",
    "regex>=2022.1.18",
    "wikipedia>=1.4.0",
    "numpy>=1.22.1",
    "pydantic>=1.9.0",
    "structlog>=21.5.0",
    "rich>=12.1.0",
    "rich-click>=1.7.3",
    "tenacity>=8.0.1",
    "simplejson>=3.17.6",
    "bugsnag>=4.2.1",
    "xlrd>=2.0.1",
    "PyPDF2>=2.11.1",
    "ruamel.yaml>=0.17.21",
    "gitpython>=3.1.30",
    "rapidfuzz>=2.13.7",
    # pinned because of frictionless, remove this pin when frictionless is updated
    "fsspec==2022.11.0",
    "openai>=1.3.6",
    "pdfplumber>=0.9.0",
    "pyhumps>=3.8.0",
    "wbgapi>=1.0.12",
    "odfpy>=1.4.1",
    "world-bank-data>=0.1.3",
    "questionary>=2.0.0",
    "typing-extensions>=4.7.1",
    "scipy>=1.11.2",
    "selenium>=4.15.1",
    "geopandas>=0.14.1",
    "shapely>=2.0.3",
    "fasteners>=0.19",
    "cdsapi>=0.7.0",
    "rioxarray>=0.15.1",
    "html2text>=2020.1.16",
    "pygithub>=2.3.0",
    "pandas==2.2.2",
    "sqlalchemy>=2.0.30",
    "pymysql>=1.1.1",
    "tiktoken>=0.7.0",
    "earthengine-api>=0.1.411",
    "python-docx>=1.1.2",
    "h5netcdf>=1.3.0",
    # we're not yet ready for frictionless 5.x.x as it raises an error for data://open_numbers/open_numbers/latest/bp__energy
    "frictionless[pandas]>=4.40.8, <5.0.0",
    "owid-catalog",
    "owid-datautils",
    "owid-repack",
    "walden",
    "deprecated>=1.2.14",
    "scikit-learn>=1.5.2",
]

[tool.uv.sources]
owid-catalog = { path = "lib/catalog", editable = true }
owid-datautils = { path = "lib/datautils", editable = true }
owid-repack = { path = "lib/repack", editable = true }
walden = { path = "lib/walden", editable = true }
sqlacodegen = { git = "https://github.com/agronholm/sqlacodegen.git" }

[tool.uv]
dev-dependencies = [
    "pytest>=8.3.2",
    "watchdog>=4.0.1",
    "pyyaml>=6.0.2",
    "argh>=0.31.3",
    "jupyter>=1.0.0",
    # For some reason, ipython 8.18.0 was not working (it seems to be incompatible with prompt_toolkit).
    "ipython>=8.14.0, <8.15.0",
    "matplotlib>=3.9.1.post1",
    "plotly>=5.23.0",
    "cookiecutter>=2.6.0",
    "types-pyyaml>=6.0.12.20240808",
    "hydra-core>=1.3.2",
    "boto3-stubs[s3]>=1.34.154",
    "gspread>=5.12.4",
    "jsonref>=1.1.0",
    "mkdocs-jupyter>=0.24.8",
    "mkdocs-exclude>=1.0.2",
    "mkdocs-gen-files>=0.5.0",
    "mkdocs-git-authors-plugin>=0.7.2",
    "mkdocs-git-revision-date-localized-plugin>=1.2.6",
    "mkdocs-click>=0.8.1",
    "mkdocs-glightbox>=0.3.7",
    "memory-profiler>=0.61.0",
    "line-profiler>=4.1.3",
    "sqlacodegen",
    # unpinning those would introduce tons of type errors
    "pyright==1.1.373",
    "pandas-stubs==1.2.0.62",
    "ruff==0.1.6",
    "ipdb>=0.13.13",
]

[project.optional-dependencies]
api = [
    "fastapi>=0.109.0",
    "uvicorn[standard]>=0.25.0",
    "slack-sdk>=3.26.2",
    "joblib>=1.3.2",
]
wizard = [
    "streamlit>=1.39.0",
    "streamlit-aggrid>=0.3.4.post3",
    "streamlit-ace>=0.1.1",
    "streamlit-extras>=0.3.6",
    "streamlit-agraph>=0.0.45",
    "plotly>=5.23.0",
    "geographiclib>=2.0",
    "pyproj>=3.6.1",
    "streamlit-feedback>=0.1.3",]

[project.scripts]
etl = 'apps.cli:cli'
etlwiz = 'apps.wizard.cli:cli'
etlr = 'etl.command:main_cli'
etl-wizard = 'apps.wizard.cli:cli'
compare = 'etl.compare:cli'
backport = 'apps.backport.backport:backport_cli'

[tool.ruff]
extend-select = [
    # isort
    "I"
]
ignore = ["E501"]
line-length = 120
target-version = "py310"
extend-exclude = [
    ".ipynb_checkpoints",
    "*cookiecutter",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.pyright]
exclude = [
    "lib/",
    "etl/steps/archive",
    "etl/snapshots/archive",
    "apps/wizard/etl_steps/cookiecutter/",
    "apps/wizard/etl_steps/cookiecutter/snapshot/**",
    "**/node_modules",
    "**/__pycache__",
    ".git",
    "**/.venv",
    "**/.ipynb_checkpoints",
    "playground/",
    ".cachedir/",
    ".cache/",
    "etl/steps",
    "snapshots/",
]

[tool.pytest.ini_options]
markers = [
    "integration: marks tests as integration tests (deselect with '-m \"not integration\"')"
]