Add Python 3.10, 3.11, 3.12 compatibility

.github/workflows/pytest_slow.yml

@@ -50,7 +50,7 @@ jobs:
           python --version
           pip --version
           python -m pip install -U pip
-          pip install torch==2.1.0 torchtext torchvision torchaudio
+          pip install torch==2.1.0 torchvision torchaudio
           pip install ray==2.3.1
           pip install '.[test]'
 

.gitignore

@@ -140,3 +140,5 @@ examples/*/visualizations/
 
 # benchmarking configs
 ludwig/benchmarking/configs/
+pytest.xml
+ludwig.code-workspace

.pre-commit-config.yaml

@@ -34,7 +34,7 @@ repos:
       - id: pyupgrade
         args: [--py36-plus]
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.5.1
+    rev: 06907d0
     hooks:
       - id: docformatter
         args: [--in-place, --wrap-summaries=115, --wrap-descriptions=120]

docker/ludwig-ray-gpu/Dockerfile

@@ -50,7 +50,7 @@ RUN pip install -U pip
 
 WORKDIR /ludwig
 
-RUN pip install --no-cache-dir torch==2.1.0 torchtext torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118
+RUN pip install --no-cache-dir torch==2.1.0  torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118
 
 COPY . .
 RUN pip install --no-cache-dir '.[full]' --extra-index-url https://download.pytorch.org/whl/cu118

docker/ludwig-ray/Dockerfile

@@ -36,7 +36,7 @@ RUN pip install -U pip
 
 WORKDIR /ludwig
 
-RUN pip install --no-cache-dir torch==2.1.0 torchtext torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+RUN pip install --no-cache-dir torch==2.1.0 torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 
 COPY . .
 RUN pip install --no-cache-dir '.[full]' --extra-index-url https://download.pytorch.org/whl/cpu

docker/ludwig/Dockerfile

@@ -24,7 +24,7 @@ RUN pip install -U pip
 
 WORKDIR /ludwig
 
-RUN pip install --no-cache-dir torch==2.0.0 torchtext torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+RUN pip install --no-cache-dir torch==2.0.0 torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 
 COPY . .
 RUN pip install --no-cache-dir '.[full]'

docker/ludwig_hatch/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12
+
+ENV PATH="/root/.local/bin:$PATH"
+RUN apt-get -y update
+RUN apt-get -y install pipx
+RUN apt-get -y install git libsndfile1 build-essential g++ cmake ffmpeg sox libsox-dev
+RUN pipx ensurepath --force
+RUN pipx install hatch
+RUN python3 -m pip install --upgrade pipx
+WORKDIR /ludwig
+#COPY /ludwig/ .
+COPY . .
+
+RUN hatch env create
+RUN hatch build
+
+ENTRYPOINT ["ludwig"]

ludwig/__about__.py

@@ -0,0 +1 @@
+__version__ = '1.13.0'

ludwig/decoders/llm_decoders.py

@@ -1,3 +1,4 @@
+# flake8: noqa: E501
 import logging
 import re
 from typing import Any, Dict, List, Union
@@ -91,7 +92,6 @@ def __init__(
             # Transformer Tokenizers
             self.tokenizer_vocab_size = self.tokenizer.tokenizer.vocab_size
         else:
-            # TorchText Tokenizers
             self.tokenizer_vocab_size = len(self.tokenizer.vocab)
 
         # Maximum number of new tokens that will be generated

ludwig/schema/trainer.py

@@ -1,8 +1,10 @@
+# flake8: noqa: E501
 import re
 from abc import ABC
 from typing import Optional, Type, Union
 
 import torch
+import torch._dynamo
 from packaging.version import parse as parse_version
 
 from ludwig.api_annotations import DeveloperAPI
@@ -33,6 +35,9 @@
 
 _torch_200 = parse_version(torch.__version__) >= parse_version("2.0")
 
+# this is a workarrpund to avoid an error regarding torch.compile.
+# TODO Fix torch.compile and dynamo problems
+torch._dynamo.config.suppress_errors = True
 
 trainer_schema_registry = Registry()
 _llm_trainer_schema_registry = Registry()