Count image urls as image modality

libs/libcommon/src/libcommon/processing_graph.py

@@ -708,7 +708,7 @@ def parse_id(id: str) -> tuple[str, str, Optional[str], Optional[str], str]:
     },
     "dataset-modalities": {
         "input_type": "dataset",
-        "triggered_by": ["dataset-info", "dataset-filetypes"],
+        "triggered_by": ["dataset-info", "dataset-filetypes", "split-image-url-columns"],
         "job_runner_version": 2,
         "difficulty": 20,
     },

libs/libcommon/tests/test_processing_graph.py

@@ -168,8 +168,19 @@ def test_graph() -> None:
         (
             "dataset-modalities",
             ["dataset-hub-cache"],
-            ["dataset-info", "dataset-filetypes"],
-            ["dataset-config-names", "config-parquet-and-info", "config-info", "dataset-info", "dataset-filetypes"],
+            ["dataset-info", "dataset-filetypes", "split-image-url-columns"],
+            [
+                "config-info",
+                "config-parquet",
+                "config-parquet-and-info",
+                "config-parquet-metadata",
+                "config-split-names",
+                "dataset-config-names",
+                "dataset-filetypes",
+                "dataset-info",
+                "split-first-rows",
+                "split-image-url-columns",
+            ],
         ),
         (
             "dataset-is-valid",
@@ -195,7 +206,7 @@ def test_graph() -> None:
         ),
         (
             "split-image-url-columns",
-            ["split-opt-in-out-urls-scan"],
+            ["dataset-modalities", "split-opt-in-out-urls-scan"],
             ["split-first-rows"],
             [
                 "dataset-config-names",
@@ -371,6 +382,7 @@ def test_graph() -> None:
                 "dataset-size",
                 "split-duckdb-index",
                 "split-first-rows",
+                "split-image-url-columns",
                 "split-is-valid",
                 "split-descriptive-statistics",
             ],

libs/libcommon/tests/test_utils.py

@@ -152,4 +152,4 @@ def test_serialize_and_truncate_raises(obj: Any, max_bytes: int) -> None:
 
 
 def test_get_duration() -> None:
-    assert get_duration(get_datetime() - timedelta(seconds=10)) == pytest.approx(10)
+    assert get_duration(get_datetime() - timedelta(seconds=10)) == pytest.approx(10, rel=0.01)

services/worker/src/worker/job_runners/dataset/modalities.py

@@ -2,12 +2,15 @@
 # Copyright 2022 The HuggingFace Authors.
 
 import logging
+from http import HTTPStatus
 
 from datasets import Audio, Features, Image, Sequence, Translation, TranslationVariableLanguages, Value
 from datasets.features.features import FeatureType, _visit
 from libcommon.exceptions import PreviousStepFormatError
 from libcommon.simple_cache import (
+    CachedArtifactNotFoundError,
     get_previous_step_or_raise,
+    get_response,
 )
 
 from worker.dtos import (
@@ -106,6 +109,53 @@ def detect_modalities_from_features(dataset: str) -> set[DatasetModality]:
     return modalities
 
 
+def detect_modalities_from_url_columns(dataset: str) -> set[DatasetModality]:
+    """
+    Detect modalities of a dataset using the type of URL columns.
+    E.g. if a column contains URLs of images.
+
+    Args:
+        dataset (`str`):
+            A namespace (user or an organization) and a repo name separated by a `/`.
+
+    Raises:
+        [~`libcommon.simple_cache.CachedArtifactError`]:
+            If the previous step gave an error.
+        [~`libcommon.exceptions.PreviousStepFormatError`]:
+            If the content of the previous step has not the expected format
+
+    Returns:
+        `set[DatasetModality]`: A set of modalities.
+    """
+    config_names_response = get_previous_step_or_raise(kind="dataset-split-names", dataset=dataset)
+    content = config_names_response["content"]
+    if "splits" not in content and not isinstance(content["splits"], list):
+        raise PreviousStepFormatError("Previous step did not return the expected content: 'splits'.")
+
+    try:
+        for split_item in content["splits"][:10]:  # no need to check all the configs
+            config = split_item["config"]
+            split = split_item["config"]
+            try:
+                response = get_response(kind="split-image-url-columns", dataset=dataset, config=config, split=split)
+            except CachedArtifactNotFoundError:
+                logging.debug("No response found in previous step for this dataset: 'split-image-url-columns'.")
+                continue
+            if response["http_status"] != HTTPStatus.OK:
+                logging.debug(f"Previous step gave an error: {response['http_status']}.")
+                continue
+            else:
+                try:
+                    if response["content"]["columns"]:
+                        return {"image"}
+                except Exception as e:
+                    raise PreviousStepFormatError("Previous step did not return the expected content.", e) from e
+    except Exception as e:
+        raise PreviousStepFormatError("Previous step did not return the expected content.", e) from e
+
+    return set()
+
+
 # from https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types
 IMAGE_EXTENSIONS = {
     ".apng",
@@ -302,6 +352,14 @@ def compute_modalities_response(dataset: str) -> DatasetModalitiesResponse:
         logging.info(f"failed to detect modalities from features of {dataset=}")
         pass
 
+    try:
+        modalities.update(detect_modalities_from_url_columns(dataset))
+    except PreviousStepFormatError:
+        raise
+    except Exception:
+        logging.info(f"failed to detect modalities from file types of {dataset=}")
+        pass
+
     try:
         modalities.update(detect_modalities_from_filetypes(dataset))
     except PreviousStepFormatError: