In the "my_datasets.py" code, how do you get the location of a local dataset.In the "my_datasets.py" code, how to get the location of the local dataset. My approach doesn't work anymore in the new version

[aihao2000]

it shows error:

ValueError: `data_dir` must be relative to a dataset directory's root: /content/drive/MyDrive/Colab Notebooks/DeepLearningContent/datasets/characters
Traceback (most recent call last):

datasets script:

import datasets
import os
from PIL import Image
import json
import torch
import cv2
import numpy as np


class ImagesConfig(datasets.BuilderConfig):
    def __init__(self, **kwargs):
        super(ImagesConfig, self).__init__(**kwargs)


class Images(datasets.GeneratorBasedBuilder):
    def __init__(self, **kwargs):
        self.DEFAULT_WRITER_BATCH_SIZE = 100
        super(Images, self).__init__(**kwargs)

    def _split_generators(self, dl_manager: datasets.DownloadManager):
        meta_data = {}
        print(self.config)
        with open(os.path.join(self.config.data_dir, "meta_data.json"), "r") as f:
            meta_data = json.load(f)
        data = []
        if (
            self.config.name == "similar_pairs"
        ):
            for image1_path in meta_data:
                for image2_path, similarity in meta_data[image1_path]["similar_images"]:
                    data.append(
                        (
                            image1_path,
                            image2_path,
                            similarity,
                        )
                    )
        elif self.config.name == "image_prompt_pairs":
            for image_path in meta_data:
                data.append(image_path, meta_data[image_path]["prompt"])
        print("data size:", len(data))
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={"split": datasets.Split.TRAIN, "data": data},
            )
        ]

    BUILDER_CONFIGS = [
        ImagesConfig(
            name="similar_pairs",
            description="simliar pair dataset,item is a pair of similar images",
        ),
        ImagesConfig(
            name="image_prompt_pairs",
            description="image prompt pairs",
        ),
    ]

    def _info(self):
        if self.config.name == "similar_pairs":
            return datasets.DatasetInfo(
                features=datasets.Features(
                    {
                        "image1": datasets.features.Image(),
                        "image1_path": datasets.Value("string"),
                        "image2": datasets.features.Image(),
                        "image2_path": datasets.Value("string"),
                        "similarity": datasets.Value("float32"),
                    }
                )
            )
        elif self.config.name == "image_prompt_pairs":
            return datasets.DatasetInfo(
                features=datasets.Features(
                    {
                        "image": datasets.features.Image(),
                        "image_path": datasets.features.Value("string"),
                        "prompt": datasets.Value("string"),
                    }
                )
            )

    def _generate_examples(self, split, data):
        if self.config.name == "similar_pairs":
            for image1_path, image2_path, similarity in data:
                yield image1_path + ":" + image2_path, {
                    "image1": Image.open(
                        os.path.join(self.config.data_dir, image1_path)
                    ),
                    "image1_path": image1_path,
                    "image2": Image.open(
                        os.path.join(self.config.data_dir, image2_path)
                    ),
                    "image2_path": image2_path,
                    "similarity": similarity,
                }

load dataset:

from datasets import load_dataset
ds = load_dataset(
    "/content/drive/MyDrive/Colab Notebooks/DeepLearningContent/datasets/characters",
    "similar_pairs",
    "/content/drive/MyDrive/Colab Notebooks/DeepLearningContent/datasets/characters",
    split="train",
)

[aihao2000]

I just tried it and it works again. I didn't pay attention to the version. It seems that it has been updated again?