diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f246215026..21ce8cf356b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,8 @@ jobs: pip install .[quality] - name: Check quality run: | - ruff check tests src benchmarks metrics utils setup.py # linter - ruff format --check tests src benchmarks metrics utils setup.py # formatter + ruff check tests src benchmarks utils setup.py # linter + ruff format --check tests src benchmarks utils setup.py # formatter test: needs: check_code_quality @@ -56,7 +56,7 @@ jobs: - name: Install uv run: pip install --upgrade uv - name: Install dependencies - run: uv pip install --system "datasets[tests,metrics-tests] @ ." + run: uv pip install --system "datasets[tests] @ ." - name: Install dependencies (latest versions) if: ${{ matrix.os == 'ubuntu-latest' }} run: uv pip install --system -r additional-tests-requirements.txt --no-deps diff --git a/.gitignore b/.gitignore index 02f80e2f0cd..7cb91d81d14 100644 --- a/.gitignore +++ b/.gitignore @@ -42,13 +42,6 @@ venv.bak/ .idea .vscode -# keep only the empty datasets and metrics directory with it's __init__.py file -/src/*/datasets/* -!/src/*/datasets/__init__.py - -/src/*/metrics/* -!/src/*/metrics/__init__.py - # Vim .*.swp diff --git a/Makefile b/Makefile index 916524e7754..61fa1f84854 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: quality style test -check_dirs := tests src benchmarks metrics utils +check_dirs := tests src benchmarks utils # Check that source code meets quality standards diff --git a/additional-tests-requirements.txt b/additional-tests-requirements.txt index 3b08bab84db..86e011e47e2 100644 --- a/additional-tests-requirements.txt +++ b/additional-tests-requirements.txt @@ -1,5 +1 @@ -unbabel-comet>=1.0.0 git+https://github.com/pytorch/data.git -git+https://github.com/google-research/bleurt.git -git+https://github.com/ns-moosavi/coval.git -git+https://github.com/hendrycks/math.git diff --git a/docs/source/_redirects.yml b/docs/source/_redirects.yml index 50373bfea2f..606a00ea1cc 100644 --- a/docs/source/_redirects.yml +++ b/docs/source/_redirects.yml @@ -8,7 +8,6 @@ splits: loading#slice-splits processing: process faiss_and_ea: faiss_es features: about_dataset_features -using_metrics: how_to_metrics exploring: access package_reference/logging_methods: package_reference/utilities # end of first_section diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 9717d5a5967..2a3f6cd8d76 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -15,8 +15,6 @@ title: Know your dataset - local: use_dataset title: Preprocess - - local: metrics - title: Evaluate predictions - local: create_dataset title: Create a dataset - local: upload_dataset @@ -48,8 +46,6 @@ title: Search index - local: cli title: CLI - - local: how_to_metrics - title: Metrics - local: troubleshoot title: Troubleshooting title: "General usage" @@ -111,8 +107,6 @@ title: Build and load - local: about_map_batch title: Batch mapping - - local: about_metrics - title: All about metrics title: "Conceptual guides" - sections: - local: package_reference/main_classes diff --git a/docs/source/about_metrics.mdx b/docs/source/about_metrics.mdx deleted file mode 100644 index 2e5b722f988..00000000000 --- a/docs/source/about_metrics.mdx +++ /dev/null @@ -1,25 +0,0 @@ -# All about metrics - - - -Metrics is deprecated in πŸ€— Datasets. To learn more about how to use metrics, take a look at the library πŸ€— [Evaluate](https://huggingface.co/docs/evaluate/index)! In addition to metrics, you can find more tools for evaluating models and datasets. - - - -πŸ€— Datasets provides access to a wide range of NLP metrics. You can load metrics associated with benchmark datasets like GLUE or SQuAD, and complex metrics like BLEURT or BERTScore, with a single command: [`load_metric`]. Once you've loaded a metric, easily compute and evaluate a model's performance. - -## ELI5: `load_metric` - -Loading a dataset and loading a metric share many similarities. This was an intentional design choice because we wanted to create a simple and unified experience. When you call [`load_metric`], the metric loading script is downloaded and imported from GitHub (if it hasn't already been downloaded before). It contains information about the metric such as it's citation, homepage, and description. - -The metric loading script will instantiate and return a [`Metric`] object. This stores the predictions and references, which you need to compute the metric values. The [`Metric`] object is stored as an Apache Arrow table. As a result, the predictions and references are stored directly on disk with memory-mapping. This enables πŸ€— Datasets to do a lazy computation of the metric, and makes it easier to gather all the predictions in a distributed setting. - -## Distributed evaluation - -Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive (`f(AuB) = f(A) + f(B)`), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive (`f(AuB) β‰  f(A) + f(B)`), it's not that simple. For example, you can't take the sum of the [F1](https://huggingface.co/metrics/f1) scores of each data subset as your **final metric**. - -A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient. - -πŸ€— Datasets solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to [`Metric.compute`] the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, [`Metric.compute`] will perform the final metric evaluation. - -This solution allows πŸ€— Datasets to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory. \ No newline at end of file diff --git a/docs/source/audio_dataset.mdx b/docs/source/audio_dataset.mdx index cf0ac45c660..9014c10a020 100644 --- a/docs/source/audio_dataset.mdx +++ b/docs/source/audio_dataset.mdx @@ -14,8 +14,6 @@ There are several methods for creating and sharing an audio dataset: * Create an audio dataset repository with the `AudioFolder` builder. This is a no-code solution for quickly creating an audio dataset with several thousand audio files. -* Create an audio dataset by writing a loading script. This method is for advanced users and requires more effort and coding, but you have greater flexibility over how a dataset is defined, downloaded, and generated which can be useful for more complex or large scale audio datasets. - @@ -175,7 +173,7 @@ Some audio datasets, like those found in [Kaggle competitions](https://www.kaggl -## Loading script +## (Legacy) Loading script Write a dataset loading script to manually create a dataset. It defines a dataset's splits and configurations, and handles downloading and generating the dataset examples. diff --git a/docs/source/cache.mdx b/docs/source/cache.mdx index 49bf550d392..56d674bbf61 100644 --- a/docs/source/cache.mdx +++ b/docs/source/cache.mdx @@ -24,13 +24,6 @@ When you load a dataset, you also have the option to change where the data is ca >>> dataset = load_dataset('LOADING_SCRIPT', cache_dir="PATH/TO/MY/CACHE/DIR") ``` -Similarly, you can change where a metric is cached with the `cache_dir` parameter: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('glue', 'mrpc', cache_dir="MY/CACHE/DIRECTORY") -``` - ## Download mode After you download a dataset, control how it is loaded by [`load_dataset`] with the `download_mode` parameter. By default, πŸ€— Datasets will reuse a dataset if it exists. But if you need the original dataset without any processing functions applied, re-download the files as shown below: @@ -77,19 +70,6 @@ If you want to reuse a dataset from scratch, try setting the `download_mode` par -You can also avoid caching your metric entirely, and keep it in CPU memory instead: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('glue', 'mrpc', keep_in_memory=True) -``` - - - -Keeping the predictions in-memory is not possible in a distributed setting since the CPU memory spaces of the various processes are not shared. - - - ## Improve performance diff --git a/docs/source/create_dataset.mdx b/docs/source/create_dataset.mdx index 29eee57f4fd..ddc9d48f85c 100644 --- a/docs/source/create_dataset.mdx +++ b/docs/source/create_dataset.mdx @@ -105,7 +105,7 @@ You can also create a dataset from local files by specifying the path to the dat ## Next steps -We didn't mention this in the tutorial, but you can also create a dataset with a loading script. A loading script is a more manual and code-intensive method for creating a dataset, but it also gives you the most flexibility and control over how a dataset is generated. It lets you configure additional options such as creating multiple configurations within a dataset, or enabling your dataset to be streamed. +We didn't mention this in the tutorial, but you can also create a dataset with a loading script. A loading script is a more manual and code-intensive method for creating a dataset, and are not well supported on Hugging Face. Though in some rare cases it can still be helpful. To learn more about how to write loading scripts, take a look at the image loading script, audio loading script, and text loading script guides. diff --git a/docs/source/filesystems.mdx b/docs/source/filesystems.mdx index 2d0e02c42e3..af262492261 100644 --- a/docs/source/filesystems.mdx +++ b/docs/source/filesystems.mdx @@ -142,14 +142,6 @@ Load a dataset builder from the Hugging Face Hub (see [how to load from the Hugg >>> builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet") ``` -Load a dataset builder using a loading script (see [how to load a local loading script](./loading#local-loading-script)): - -```py ->>> output_dir = "s3://my-bucket/imdb" ->>> builder = load_dataset_builder("path/to/local/loading_script/loading_script.py") ->>> builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet") -``` - Use your own data files (see [how to load local and remote files](./loading#local-and-remote-files)): ```py diff --git a/docs/source/how_to_metrics.mdx b/docs/source/how_to_metrics.mdx deleted file mode 100644 index 01b8dcbcf82..00000000000 --- a/docs/source/how_to_metrics.mdx +++ /dev/null @@ -1,232 +0,0 @@ -# Metrics - - - -Metrics is deprecated in πŸ€— Datasets. To learn more about how to use metrics, take a look at the library πŸ€— [Evaluate](https://huggingface.co/docs/evaluate/index)! In addition to metrics, you can find more tools for evaluating models and datasets. - - - -Metrics are important for evaluating a model's predictions. In the tutorial, you learned how to compute a metric over an entire evaluation set. You have also seen how to load a metric. - -This guide will show you how to: - -- Add predictions and references. -- Compute metrics using different methods. -- Write your own metric loading script. - -## Add predictions and references - -When you want to add model predictions and references to a [`Metric`] instance, you have two options: - -- [`Metric.add`] adds a single `prediction` and `reference`. - -- [`Metric.add_batch`] adds a batch of `predictions` and `references`. - -Use [`Metric.add_batch`] by passing it your model predictions, and the references the model predictions should be evaluated against: - -```py ->>> import datasets ->>> metric = datasets.load_metric('my_metric') ->>> for model_input, gold_references in evaluation_dataset: -... model_predictions = model(model_inputs) -... metric.add_batch(predictions=model_predictions, references=gold_references) ->>> final_score = metric.compute() -``` - - - -Metrics accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation. - - - -## Compute scores - -The most straightforward way to calculate a metric is to call [`Metric.compute`]. But some metrics have additional arguments that allow you to modify the metrics behavior. - -Let's load the [SacreBLEU](https://huggingface.co/metrics/sacrebleu) metric, and compute it with a different smoothing method. - -1. Load the SacreBLEU metric: - -```py ->>> import datasets ->>> metric = datasets.load_metric('sacrebleu') -``` - -2. Inspect the different argument methods for computing the metric: - -```py ->>> print(metric.inputs_description) -Produces BLEU scores along with its sufficient statistics -from a source against one or more references. - -Args: - predictions: The system stream (a sequence of segments). - references: A list of one or more reference streams (each a sequence of segments). - smooth_method: The smoothing method to use. (Default: 'exp'). - smooth_value: The smoothing value. Only valid for 'floor' and 'add-k'. (Defaults: floor: 0.1, add-k: 1). - tokenize: Tokenization method to use for BLEU. If not provided, defaults to 'zh' for Chinese, 'ja-mecab' for Japanese and '13a' (mteval) otherwise. - lowercase: Lowercase the data. If True, enables case-insensitivity. (Default: False). - force: Insist that your tokenized input is actually detokenized. -... -``` - -3. Compute the metric with the `floor` method, and a different `smooth_value`: - -```py ->>> score = metric.compute(smooth_method="floor", smooth_value=0.2) -``` - - - -## Custom metric loading script - -Write a metric loading script to use your own custom metric (or one that is not on the Hub). Then you can load it as usual with [`load_metric`]. - -To help you get started, open the [SQuAD metric loading script](https://github.com/huggingface/datasets/blob/main/metrics/squad/squad.py) and follow along. - - - -Get jump started with our metric loading script [template](https://github.com/huggingface/datasets/blob/f9713d2e23813142a02f1b0e965095f528785cff/templates/new_metric_script.py)! - - - -### Add metric attributes - -Start by adding some information about your metric in [`Metric._info`]. The most important attributes you should specify are: - -1. [`MetricInfo.description`] provides a brief description about your metric. - -2. [`MetricInfo.citation`] contains a BibTex citation for the metric. - -3. [`MetricInfo.inputs_description`] describes the expected inputs and outputs. It may also provide an example usage of the metric. - -4. [`MetricInfo.features`] defines the name and type of the predictions and references. - -After you've filled out all these fields in the template, it should look like the following example from the SQuAD metric script: - -```py -class Squad(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")}, - "references": { - "id": datasets.Value("string"), - "answers": datasets.features.Sequence( - { - "text": datasets.Value("string"), - "answer_start": datasets.Value("int32"), - } - ), - }, - } - ), - codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - ) -``` - -### Download metric files - -If your metric needs to download, or retrieve local files, you will need to use the [`Metric._download_and_prepare`] method. For this example, let's examine the [BLEURT metric loading script](https://github.com/huggingface/datasets/blob/main/metrics/bleurt/bleurt.py). - -1. Provide a dictionary of URLs that point to the metric files: - -```py -CHECKPOINT_URLS = { - "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip", - "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip", - "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip", - "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip", - "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip", - "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip", -} -``` - - - -If the files are stored locally, provide a dictionary of path(s) instead of URLs. - - - -2. [`Metric._download_and_prepare`] will take the URLs and download the metric files specified: - -```py -def _download_and_prepare(self, dl_manager): - - # check that config name specifies a valid BLEURT model - if self.config_name == "default": - logger.warning( - "Using default BLEURT-Base checkpoint for sequence maximum length 128. " - "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512')." - ) - self.config_name = "bleurt-base-128" - if self.config_name not in CHECKPOINT_URLS.keys(): - raise KeyError( - f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}" - ) - - # download the model checkpoint specified by self.config_name and set up the scorer - model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name]) - self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name)) -``` - -### Compute score - -[`DatasetBuilder._compute`] provides the actual instructions for how to compute a metric given the predictions and references. Now let's take a look at the [GLUE metric loading script](https://github.com/huggingface/datasets/blob/main/metrics/glue/glue.py). - -1. Provide the functions for [`DatasetBuilder._compute`] to calculate your metric: - -```py -def simple_accuracy(preds, labels): - return (preds == labels).mean().item() - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = f1_score(y_true=labels, y_pred=preds).item() - return { - "accuracy": acc, - "f1": f1, - } - -def pearson_and_spearman(preds, labels): - pearson_corr = pearsonr(preds, labels)[0].item() - spearman_corr = spearmanr(preds, labels)[0].item() - return { - "pearson": pearson_corr, - "spearmanr": spearman_corr, - } -``` - -2. Create [`DatasetBuilder._compute`] with instructions for what metric to calculate for each configuration: - -```py -def _compute(self, predictions, references): - if self.config_name == "cola": - return {"matthews_correlation": matthews_corrcoef(references, predictions)} - elif self.config_name == "stsb": - return pearson_and_spearman(predictions, references) - elif self.config_name in ["mrpc", "qqp"]: - return acc_and_f1(predictions, references) - elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]: - return {"accuracy": simple_accuracy(predictions, references)} - else: - raise KeyError( - "You should supply a configuration name selected in " - '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' - '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' - ) -``` - -### Test - -Once you're finished writing your metric loading script, try to load it locally: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('PATH/TO/MY/SCRIPT.py') -``` diff --git a/docs/source/image_dataset.mdx b/docs/source/image_dataset.mdx index 8ec6d867b1f..6a41e60b9df 100644 --- a/docs/source/image_dataset.mdx +++ b/docs/source/image_dataset.mdx @@ -2,8 +2,9 @@ There are two methods for creating and sharing an image dataset. This guide will show you how to: +* Create an audio dataset from local files in python with [`Dataset.push_to_hub`]. This is an easy way that requires only a few steps in python. + * Create an image dataset with `ImageFolder` and some metadata. This is a no-code solution for quickly creating an image dataset with several thousand images. -* Create an image dataset by writing a loading script. This method is a bit more involved, but you have greater flexibility over how a dataset is defined, downloaded, and generated which can be useful for more complex or large scale image datasets. @@ -188,7 +189,7 @@ Load your WebDataset and it will create on column per file suffix (here "jpg" an {"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]} ``` -## Loading script +## (Legacy) Loading script Write a dataset loading script to share a dataset. It defines a dataset's splits and configurations, and handles downloading and generating a dataset. The script is located in the same folder or repository as the dataset and should have the same name. diff --git a/docs/source/load_hub.mdx b/docs/source/load_hub.mdx index 4d3796ae810..87eea5a80aa 100644 --- a/docs/source/load_hub.mdx +++ b/docs/source/load_hub.mdx @@ -102,11 +102,7 @@ Then load the configuration you want: ## Remote code -Certain datasets repositories contain a loading script with the Python code used to generate the dataset. -Those datasets are generally exported to Parquet by Hugging Face, so that πŸ€— Datasets can load the dataset fast and without running a loading script. - -Even if a Parquet export is not available, you can still use any dataset with Python code in its repository with `load_dataset`. -All files and code uploaded to the Hub are scanned for malware (refer to the Hub security documentation for more information), but you should still review the dataset loading scripts and authors to avoid executing malicious code on your machine. You should set `trust_remote_code=True` to use a dataset with a loading script, or you will get an error: +Certain datasets repositories contain a loading script with the Python code used to generate the dataset. All files and code uploaded to the Hub are scanned for malware (refer to the Hub security documentation for more information), but you should still review the dataset loading scripts and authors to avoid executing malicious code on your machine. You should set `trust_remote_code=True` to use a dataset with a loading script, or you will get an error: ```py >>> from datasets import get_dataset_config_names, get_dataset_split_names, load_dataset diff --git a/docs/source/loading.mdx b/docs/source/loading.mdx index 5dc51554ecc..7df58ea730b 100644 --- a/docs/source/loading.mdx +++ b/docs/source/loading.mdx @@ -4,12 +4,12 @@ Your data can be stored in various places; they can be on your local machine's d This guide will show you how to load a dataset from: -- The Hub without a dataset loading script -- Local loading script +- The Hugging Face Hub - Local files - In-memory data - Offline - A specific slice of a split +- Local loading script (legacy) For more details specific to loading other dataset modalities, take a look at the load audio dataset guide, the load image dataset guide, or the load text dataset guide. @@ -73,35 +73,6 @@ The `split` parameter can also map a data file to a specific split: >>> c4_validation = load_dataset("allenai/c4", data_files=data_files, split="validation") ``` -## Local loading script - -You may have a πŸ€— Datasets loading script locally on your computer. In this case, load the dataset by passing one of the following paths to [`load_dataset`]: - -- The local path to the loading script file. -- The local path to the directory containing the loading script file (only if the script file has the same name as the directory). - -Pass `trust_remote_code=True` to allow πŸ€— Datasets to execute the loading script: - -```py ->>> dataset = load_dataset("path/to/local/loading_script/loading_script.py", split="train", trust_remote_code=True) ->>> dataset = load_dataset("path/to/local/loading_script", split="train", trust_remote_code=True) # equivalent because the file has the same name as the directory -``` - -### Edit loading script - -You can also edit a loading script from the Hub to add your own modifications. Download the dataset repository locally so any data files referenced by a relative path in the loading script can be loaded: - -```bash -git clone https://huggingface.co/datasets/eli5 -``` - -Make your edits to the loading script and then load it by passing its local path to [`~datasets.load_dataset`]: - -```py ->>> from datasets import load_dataset ->>> eli5 = load_dataset("path/to/local/eli5") -``` - ## Local and remote files Datasets can be loaded from local files stored on your computer and from remote files. The datasets are most likely stored as a `csv`, `json`, `txt` or `parquet` file. The [`load_dataset`] function can load each of these file types. @@ -467,70 +438,16 @@ Now when you look at your dataset features, you can see it uses the custom label 'label': ClassLabel(num_classes=6, names=['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'], names_file=None, id=None)} ``` -## Metrics - - - -Metrics is deprecated in πŸ€— Datasets. To learn more about how to use metrics, take a look at the library πŸ€— [Evaluate](https://huggingface.co/docs/evaluate/index)! In addition to metrics, you can find more tools for evaluating models and datasets. - - - -When the metric you want to use is not supported by πŸ€— Datasets, you can write and use your own metric script. Load your metric by providing the path to your local metric loading script: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('PATH/TO/MY/METRIC/SCRIPT') - ->>> # Example of typical usage ->>> for batch in dataset: -... inputs, references = batch -... predictions = model(inputs) -... metric.add_batch(predictions=predictions, references=references) ->>> score = metric.compute() -``` - - - -See the [Metrics](./how_to_metrics#custom-metric-loading-script) guide for more details on how to write your own metric loading script. - - - -### Load configurations - -It is possible for a metric to have different configurations. The configurations are stored in the `config_name` parameter in [`MetricInfo`] attribute. When you load a metric, provide the configuration name as shown in the following: - -``` ->>> from datasets import load_metric ->>> metric = load_metric('bleurt', name='bleurt-base-128') ->>> metric = load_metric('bleurt', name='bleurt-base-512') -``` - -### Distributed setup - -When working in a distributed or parallel processing environment, loading and computing a metric can be tricky because these processes are executed in parallel on separate subsets of the data. πŸ€— Datasets supports distributed usage with a few additional arguments when you load a metric. - -For example, imagine you are training and evaluating on eight parallel processes. Here's how you would load a metric in this distributed setting: - -1. Define the total number of processes with the `num_process` argument. - -2. Set the process `rank` as an integer between zero and `num_process - 1`. - -3. Load your metric with [`load_metric`] with these arguments: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('glue', 'mrpc', num_process=num_process, process_id=rank) -``` - - +## (Legacy) Local loading script -Once you've loaded a metric for distributed usage, you can compute the metric as usual. Behind the scenes, [`Metric.compute`] gathers all the predictions and references from the nodes, and computes the final metric. +You may have a πŸ€— Datasets loading script locally on your computer. In this case, load the dataset by passing one of the following paths to [`load_dataset`]: - +- The local path to the loading script file. +- The local path to the directory containing the loading script file (only if the script file has the same name as the directory). -In some instances, you may be simultaneously running multiple independent distributed evaluations on the same server and files. To avoid any conflicts, it is important to provide an `experiment_id` to distinguish the separate evaluations: +Pass `trust_remote_code=True` to allow πŸ€— Datasets to execute the loading script: ```py ->>> from datasets import load_metric ->>> metric = load_metric('glue', 'mrpc', num_process=num_process, process_id=process_id, experiment_id="My_experiment_10") +>>> dataset = load_dataset("path/to/local/loading_script/loading_script.py", split="train", trust_remote_code=True) +>>> dataset = load_dataset("path/to/local/loading_script", split="train", trust_remote_code=True) # equivalent because the file has the same name as the directory ``` diff --git a/docs/source/metrics.mdx b/docs/source/metrics.mdx deleted file mode 100644 index 3342fa847c4..00000000000 --- a/docs/source/metrics.mdx +++ /dev/null @@ -1,85 +0,0 @@ -# Evaluate predictions - - - -Metrics is deprecated in πŸ€— Datasets. To learn more about how to use metrics, take a look at the library πŸ€— [Evaluate](https://huggingface.co/docs/evaluate/index)! In addition to metrics, you can find more tools for evaluating models and datasets. - - - -πŸ€— Datasets provides various common and NLP-specific [metrics](https://huggingface.co/metrics) for you to measure your models performance. In this section of the tutorials, you will load a metric and use it to evaluate your models predictions. - -You can see what metrics are available with [`list_metrics`]: - -```py ->>> from datasets import list_metrics ->>> metrics_list = list_metrics() ->>> len(metrics_list) -28 ->>> print(metrics_list) -['accuracy', 'bertscore', 'bleu', 'bleurt', 'cer', 'comet', 'coval', 'cuad', 'f1', 'gleu', 'glue', 'indic_glue', 'matthews_correlation', 'meteor', 'pearsonr', 'precision', 'recall', 'rouge', 'sacrebleu', 'sari', 'seqeval', 'spearmanr', 'squad', 'squad_v2', 'super_glue', 'wer', 'wiki_split', 'xnli'] -``` - -## Load metric - -It is very easy to load a metric with πŸ€— Datasets. In fact, you will notice that it is very similar to loading a dataset! Load a metric from the Hub with [`load_metric`]: - -```py ->>> from datasets import load_metric ->>> metric = load_metric('glue', 'mrpc') -``` - -This will load the metric associated with the MRPC dataset from the GLUE benchmark. - -## Select a configuration - -If you are using a benchmark dataset, you need to select a metric that is associated with the configuration you are using. Select a metric configuration by providing the configuration name: - -```py ->>> metric = load_metric('glue', 'mrpc') -``` - -## Metrics object - -Before you begin using a [`Metric`] object, you should get to know it a little better. As with a dataset, you can return some basic information about a metric. For example, access the `inputs_description` parameter in [`datasets.MetricInfo`] to get more information about a metrics expected input format and some usage examples: - -```py ->>> print(metric.inputs_description) -Compute GLUE evaluation metric associated to each GLUE dataset. -Args: - predictions: list of predictions to score. - Each translation should be tokenized into a list of tokens. - references: list of lists of references for each translation. - Each reference should be tokenized into a list of tokens. -Returns: depending on the GLUE subset, one or several of: - "accuracy": Accuracy - "f1": F1 score - "pearson": Pearson Correlation - "spearmanr": Spearman Correlation - "matthews_correlation": Matthew Correlation -Examples: - >>> glue_metric = datasets.load_metric('glue', 'sst2') # 'sst2' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"] - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0} - ... - >>> glue_metric = datasets.load_metric('glue', 'mrpc') # 'mrpc' or 'qqp' - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0, 'f1': 1.0} - ... -``` - -Notice for the MRPC configuration, the metric expects the input format to be zero or one. For a complete list of attributes you can return with your metric, take a look at [`MetricInfo`]. - -## Compute metric - -Once you have loaded a metric, you are ready to use it to evaluate a models predictions. Provide the model predictions and references to [`~datasets.Metric.compute`]: - -```py ->>> model_predictions = model(model_inputs) ->>> final_score = metric.compute(predictions=model_predictions, references=gold_references) -``` diff --git a/docs/source/package_reference/loading_methods.mdx b/docs/source/package_reference/loading_methods.mdx index 4d1baee2a75..b17cbed8a3b 100644 --- a/docs/source/package_reference/loading_methods.mdx +++ b/docs/source/package_reference/loading_methods.mdx @@ -1,6 +1,6 @@ # Loading methods -Methods for listing and loading datasets and metrics: +Methods for listing and loading datasets: ## Datasets @@ -16,20 +16,6 @@ Methods for listing and loading datasets and metrics: [[autodoc]] datasets.get_dataset_split_names -## Metrics - - - -Metrics is deprecated in πŸ€— Datasets. To learn more about how to use metrics, take a look at the library πŸ€— [Evaluate](https://huggingface.co/docs/evaluate/index)! In addition to metrics, you can find more tools for evaluating models and datasets. - - - -[[autodoc]] datasets.list_metrics - -[[autodoc]] datasets.load_metric - -[[autodoc]] datasets.inspect_metric - ## From files Configurations used to load data files. diff --git a/docs/source/package_reference/main_classes.mdx b/docs/source/package_reference/main_classes.mdx index e54f5d737da..49a4fed84f0 100644 --- a/docs/source/package_reference/main_classes.mdx +++ b/docs/source/package_reference/main_classes.mdx @@ -232,16 +232,6 @@ Dictionary with split names as keys ('train', 'test' for example), and `Iterable [[autodoc]] datasets.Image -## MetricInfo - -[[autodoc]] datasets.MetricInfo - -## Metric - -The base class `Metric` implements a Metric backed by one or several [`Dataset`]. - -[[autodoc]] datasets.Metric - ## Filesystems [[autodoc]] datasets.filesystems.is_remote_filesystem diff --git a/docs/source/repository_structure.mdx b/docs/source/repository_structure.mdx index 80b5da95d28..c527a3ca730 100644 --- a/docs/source/repository_structure.mdx +++ b/docs/source/repository_structure.mdx @@ -277,5 +277,3 @@ my_dataset_repository/ β”œβ”€β”€ shard_0.csv └── shard_1.csv ``` - -For more flexibility over how to load and generate a dataset, you can also write a [dataset loading script](./dataset_script). diff --git a/docs/source/upload_dataset.mdx b/docs/source/upload_dataset.mdx index 59b6f2bdf19..ff35a09b74b 100644 --- a/docs/source/upload_dataset.mdx +++ b/docs/source/upload_dataset.mdx @@ -129,6 +129,6 @@ From here, you can go on to: - Learn more about how to use πŸ€— Datasets other functions to [process your dataset](process). - [Stream large datasets](stream) without downloading it locally. -- [Define your dataset splits and configurations](repository_structure) or [loading script](dataset_script) and share your dataset with the community. +- [Define your dataset splits and configurations](repository_structure) and share your dataset with the community. If you have any questions about πŸ€— Datasets, feel free to join and ask the community on our [forum](https://discuss.huggingface.co/c/datasets/10). diff --git a/metrics/accuracy/README.md b/metrics/accuracy/README.md deleted file mode 100644 index 3c4805d29ef..00000000000 --- a/metrics/accuracy/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Metric Card for Accuracy - - -## Metric Description - -Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: -Accuracy = (TP + TN) / (TP + TN + FP + FN) - Where: -TP: True positive -TN: True negative -FP: False positive -FN: False negative - - -## How to Use - -At minimum, this metric requires predictions and references as inputs. - -```python ->>> accuracy_metric = datasets.load_metric("accuracy") ->>> results = accuracy_metric.compute(references=[0, 1], predictions=[0, 1]) ->>> print(results) -{'accuracy': 1.0} -``` - - -### Inputs -- **predictions** (`list` of `int`): Predicted labels. -- **references** (`list` of `int`): Ground truth labels. -- **normalize** (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. -- **sample_weight** (`list` of `float`): Sample weights Defaults to None. - - -### Output Values -- **accuracy**(`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy. - -Output Example(s): -```python -{'accuracy': 1.0} -``` - -This metric outputs a dictionary, containing the accuracy score. - - -#### Values from Popular Papers - -Top-1 or top-5 accuracy is often used to report performance on supervised classification tasks such as image classification (e.g. on [ImageNet](https://paperswithcode.com/sota/image-classification-on-imagenet)) or sentiment analysis (e.g. on [IMDB](https://paperswithcode.com/sota/text-classification-on-imdb)). - - -### Examples - -Example 1-A simple example -```python ->>> accuracy_metric = datasets.load_metric("accuracy") ->>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) ->>> print(results) -{'accuracy': 0.5} -``` - -Example 2-The same as Example 1, except with `normalize` set to `False`. -```python ->>> accuracy_metric = datasets.load_metric("accuracy") ->>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) ->>> print(results) -{'accuracy': 3.0} -``` - -Example 3-The same as Example 1, except with `sample_weight` set. -```python ->>> accuracy_metric = datasets.load_metric("accuracy") ->>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) ->>> print(results) -{'accuracy': 0.8778625954198473} -``` - - -## Limitations and Bias -This metric can be easily misleading, especially in the case of unbalanced classes. For example, a high accuracy might be because a model is doing well, but if the data is unbalanced, it might also be because the model is only accurately labeling the high-frequency class. In such cases, a more detailed analysis of the model's behavior, or the use of a different metric entirely, is necessary to determine how well the model is actually performing. - - -## Citation(s) -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - - -## Further References diff --git a/metrics/accuracy/accuracy.py b/metrics/accuracy/accuracy.py deleted file mode 100644 index b9b0e334772..00000000000 --- a/metrics/accuracy/accuracy.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Accuracy metric.""" - -from sklearn.metrics import accuracy_score - -import datasets - - -_DESCRIPTION = """ -Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: -Accuracy = (TP + TN) / (TP + TN + FP + FN) - Where: -TP: True positive -TN: True negative -FP: False positive -FN: False negative -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`list` of `int`): Predicted labels. - references (`list` of `int`): Ground truth labels. - normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. - sample_weight (`list` of `float`): Sample weights Defaults to None. - -Returns: - accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy. - -Examples: - - Example 1-A simple example - >>> accuracy_metric = datasets.load_metric("accuracy") - >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) - >>> print(results) - {'accuracy': 0.5} - - Example 2-The same as Example 1, except with `normalize` set to `False`. - >>> accuracy_metric = datasets.load_metric("accuracy") - >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) - >>> print(results) - {'accuracy': 3.0} - - Example 3-The same as Example 1, except with `sample_weight` set. - >>> accuracy_metric = datasets.load_metric("accuracy") - >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) - >>> print(results) - {'accuracy': 0.8778625954198473} -""" - - -_CITATION = """ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Accuracy(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("int32")), - "references": datasets.Sequence(datasets.Value("int32")), - } - if self.config_name == "multilabel" - else { - "predictions": datasets.Value("int32"), - "references": datasets.Value("int32"), - } - ), - reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html"], - ) - - def _compute(self, predictions, references, normalize=True, sample_weight=None): - return { - "accuracy": float( - accuracy_score(references, predictions, normalize=normalize, sample_weight=sample_weight) - ) - } diff --git a/metrics/bertscore/README.md b/metrics/bertscore/README.md deleted file mode 100644 index cf43849a7fc..00000000000 --- a/metrics/bertscore/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# Metric Card for BERT Score - -## Metric description - -BERTScore is an automatic evaluation metric for text generation that computes a similarity score for each token in the candidate sentence with each token in the reference sentence. It leverages the pre-trained contextual embeddings from [BERT](https://huggingface.co/bert-base-uncased) models and matches words in candidate and reference sentences by cosine similarity. - -Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language generation tasks. - -## How to use - -BERTScore takes 3 mandatory arguments : `predictions` (a list of string of candidate sentences), `references` (a list of strings or list of list of strings of reference sentences) and either `lang` (a string of two letters indicating the language of the sentences, in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) or `model_type` (a string specififying which model to use, according to the BERT specification). The default behavior of the metric is to use the suggested model for the target language when one is specified, otherwise to use the `model_type` indicated. - -```python -from datasets import load_metric -bertscore = load_metric("bertscore") -predictions = ["hello there", "general kenobi"] -references = ["hello there", "general kenobi"] -results = bertscore.compute(predictions=predictions, references=references, lang="en") -``` - -BERTScore also accepts multiple optional arguments: - - -`num_layers` (int): The layer of representation to use. The default is the number of layers tuned on WMT16 correlation data, which depends on the `model_type` used. - -`verbose` (bool): Turn on intermediate status update. The default value is `False`. - -`idf` (bool or dict): Use idf weighting; can also be a precomputed idf_dict. - -`device` (str): On which the contextual embedding model will be allocated on. If this argument is `None`, the model lives on `cuda:0` if cuda is available. - -`nthreads` (int): Number of threads used for computation. The default value is `4`. - -`rescale_with_baseline` (bool): Rescale BERTScore with the pre-computed baseline. The default value is `False`. - -`batch_size` (int): BERTScore processing batch size, at least one of `model_type` or `lang`. `lang` needs to be specified when `rescale_with_baseline` is `True`. - -`baseline_path` (str): Customized baseline file. - -`use_fast_tokenizer` (bool): `use_fast` parameter passed to HF tokenizer. The default value is `False`. - - -## Output values - -BERTScore outputs a dictionary with the following values: - -`precision`: The [precision](https://huggingface.co/metrics/precision) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. - -`recall`: The [recall](https://huggingface.co/metrics/recall) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. - -`f1`: The [F1 score](https://huggingface.co/metrics/f1) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. - -`hashcode:` The hashcode of the library. - - -### Values from popular papers -The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) reported average model selection accuracies (Hits@1) on WMT18 hybrid systems for different language pairs, which ranged from 0.004 for `en<->tr` to 0.824 for `en<->de`. - -For more recent model performance, see the [metric leaderboard](https://paperswithcode.com/paper/bertscore-evaluating-text-generation-with). - -## Examples - -Maximal values with the `distilbert-base-uncased` model: - -```python -from datasets import load_metric -bertscore = load_metric("bertscore") -predictions = ["hello world", "general kenobi"] -references = ["hello world", "general kenobi"] -results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") -print(results) -{'precision': [1.0, 1.0], 'recall': [1.0, 1.0], 'f1': [1.0, 1.0], 'hashcode': 'distilbert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} -``` - -Partial match with the `bert-base-uncased` model: - -```python -from datasets import load_metric -bertscore = load_metric("bertscore") -predictions = ["hello world", "general kenobi"] -references = ["goodnight moon", "the sun is shining"] -results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") -print(results) -{'precision': [0.7380737066268921, 0.5584042072296143], 'recall': [0.7380737066268921, 0.5889028906822205], 'f1': [0.7380737066268921, 0.5732481479644775], 'hashcode': 'bert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} -``` - -## Limitations and bias - -The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) showed that BERTScore correlates well with human judgment on sentence-level and system-level evaluation, but this depends on the model and language pair selected. - -Furthermore, not all languages are supported by the metric -- see the [BERTScore supported language list](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages) for more information. - -Finally, calculating the BERTScore metric involves downloading the BERT model that is used to compute the score-- the default model for `en`, `roberta-large`, takes over 1.4GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `distilbert-base-uncased` is 268MB. A full list of compatible models can be found [here](https://docs.google.com/spreadsheets/d/1RKOVpselB98Nnh_EOC4A2BYn8_201tmPODpNWu4w7xI/edit#gid=0). - - -## Citation - -```bibtex -@inproceedings{bert-score, - title={BERTScore: Evaluating Text Generation with BERT}, - author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, - booktitle={International Conference on Learning Representations}, - year={2020}, - url={https://openreview.net/forum?id=SkeHuCVFDr} -} -``` - -## Further References - -- [BERTScore Project README](https://github.com/Tiiiger/bert_score#readme) -- [BERTScore ICLR 2020 Poster Presentation](https://iclr.cc/virtual_2020/poster_SkeHuCVFDr.html) diff --git a/metrics/bertscore/bertscore.py b/metrics/bertscore/bertscore.py deleted file mode 100644 index 3836bdea126..00000000000 --- a/metrics/bertscore/bertscore.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""BERTScore metric.""" - -import functools -from contextlib import contextmanager - -import bert_score -from packaging import version - -import datasets - - -@contextmanager -def filter_logging_context(): - def filter_log(record): - return False if "This IS expected if you are initializing" in record.msg else True - - logger = datasets.utils.logging.get_logger("transformers.modeling_utils") - logger.addFilter(filter_log) - try: - yield - finally: - logger.removeFilter(filter_log) - - -_CITATION = """\ -@inproceedings{bert-score, - title={BERTScore: Evaluating Text Generation with BERT}, - author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, - booktitle={International Conference on Learning Representations}, - year={2020}, - url={https://openreview.net/forum?id=SkeHuCVFDr} -} -""" - -_DESCRIPTION = """\ -BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference -sentences by cosine similarity. -It has been shown to correlate with human judgment on sentence-level and system-level evaluation. -Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language -generation tasks. - -See the project's README at https://github.com/Tiiiger/bert_score#readme for more information. -""" - -_KWARGS_DESCRIPTION = """ -BERTScore Metrics with the hashcode from a source against one or more references. - -Args: - predictions (list of str): Prediction/candidate sentences. - references (list of str or list of list of str): Reference sentences. - lang (str): Language of the sentences; required (e.g. 'en'). - model_type (str): Bert specification, default using the suggested - model for the target language; has to specify at least one of - `model_type` or `lang`. - num_layers (int): The layer of representation to use, - default using the number of layers tuned on WMT16 correlation data. - verbose (bool): Turn on intermediate status update. - idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict. - device (str): On which the contextual embedding model will be allocated on. - If this argument is None, the model lives on cuda:0 if cuda is available. - nthreads (int): Number of threads. - batch_size (int): Bert score processing batch size, - at least one of `model_type` or `lang`. `lang` needs to be - specified when `rescale_with_baseline` is True. - rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline. - baseline_path (str): Customized baseline file. - use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10. - -Returns: - precision: Precision. - recall: Recall. - f1: F1 score. - hashcode: Hashcode of the library. - -Examples: - - >>> predictions = ["hello there", "general kenobi"] - >>> references = ["hello there", "general kenobi"] - >>> bertscore = datasets.load_metric("bertscore") - >>> results = bertscore.compute(predictions=predictions, references=references, lang="en") - >>> print([round(v, 2) for v in results["f1"]]) - [1.0, 1.0] -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class BERTScore(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/Tiiiger/bert_score", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=["https://github.com/Tiiiger/bert_score"], - reference_urls=[ - "https://github.com/Tiiiger/bert_score", - "https://arxiv.org/abs/1904.09675", - ], - ) - - def _compute( - self, - predictions, - references, - lang=None, - model_type=None, - num_layers=None, - verbose=False, - idf=False, - device=None, - batch_size=64, - nthreads=4, - all_layers=False, - rescale_with_baseline=False, - baseline_path=None, - use_fast_tokenizer=False, - ): - get_hash = bert_score.utils.get_hash - scorer = bert_score.BERTScorer - - if version.parse(bert_score.__version__) >= version.parse("0.3.10"): - get_hash = functools.partial(get_hash, use_fast_tokenizer=use_fast_tokenizer) - scorer = functools.partial(scorer, use_fast_tokenizer=use_fast_tokenizer) - elif use_fast_tokenizer: - raise ImportWarning( - "To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of `bert-score` doesn't match this condition.\n" - 'You can install it with `pip install "bert-score>=0.3.10"`.' - ) - - if model_type is None: - assert lang is not None, "either lang or model_type should be specified" - model_type = bert_score.utils.lang2model[lang.lower()] - - if num_layers is None: - num_layers = bert_score.utils.model2layers[model_type] - - hashcode = get_hash( - model=model_type, - num_layers=num_layers, - idf=idf, - rescale_with_baseline=rescale_with_baseline, - use_custom_baseline=baseline_path is not None, - ) - - with filter_logging_context(): - if not hasattr(self, "cached_bertscorer") or self.cached_bertscorer.hash != hashcode: - self.cached_bertscorer = scorer( - model_type=model_type, - num_layers=num_layers, - batch_size=batch_size, - nthreads=nthreads, - all_layers=all_layers, - idf=idf, - device=device, - lang=lang, - rescale_with_baseline=rescale_with_baseline, - baseline_path=baseline_path, - ) - - (P, R, F) = self.cached_bertscorer.score( - cands=predictions, - refs=references, - verbose=verbose, - batch_size=batch_size, - ) - output_dict = { - "precision": P.tolist(), - "recall": R.tolist(), - "f1": F.tolist(), - "hashcode": hashcode, - } - return output_dict - - def add_batch(self, predictions=None, references=None, **kwargs): - """Add a batch of predictions and references for the metric's stack.""" - # References can be strings or lists of strings - # Let's change strings to lists of strings with one element - if references is not None: - references = [[ref] if isinstance(ref, str) else ref for ref in references] - super().add_batch(predictions=predictions, references=references, **kwargs) - - def add(self, prediction=None, reference=None, **kwargs): - """Add one prediction and reference for the metric's stack.""" - # References can be strings or lists of strings - # Let's change strings to lists of strings with one element - if isinstance(reference, str): - reference = [reference] - super().add(prediction=prediction, reference=reference, **kwargs) diff --git a/metrics/bleu/README.md b/metrics/bleu/README.md deleted file mode 100644 index 0191a1a1f9d..00000000000 --- a/metrics/bleu/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# Metric Card for BLEU - - -## Metric Description -BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. - -Scores are calculated for individual translated segmentsβ€”generally sentencesβ€”by comparing them with a set of good quality reference translations. Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. Neither intelligibility nor grammatical correctness are not taken into account. - -## Intended Uses -BLEU and BLEU-derived metrics are most often used for machine translation. - -## How to Use - -This metric takes as input lists of predicted sentences and reference sentences: - -```python ->>> predictions = [ -... ["hello", "there", "general", "kenobi"], -... ["foo", "bar", "foobar"] -... ] ->>> references = [ -... [["hello", "there", "general", "kenobi"]], -... [["foo", "bar", "foobar"]] -... ] ->>> bleu = datasets.load_metric("bleu") ->>> results = bleu.compute(predictions=predictions, references=references) ->>> print(results) -{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.0, 'translation_length': 7, 'reference_length': 7} -``` - -### Inputs -- **predictions** (`list`): Translations to score. Each translation should be tokenized into a list of tokens. -- **references** (`list` of `list`s): references for each translation. Each reference should be tokenized into a list of tokens. -- **max_order** (`int`): Maximum n-gram order to use when computing BLEU score. Defaults to `4`. -- **smooth** (`boolean`): Whether or not to apply Lin et al. 2004 smoothing. Defaults to `False`. - -### Output Values -- **bleu** (`float`): bleu score -- **precisions** (`list` of `float`s): geometric mean of n-gram precisions, -- **brevity_penalty** (`float`): brevity penalty, -- **length_ratio** (`float`): ratio of lengths, -- **translation_length** (`int`): translation_length, -- **reference_length** (`int`): reference_length - -Output Example: -```python -{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.167, 'translation_length': 7, 'reference_length': 6} -``` - -BLEU's output is always a number between 0 and 1. This value indicates how similar the candidate text is to the reference texts, with values closer to 1 representing more similar texts. Few human translations will attain a score of 1, since this would indicate that the candidate is identical to one of the reference translations. For this reason, it is not necessary to attain a score of 1. Because there are more opportunities to match, adding additional reference translations will increase the BLEU score. - -#### Values from Popular Papers -The [original BLEU paper](https://aclanthology.org/P02-1040/) (Papineni et al. 2002) compares BLEU scores of five different models on the same 500-sentence corpus. These scores ranged from 0.0527 to 0.2571. - -The [Attention is All you Need paper](https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf) (Vaswani et al. 2017) got a BLEU score of 0.284 on the WMT 2014 English-to-German translation task, and 0.41 on the WMT 2014 English-to-French translation task. - -### Examples - -Example where each sample has 1 reference: -```python ->>> predictions = [ -... ["hello", "there", "general", "kenobi"], -... ["foo", "bar", "foobar"] -... ] ->>> references = [ -... [["hello", "there", "general", "kenobi"]], -... [["foo", "bar", "foobar"]] -... ] ->>> bleu = datasets.load_metric("bleu") ->>> results = bleu.compute(predictions=predictions, references=references) ->>> print(results) -{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.0, 'translation_length': 7, 'reference_length': 7} -``` - -Example where the first sample has 2 references: -```python ->>> predictions = [ -... ["hello", "there", "general", "kenobi"], -... ["foo", "bar", "foobar"] -... ] ->>> references = [ -... [["hello", "there", "general", "kenobi"], ["hello", "there", "!"]], -... [["foo", "bar", "foobar"]] -... ] ->>> bleu = datasets.load_metric("bleu") ->>> results = bleu.compute(predictions=predictions, references=references) ->>> print(results) -{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} -``` - -## Limitations and Bias -This metric hase multiple known limitations and biases: -- BLEU compares overlap in tokens from the predictions and references, instead of comparing meaning. This can lead to discrepencies between BLEU scores and human ratings. -- BLEU scores are not comparable across different datasets, nor are they comparable across different languages. -- BLEU scores can vary greatly depending on which parameters are used to generate the scores, especially when different tokenization and normalization techniques are used. It is therefore not possible to compare BLEU scores generated using different parameters, or when these parameters are unknown. -- Shorter predicted translations achieve higher scores than longer ones, simply due to how the score is calculated. A brevity penalty is introduced to attempt to counteract this. - - -## Citation -```bibtex -@INPROCEEDINGS{Papineni02bleu:a, - author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, - title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, - booktitle = {}, - year = {2002}, - pages = {311--318} -} -@inproceedings{lin-och-2004-orange, - title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", - author = "Lin, Chin-Yew and - Och, Franz Josef", - booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", - month = "aug 23{--}aug 27", - year = "2004", - address = "Geneva, Switzerland", - publisher = "COLING", - url = "https://www.aclweb.org/anthology/C04-1072", - pages = "501--507", -} -``` - -## Further References -- This Hugging Face implementation uses [this Tensorflow implementation](https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py) diff --git a/metrics/bleu/bleu.py b/metrics/bleu/bleu.py deleted file mode 100644 index d5a73a3b84c..00000000000 --- a/metrics/bleu/bleu.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""BLEU metric.""" - -import datasets - -from .nmt_bleu import compute_bleu # From: https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py - - -_CITATION = """\ -@INPROCEEDINGS{Papineni02bleu:a, - author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, - title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, - booktitle = {}, - year = {2002}, - pages = {311--318} -} -@inproceedings{lin-och-2004-orange, - title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", - author = "Lin, Chin-Yew and - Och, Franz Josef", - booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", - month = "aug 23{--}aug 27", - year = "2004", - address = "Geneva, Switzerland", - publisher = "COLING", - url = "https://www.aclweb.org/anthology/C04-1072", - pages = "501--507", -} -""" - -_DESCRIPTION = """\ -BLEU (bilingual evaluation understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. -Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, -the better it is" – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and -remains one of the most popular automated and inexpensive metrics. - -Scores are calculated for individual translated segmentsβ€”generally sentencesβ€”by comparing them with a set of good quality reference translations. -Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. Intelligibility or grammatical correctness -are not taken into account[citation needed]. - -BLEU's output is always a number between 0 and 1. This value indicates how similar the candidate text is to the reference texts, with values closer to 1 -representing more similar texts. Few human translations will attain a score of 1, since this would indicate that the candidate is identical to one of the -reference translations. For this reason, it is not necessary to attain a score of 1. Because there are more opportunities to match, adding additional -reference translations will increase the BLEU score. -""" - -_KWARGS_DESCRIPTION = """ -Computes BLEU score of translated segments against one or more references. -Args: - predictions: list of translations to score. - Each translation should be tokenized into a list of tokens. - references: list of lists of references for each translation. - Each reference should be tokenized into a list of tokens. - max_order: Maximum n-gram order to use when computing BLEU score. - smooth: Whether or not to apply Lin et al. 2004 smoothing. -Returns: - 'bleu': bleu score, - 'precisions': geometric mean of n-gram precisions, - 'brevity_penalty': brevity penalty, - 'length_ratio': ratio of lengths, - 'translation_length': translation_length, - 'reference_length': reference_length -Examples: - - >>> predictions = [ - ... ["hello", "there", "general", "kenobi"], # tokenized prediction of the first sample - ... ["foo", "bar", "foobar"] # tokenized prediction of the second sample - ... ] - >>> references = [ - ... [["hello", "there", "general", "kenobi"], ["hello", "there", "!"]], # tokenized references for the first sample (2 references) - ... [["foo", "bar", "foobar"]] # tokenized references for the second sample (1 reference) - ... ] - >>> bleu = datasets.load_metric("bleu") - >>> results = bleu.compute(predictions=predictions, references=references) - >>> print(results["bleu"]) - 1.0 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Bleu(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("string", id="token"), id="sequence"), - "references": datasets.Sequence( - datasets.Sequence(datasets.Value("string", id="token"), id="sequence"), id="references" - ), - } - ), - codebase_urls=["https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py"], - reference_urls=[ - "https://en.wikipedia.org/wiki/BLEU", - "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", - ], - ) - - def _compute(self, predictions, references, max_order=4, smooth=False): - score = compute_bleu( - reference_corpus=references, translation_corpus=predictions, max_order=max_order, smooth=smooth - ) - (bleu, precisions, bp, ratio, translation_length, reference_length) = score - return { - "bleu": bleu, - "precisions": precisions, - "brevity_penalty": bp, - "length_ratio": ratio, - "translation_length": translation_length, - "reference_length": reference_length, - } diff --git a/metrics/bleurt/bleurt.py b/metrics/bleurt/bleurt.py deleted file mode 100644 index 0f0cef32915..00000000000 --- a/metrics/bleurt/bleurt.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""BLEURT metric.""" - -import os - -from bleurt import score # From: git+https://github.com/google-research/bleurt.git - -import datasets - - -logger = datasets.logging.get_logger(__name__) - - -_CITATION = """\ -@inproceedings{bleurt, - title={BLEURT: Learning Robust Metrics for Text Generation}, - author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh}, - booktitle={ACL}, - year={2020}, - url={https://arxiv.org/abs/2004.04696} -} -""" - -_DESCRIPTION = """\ -BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) -and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune -it for your specific application (the latter is expected to perform better). - -See the project's README at https://github.com/google-research/bleurt#readme for more information. -""" - -_KWARGS_DESCRIPTION = """ -BLEURT score. - -Args: - `predictions` (list of str): prediction/candidate sentences - `references` (list of str): reference sentences - `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None. - -Returns: - 'scores': List of scores. -Examples: - - >>> predictions = ["hello there", "general kenobi"] - >>> references = ["hello there", "general kenobi"] - >>> bleurt = datasets.load_metric("bleurt") - >>> results = bleurt.compute(predictions=predictions, references=references) - >>> print([round(v, 2) for v in results["scores"]]) - [1.03, 1.04] -""" - -CHECKPOINT_URLS = { - "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip", - "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip", - "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip", - "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip", - "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip", - "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip", - "BLEURT-20-D3": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip", - "BLEURT-20-D6": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip", - "BLEURT-20-D12": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip", - "BLEURT-20": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip", -} - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class BLEURT(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/google-research/bleurt", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/google-research/bleurt"], - reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"], - ) - - def _download_and_prepare(self, dl_manager): - # check that config name specifies a valid BLEURT model - if self.config_name == "default": - logger.warning( - "Using default BLEURT-Base checkpoint for sequence maximum length 128. " - "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512')." - ) - self.config_name = "bleurt-base-128" - - if self.config_name.lower() in CHECKPOINT_URLS: - checkpoint_name = self.config_name.lower() - - elif self.config_name.upper() in CHECKPOINT_URLS: - checkpoint_name = self.config_name.upper() - - else: - raise KeyError( - f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}" - ) - - # download the model checkpoint specified by self.config_name and set up the scorer - model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]) - self.scorer = score.BleurtScorer(os.path.join(model_path, checkpoint_name)) - - def _compute(self, predictions, references): - scores = self.scorer.score(references=references, candidates=predictions) - return {"scores": scores} diff --git a/metrics/cer/README.md b/metrics/cer/README.md deleted file mode 100644 index f3661a301f1..00000000000 --- a/metrics/cer/README.md +++ /dev/null @@ -1,124 +0,0 @@ -# Metric Card for CER - -## Metric description - -Character error rate (CER) is a common metric of the performance of an automatic speech recognition (ASR) system. CER is similar to Word Error Rate (WER), but operates on character instead of word. - -Character error rate can be computed as: - -`CER = (S + D + I) / N = (S + D + I) / (S + D + C)` - -where - -`S` is the number of substitutions, - -`D` is the number of deletions, - -`I` is the number of insertions, - -`C` is the number of correct characters, - -`N` is the number of characters in the reference (`N=S+D+C`). - - -## How to use - -The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). - -```python -from datasets import load_metric -cer = load_metric("cer") -cer_score = cer.compute(predictions=predictions, references=references) -``` -## Output values - -This metric outputs a float representing the character error rate. - -``` -print(cer_score) -0.34146341463414637 -``` - -The **lower** the CER value, the **better** the performance of the ASR system, with a CER of 0 being a perfect score. - -However, CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions (see [Examples](#Examples) below). - -### Values from popular papers - -This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. - -Multilingual datasets such as [Common Voice](https://huggingface.co/datasets/common_voice) report different CERs depending on the language, ranging from 0.02-0.03 for languages such as French and Italian, to 0.05-0.07 for English (see [here](https://github.com/speechbrain/speechbrain/tree/develop/recipes/CommonVoice/ASR/CTC) for more values). - -## Examples - -Perfect match between prediction and reference: - -```python -from datasets import load_metric -cer = load_metric("cer") -predictions = ["hello world", "good night moon"] -references = ["hello world", "good night moon"] -cer_score = cer.compute(predictions=predictions, references=references) -print(cer_score) -0.0 -``` - -Partial match between prediction and reference: - -```python -from datasets import load_metric -cer = load_metric("cer") -predictions = ["this is the prediction", "there is an other sample"] -references = ["this is the reference", "there is another one"] -cer_score = cer.compute(predictions=predictions, references=references) -print(cer_score) -0.34146341463414637 -``` - -No match between prediction and reference: - -```python -from datasets import load_metric -cer = load_metric("cer") -predictions = ["hello"] -references = ["gracias"] -cer_score = cer.compute(predictions=predictions, references=references) -print(cer_score) -1.0 -``` - -CER above 1 due to insertion errors: - -```python -from datasets import load_metric -cer = load_metric("cer") -predictions = ["hello world"] -references = ["hello"] -cer_score = cer.compute(predictions=predictions, references=references) -print(cer_score) -1.2 -``` - -## Limitations and bias - -CER is useful for comparing different models for tasks such as automatic speech recognition (ASR) and optic character recognition (OCR), especially for multilingual datasets where WER is not suitable given the diversity of languages. However, CER provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. - -Also, in some cases, instead of reporting the raw CER, a normalized CER is reported where the number of mistakes is divided by the sum of the number of edit operations (`I` + `S` + `D`) and `C` (the number of correct characters), which results in CER values that fall within the range of 0–100%. - - -## Citation - - -```bibtex -@inproceedings{morris2004, -author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, -year = {2004}, -month = {01}, -pages = {}, -title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} -} -``` - -## Further References - -- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/metrics/cer/cer.py b/metrics/cer/cer.py deleted file mode 100644 index 7934619bc94..00000000000 --- a/metrics/cer/cer.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Character Error Ratio (CER) metric.""" - -from typing import List - -import jiwer -import jiwer.transforms as tr -from packaging import version - -import datasets -from datasets.config import PY_VERSION - - -if PY_VERSION < version.parse("3.8"): - import importlib_metadata -else: - import importlib.metadata as importlib_metadata - - -SENTENCE_DELIMITER = "" - - -if version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): - - class SentencesToListOfCharacters(tr.AbstractTransform): - def __init__(self, sentence_delimiter: str = " "): - self.sentence_delimiter = sentence_delimiter - - def process_string(self, s: str): - return list(s) - - def process_list(self, inp: List[str]): - chars = [] - for sent_idx, sentence in enumerate(inp): - chars.extend(self.process_string(sentence)) - if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: - chars.append(self.sentence_delimiter) - return chars - - cer_transform = tr.Compose( - [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] - ) -else: - cer_transform = tr.Compose( - [ - tr.RemoveMultipleSpaces(), - tr.Strip(), - tr.ReduceToSingleSentence(SENTENCE_DELIMITER), - tr.ReduceToListOfListOfChars(), - ] - ) - - -_CITATION = """\ -@inproceedings{inproceedings, - author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, - year = {2004}, - month = {01}, - pages = {}, - title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} -} -""" - -_DESCRIPTION = """\ -Character error rate (CER) is a common metric of the performance of an automatic speech recognition system. - -CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information. - -Character error rate can be computed as: - -CER = (S + D + I) / N = (S + D + I) / (S + D + C) - -where - -S is the number of substitutions, -D is the number of deletions, -I is the number of insertions, -C is the number of correct characters, -N is the number of characters in the reference (N=S+D+C). - -CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the -performance of the ASR system with a CER of 0 being a perfect score. -""" - -_KWARGS_DESCRIPTION = """ -Computes CER score of transcribed segments against references. -Args: - references: list of references for each speech input. - predictions: list of transcribtions to score. - concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result. -Returns: - (float): the character error rate - -Examples: - - >>> predictions = ["this is the prediction", "there is an other sample"] - >>> references = ["this is the reference", "there is another one"] - >>> cer = datasets.load_metric("cer") - >>> cer_score = cer.compute(predictions=predictions, references=references) - >>> print(cer_score) - 0.34146341463414637 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class CER(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/jitsi/jiwer/"], - reference_urls=[ - "https://en.wikipedia.org/wiki/Word_error_rate", - "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates", - ], - ) - - def _compute(self, predictions, references, concatenate_texts=False): - if concatenate_texts: - return jiwer.compute_measures( - references, - predictions, - truth_transform=cer_transform, - hypothesis_transform=cer_transform, - )["wer"] - - incorrect = 0 - total = 0 - for prediction, reference in zip(predictions, references): - measures = jiwer.compute_measures( - reference, - prediction, - truth_transform=cer_transform, - hypothesis_transform=cer_transform, - ) - incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] - total += measures["substitutions"] + measures["deletions"] + measures["hits"] - - return incorrect / total diff --git a/metrics/cer/test_cer.py b/metrics/cer/test_cer.py deleted file mode 100644 index 3344ce32926..00000000000 --- a/metrics/cer/test_cer.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from cer import CER - - -cer = CER() - - -class TestCER(unittest.TestCase): - def test_cer_case_senstive(self): - refs = ["White House"] - preds = ["white house"] - # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.1818181818) < 1e-6) - - def test_cer_whitespace(self): - refs = ["were wolf"] - preds = ["werewolf"] - # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.1111111) < 1e-6) - - refs = ["werewolf"] - preds = ["weae wolf"] - # S = 1, D = 1, I = 0, N = 8, CER = 0.25 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.25) < 1e-6) - - # consecutive whitespaces case 1 - refs = ["were wolf"] - preds = ["were wolf"] - # S = 0, D = 0, I = 0, N = 9, CER = 0 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) - - # consecutive whitespaces case 2 - refs = ["were wolf"] - preds = ["were wolf"] - # S = 0, D = 0, I = 0, N = 9, CER = 0 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) - - def test_cer_sub(self): - refs = ["werewolf"] - preds = ["weaewolf"] - # S = 1, D = 0, I = 0, N = 8, CER = 0.125 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) - - def test_cer_del(self): - refs = ["werewolf"] - preds = ["wereawolf"] - # S = 0, D = 1, I = 0, N = 8, CER = 0.125 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) - - def test_cer_insert(self): - refs = ["werewolf"] - preds = ["wereolf"] - # S = 0, D = 0, I = 1, N = 8, CER = 0.125 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) - - def test_cer_equal(self): - refs = ["werewolf"] - char_error_rate = cer.compute(predictions=refs, references=refs) - self.assertEqual(char_error_rate, 0.0) - - def test_cer_list_of_seqs(self): - refs = ["werewolf", "I am your father"] - char_error_rate = cer.compute(predictions=refs, references=refs) - self.assertEqual(char_error_rate, 0.0) - - refs = ["werewolf", "I am your father", "doge"] - preds = ["werxwolf", "I am your father", "doge"] - # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.03571428) < 1e-6) - - def test_correlated_sentences(self): - refs = ["My hovercraft", "is full of eels"] - preds = ["My hovercraft is full", " of eels"] - # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28 - # whitespace at the front of " of eels" will be strip during preporcessing - # so need to insert 2 whitespaces - char_error_rate = cer.compute(predictions=preds, references=refs, concatenate_texts=True) - self.assertTrue(abs(char_error_rate - 0.071428) < 1e-6) - - def test_cer_unicode(self): - refs = ["ζˆ‘θƒ½εžδΈ‹ηŽ»η’ƒθ€ŒδΈδΌ€θΊ«δ½“"] - preds = [" θƒ½εžθ™ΎηŽ»η’ƒθ€Œ δΈιœœθΊ«δ½“ε•¦"] - # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.4545454545) < 1e-6) - - refs = ["ζˆ‘θƒ½εžδΈ‹ηŽ»η’ƒ", "θ€ŒδΈδΌ€θΊ«δ½“"] - preds = ["ζˆ‘ 能 吞 δΈ‹ 玻 η’ƒ", "θ€ŒδΈδΌ€θΊ«δ½“"] - # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11 - char_error_rate = cer.compute(predictions=preds, references=refs) - self.assertTrue(abs(char_error_rate - 0.454545454545) < 1e-6) - - refs = ["ζˆ‘θƒ½εžδΈ‹ηŽ»η’ƒθ€ŒδΈδΌ€θΊ«δ½“"] - char_error_rate = cer.compute(predictions=refs, references=refs) - self.assertFalse(char_error_rate, 0.0) - - def test_cer_empty(self): - refs = [""] - preds = ["Hypothesis"] - with self.assertRaises(ValueError): - cer.compute(predictions=preds, references=refs) - - -if __name__ == "__main__": - unittest.main() diff --git a/metrics/chrf/README.md b/metrics/chrf/README.md deleted file mode 100644 index a3a602a5f11..00000000000 --- a/metrics/chrf/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# Metric Card for chrF(++) - - -## Metric Description -ChrF and ChrF++ are two MT evaluation metrics that use the F-score statistic for character n-gram matches. ChrF++ additionally includes word n-grams, which correlate more strongly with direct assessment. We use the implementation that is already present in sacrebleu. - -While this metric is included in sacreBLEU, the implementation here is slightly different from sacreBLEU in terms of the required input format. Here, the length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 - -See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information. - - -## How to Use -At minimum, this metric requires a `list` of predictions and a `list` of `list`s of references: -```python ->>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] ->>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] ->>> chrf = datasets.load_metric("chrf") ->>> results = chrf.compute(predictions=prediction, references=reference) ->>> print(results) -{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} -``` - -### Inputs -- **`predictions`** (`list` of `str`): The predicted sentences. -- **`references`** (`list` of `list` of `str`): The references. There should be one reference sub-list for each prediction sentence. -- **`char_order`** (`int`): Character n-gram order. Defaults to `6`. -- **`word_order`** (`int`): Word n-gram order. If equals to 2, the metric is referred to as chrF++. Defaults to `0`. -- **`beta`** (`int`): Determine the importance of recall w.r.t precision. Defaults to `2`. -- **`lowercase`** (`bool`): If `True`, enables case-insensitivity. Defaults to `False`. -- **`whitespace`** (`bool`): If `True`, include whitespaces when extracting character n-grams. Defaults to `False`. -- **`eps_smoothing`** (`bool`): If `True`, applies epsilon smoothing similar to reference chrF++.py, NLTK, and Moses implementations. If `False`, takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. - - - -### Output Values -The output is a dictionary containing the following fields: -- **`'score'`** (`float`): The chrF (chrF++) score. -- **`'char_order'`** (`int`): The character n-gram order. -- **`'word_order'`** (`int`): The word n-gram order. If equals to `2`, the metric is referred to as chrF++. -- **`'beta'`** (`int`): Determine the importance of recall w.r.t precision. - - -The output is formatted as below: -```python -{'score': 61.576379378113785, 'char_order': 6, 'word_order': 0, 'beta': 2} -``` - -The chrF(++) score can be any value between `0.0` and `100.0`, inclusive. - -#### Values from Popular Papers - - -### Examples -A simple example of calculating chrF: -```python ->>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] ->>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] ->>> chrf = datasets.load_metric("chrf") ->>> results = chrf.compute(predictions=prediction, references=reference) ->>> print(results) -{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} -``` - -The same example, but with the argument `word_order=2`, to calculate chrF++ instead of chrF: -```python ->>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] ->>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] ->>> chrf = datasets.load_metric("chrf") ->>> results = chrf.compute(predictions=prediction, -... references=reference, -... word_order=2) ->>> print(results) -{'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} -``` - -The same chrF++ example as above, but with `lowercase=True` to normalize all case: -```python ->>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] ->>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] ->>> chrf = datasets.load_metric("chrf") ->>> results = chrf.compute(predictions=prediction, -... references=reference, -... word_order=2, -... lowercase=True) ->>> print(results) -{'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} -``` - - -## Limitations and Bias -- According to [PopoviΔ‡ 2017](https://www.statmt.org/wmt17/pdf/WMT70.pdf), chrF+ (where `word_order=1`) and chrF++ (where `word_order=2`) produce scores that correlate better with human judgements than chrF (where `word_order=0`) does. - -## Citation -```bibtex -@inproceedings{popovic-2015-chrf, - title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", - author = "Popovi{\'c}, Maja", - booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", - month = sep, - year = "2015", - address = "Lisbon, Portugal", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/W15-3049", - doi = "10.18653/v1/W15-3049", - pages = "392--395", -} -@inproceedings{popovic-2017-chrf, - title = "chr{F}++: words helping character n-grams", - author = "Popovi{\'c}, Maja", - booktitle = "Proceedings of the Second Conference on Machine Translation", - month = sep, - year = "2017", - address = "Copenhagen, Denmark", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/W17-4770", - doi = "10.18653/v1/W17-4770", - pages = "612--618", -} -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -``` - -## Further References -- See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information on this implementation. \ No newline at end of file diff --git a/metrics/chrf/chrf.py b/metrics/chrf/chrf.py deleted file mode 100644 index 4b8b0213b1c..00000000000 --- a/metrics/chrf/chrf.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Chrf(++) metric as available in sacrebleu.""" - -import sacrebleu as scb -from packaging import version -from sacrebleu import CHRF - -import datasets - - -_CITATION = """\ -@inproceedings{popovic-2015-chrf, - title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", - author = "Popovi{\'c}, Maja", - booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", - month = sep, - year = "2015", - address = "Lisbon, Portugal", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/W15-3049", - doi = "10.18653/v1/W15-3049", - pages = "392--395", -} -@inproceedings{popovic-2017-chrf, - title = "chr{F}++: words helping character n-grams", - author = "Popovi{\'c}, Maja", - booktitle = "Proceedings of the Second Conference on Machine Translation", - month = sep, - year = "2017", - address = "Copenhagen, Denmark", - publisher = "Association for Computational Linguistics", - url = "https://aclanthology.org/W17-4770", - doi = "10.18653/v1/W17-4770", - pages = "612--618", -} -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -""" - -_DESCRIPTION = """\ -ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, -and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation -that is already present in sacrebleu. - -The implementation here is slightly different from sacrebleu in terms of the required input format. The length of -the references and hypotheses lists need to be the same, so you may need to transpose your references compared to -sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 - -See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information. -""" - -_KWARGS_DESCRIPTION = """ -Produces ChrF(++) scores for hypotheses given reference translations. - -Args: - predictions (list of str): The predicted sentences. - references (list of list of str): The references. There should be one reference sub-list for each prediction sentence. - char_order (int): Character n-gram order. Defaults to `6`. - word_order (int): Word n-gram order. If equals to `2`, the metric is referred to as chrF++. Defaults to `0`. - beta (int): Determine the importance of recall w.r.t precision. Defaults to `2`. - lowercase (bool): if `True`, enables case-insensitivity. Defaults to `False`. - whitespace (bool): If `True`, include whitespaces when extracting character n-grams. - eps_smoothing (bool): If `True`, applies epsilon smoothing similar - to reference chrF++.py, NLTK and Moses implementations. If `False`, - it takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. - -Returns: - 'score' (float): The chrF (chrF++) score, - 'char_order' (int): The character n-gram order, - 'word_order' (int): The word n-gram order. If equals to 2, the metric is referred to as chrF++, - 'beta' (int): Determine the importance of recall w.r.t precision - -Examples: - Example 1--a simple example of calculating chrF: - >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] - >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] - >>> chrf = datasets.load_metric("chrf") - >>> results = chrf.compute(predictions=prediction, references=reference) - >>> print(results) - {'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} - - Example 2--the same example, but with the argument word_order=2, to calculate chrF++ instead of chrF: - >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] - >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] - >>> chrf = datasets.load_metric("chrf") - >>> results = chrf.compute(predictions=prediction, - ... references=reference, - ... word_order=2) - >>> print(results) - {'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} - - Example 3--the same chrF++ example as above, but with `lowercase=True` to normalize all case: - >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] - >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] - >>> chrf = datasets.load_metric("chrf") - >>> results = chrf.compute(predictions=prediction, - ... references=reference, - ... word_order=2, - ... lowercase=True) - >>> print(results) - {'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class ChrF(datasets.Metric): - def _info(self): - if version.parse(scb.__version__) < version.parse("1.4.12"): - raise ImportWarning( - "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" - 'You can install it with `pip install "sacrebleu>=1.4.12"`.' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/mjpost/sacreBLEU#chrf--chrf", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=["https://github.com/mjpost/sacreBLEU#chrf--chrf"], - reference_urls=[ - "https://github.com/m-popovic/chrF", - ], - ) - - def _compute( - self, - predictions, - references, - char_order: int = CHRF.CHAR_ORDER, - word_order: int = CHRF.WORD_ORDER, - beta: int = CHRF.BETA, - lowercase: bool = False, - whitespace: bool = False, - eps_smoothing: bool = False, - ): - references_per_prediction = len(references[0]) - if any(len(refs) != references_per_prediction for refs in references): - raise ValueError("Sacrebleu requires the same number of references for each prediction") - transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] - - sb_chrf = CHRF(char_order, word_order, beta, lowercase, whitespace, eps_smoothing) - output = sb_chrf.corpus_score(predictions, transformed_references) - - return { - "score": output.score, - "char_order": output.char_order, - "word_order": output.word_order, - "beta": output.beta, - } diff --git a/metrics/code_eval/README.md b/metrics/code_eval/README.md deleted file mode 100644 index 01c7659822b..00000000000 --- a/metrics/code_eval/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# Metric Card for Code Eval - -## Metric description - -The CodeEval metric estimates the pass@k metric for code synthesis. - -It implements the evaluation harness for the HumanEval problem solving dataset described in the paper ["Evaluating Large Language Models Trained on Code"](https://arxiv.org/abs/2107.03374). - - -## How to use - -The Code Eval metric calculates how good are predictions given a set of references. Its arguments are: - -`predictions`: a list of candidates to evaluate. Each candidate should be a list of strings with several code candidates to solve the problem. - -`references`: a list with a test for each prediction. Each test should evaluate the correctness of a code candidate. - -`k`: number of code candidates to consider in the evaluation. The default value is `[1, 10, 100]`. - -`num_workers`: the number of workers used to evaluate the candidate programs (The default value is `4`). - -`timeout`: The maximum time taken to produce a prediction before it is considered a "timeout". The default value is `3.0` (i.e. 3 seconds). - -```python -from datasets import load_metric -code_eval = load_metric("code_eval") -test_cases = ["assert add(2,3)==5"] -candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] -pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) -``` - -N.B. -This metric exists to run untrusted model-generated code. Users are strongly encouraged not to do so outside of a robust security sandbox. Before running this metric and once you've taken the necessary precautions, you will need to set the `HF_ALLOW_CODE_EVAL` environment variable. Use it at your own risk: -```python -import os -os.environ["HF_ALLOW_CODE_EVAL"] = "1"` -``` - -## Output values - -The Code Eval metric outputs two things: - -`pass_at_k`: a dictionary with the pass rates for each k value defined in the arguments. - -`results`: a dictionary with granular results of each unit test. - -### Values from popular papers -The [original CODEX paper](https://arxiv.org/pdf/2107.03374.pdf) reported that the CODEX-12B model had a pass@k score of 28.8% at `k=1`, 46.8% at `k=10` and 72.3% at `k=100`. However, since the CODEX model is not open source, it is hard to verify these numbers. - - - -## Examples - -Full match at `k=1`: - -```python -from datasets import load_metric -code_eval = load_metric("code_eval") -test_cases = ["assert add(2,3)==5"] -candidates = [["def add(a, b): return a+b"]] -pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) -print(pass_at_k) -{'pass@1': 1.0} -``` - -No match for k = 1: - -```python -from datasets import load_metric -code_eval = load_metric("code_eval") -test_cases = ["assert add(2,3)==5"] -candidates = [["def add(a,b): return a*b"]] -pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) -print(pass_at_k) -{'pass@1': 0.0} -``` - -Partial match at k=1, full match at k=2: - -```python -from datasets import load_metric -code_eval = load_metric("code_eval") -test_cases = ["assert add(2,3)==5"] -candidates = [["def add(a, b): return a+b", "def add(a,b): return a*b"]] -pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) -print(pass_at_k) -{'pass@1': 0.5, 'pass@2': 1.0} -``` - -## Limitations and bias - -As per the warning included in the metric code itself: -> This program exists to execute untrusted model-generated code. Although it is highly unlikely that model-generated code will do something overtly malicious in response to this test suite, model-generated code may act destructively due to a lack of model capability or alignment. Users are strongly encouraged to sandbox this evaluation suite so that it does not perform destructive actions on their host or network. For more information on how OpenAI sandboxes its code, see the accompanying paper. Once you have read this disclaimer and taken appropriate precautions, uncomment the following line and proceed at your own risk: - -More information about the limitations of the code can be found on the [Human Eval Github repository](https://github.com/openai/human-eval). - -## Citation - -```bibtex -@misc{chen2021evaluating, - title={Evaluating Large Language Models Trained on Code}, - author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ -and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ -and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ -and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ -and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ -and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ -and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ -and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ -and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ -and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ -and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ -and William Saunders and Christopher Hesse and Andrew N. Carr \ -and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ -and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ -and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ -and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, - year={2021}, - eprint={2107.03374}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - -## Further References - -- [Human Eval Github repository](https://github.com/openai/human-eval) -- [OpenAI Codex website](https://openai.com/blog/openai-codex/) diff --git a/metrics/code_eval/code_eval.py b/metrics/code_eval/code_eval.py deleted file mode 100644 index 444685a376f..00000000000 --- a/metrics/code_eval/code_eval.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The CodeEval metric estimates the pass@k metric for code synthesis. -This is an evaluation harness for the HumanEval problem solving dataset -described in the paper "Evaluating Large Language Models Trained on Code" -(https://arxiv.org/abs/2107.03374).""" - -import itertools -import os -from collections import Counter, defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed - -import numpy as np - -import datasets - -from .execute import check_correctness - - -_CITATION = """\ -@misc{chen2021evaluating, - title={Evaluating Large Language Models Trained on Code}, - author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ -and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ -and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ -and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ -and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ -and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ -and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ -and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ -and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ -and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ -and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ -and William Saunders and Christopher Hesse and Andrew N. Carr \ -and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ -and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ -and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ -and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, - year={2021}, - eprint={2107.03374}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -""" - -_DESCRIPTION = """\ -This metric implements the evaluation harness for the HumanEval problem solving dataset -described in the paper "Evaluating Large Language Models Trained on Code" -(https://arxiv.org/abs/2107.03374). -""" - - -_KWARGS_DESCRIPTION = """ -Calculates how good are predictions given some references, using certain scores -Args: - predictions: list of candidates to evaluate. Each candidates should be a list - of strings with several code candidates to solve the problem. - references: a list with a test for each prediction. Each test should evaluate the - correctness of a code candidate. - k: number of code candidates to consider in the evaluation (Default: [1, 10, 100]) - num_workers: number of workers used to evaluate the canidate programs (Default: 4). - timeout: -Returns: - pass_at_k: dict with pass rates for each k - results: dict with granular results of each unittest -Examples: - >>> code_eval = datasets.load_metric("code_eval") - >>> test_cases = ["assert add(2,3)==5"] - >>> candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] - >>> pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) - >>> print(pass_at_k) - {'pass@1': 0.5, 'pass@2': 1.0} -""" - - -_WARNING = """ -################################################################################ - !!!WARNING!!! -################################################################################ -The "code_eval" metric executes untrusted model-generated code in Python. -Although it is highly unlikely that model-generated code will do something -overtly malicious in response to this test suite, model-generated code may act -destructively due to a lack of model capability or alignment. -Users are strongly encouraged to sandbox this evaluation suite so that it -does not perform destructive actions on their host or network. For more -information on how OpenAI sandboxes its code, see the paper "Evaluating Large -Language Models Trained on Code" (https://arxiv.org/abs/2107.03374). - -Once you have read this disclaimer and taken appropriate precautions, -set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this -with: - ->>> import os ->>> os.environ["HF_ALLOW_CODE_EVAL"] = "1" - -################################################################################\ -""" - -_LICENSE = """The MIT License - -Copyright (c) OpenAI (https://openai.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE.""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class CodeEval(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - # This is the description that will appear on the metrics page. - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - # This defines the format of each prediction and reference - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("string")), - "references": datasets.Value("string"), - } - ), - homepage="https://github.com/openai/human-eval", - codebase_urls=["https://github.com/openai/human-eval"], - reference_urls=["https://github.com/openai/human-eval"], - license=_LICENSE, - ) - - def _compute(self, predictions, references, k=[1, 10, 100], num_workers=4, timeout=3.0): - """Returns the scores""" - - if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1": - raise ValueError(_WARNING) - - if os.name == "nt": - raise NotImplementedError("This metric is currently not supported on Windows.") - - with ThreadPoolExecutor(max_workers=num_workers) as executor: - futures = [] - completion_id = Counter() - n_samples = 0 - results = defaultdict(list) - - for task_id, (candidates, test_case) in enumerate(zip(predictions, references)): - for candidate in candidates: - test_program = candidate + "\n" + test_case - args = (test_program, timeout, task_id, completion_id[task_id]) - future = executor.submit(check_correctness, *args) - futures.append(future) - completion_id[task_id] += 1 - n_samples += 1 - - for future in as_completed(futures): - result = future.result() - results[result["task_id"]].append((result["completion_id"], result)) - - total, correct = [], [] - for result in results.values(): - result.sort() - passed = [r[1]["passed"] for r in result] - total.append(len(passed)) - correct.append(sum(passed)) - total = np.array(total) - correct = np.array(correct) - - ks = k - pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean() for k in ks if (total >= k).all()} - - return pass_at_k, results - - -def estimate_pass_at_k(num_samples, num_correct, k): - """Estimates pass@k of each problem and returns them in an array.""" - - def estimator(n: int, c: int, k: int) -> float: - """Calculates 1 - comb(n - c, k) / comb(n, k).""" - if n - c < k: - return 1.0 - return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) - - if isinstance(num_samples, int): - num_samples_it = itertools.repeat(num_samples, len(num_correct)) - else: - assert len(num_samples) == len(num_correct) - num_samples_it = iter(num_samples) - - return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) diff --git a/metrics/code_eval/execute.py b/metrics/code_eval/execute.py deleted file mode 100644 index 4f1b4e1aced..00000000000 --- a/metrics/code_eval/execute.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code is adapted from OpenAI's release -# https://github.com/openai/human-eval/blob/master/human_eval/execution.py - -import contextlib -import faulthandler -import io -import multiprocessing -import os -import platform -import signal -import tempfile - - -def check_correctness(check_program, timeout, task_id, completion_id): - """ - Evaluates the functional correctness of a completion by running the test - suite provided in the problem. - - :param completion_id: an optional completion ID so we can match - the results later even if execution finishes asynchronously. - """ - manager = multiprocessing.Manager() - result = manager.list() - - p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout)) - p.start() - p.join(timeout=timeout + 1) - if p.is_alive(): - p.kill() - - if not result: - result.append("timed out") - - return { - "task_id": task_id, - "passed": result[0] == "passed", - "result": result[0], - "completion_id": completion_id, - } - - -def unsafe_execute(check_program, result, timeout): - with create_tempdir(): - # These system calls are needed when cleaning up tempdir. - import os - import shutil - - rmtree = shutil.rmtree - rmdir = os.rmdir - chdir = os.chdir - - # Disable functionalities that can make destructive changes to the test. - reliability_guard() - - # Run program. - try: - exec_globals = {} - with swallow_io(): - with time_limit(timeout): - exec(check_program, exec_globals) - result.append("passed") - except TimeoutException: - result.append("timed out") - except BaseException as e: - result.append(f"failed: {e}") - - # Needed for cleaning up. - shutil.rmtree = rmtree - os.rmdir = rmdir - os.chdir = chdir - - -@contextlib.contextmanager -def time_limit(seconds): - def signal_handler(signum, frame): - raise TimeoutException("Timed out!") - - signal.setitimer(signal.ITIMER_REAL, seconds) - signal.signal(signal.SIGALRM, signal_handler) - try: - yield - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - - -@contextlib.contextmanager -def swallow_io(): - stream = WriteOnlyStringIO() - with contextlib.redirect_stdout(stream): - with contextlib.redirect_stderr(stream): - with redirect_stdin(stream): - yield - - -@contextlib.contextmanager -def create_tempdir(): - with tempfile.TemporaryDirectory() as dirname: - with chdir(dirname): - yield dirname - - -class TimeoutException(Exception): - pass - - -class WriteOnlyStringIO(io.StringIO): - """StringIO that throws an exception when it's read from""" - - def read(self, *args, **kwargs): - raise OSError - - def readline(self, *args, **kwargs): - raise OSError - - def readlines(self, *args, **kwargs): - raise OSError - - def readable(self, *args, **kwargs): - """Returns True if the IO object can be read.""" - return False - - -class redirect_stdin(contextlib._RedirectStream): # type: ignore - _stream = "stdin" - - -@contextlib.contextmanager -def chdir(root): - if root == ".": - yield - return - cwd = os.getcwd() - os.chdir(root) - try: - yield - except BaseException as exc: - raise exc - finally: - os.chdir(cwd) - - -def reliability_guard(maximum_memory_bytes=None): - """ - This disables various destructive functions and prevents the generated code - from interfering with the test (e.g. fork bomb, killing other processes, - removing filesystem files, etc.) - - WARNING - This function is NOT a security sandbox. Untrusted code, including, model- - generated code, should not be blindly executed outside of one. See the - Codex paper for more information about OpenAI's code sandbox, and proceed - with caution. - """ - - if maximum_memory_bytes is not None: - import resource - - resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) - resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) - if not platform.uname().system == "Darwin": - resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) - - faulthandler.disable() - - import builtins - - builtins.exit = None - builtins.quit = None - - import os - - os.environ["OMP_NUM_THREADS"] = "1" - - os.kill = None - os.system = None - os.putenv = None - os.remove = None - os.removedirs = None - os.rmdir = None - os.fchdir = None - os.setuid = None - os.fork = None - os.forkpty = None - os.killpg = None - os.rename = None - os.renames = None - os.truncate = None - os.replace = None - os.unlink = None - os.fchmod = None - os.fchown = None - os.chmod = None - os.chown = None - os.chroot = None - os.fchdir = None - os.lchflags = None - os.lchmod = None - os.lchown = None - os.getcwd = None - os.chdir = None - - import shutil - - shutil.rmtree = None - shutil.move = None - shutil.chown = None - - import subprocess - - subprocess.Popen = None # type: ignore - - __builtins__["help"] = None - - import sys - - sys.modules["ipdb"] = None - sys.modules["joblib"] = None - sys.modules["resource"] = None - sys.modules["psutil"] = None - sys.modules["tkinter"] = None diff --git a/metrics/comet/README.md b/metrics/comet/README.md deleted file mode 100644 index fb6b24e8c45..00000000000 --- a/metrics/comet/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# Metric Card for COMET - -## Metric description - -Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments. - -## How to use - -COMET takes 3 lists of strings as input: `sources` (a list of source sentences), `predictions` (a list of candidate translations) and `references` (a list of reference translations). - -```python -from datasets import load_metric -comet_metric = load_metric('comet') -source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] -hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] -reference = ["They were able to control the fire.", "Schools and kindergartens opened"] -comet_score = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) -``` - -It has several configurations, named after the COMET model to be used. It will default to `wmt20-comet-da` (previously known as `wmt-large-da-estimator-1719`). Alternate models that can be chosen include `wmt20-comet-qe-da`, `wmt21-comet-mqm`, `wmt21-cometinho-da`, `wmt21-comet-qe-mqm` and `emnlp20-comet-rank`. - -It also has several optional arguments: - -`gpus`: optional, an integer (number of GPUs to train on) or a list of integers (which GPUs to train on). Set to 0 to use CPU. The default value is `None` (uses one GPU if possible, else use CPU). - -`progress_bar`a boolean -- if set to `True`, progress updates will be printed out. The default value is `False`. - -More information about model characteristics can be found on the [COMET website](https://unbabel.github.io/COMET/html/models.html). - -## Output values - -The COMET metric outputs two lists: - -`scores`: a list of COMET scores for each of the input sentences, ranging from 0-1. - -`mean_score`: the mean value of COMET scores `scores` over all the input sentences, ranging from 0-1. - -### Values from popular papers - -The [original COMET paper](https://arxiv.org/pdf/2009.09025.pdf) reported average COMET scores ranging from 0.4 to 0.6, depending on the language pairs used for evaluating translation models. They also illustrate that COMET correlates well with human judgements compared to other metrics such as [BLEU](https://huggingface.co/metrics/bleu) and [CHRF](https://huggingface.co/metrics/chrf). - -## Examples - -Full match: - -```python -from datasets import load_metric -comet_metric = load_metric('comet') -source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] -hypothesis = ["They were able to control the fire.", "Schools and kindergartens opened"] -reference = ["They were able to control the fire.", "Schools and kindergartens opened"] -results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) -print([round(v, 1) for v in results["scores"]]) -[1.0, 1.0] -``` - -Partial match: - -```python -from datasets import load_metric -comet_metric = load_metric('comet') -source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] -hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] -reference = ["They were able to control the fire", "Schools and kindergartens opened"] -results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) -print([round(v, 2) for v in results["scores"]]) -[0.19, 0.92] -``` - -No match: - -```python -from datasets import load_metric -comet_metric = load_metric('comet') -source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] -hypothesis = ["The girl went for a walk", "The boy was sleeping"] -reference = ["They were able to control the fire", "Schools and kindergartens opened"] -results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) -print([round(v, 2) for v in results["scores"]]) -[0.00, 0.00] -``` - -## Limitations and bias - -The models provided for calculating the COMET metric are built on top of XLM-R and cover the following languages: - -Afrikaans, Albanian, Amharic, Arabic, Armenian, Assamese, Azerbaijani, Basque, Belarusian, Bengali, Bengali Romanized, Bosnian, Breton, Bulgarian, Burmese, Burmese, Catalan, Chinese (Simplified), Chinese (Traditional), Croatian, Czech, Danish, Dutch, English, Esperanto, Estonian, Filipino, Finnish, French, Galician, Georgian, German, Greek, Gujarati, Hausa, Hebrew, Hindi, Hindi Romanized, Hungarian, Icelandic, Indonesian, Irish, Italian, Japanese, Javanese, Kannada, Kazakh, Khmer, Korean, Kurdish (Kurmanji), Kyrgyz, Lao, Latin, Latvian, Lithuanian, Macedonian, Malagasy, Malay, Malayalam, Marathi, Mongolian, Nepali, Norwegian, Oriya, Oromo, Pashto, Persian, Polish, Portuguese, Punjabi, Romanian, Russian, Sanskri, Scottish, Gaelic, Serbian, Sindhi, Sinhala, Slovak, Slovenian, Somali, Spanish, Sundanese, Swahili, Swedish, Tamil, Tamil Romanized, Telugu, Telugu Romanized, Thai, Turkish, Ukrainian, Urdu, Urdu Romanized, Uyghur, Uzbek, Vietnamese, Welsh, Western, Frisian, Xhosa, Yiddish. - -Thus, results for language pairs containing uncovered languages are unreliable, as per the [COMET website](https://github.com/Unbabel/COMET) - -Also, calculating the COMET metric involves downloading the model from which features are obtained -- the default model, `wmt20-comet-da`, takes over 1.79GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `wmt21-cometinho-da` is 344MB. - -## Citation - -```bibtex -@inproceedings{rei-EtAl:2020:WMT, - author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, - title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, - booktitle = {Proceedings of the Fifth Conference on Machine Translation}, - month = {November}, - year = {2020}, - address = {Online}, - publisher = {Association for Computational Linguistics}, - pages = {909--918}, -} -``` - -```bibtex -@inproceedings{rei-etal-2020-comet, - title = "{COMET}: A Neural Framework for {MT} Evaluation", - author = "Rei, Ricardo and - Stewart, Craig and - Farinha, Ana C and - Lavie, Alon", - booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", - month = nov, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", - pages = "2685--2702", - -``` - -## Further References - -- [COMET website](https://unbabel.github.io/COMET/html/index.html) -- [Hugging Face Tasks - Machine Translation](https://huggingface.co/tasks/translation) diff --git a/metrics/comet/comet.py b/metrics/comet/comet.py deleted file mode 100644 index 799a8d5a254..00000000000 --- a/metrics/comet/comet.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""COMET metric. - -Requirements: -pip install unbabel-comet - -Usage: - -```python -from datasets import load_metric -comet_metric = load_metric('metrics/comet/comet.py') -#comet_metric = load_metric('comet') -#comet_metric = load_metric('comet', 'wmt-large-hter-estimator') - - -source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] -hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] -reference = ["They were able to control the fire.", "Schools and kindergartens opened"] - -predictions = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) -predictions['scores'] -``` -""" - -import comet # From: unbabel-comet -import torch - -import datasets - - -logger = datasets.logging.get_logger(__name__) - -_CITATION = """\ -@inproceedings{rei-EtAl:2020:WMT, - author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, - title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, - booktitle = {Proceedings of the Fifth Conference on Machine Translation}, - month = {November}, - year = {2020}, - address = {Online}, - publisher = {Association for Computational Linguistics}, - pages = {909--918}, -} -@inproceedings{rei-etal-2020-comet, - title = "{COMET}: A Neural Framework for {MT} Evaluation", - author = "Rei, Ricardo and - Stewart, Craig and - Farinha, Ana C and - Lavie, Alon", - booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", - month = nov, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", - pages = "2685--2702", -} -""" - -_DESCRIPTION = """\ -Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM). -With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition. - -See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information. -""" - -_KWARGS_DESCRIPTION = """ -COMET score. - -Args: - -`sources` (list of str): Source sentences -`predictions` (list of str): candidate translations -`references` (list of str): reference translations -`cuda` (bool): If set to True, runs COMET using GPU -`show_progress` (bool): Shows progress -`model`: COMET model to be used. Will default to `wmt-large-da-estimator-1719` if None. - -Returns: - `samples`: List of dictionaries with `src`, `mt`, `ref` and `score`. - `scores`: List of scores. - -Examples: - - >>> comet_metric = datasets.load_metric('comet') - >>> # comet_metric = load_metric('comet', 'wmt20-comet-da') # you can also choose which model to use - >>> source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und KindergΓ€rten wurden erΓΆffnet."] - >>> hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] - >>> reference = ["They were able to control the fire.", "Schools and kindergartens opened"] - >>> results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) - >>> print([round(v, 2) for v in results["scores"]]) - [0.19, 0.92] -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class COMET(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://unbabel.github.io/COMET/html/index.html", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "sources": datasets.Value("string", id="sequence"), - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/Unbabel/COMET"], - reference_urls=[ - "https://github.com/Unbabel/COMET", - "https://www.aclweb.org/anthology/2020.emnlp-main.213/", - "http://www.statmt.org/wmt20/pdf/2020.wmt-1.101.pdf6", - ], - ) - - def _download_and_prepare(self, dl_manager): - if self.config_name == "default": - self.scorer = comet.load_from_checkpoint(comet.download_model("wmt20-comet-da")) - else: - self.scorer = comet.load_from_checkpoint(comet.download_model(self.config_name)) - - def _compute(self, sources, predictions, references, gpus=None, progress_bar=False): - if gpus is None: - gpus = 1 if torch.cuda.is_available() else 0 - data = {"src": sources, "mt": predictions, "ref": references} - data = [dict(zip(data, t)) for t in zip(*data.values())] - scores, mean_score = self.scorer.predict(data, gpus=gpus, progress_bar=progress_bar) - return {"mean_score": mean_score, "scores": scores} diff --git a/metrics/competition_math/README.md b/metrics/competition_math/README.md deleted file mode 100644 index a7d9893e62d..00000000000 --- a/metrics/competition_math/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Metric Card for Competition MATH - -## Metric description - -This metric is used to assess performance on the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math). - -It first canonicalizes the inputs (e.g., converting `1/2` to `\\frac{1}{2}`) and then computes accuracy. - -## How to use - -This metric takes two arguments: - -`predictions`: a list of predictions to score. Each prediction is a string that contains natural language and LaTeX. - -`references`: list of reference for each prediction. Each reference is a string that contains natural language and LaTeX. - - -```python ->>> from datasets import load_metric ->>> math = load_metric("competition_math") ->>> references = ["\\frac{1}{2}"] ->>> predictions = ["1/2"] ->>> results = math.compute(references=references, predictions=predictions) -``` - -N.B. To be able to use Competition MATH, you need to install the `math_equivalence` dependency using `pip install git+https://github.com/hendrycks/math.git`. - - -## Output values - -This metric returns a dictionary that contains the [accuracy](https://huggingface.co/metrics/accuracy) after canonicalizing inputs, on a scale between 0.0 and 1.0. - -### Values from popular papers -The [original MATH dataset paper](https://arxiv.org/abs/2103.03874) reported accuracies ranging from 3.0% to 6.9% by different large language models. - -More recent progress on the dataset can be found on the [dataset leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math). - -## Examples - -Maximal values (full match): - -```python ->>> from datasets import load_metric ->>> math = load_metric("competition_math") ->>> references = ["\\frac{1}{2}"] ->>> predictions = ["1/2"] ->>> results = math.compute(references=references, predictions=predictions) ->>> print(results) -{'accuracy': 1.0} -``` - -Minimal values (no match): - -```python ->>> from datasets import load_metric ->>> math = load_metric("competition_math") ->>> references = ["\\frac{1}{2}"] ->>> predictions = ["3/4"] ->>> results = math.compute(references=references, predictions=predictions) ->>> print(results) -{'accuracy': 0.0} -``` - -Partial match: - -```python ->>> from datasets import load_metric ->>> math = load_metric("competition_math") ->>> references = ["\\frac{1}{2}","\\frac{3}{4}"] ->>> predictions = ["1/5", "3/4"] ->>> results = math.compute(references=references, predictions=predictions) ->>> print(results) -{'accuracy': 0.5} -``` - -## Limitations and bias - -This metric is limited to datasets with the same format as the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math), and is meant to evaluate the performance of large language models at solving mathematical problems. - -N.B. The MATH dataset also assigns levels of difficulty to different problems, so disagregating model performance by difficulty level (similarly to what was done in the [original paper](https://arxiv.org/abs/2103.03874) can give a better indication of how a given model does on a given difficulty of math problem, compared to overall accuracy. - -## Citation - -```bibtex -@article{hendrycksmath2021, - title={Measuring Mathematical Problem Solving With the MATH Dataset}, - author={Dan Hendrycks - and Collin Burns - and Saurav Kadavath - and Akul Arora - and Steven Basart - and Eric Tang - and Dawn Song - and Jacob Steinhardt}, - journal={arXiv preprint arXiv:2103.03874}, - year={2021} -} -``` - -## Further References -- [MATH dataset](https://huggingface.co/datasets/competition_math) -- [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math) -- [MATH paper](https://arxiv.org/abs/2103.03874) diff --git a/metrics/competition_math/competition_math.py b/metrics/competition_math/competition_math.py deleted file mode 100644 index baca345913d..00000000000 --- a/metrics/competition_math/competition_math.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Accuracy metric for the Mathematics Aptitude Test of Heuristics (MATH) dataset.""" - -import math_equivalence # From: git+https://github.com/hendrycks/math.git - -import datasets - - -_CITATION = """\ -@article{hendrycksmath2021, - title={Measuring Mathematical Problem Solving With the MATH Dataset}, - author={Dan Hendrycks - and Collin Burns - and Saurav Kadavath - and Akul Arora - and Steven Basart - and Eric Tang - and Dawn Song - and Jacob Steinhardt}, - journal={arXiv preprint arXiv:2103.03874}, - year={2021} -} -""" - - -_DESCRIPTION = """\ -This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset. -It first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy. -""" - - -_KWARGS_DESCRIPTION = r""" -Calculates accuracy after canonicalizing inputs. - -Args: - predictions: list of predictions to score. Each prediction - is a string that contains natural language and LaTex. - references: list of reference for each prediction. Each - reference is a string that contains natural language - and LaTex. -Returns: - accuracy: accuracy after canonicalizing inputs - (e.g., converting "1/2" to "\\frac{1}{2}") - -Examples: - >>> metric = datasets.load_metric("competition_math") - >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"]) - >>> print(results) - {'accuracy': 1.0} -""" - - -@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class CompetitionMathMetric(datasets.Metric): - """Accuracy metric for the MATH dataset.""" - - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string"), - "references": datasets.Value("string"), - } - ), - # Homepage of the metric for documentation - homepage="https://github.com/hendrycks/math", - # Additional links to the codebase or references - codebase_urls=["https://github.com/hendrycks/math"], - ) - - def _compute(self, predictions, references): - """Returns the scores""" - n_correct = 0.0 - for i, j in zip(predictions, references): - n_correct += 1.0 if math_equivalence.is_equiv(i, j) else 0.0 - accuracy = n_correct / len(predictions) - return { - "accuracy": accuracy, - } diff --git a/metrics/coval/README.md b/metrics/coval/README.md deleted file mode 100644 index bd9fc0f8135..00000000000 --- a/metrics/coval/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# Metric Card for COVAL - -## Metric description - -CoVal is a coreference evaluation tool for the [CoNLL](https://huggingface.co/datasets/conll2003) and [ARRAU](https://catalog.ldc.upenn.edu/LDC2013T22) datasets which implements of the common evaluation metrics including MUC [Vilain et al, 1995](https://aclanthology.org/M95-1005.pdf), B-cubed [Bagga and Baldwin, 1998](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.34.2578&rep=rep1&type=pdf), CEAFe [Luo et al., 2005](https://aclanthology.org/H05-1004.pdf), LEA [Moosavi and Strube, 2016](https://aclanthology.org/P16-1060.pdf) and the averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe). - -CoVal code was written by [`@ns-moosavi`](https://github.com/ns-moosavi), with some parts borrowed from [Deep Coref](https://github.com/clarkkev/deep-coref/blob/master/evaluation.py). The test suite is taken from the [official CoNLL code](https://github.com/conll/reference-coreference-scorers/), with additions by [`@andreasvc`](https://github.com/andreasvc) and file parsing developed by Leo Born. - -## How to use - -The metric takes two lists of sentences as input: one representing `predictions` and `references`, with the sentences consisting of words in the CoNLL format (see the [Limitations and bias](#Limitations-and-bias) section below for more details on the CoNLL format). - -```python -from datasets import load_metric -coval = load_metric('coval') -words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', -... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', -... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', -... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', -... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', -... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] -references = [words] -predictions = [words] -results = coval.compute(predictions=predictions, references=references) -``` -It also has several optional arguments: - -`keep_singletons`: After extracting all mentions of key or system file mentions whose corresponding coreference chain is of size one are considered as singletons. The default evaluation mode will include singletons in evaluations if they are included in the key or the system files. By setting `keep_singletons=False`, all singletons in the key and system files will be excluded from the evaluation. - -`NP_only`: Most of the recent coreference resolvers only resolve NP mentions and leave out the resolution of VPs. By setting the `NP_only` option, the scorer will only evaluate the resolution of NPs. - -`min_span`: By setting `min_span`, the scorer reports the results based on automatically detected minimum spans. Minimum spans are determined using the [MINA algorithm](https://arxiv.org/pdf/1906.06703.pdf). - - -## Output values - -The metric outputs a dictionary with the following key-value pairs: - -`mentions`: number of mentions, ranges from 0-1 - -`muc`: MUC metric, which expresses performance in terms of recall and precision, ranging from 0-1. - -`bcub`: B-cubed metric, which is the averaged precision of all items in the distribution, ranging from 0-1. - -`ceafe`: CEAFe (Constrained Entity Alignment F-Measure) is computed by aligning reference and system entities with the constraint that a reference entity is aligned with at most one reference entity. It ranges from 0-1 - -`lea`: LEA is a Link-Based Entity-Aware metric which, for each entity, considers how important the entity is and how well it is resolved. It ranges from 0-1. - -`conll_score`: averaged CoNLL score (the average of the F1 values of `muc`, `bcub` and `ceafe`), ranging from 0 to 100. - - -### Values from popular papers - -Given that many of the metrics returned by COVAL come from different sources, is it hard to cite reference values for all of them. - -The CoNLL score is used to track progress on different datasets such as the [ARRAU corpus](https://paperswithcode.com/sota/coreference-resolution-on-the-arrau-corpus) and [CoNLL 2012](https://paperswithcode.com/sota/coreference-resolution-on-conll-2012). - -## Examples - -Maximal values - -```python -from datasets import load_metric -coval = load_metric('coval') -words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', -... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', -... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', -... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', -... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', -... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] -references = [words] -predictions = [words] -results = coval.compute(predictions=predictions, references=references) -print(results) -{'mentions/recall': 1.0, 'mentions/precision': 1.0, 'mentions/f1': 1.0, 'muc/recall': 1.0, 'muc/precision': 1.0, 'muc/f1': 1.0, 'bcub/recall': 1.0, 'bcub/precision': 1.0, 'bcub/f1': 1.0, 'ceafe/recall': 1.0, 'ceafe/precision': 1.0, 'ceafe/f1': 1.0, 'lea/recall': 1.0, 'lea/precision': 1.0, 'lea/f1': 1.0, 'conll_score': 100.0} -``` - -## Limitations and bias - -This wrapper of CoVal currently only works with [CoNLL line format](https://huggingface.co/datasets/conll2003), which has one word per line with all the annotation for this word in column separated by spaces: - -| Column | Type | Description | -|:-------|:----------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1 | Document ID | This is a variation on the document filename | -| 2 | Part number | Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. | -| 3 | Word number | | -| 4 | Word | This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. | -| 5 | Part-of-Speech | | -| 6 | Parse bit | This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. | -| 7 | Predicate lemma | The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-". | -| 8 | Predicate Frameset ID | This is the PropBank frameset ID of the predicate in Column 7. | -| 9 | Word sense | This is the word sense of the word in Column 3. | -| 10 | Speaker/Author | This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. | -| 11 | Named Entities | These columns identifies the spans representing various named entities. | -| 12:N | Predicate Arguments | There is one column each of predicate argument structure information for the predicate mentioned in Column 7. | -| N | Coreference | Coreference chain information encoded in a parenthesis structure. | - -## Citations - -```bibtex -@InProceedings{moosavi2019minimum, - author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, - title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, - year = {2019}, - booktitle = {Proceedings of the 57th Annual Meeting of - the Association for Computational Linguistics (Volume 1: Long Papers)}, - publisher = {Association for Computational Linguistics}, - address = {Florence, Italy}, -} -``` -```bibtex -@inproceedings{10.3115/1072399.1072405, - author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, - title = {A Model-Theoretic Coreference Scoring Scheme}, - year = {1995}, - isbn = {1558604022}, - publisher = {Association for Computational Linguistics}, - address = {USA}, - url = {https://doi.org/10.3115/1072399.1072405}, - doi = {10.3115/1072399.1072405}, - booktitle = {Proceedings of the 6th Conference on Message Understanding}, - pages = {45–52}, - numpages = {8}, - location = {Columbia, Maryland}, - series = {MUC6 ’95} -} -``` - -```bibtex -@INPROCEEDINGS{Bagga98algorithmsfor, - author = {Amit Bagga and Breck Baldwin}, - title = {Algorithms for Scoring Coreference Chains}, - booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, - year = {1998}, - pages = {563--566} -} -``` -```bibtex -@INPROCEEDINGS{Luo05oncoreference, - author = {Xiaoqiang Luo}, - title = {On coreference resolution performance metrics}, - booktitle = {In Proc. of HLT/EMNLP}, - year = {2005}, - pages = {25--32}, - publisher = {URL} -} -``` - -```bibtex -@inproceedings{moosavi-strube-2016-coreference, - title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", - author = "Moosavi, Nafise Sadat and - Strube, Michael", - booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", - month = aug, - year = "2016", - address = "Berlin, Germany", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/P16-1060", - doi = "10.18653/v1/P16-1060", - pages = "632--642", -} -``` - -## Further References - -- [CoNLL 2012 Task Description](http://www.conll.cemantix.org/2012/data.html): for information on the format (section "*_conll File Format") -- [CoNLL Evaluation details](https://github.com/ns-moosavi/coval/blob/master/conll/README.md) -- [Hugging Face - Neural Coreference Resolution (Neuralcoref)](https://huggingface.co/coref/) - diff --git a/metrics/coval/coval.py b/metrics/coval/coval.py deleted file mode 100644 index 9ec133aa044..00000000000 --- a/metrics/coval/coval.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""CoVal metric.""" - -import coval # From: git+https://github.com/ns-moosavi/coval.git # noqa: F401 -from coval.conll import reader, util -from coval.eval import evaluator - -import datasets - - -logger = datasets.logging.get_logger(__name__) - - -_CITATION = """\ -@InProceedings{moosavi2019minimum, - author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, - title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, - year = {2019}, - booktitle = {Proceedings of the 57th Annual Meeting of - the Association for Computational Linguistics (Volume 1: Long Papers)}, - publisher = {Association for Computational Linguistics}, - address = {Florence, Italy}, -} - -@inproceedings{10.3115/1072399.1072405, -author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, -title = {A Model-Theoretic Coreference Scoring Scheme}, -year = {1995}, -isbn = {1558604022}, -publisher = {Association for Computational Linguistics}, -address = {USA}, -url = {https://doi.org/10.3115/1072399.1072405}, -doi = {10.3115/1072399.1072405}, -booktitle = {Proceedings of the 6th Conference on Message Understanding}, -pages = {45–52}, -numpages = {8}, -location = {Columbia, Maryland}, -series = {MUC6 ’95} -} - -@INPROCEEDINGS{Bagga98algorithmsfor, - author = {Amit Bagga and Breck Baldwin}, - title = {Algorithms for Scoring Coreference Chains}, - booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, - year = {1998}, - pages = {563--566} -} - -@INPROCEEDINGS{Luo05oncoreference, - author = {Xiaoqiang Luo}, - title = {On coreference resolution performance metrics}, - booktitle = {In Proc. of HLT/EMNLP}, - year = {2005}, - pages = {25--32}, - publisher = {URL} -} - -@inproceedings{moosavi-strube-2016-coreference, - title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", - author = "Moosavi, Nafise Sadat and - Strube, Michael", - booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", - month = aug, - year = "2016", - address = "Berlin, Germany", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/P16-1060", - doi = "10.18653/v1/P16-1060", - pages = "632--642", -} - -""" - -_DESCRIPTION = """\ -CoVal is a coreference evaluation tool for the CoNLL and ARRAU datasets which -implements of the common evaluation metrics including MUC [Vilain et al, 1995], -B-cubed [Bagga and Baldwin, 1998], CEAFe [Luo et al., 2005], -LEA [Moosavi and Strube, 2016] and the averaged CoNLL score -(the average of the F1 values of MUC, B-cubed and CEAFe) -[Denis and Baldridge, 2009a; Pradhan et al., 2011]. - -This wrapper of CoVal currently only work with CoNLL line format: -The CoNLL format has one word per line with all the annotation for this word in column separated by spaces: -Column Type Description -1 Document ID This is a variation on the document filename -2 Part number Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. -3 Word number -4 Word itself This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. -5 Part-of-Speech -6 Parse bit This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. -7 Predicate lemma The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-" -8 Predicate Frameset ID This is the PropBank frameset ID of the predicate in Column 7. -9 Word sense This is the word sense of the word in Column 3. -10 Speaker/Author This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. -11 Named Entities These columns identifies the spans representing various named entities. -12:N Predicate Arguments There is one column each of predicate argument structure information for the predicate mentioned in Column 7. -N Coreference Coreference chain information encoded in a parenthesis structure. -More informations on the format can be found here (section "*_conll File Format"): http://www.conll.cemantix.org/2012/data.html - -Details on the evaluation on CoNLL can be found here: https://github.com/ns-moosavi/coval/blob/master/conll/README.md - -CoVal code was written by @ns-moosavi. -Some parts are borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py -The test suite is taken from https://github.com/conll/reference-coreference-scorers/ -Mention evaluation and the test suite are added by @andreasvc. -Parsing CoNLL files is developed by Leo Born. -""" - -_KWARGS_DESCRIPTION = """ -Calculates coreference evaluation metrics. -Args: - predictions: list of sentences. Each sentence is a list of word predictions to score in the CoNLL format. - Each prediction is a word with its annotations as a string made of columns joined with spaces. - Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) - See the details on the format in the description of the metric. - references: list of sentences. Each sentence is a list of word reference to score in the CoNLL format. - Each reference is a word with its annotations as a string made of columns joined with spaces. - Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) - See the details on the format in the description of the metric. - keep_singletons: After extracting all mentions of key or system files, - mentions whose corresponding coreference chain is of size one, - are considered as singletons. The default evaluation mode will include - singletons in evaluations if they are included in the key or the system files. - By setting 'keep_singletons=False', all singletons in the key and system files - will be excluded from the evaluation. - NP_only: Most of the recent coreference resolvers only resolve NP mentions and - leave out the resolution of VPs. By setting the 'NP_only' option, the scorer will only evaluate the resolution of NPs. - min_span: By setting 'min_span', the scorer reports the results based on automatically detected minimum spans. - Minimum spans are determined using the MINA algorithm. - -Returns: - 'mentions': mentions - 'muc': MUC metric [Vilain et al, 1995] - 'bcub': B-cubed [Bagga and Baldwin, 1998] - 'ceafe': CEAFe [Luo et al., 2005] - 'lea': LEA [Moosavi and Strube, 2016] - 'conll_score': averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe) - -Examples: - - >>> coval = datasets.load_metric('coval') - >>> words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', - ... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', - ... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', - ... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', - ... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', - ... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] - >>> references = [words] - >>> predictions = [words] - >>> results = coval.compute(predictions=predictions, references=references) - >>> print(results) # doctest:+ELLIPSIS - {'mentions/recall': 1.0,[...] 'conll_score': 100.0} -""" - - -def get_coref_infos( - key_lines, sys_lines, NP_only=False, remove_nested=False, keep_singletons=True, min_span=False, doc="dummy_doc" -): - key_doc_lines = {doc: key_lines} - sys_doc_lines = {doc: sys_lines} - - doc_coref_infos = {} - - key_nested_coref_num = 0 - sys_nested_coref_num = 0 - key_removed_nested_clusters = 0 - sys_removed_nested_clusters = 0 - key_singletons_num = 0 - sys_singletons_num = 0 - - key_clusters, singletons_num = reader.get_doc_mentions(doc, key_doc_lines[doc], keep_singletons) - key_singletons_num += singletons_num - - if NP_only or min_span: - key_clusters = reader.set_annotated_parse_trees(key_clusters, key_doc_lines[doc], NP_only, min_span) - - sys_clusters, singletons_num = reader.get_doc_mentions(doc, sys_doc_lines[doc], keep_singletons) - sys_singletons_num += singletons_num - - if NP_only or min_span: - sys_clusters = reader.set_annotated_parse_trees(sys_clusters, key_doc_lines[doc], NP_only, min_span) - - if remove_nested: - nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(key_clusters, keep_singletons) - key_nested_coref_num += nested_mentions - key_removed_nested_clusters += removed_clusters - - nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(sys_clusters, keep_singletons) - sys_nested_coref_num += nested_mentions - sys_removed_nested_clusters += removed_clusters - - sys_mention_key_cluster = reader.get_mention_assignments(sys_clusters, key_clusters) - key_mention_sys_cluster = reader.get_mention_assignments(key_clusters, sys_clusters) - - doc_coref_infos[doc] = (key_clusters, sys_clusters, key_mention_sys_cluster, sys_mention_key_cluster) - - if remove_nested: - logger.info( - "Number of removed nested coreferring mentions in the key " - f"annotation: {key_nested_coref_num}; and system annotation: {sys_nested_coref_num}" - ) - logger.info( - "Number of resulting singleton clusters in the key " - f"annotation: {key_removed_nested_clusters}; and system annotation: {sys_removed_nested_clusters}" - ) - - if not keep_singletons: - logger.info( - f"{key_singletons_num:d} and {sys_singletons_num:d} singletons are removed from the key and system " - "files, respectively" - ) - - return doc_coref_infos - - -def evaluate(key_lines, sys_lines, metrics, NP_only, remove_nested, keep_singletons, min_span): - doc_coref_infos = get_coref_infos(key_lines, sys_lines, NP_only, remove_nested, keep_singletons, min_span) - - output_scores = {} - conll = 0 - conll_subparts_num = 0 - - for name, metric in metrics: - recall, precision, f1 = evaluator.evaluate_documents(doc_coref_infos, metric, beta=1) - if name in ["muc", "bcub", "ceafe"]: - conll += f1 - conll_subparts_num += 1 - output_scores.update({f"{name}/recall": recall, f"{name}/precision": precision, f"{name}/f1": f1}) - - logger.info( - name.ljust(10), - f"Recall: {recall * 100:.2f}", - f" Precision: {precision * 100:.2f}", - f" F1: {f1 * 100:.2f}", - ) - - if conll_subparts_num == 3: - conll = (conll / 3) * 100 - logger.info(f"CoNLL score: {conll:.2f}") - output_scores.update({"conll_score": conll}) - - return output_scores - - -def check_gold_parse_annotation(key_lines): - has_gold_parse = False - for line in key_lines: - if not line.startswith("#"): - if len(line.split()) > 6: - parse_col = line.split()[5] - if not parse_col == "-": - has_gold_parse = True - break - else: - break - return has_gold_parse - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Coval(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("string")), - "references": datasets.Sequence(datasets.Value("string")), - } - ), - codebase_urls=["https://github.com/ns-moosavi/coval"], - reference_urls=[ - "https://github.com/ns-moosavi/coval", - "https://www.aclweb.org/anthology/P16-1060", - "http://www.conll.cemantix.org/2012/data.html", - ], - ) - - def _compute( - self, predictions, references, keep_singletons=True, NP_only=False, min_span=False, remove_nested=False - ): - allmetrics = [ - ("mentions", evaluator.mentions), - ("muc", evaluator.muc), - ("bcub", evaluator.b_cubed), - ("ceafe", evaluator.ceafe), - ("lea", evaluator.lea), - ] - - if min_span: - has_gold_parse = util.check_gold_parse_annotation(references) - if not has_gold_parse: - raise NotImplementedError("References should have gold parse annotation to use 'min_span'.") - # util.parse_key_file(key_file) - # key_file = key_file + ".parsed" - - score = evaluate( - key_lines=references, - sys_lines=predictions, - metrics=allmetrics, - NP_only=NP_only, - remove_nested=remove_nested, - keep_singletons=keep_singletons, - min_span=min_span, - ) - - return score diff --git a/metrics/cuad/README.md b/metrics/cuad/README.md deleted file mode 100644 index 309cc97a638..00000000000 --- a/metrics/cuad/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Metric Card for CUAD - -## Metric description - -This metric wraps the official scoring script for version 1 of the [Contract Understanding Atticus Dataset (CUAD)](https://huggingface.co/datasets/cuad), which is a corpus of more than 13,000 labels in 510 commercial legal contracts that have been manually labeled to identify 41 categories of important clauses that lawyers look for when reviewing contracts in connection with corporate transactions. - -The CUAD metric computes several scores: [Exact Match](https://huggingface.co/metrics/exact_match), [F1 score](https://huggingface.co/metrics/f1), Area Under the Precision-Recall Curve, [Precision](https://huggingface.co/metrics/precision) at 80% [recall](https://huggingface.co/metrics/recall) and Precision at 90% recall. - -## How to use - -The CUAD metric takes two inputs : - - -`predictions`, a list of question-answer dictionaries with the following key-values: -- `id`: the id of the question-answer pair as given in the references. -- `prediction_text`: a list of possible texts for the answer, as a list of strings depending on a threshold on the confidence probability of each prediction. - - -`references`: a list of question-answer dictionaries with the following key-values: - - `id`: the id of the question-answer pair (the same as above). - - `answers`: a dictionary *in the CUAD dataset format* with the following keys: - - `text`: a list of possible texts for the answer, as a list of strings. - - `answer_start`: a list of start positions for the answer, as a list of ints. - - Note that `answer_start` values are not taken into account to compute the metric. - -```python -from datasets import load_metric -cuad_metric = load_metric("cuad") -predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -results = cuad_metric.compute(predictions=predictions, references=references) -``` -## Output values - -The output of the CUAD metric consists of a dictionary that contains one or several of the following metrics: - -`exact_match`: The normalized answers that exactly match the reference answer, with a range between 0.0 and 1.0 (see [exact match](https://huggingface.co/metrics/exact_match) for more information). - -`f1`: The harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is between 0.0 and 1.0 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -`aupr`: The Area Under the Precision-Recall curve, with a range between 0.0 and 1.0, with a higher value representing both high recall and high precision, and a low value representing low values for both. See the [Wikipedia article](https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve) for more information. - -`prec_at_80_recall`: The fraction of true examples among the predicted examples at a recall rate of 80%. Its range is between 0.0 and 1.0. For more information, see [precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall). - -`prec_at_90_recall`: The fraction of true examples among the predicted examples at a recall rate of 90%. Its range is between 0.0 and 1.0. - - -### Values from popular papers -The [original CUAD paper](https://arxiv.org/pdf/2103.06268.pdf) reports that a [DeBERTa model](https://huggingface.co/microsoft/deberta-base) attains -an AUPR of 47.8%, a Precision at 80% Recall of 44.0%, and a Precision at 90% Recall of 17.8% (they do not report F1 or Exact Match separately). - -For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/cuad). - -## Examples - -Maximal values : - -```python -from datasets import load_metric -cuad_metric = load_metric("cuad") -predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -results = cuad_metric.compute(predictions=predictions, references=references) -print(results) -{'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} -``` - -Minimal values: - -```python -from datasets import load_metric -cuad_metric = load_metric("cuad") -predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.'], 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] -references = [{'answers': {'answer_start': [143], 'text': 'The seller'}, 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] -results = cuad_metric.compute(predictions=predictions, references=references) -print(results) -{'exact_match': 0.0, 'f1': 0.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} -``` - -Partial match: - -```python -from datasets import load_metric -cuad_metric = load_metric("cuad") -predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] -results = cuad_metric.compute(predictions=predictions, references=references) -print(results) -{'exact_match': 100.0, 'f1': 50.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as the [CUAD dataset](https://huggingface.co/datasets/cuad). The limitations of the biases of this dataset are not discussed, but could exhibit annotation bias given the homogeneity of annotators for this dataset. - -In terms of the metric itself, the accuracy of AUPR has been debated because its estimates are quite noisy and because of the fact that reducing the Precision-Recall Curve to a single number ignores the fact that it is about the tradeoffs between the different systems or performance points plotted and not the performance of an individual system. Reporting the original F1 and exact match scores is therefore useful to ensure a more complete representation of system performance. - - -## Citation - -```bibtex -@article{hendrycks2021cuad, - title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, - author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, - journal={arXiv preprint arXiv:2103.06268}, - year={2021} -} -``` - -## Further References - -- [CUAD dataset homepage](https://www.atticusprojectai.org/cuad-v1-performance-announcements) diff --git a/metrics/cuad/cuad.py b/metrics/cuad/cuad.py deleted file mode 100644 index 64f22c7d8ce..00000000000 --- a/metrics/cuad/cuad.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""CUAD metric.""" - -import datasets - -from .evaluate import evaluate - - -_CITATION = """\ -@article{hendrycks2021cuad, - title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, - author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, - journal={arXiv preprint arXiv:2103.06268}, - year={2021} -} -""" - -_DESCRIPTION = """ -This metric wrap the official scoring script for version 1 of the Contract -Understanding Atticus Dataset (CUAD). -Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510 -commercial legal contracts that have been manually labeled to identify 41 categories of important -clauses that lawyers look for when reviewing contracts in connection with corporate transactions. -""" - -_KWARGS_DESCRIPTION = """ -Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall). -Args: - predictions: List of question-answers dictionaries with the following key-values: - - 'id': id of the question-answer pair as given in the references (see below) - - 'prediction_text': list of possible texts for the answer, as a list of strings - depending on a threshold on the confidence probability of each prediction. - references: List of question-answers dictionaries with the following key-values: - - 'id': id of the question-answer pair (see above), - - 'answers': a Dict in the CUAD dataset format - { - 'text': list of possible texts for the answer, as a list of strings - 'answer_start': list of start positions for the answer, as a list of ints - } - Note that answer_start values are not taken into account to compute the metric. -Returns: - 'exact_match': Exact match (the normalized answer exactly match the gold answer) - 'f1': The F-score of predicted tokens versus the gold answer - 'aupr': Area Under the Precision-Recall curve - 'prec_at_80_recall': Precision at 80% recall - 'prec_at_90_recall': Precision at 90% recall -Examples: - >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] - >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] - >>> cuad_metric = datasets.load_metric("cuad") - >>> results = cuad_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class CUAD(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": { - "id": datasets.Value("string"), - "prediction_text": datasets.features.Sequence(datasets.Value("string")), - }, - "references": { - "id": datasets.Value("string"), - "answers": datasets.features.Sequence( - { - "text": datasets.Value("string"), - "answer_start": datasets.Value("int32"), - } - ), - }, - } - ), - codebase_urls=["https://www.atticusprojectai.org/cuad"], - reference_urls=["https://www.atticusprojectai.org/cuad"], - ) - - def _compute(self, predictions, references): - pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} - dataset = [ - { - "paragraphs": [ - { - "qas": [ - { - "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], - "id": ref["id"], - } - for ref in references - ] - } - ] - } - ] - score = evaluate(dataset=dataset, predictions=pred_dict) - return score diff --git a/metrics/cuad/evaluate.py b/metrics/cuad/evaluate.py deleted file mode 100644 index 36ee4628f0f..00000000000 --- a/metrics/cuad/evaluate.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Official evaluation script for CUAD dataset.""" - -import argparse -import json -import re -import string -import sys - -import numpy as np - - -IOU_THRESH = 0.5 - - -def get_jaccard(prediction, ground_truth): - remove_tokens = [".", ",", ";", ":"] - - for token in remove_tokens: - ground_truth = ground_truth.replace(token, "") - prediction = prediction.replace(token, "") - - ground_truth, prediction = ground_truth.lower(), prediction.lower() - ground_truth, prediction = ground_truth.replace("/", " "), prediction.replace("/", " ") - ground_truth, prediction = set(ground_truth.split(" ")), set(prediction.split(" ")) - - intersection = ground_truth.intersection(prediction) - union = ground_truth.union(prediction) - jaccard = len(intersection) / len(union) - return jaccard - - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - return re.sub(r"\b(a|an|the)\b", " ", text) - - def white_space_fix(text): - return " ".join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def compute_precision_recall(predictions, ground_truths, qa_id): - tp, fp, fn = 0, 0, 0 - - substr_ok = "Parties" in qa_id - - # first check if ground truth is empty - if len(ground_truths) == 0: - if len(predictions) > 0: - fp += len(predictions) # false positive for each one - else: - for ground_truth in ground_truths: - assert len(ground_truth) > 0 - # check if there is a match - match_found = False - for pred in predictions: - if substr_ok: - is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred - else: - is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH - if is_match: - match_found = True - - if match_found: - tp += 1 - else: - fn += 1 - - # now also get any fps by looping through preds - for pred in predictions: - # Check if there's a match. if so, don't count (don't want to double count based on the above) - # but if there's no match, then this is a false positive. - # (Note: we get the true positives in the above loop instead of this loop so that we don't double count - # multiple predictions that are matched with the same answer.) - match_found = False - for ground_truth in ground_truths: - assert len(ground_truth) > 0 - if substr_ok: - is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred - else: - is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH - if is_match: - match_found = True - - if not match_found: - fp += 1 - - precision = tp / (tp + fp) if tp + fp > 0 else np.nan - recall = tp / (tp + fn) if tp + fn > 0 else np.nan - - return precision, recall - - -def process_precisions(precisions): - """ - Processes precisions to ensure that precision and recall don't both get worse. - Assumes the list precision is sorted in order of recalls - """ - precision_best = precisions[::-1] - for i in range(1, len(precision_best)): - precision_best[i] = max(precision_best[i - 1], precision_best[i]) - precisions = precision_best[::-1] - return precisions - - -def get_aupr(precisions, recalls): - processed_precisions = process_precisions(precisions) - aupr = np.trapz(processed_precisions, recalls) - if np.isnan(aupr): - return 0 - return aupr - - -def get_prec_at_recall(precisions, recalls, recall_thresh): - """Assumes recalls are sorted in increasing order""" - processed_precisions = process_precisions(precisions) - prec_at_recall = 0 - for prec, recall in zip(processed_precisions, recalls): - if recall >= recall_thresh: - prec_at_recall = prec - break - return prec_at_recall - - -def exact_match_score(prediction, ground_truth): - return normalize_answer(prediction) == normalize_answer(ground_truth) - - -def metric_max_over_ground_truths(metric_fn, predictions, ground_truths): - score = 0 - for pred in predictions: - for ground_truth in ground_truths: - score = metric_fn(pred, ground_truth) - if score == 1: # break the loop when one prediction matches the ground truth - break - if score == 1: - break - return score - - -def evaluate(dataset, predictions): - f1 = exact_match = total = 0 - precisions = [] - recalls = [] - for article in dataset: - for paragraph in article["paragraphs"]: - for qa in paragraph["qas"]: - total += 1 - if qa["id"] not in predictions: - message = "Unanswered question " + qa["id"] + " will receive score 0." - print(message, file=sys.stderr) - continue - ground_truths = [x["text"] for x in qa["answers"]] - prediction = predictions[qa["id"]] - precision, recall = compute_precision_recall(prediction, ground_truths, qa["id"]) - - precisions.append(precision) - recalls.append(recall) - - if precision == 0 and recall == 0: - f1 += 0 - else: - f1 += 2 * (precision * recall) / (precision + recall) - - exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) - - precisions = [x for _, x in sorted(zip(recalls, precisions))] - recalls.sort() - - f1 = 100.0 * f1 / total - exact_match = 100.0 * exact_match / total - aupr = get_aupr(precisions, recalls) - - prec_at_90_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.9) - prec_at_80_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.8) - - return { - "exact_match": exact_match, - "f1": f1, - "aupr": aupr, - "prec_at_80_recall": prec_at_80_recall, - "prec_at_90_recall": prec_at_90_recall, - } - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Evaluation for CUAD") - parser.add_argument("dataset_file", help="Dataset file") - parser.add_argument("prediction_file", help="Prediction File") - args = parser.parse_args() - with open(args.dataset_file) as dataset_file: - dataset_json = json.load(dataset_file) - dataset = dataset_json["data"] - with open(args.prediction_file) as prediction_file: - predictions = json.load(prediction_file) - print(json.dumps(evaluate(dataset, predictions))) diff --git a/metrics/exact_match/README.md b/metrics/exact_match/README.md deleted file mode 100644 index 47df0ba8943..00000000000 --- a/metrics/exact_match/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Metric Card for Exact Match - - -## Metric Description -A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. - -- **Example 1**: The exact match score of prediction "Happy Birthday!" is 0, given its reference is "Happy New Year!". -- **Example 2**: The exact match score of prediction "The Colour of Magic (1983)" is 1, given its reference is also "The Colour of Magic (1983)". - -The exact match score of a set of predictions is the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. - -- **Example**: The exact match score of the set {Example 1, Example 2} (above) is 0.5. - - -## How to Use -At minimum, this metric takes as input predictions and references: -```python ->>> from datasets import load_metric ->>> exact_match_metric = load_metric("exact_match") ->>> results = exact_match_metric.compute(predictions=predictions, references=references) -``` - -### Inputs -- **`predictions`** (`list` of `str`): List of predicted texts. -- **`references`** (`list` of `str`): List of reference texts. -- **`regexes_to_ignore`** (`list` of `str`): Regex expressions of characters to ignore when calculating the exact matches. Defaults to `None`. Note: the regex changes are applied before capitalization is normalized. -- **`ignore_case`** (`bool`): If `True`, turns everything to lowercase so that capitalization differences are ignored. Defaults to `False`. -- **`ignore_punctuation`** (`bool`): If `True`, removes punctuation before comparing strings. Defaults to `False`. -- **`ignore_numbers`** (`bool`): If `True`, removes all digits before comparing strings. Defaults to `False`. - - -### Output Values -This metric outputs a dictionary with one value: the average exact match score. - -```python -{'exact_match': 100.0} -``` - -This metric's range is 0-100, inclusive. Here, 0.0 means no prediction/reference pairs were matches, while 100.0 means they all were. - -#### Values from Popular Papers -The exact match metric is often included in other metrics, such as SQuAD. For example, the [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an Exact Match score of 40.0%. They also report that the human performance Exact Match score on the dataset was 80.3%. - -### Examples -Without including any regexes to ignore: -```python ->>> exact_match = datasets.load_metric("exact_match") ->>> refs = ["the cat", "theater", "YELLING", "agent007"] ->>> preds = ["cat?", "theater", "yelling", "agent"] ->>> results = exact_match.compute(references=refs, predictions=preds) ->>> print(round(results["exact_match"], 1)) -25.0 -``` - -Ignoring regexes "the" and "yell", as well as ignoring case and punctuation: -```python ->>> exact_match = datasets.load_metric("exact_match") ->>> refs = ["the cat", "theater", "YELLING", "agent007"] ->>> preds = ["cat?", "theater", "yelling", "agent"] ->>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) ->>> print(round(results["exact_match"], 1)) -50.0 -``` -Note that in the example above, because the regexes are ignored before the case is normalized, "yell" from "YELLING" is not deleted. - -Ignoring "the", "yell", and "YELL", as well as ignoring case and punctuation: -```python ->>> exact_match = datasets.load_metric("exact_match") ->>> refs = ["the cat", "theater", "YELLING", "agent007"] ->>> preds = ["cat?", "theater", "yelling", "agent"] ->>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) ->>> print(round(results["exact_match"], 1)) -75.0 -``` - -Ignoring "the", "yell", and "YELL", as well as ignoring case, punctuation, and numbers: -```python ->>> exact_match = datasets.load_metric("exact_match") ->>> refs = ["the cat", "theater", "YELLING", "agent007"] ->>> preds = ["cat?", "theater", "yelling", "agent"] ->>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) ->>> print(round(results["exact_match"], 1)) -100.0 -``` - -An example that includes sentences: -```python ->>> exact_match = datasets.load_metric("exact_match") ->>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] ->>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] ->>> results = exact_match.compute(references=refs, predictions=preds) ->>> print(round(results["exact_match"], 1)) -33.3 -``` - - -## Limitations and Bias -This metric is limited in that it outputs the same score for something that is completely wrong as for something that is correct except for a single character. In other words, there is no award for being *almost* right. - -## Citation - -## Further References -- Also used in the [SQuAD metric](https://github.com/huggingface/datasets/tree/main/metrics/squad) diff --git a/metrics/exact_match/exact_match.py b/metrics/exact_match/exact_match.py deleted file mode 100644 index ad3ef7806b1..00000000000 --- a/metrics/exact_match/exact_match.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Exact Match metric.""" - -import re -import string - -import numpy as np - -import datasets - - -_DESCRIPTION = """ -Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list. -""" - -_KWARGS_DESCRIPTION = """ -Args: - predictions: List of predicted texts. - references: List of reference texts. - regexes_to_ignore: List, defaults to None. Regex expressions of characters to - ignore when calculating the exact matches. Note: these regexes are removed - from the input data before the changes based on the options below (e.g. ignore_case, - ignore_punctuation, ignore_numbers) are applied. - ignore_case: Boolean, defaults to False. If true, turns everything - to lowercase so that capitalization differences are ignored. - ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before - comparing predictions and references. - ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before - comparing predictions and references. -Returns: - exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive. -Examples: - >>> exact_match = datasets.load_metric("exact_match") - >>> refs = ["the cat", "theater", "YELLING", "agent007"] - >>> preds = ["cat?", "theater", "yelling", "agent"] - >>> results = exact_match.compute(references=refs, predictions=preds) - >>> print(round(results["exact_match"], 1)) - 25.0 - - >>> exact_match = datasets.load_metric("exact_match") - >>> refs = ["the cat", "theater", "YELLING", "agent007"] - >>> preds = ["cat?", "theater", "yelling", "agent"] - >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) - >>> print(round(results["exact_match"], 1)) - 50.0 - - - >>> exact_match = datasets.load_metric("exact_match") - >>> refs = ["the cat", "theater", "YELLING", "agent007"] - >>> preds = ["cat?", "theater", "yelling", "agent"] - >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) - >>> print(round(results["exact_match"], 1)) - 75.0 - - >>> exact_match = datasets.load_metric("exact_match") - >>> refs = ["the cat", "theater", "YELLING", "agent007"] - >>> preds = ["cat?", "theater", "yelling", "agent"] - >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) - >>> print(round(results["exact_match"], 1)) - 100.0 - - >>> exact_match = datasets.load_metric("exact_match") - >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] - >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] - >>> results = exact_match.compute(references=refs, predictions=preds) - >>> print(round(results["exact_match"], 1)) - 33.3 - -""" - -_CITATION = """ -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class ExactMatch(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - reference_urls=[], - ) - - def _compute( - self, - predictions, - references, - regexes_to_ignore=None, - ignore_case=False, - ignore_punctuation=False, - ignore_numbers=False, - ): - if regexes_to_ignore is not None: - for s in regexes_to_ignore: - predictions = np.array([re.sub(s, "", x) for x in predictions]) - references = np.array([re.sub(s, "", x) for x in references]) - else: - predictions = np.asarray(predictions) - references = np.asarray(references) - - if ignore_case: - predictions = np.char.lower(predictions) - references = np.char.lower(references) - - if ignore_punctuation: - repl_table = string.punctuation.maketrans("", "", string.punctuation) - predictions = np.char.translate(predictions, table=repl_table) - references = np.char.translate(references, table=repl_table) - - if ignore_numbers: - repl_table = string.digits.maketrans("", "", string.digits) - predictions = np.char.translate(predictions, table=repl_table) - references = np.char.translate(references, table=repl_table) - - score_list = predictions == references - - return {"exact_match": np.mean(score_list) * 100} diff --git a/metrics/f1/README.md b/metrics/f1/README.md deleted file mode 100644 index 5cf4f25b25e..00000000000 --- a/metrics/f1/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Metric Card for F1 - - -## Metric Description - -The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: -F1 = 2 * (precision * recall) / (precision + recall) - - -## How to Use - -At minimum, this metric requires predictions and references as input - -```python ->>> f1_metric = datasets.load_metric("f1") ->>> results = f1_metric.compute(predictions=[0, 1], references=[0, 1]) ->>> print(results) -["{'f1': 1.0}"] -``` - - -### Inputs -- **predictions** (`list` of `int`): Predicted labels. -- **references** (`list` of `int`): Ground truth labels. -- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. -- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. -- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. - - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. - - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. - - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). -- **sample_weight** (`list` of `float`): Sample weights Defaults to None. - - -### Output Values -- **f1**(`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. - -Output Example(s): -```python -{'f1': 0.26666666666666666} -``` -```python -{'f1': array([0.8, 0.0, 0.0])} -``` - -This metric outputs a dictionary, with either a single f1 score, of type `float`, or an array of f1 scores, with entries of type `float`. - - -#### Values from Popular Papers - - - - -### Examples - -Example 1-A simple binary example -```python ->>> f1_metric = datasets.load_metric("f1") ->>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) ->>> print(results) -{'f1': 0.5} -``` - -Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. -```python ->>> f1_metric = datasets.load_metric("f1") ->>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) ->>> print(round(results['f1'], 2)) -0.67 -``` - -Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. -```python ->>> f1_metric = datasets.load_metric("f1") ->>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) ->>> print(round(results['f1'], 2)) -0.35 -``` - -Example 4-A multiclass example, with different values for the `average` input. -```python ->>> predictions = [0, 2, 1, 0, 0, 1] ->>> references = [0, 1, 2, 0, 1, 2] ->>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") ->>> print(round(results['f1'], 2)) -0.27 ->>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") ->>> print(round(results['f1'], 2)) -0.33 ->>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") ->>> print(round(results['f1'], 2)) -0.27 ->>> results = f1_metric.compute(predictions=predictions, references=references, average=None) ->>> print(results) -{'f1': array([0.8, 0. , 0. ])} -``` - - -## Limitations and Bias - - - -## Citation(s) -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - - -## Further References \ No newline at end of file diff --git a/metrics/f1/f1.py b/metrics/f1/f1.py deleted file mode 100644 index 513c8fbd57e..00000000000 --- a/metrics/f1/f1.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""F1 metric.""" - -from sklearn.metrics import f1_score - -import datasets - - -_DESCRIPTION = """ -The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: -F1 = 2 * (precision * recall) / (precision + recall) -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`list` of `int`): Predicted labels. - references (`list` of `int`): Ground truth labels. - labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. - pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. - average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. - - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. - - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. - - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). - sample_weight (`list` of `float`): Sample weights Defaults to None. - -Returns: - f1 (`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. - -Examples: - - Example 1-A simple binary example - >>> f1_metric = datasets.load_metric("f1") - >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) - >>> print(results) - {'f1': 0.5} - - Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. - >>> f1_metric = datasets.load_metric("f1") - >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) - >>> print(round(results['f1'], 2)) - 0.67 - - Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. - >>> f1_metric = datasets.load_metric("f1") - >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) - >>> print(round(results['f1'], 2)) - 0.35 - - Example 4-A multiclass example, with different values for the `average` input. - >>> predictions = [0, 2, 1, 0, 0, 1] - >>> references = [0, 1, 2, 0, 1, 2] - >>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") - >>> print(round(results['f1'], 2)) - 0.27 - >>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") - >>> print(round(results['f1'], 2)) - 0.33 - >>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") - >>> print(round(results['f1'], 2)) - 0.27 - >>> results = f1_metric.compute(predictions=predictions, references=references, average=None) - >>> print(results) - {'f1': array([0.8, 0. , 0. ])} -""" - - -_CITATION = """ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class F1(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("int32")), - "references": datasets.Sequence(datasets.Value("int32")), - } - if self.config_name == "multilabel" - else { - "predictions": datasets.Value("int32"), - "references": datasets.Value("int32"), - } - ), - reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html"], - ) - - def _compute(self, predictions, references, labels=None, pos_label=1, average="binary", sample_weight=None): - score = f1_score( - references, predictions, labels=labels, pos_label=pos_label, average=average, sample_weight=sample_weight - ) - return {"f1": float(score) if score.size == 1 else score} diff --git a/metrics/frugalscore/README.md b/metrics/frugalscore/README.md deleted file mode 100644 index 977731a569d..00000000000 --- a/metrics/frugalscore/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# Metric Card for FrugalScore - - -## Metric Description -FrugalScore is a reference-based metric for Natural Language Generation (NLG) model evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. - -The FrugalScore models are obtained by continuing the pretraining of small models on a synthetic dataset constructed using summarization, backtranslation and denoising models. During the training, the small models learn the internal mapping of the expensive metric, including any similarity function. - -## How to use - -When loading FrugalScore, you can indicate the model you wish to use to compute the score. The default model is `moussaKam/frugalscore_tiny_bert-base_bert-score`, and a full list of models can be found in the [Limitations and bias](#Limitations-and-bias) section. - -```python ->>> from datasets import load_metric ->>> frugalscore = load_metric("frugalscore", "moussaKam/frugalscore_medium_bert-base_mover-score") -``` - -FrugalScore calculates how good are the predictions given some references, based on a set of scores. - -The inputs it takes are: - -`predictions`: a list of strings representing the predictions to score. - -`references`: a list of string representing the references for each prediction. - -Its optional arguments are: - -`batch_size`: the batch size for predictions (default value is `32`). - -`max_length`: the maximum sequence length (default value is `128`). - -`device`: either "gpu" or "cpu" (default value is `None`). - -```python ->>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face'], batch_size=16, max_length=64, device="gpu") -``` - -## Output values - -The output of FrugalScore is a dictionary with the list of scores for each prediction-reference pair: -```python -{'scores': [0.6307541, 0.6449357]} -``` - -### Values from popular papers -The [original FrugalScore paper](https://arxiv.org/abs/2110.08559) reported that FrugalScore-Tiny retains 97.7/94.7% of the original performance compared to [BertScore](https://huggingface.co/metrics/bertscore) while running 54 times faster and having 84 times less parameters. - -## Examples - -Maximal values (exact match between `references` and `predictions`): - -```python ->>> from datasets import load_metric ->>> frugalscore = load_metric("frugalscore") ->>> results = frugalscore.compute(predictions=['hello world'], references=['hello world']) ->>> print(results) -{'scores': [0.9891098]} -``` - -Partial values: - -```python ->>> from datasets import load_metric ->>> frugalscore = load_metric("frugalscore") ->>> results = frugalscore.compute(predictions=['hello world'], references=['hugging face']) ->>> print(results) -{'scores': [0.42482382]} -``` - -## Limitations and bias - -FrugalScore is based on [BertScore](https://huggingface.co/metrics/bertscore) and [MoverScore](https://arxiv.org/abs/1909.02622), and the models used are based on the original models used for these scores. - -The full list of available models for FrugalScore is: - -| FrugalScore | Student | Teacher | Method | -|----------------------------------------------------|-------------|----------------|------------| -| [moussaKam/frugalscore_tiny_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_bert-score) | BERT-tiny | BERT-Base | BERTScore | -| [moussaKam/frugalscore_small_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_bert-score) | BERT-small | BERT-Base | BERTScore | -| [moussaKam/frugalscore_medium_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_bert-score) | BERT-medium | BERT-Base | BERTScore | -| [moussaKam/frugalscore_tiny_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_roberta_bert-score) | BERT-tiny | RoBERTa-Large | BERTScore | -| [moussaKam/frugalscore_small_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_roberta_bert-score) | BERT-small | RoBERTa-Large | BERTScore | -| [moussaKam/frugalscore_medium_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_roberta_bert-score) | BERT-medium | RoBERTa-Large | BERTScore | -| [moussaKam/frugalscore_tiny_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_deberta_bert-score) | BERT-tiny | DeBERTa-XLarge | BERTScore | -| [moussaKam/frugalscore_small_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_deberta_bert-score) | BERT-small | DeBERTa-XLarge | BERTScore | -| [moussaKam/frugalscore_medium_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_deberta_bert-score) | BERT-medium | DeBERTa-XLarge | BERTScore | -| [moussaKam/frugalscore_tiny_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_mover-score) | BERT-tiny | BERT-Base | MoverScore | -| [moussaKam/frugalscore_small_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_mover-score) | BERT-small | BERT-Base | MoverScore | -| [moussaKam/frugalscore_medium_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_mover-score) | BERT-medium | BERT-Base | MoverScore | - -Depending on the size of the model picked, the loading time will vary: the `tiny` models will load very quickly, whereas the `medium` ones can take several minutes, depending on your Internet connection. - -## Citation -```bibtex -@article{eddine2021frugalscore, - title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, - author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, - journal={arXiv preprint arXiv:2110.08559}, - year={2021} -} -``` - -## Further References -- [Original FrugalScore code](https://github.com/moussaKam/FrugalScore) -- [FrugalScore paper](https://arxiv.org/abs/2110.08559) diff --git a/metrics/frugalscore/frugalscore.py b/metrics/frugalscore/frugalscore.py deleted file mode 100644 index 35180b17593..00000000000 --- a/metrics/frugalscore/frugalscore.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors and the current metric script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""FrugalScore metric.""" - -import torch -from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments - -import datasets - - -_CITATION = """\ -@article{eddine2021frugalscore, - title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, - author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, - journal={arXiv preprint arXiv:2110.08559}, - year={2021} -} -""" - -_DESCRIPTION = """\ -FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. -""" - - -_KWARGS_DESCRIPTION = """ -Calculates how good are predictions given some references, using certain scores. -Args: - predictions (list of str): list of predictions to score. Each predictions - should be a string. - references (list of str): list of reference for each prediction. Each - reference should be a string. - batch_size (int): the batch size for predictions. - max_length (int): maximum sequence length. - device (str): either gpu or cpu -Returns: - scores (list of int): list of scores. -Examples: - >>> frugalscore = datasets.load_metric("frugalscore") - >>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face']) - >>> print([round(s, 3) for s in results["scores"]]) - [0.631, 0.645] -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class FRUGALSCORE(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string"), - "references": datasets.Value("string"), - } - ), - homepage="https://github.com/moussaKam/FrugalScore", - ) - - def _download_and_prepare(self, dl_manager): - if self.config_name == "default": - checkpoint = "moussaKam/frugalscore_tiny_bert-base_bert-score" - else: - checkpoint = self.config_name - self.model = AutoModelForSequenceClassification.from_pretrained(checkpoint) - self.tokenizer = AutoTokenizer.from_pretrained(checkpoint) - - def _compute( - self, - predictions, - references, - batch_size=32, - max_length=128, - device=None, - ): - """Returns the scores""" - assert len(predictions) == len( - references - ), "predictions and references should have the same number of sentences." - if device is not None: - assert device in ["gpu", "cpu"], "device should be either gpu or cpu." - else: - device = "gpu" if torch.cuda.is_available() else "cpu" - training_args = TrainingArguments( - "trainer", - fp16=(device == "gpu"), - per_device_eval_batch_size=batch_size, - report_to="all", - no_cuda=(device == "cpu"), - log_level="warning", - ) - dataset = {"sentence1": predictions, "sentence2": references} - raw_datasets = datasets.Dataset.from_dict(dataset) - - def tokenize_function(data): - return self.tokenizer( - data["sentence1"], data["sentence2"], max_length=max_length, truncation=True, padding=True - ) - - tokenized_datasets = raw_datasets.map(tokenize_function, batched=True) - tokenized_datasets.remove_columns(["sentence1", "sentence2"]) - trainer = Trainer(self.model, training_args, tokenizer=self.tokenizer) - predictions = trainer.predict(tokenized_datasets) - return {"scores": list(predictions.predictions.squeeze(-1))} diff --git a/metrics/glue/README.md b/metrics/glue/README.md deleted file mode 100644 index f7f60a733db..00000000000 --- a/metrics/glue/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# Metric Card for GLUE - -## Metric description -This metric is used to compute the GLUE evaluation metric associated to each [GLUE dataset](https://huggingface.co/datasets/glue). - -GLUE, the General Language Understanding Evaluation benchmark is a collection of resources for training, evaluating, and analyzing natural language understanding systems. - -## How to use - -There are two steps: (1) loading the GLUE metric relevant to the subset of the GLUE dataset being used for evaluation; and (2) calculating the metric. - -1. **Loading the relevant GLUE metric** : the subsets of GLUE are the following: `sst2`, `mnli`, `mnli_mismatched`, `mnli_matched`, `qnli`, `rte`, `wnli`, `cola`,`stsb`, `mrpc`, `qqp`, and `hans`. - -More information about the different subsets of the GLUE dataset can be found on the [GLUE dataset page](https://huggingface.co/datasets/glue). - -2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation. - -```python -from datasets import load_metric -glue_metric = load_metric('glue', 'sst2') -references = [0, 1] -predictions = [0, 1] -results = glue_metric.compute(predictions=predictions, references=references) -``` -## Output values - -The output of the metric depends on the GLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: - -`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). - -`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -`pearson`: a measure of the linear relationship between two datasets (see [Pearson correlation](https://huggingface.co/metrics/pearsonr) for more information). Its range is between -1 and +1, with 0 implying no correlation, and -1/+1 implying an exact linear relationship. Positive correlations imply that as x increases, so does y, whereas negative correlations imply that as x increases, y decreases. - -`spearmanr`: a nonparametric measure of the monotonicity of the relationship between two datasets(see [Spearman Correlation](https://huggingface.co/metrics/spearmanr) for more information). `spearmanr` has the same range as `pearson`. - -`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. - -The `cola` subset returns `matthews_correlation`, the `stsb` subset returns `pearson` and `spearmanr`, the `mrpc` and `qqp` subsets return both `accuracy` and `f1`, and all other subsets of GLUE return only accuracy. - -### Values from popular papers -The [original GLUE paper](https://huggingface.co/datasets/glue) reported average scores ranging from 58 to 64%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). - -For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/glue). - -## Examples - -Maximal values for the MRPC subset (which outputs `accuracy` and `f1`): - -```python -from datasets import load_metric -glue_metric = load_metric('glue', 'mrpc') # 'mrpc' or 'qqp' -references = [0, 1] -predictions = [0, 1] -results = glue_metric.compute(predictions=predictions, references=references) -print(results) -{'accuracy': 1.0, 'f1': 1.0} -``` - -Minimal values for the STSB subset (which outputs `pearson` and `spearmanr`): - -```python -from datasets import load_metric -glue_metric = load_metric('glue', 'stsb') -references = [0., 1., 2., 3., 4., 5.] -predictions = [-10., -11., -12., -13., -14., -15.] -results = glue_metric.compute(predictions=predictions, references=references) -print(results) -{'pearson': -1.0, 'spearmanr': -1.0} -``` - -Partial match for the COLA subset (which outputs `matthews_correlation`) - -```python -from datasets import load_metric -glue_metric = load_metric('glue', 'cola') -references = [0, 1] -predictions = [1, 1] -results = glue_metric.compute(predictions=predictions, references=references) -results -{'matthews_correlation': 0.0} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as the [GLUE dataset](https://huggingface.co/datasets/glue). - -While the GLUE dataset is meant to represent "General Language Understanding", the tasks represented in it are not necessarily representative of language understanding, and should not be interpreted as such. - -Also, while the GLUE subtasks were considered challenging during its creation in 2019, they are no longer considered as such given the impressive progress made since then. A more complex (or "stickier") version of it, called [SuperGLUE](https://huggingface.co/datasets/super_glue), was subsequently created. - -## Citation - -```bibtex - @inproceedings{wang2019glue, - title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, - author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, - note={In the Proceedings of ICLR.}, - year={2019} -} -``` - -## Further References - -- [GLUE benchmark homepage](https://gluebenchmark.com/) -- [Fine-tuning a model with the Trainer API](https://huggingface.co/course/chapter3/3?) diff --git a/metrics/glue/glue.py b/metrics/glue/glue.py deleted file mode 100644 index 7c2826d29fa..00000000000 --- a/metrics/glue/glue.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""GLUE benchmark metric.""" - -from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import f1_score, matthews_corrcoef - -import datasets - - -_CITATION = """\ -@inproceedings{wang2019glue, - title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, - author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, - note={In the Proceedings of ICLR.}, - year={2019} -} -""" - -_DESCRIPTION = """\ -GLUE, the General Language Understanding Evaluation benchmark -(https://gluebenchmark.com/) is a collection of resources for training, -evaluating, and analyzing natural language understanding systems. -""" - -_KWARGS_DESCRIPTION = """ -Compute GLUE evaluation metric associated to each GLUE dataset. -Args: - predictions: list of predictions to score. - Each translation should be tokenized into a list of tokens. - references: list of lists of references for each translation. - Each reference should be tokenized into a list of tokens. -Returns: depending on the GLUE subset, one or several of: - "accuracy": Accuracy - "f1": F1 score - "pearson": Pearson Correlation - "spearmanr": Spearman Correlation - "matthews_correlation": Matthew Correlation -Examples: - - >>> glue_metric = datasets.load_metric('glue', 'sst2') # 'sst2' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"] - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0} - - >>> glue_metric = datasets.load_metric('glue', 'mrpc') # 'mrpc' or 'qqp' - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0, 'f1': 1.0} - - >>> glue_metric = datasets.load_metric('glue', 'stsb') - >>> references = [0., 1., 2., 3., 4., 5.] - >>> predictions = [0., 1., 2., 3., 4., 5.] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)}) - {'pearson': 1.0, 'spearmanr': 1.0} - - >>> glue_metric = datasets.load_metric('glue', 'cola') - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'matthews_correlation': 1.0} -""" - - -def simple_accuracy(preds, labels): - return float((preds == labels).mean()) - - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = float(f1_score(y_true=labels, y_pred=preds)) - return { - "accuracy": acc, - "f1": f1, - } - - -def pearson_and_spearman(preds, labels): - pearson_corr = float(pearsonr(preds, labels)[0]) - spearman_corr = float(spearmanr(preds, labels)[0]) - return { - "pearson": pearson_corr, - "spearmanr": spearman_corr, - } - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Glue(datasets.Metric): - def _info(self): - if self.config_name not in [ - "sst2", - "mnli", - "mnli_mismatched", - "mnli_matched", - "cola", - "stsb", - "mrpc", - "qqp", - "qnli", - "rte", - "wnli", - "hans", - ]: - raise KeyError( - "You should supply a configuration name selected in " - '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' - '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("int64" if self.config_name != "stsb" else "float32"), - "references": datasets.Value("int64" if self.config_name != "stsb" else "float32"), - } - ), - codebase_urls=[], - reference_urls=[], - format="numpy", - ) - - def _compute(self, predictions, references): - if self.config_name == "cola": - return {"matthews_correlation": matthews_corrcoef(references, predictions)} - elif self.config_name == "stsb": - return pearson_and_spearman(predictions, references) - elif self.config_name in ["mrpc", "qqp"]: - return acc_and_f1(predictions, references) - elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]: - return {"accuracy": simple_accuracy(predictions, references)} - else: - raise KeyError( - "You should supply a configuration name selected in " - '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' - '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' - ) diff --git a/metrics/google_bleu/README.md b/metrics/google_bleu/README.md deleted file mode 100644 index 08659e69b86..00000000000 --- a/metrics/google_bleu/README.md +++ /dev/null @@ -1,186 +0,0 @@ -# Metric Card for Google BLEU (GLEU) - - -## Metric Description -The BLEU score has some undesirable properties when used for single -sentences, as it was designed to be a corpus measure. The Google BLEU score, also known as GLEU score, is designed to limit these undesirable properties when used for single sentences. - -To calculate this score, all sub-sequences of 1, 2, 3 or 4 tokens in output and target sequence (n-grams) are recorded. The precision and recall, described below, are then computed. - -- **precision:** the ratio of the number of matching n-grams to the number of total n-grams in the generated output sequence -- **recall:** the ratio of the number of matching n-grams to the number of total n-grams in the target (ground truth) sequence - -The minimum value of precision and recall is then returned as the score. - - -## Intended Uses -This metric is generally used to evaluate machine translation models. It is especially used when scores of individual (prediction, reference) sentence pairs are needed, as opposed to when averaging over the (prediction, reference) scores for a whole corpus. That being said, it can also be used when averaging over the scores for a whole corpus. - -Because it performs better on individual sentence pairs as compared to BLEU, Google BLEU has also been used in RL experiments. - -## How to Use -At minimum, this metric takes predictions and references: -```python ->>> sentence1 = "the cat sat on the mat".split() ->>> sentence2 = "the cat ate the mat".split() ->>> google_bleu = datasets.load_metric("google_bleu") ->>> result = google_bleu.compute(predictions=[sentence1], references=[[sentence2]]) ->>> print(result) -{'google_bleu': 0.3333333333333333} -``` - -### Inputs -- **predictions** (list of list of str): list of translations to score. Each translation should be tokenized into a list of tokens. -- **references** (list of list of list of str): list of lists of references for each translation. Each reference should be tokenized into a list of tokens. -- **min_len** (int): The minimum order of n-gram this function should extract. Defaults to 1. -- **max_len** (int): The maximum order of n-gram this function should extract. Defaults to 4. - -### Output Values -This metric returns the following in a dict: -- **google_bleu** (float): google_bleu score - -The output format is as follows: -```python -{'google_bleu': google_bleu score} -``` - -This metric can take on values from 0 to 1, inclusive. Higher scores are better, with 0 indicating no matches, and 1 indicating a perfect match. - -Note that this score is symmetrical when switching output and target. This means that, given two sentences, `sentence1` and `sentence2`, whatever score is output when `sentence1` is the predicted sentence and `sencence2` is the reference sentence will be the same as when the sentences are swapped and `sentence2` is the predicted sentence while `sentence1` is the reference sentence. In code, this looks like: - -```python -sentence1 = "the cat sat on the mat".split() -sentence2 = "the cat ate the mat".split() -google_bleu = datasets.load_metric("google_bleu") -result_a = google_bleu.compute(predictions=[sentence1], references=[[sentence2]]) -result_b = google_bleu.compute(predictions=[sentence2], references=[[sentence1]]) -print(result_a == result_b) ->>> True -``` - -#### Values from Popular Papers - - -### Examples -Example with one reference per sample: -```python ->>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', -... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] ->>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', -... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', -... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] - ->>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', -... 'interested', 'in', 'world', 'history'] ->>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', -... 'because', 'he', 'read', 'the', 'book'] - ->>> list_of_references = [[ref1a], [ref2a]] ->>> hypotheses = [hyp1, hyp2] ->>> google_bleu = datasets.load_metric("google_bleu") ->>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references) ->>> print(round(results["google_bleu"], 2)) -0.44 -``` - -Example with multiple references for the first sample: -```python ->>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', -... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] ->>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', -... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', -... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] ->>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', -... 'heed', 'the', 'cat', 'commands'] ->>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', -... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', -... 'of', 'the', 'cat'] - ->>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', -... 'interested', 'in', 'world', 'history'] ->>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', -... 'because', 'he', 'read', 'the', 'book'] - ->>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] ->>> hypotheses = [hyp1, hyp2] ->>> google_bleu = datasets.load_metric("google_bleu") ->>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references) ->>> print(round(results["google_bleu"], 2)) -0.61 -``` - -Example with multiple references for the first sample, and with `min_len` adjusted to `2`, instead of the default `1`: -```python ->>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', -... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] ->>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', -... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', -... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] ->>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', -... 'heed', 'the', 'cat', 'commands'] ->>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', -... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', -... 'of', 'the', 'cat'] - ->>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', -... 'interested', 'in', 'world', 'history'] ->>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', -... 'because', 'he', 'read', 'the', 'book'] - ->>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] ->>> hypotheses = [hyp1, hyp2] ->>> google_bleu = datasets.load_metric("google_bleu") ->>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references, min_len=2) ->>> print(round(results["google_bleu"], 2)) -0.53 -``` - -Example with multiple references for the first sample, with `min_len` adjusted to `2`, instead of the default `1`, and `max_len` adjusted to `6` instead of the default `4`: -```python ->>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', -... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] ->>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', -... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', -... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] ->>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', -... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', -... 'heed', 'the', 'cat', 'commands'] ->>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', -... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', -... 'of', 'the', 'cat'] - ->>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', -... 'interested', 'in', 'world', 'history'] ->>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', -... 'because', 'he', 'read', 'the', 'book'] - ->>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] ->>> hypotheses = [hyp1, hyp2] ->>> google_bleu = datasets.load_metric("google_bleu") ->>> results = google_bleu.compute(predictions=hypotheses,references=list_of_references, min_len=2, max_len=6) ->>> print(round(results["google_bleu"], 2)) -0.4 -``` - -## Limitations and Bias - - -## Citation -```bibtex -@misc{wu2016googles, -title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, -author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes and Jeffrey Dean}, -year={2016}, -eprint={1609.08144}, -archivePrefix={arXiv}, -primaryClass={cs.CL} -} -``` -## Further References -- This Hugging Face implementation uses the [nltk.translate.gleu_score implementation](https://www.nltk.org/_modules/nltk/translate/gleu_score.html) \ No newline at end of file diff --git a/metrics/google_bleu/google_bleu.py b/metrics/google_bleu/google_bleu.py deleted file mode 100644 index fece34b8b4f..00000000000 --- a/metrics/google_bleu/google_bleu.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Google BLEU (aka GLEU) metric.""" - -from typing import Dict, List - -from nltk.translate import gleu_score - -import datasets -from datasets import MetricInfo - - -_CITATION = """\ -@misc{wu2016googles, - title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, - author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey - and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin - Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto - Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and - Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes - and Jeffrey Dean}, - year={2016}, - eprint={1609.08144}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -""" - -_DESCRIPTION = """\ -The BLEU score has some undesirable properties when used for single -sentences, as it was designed to be a corpus measure. We therefore -use a slightly different score for our RL experiments which we call -the 'GLEU score'. For the GLEU score, we record all sub-sequences of -1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then -compute a recall, which is the ratio of the number of matching n-grams -to the number of total n-grams in the target (ground truth) sequence, -and a precision, which is the ratio of the number of matching n-grams -to the number of total n-grams in the generated output sequence. Then -GLEU score is simply the minimum of recall and precision. This GLEU -score's range is always between 0 (no matches) and 1 (all match) and -it is symmetrical when switching output and target. According to -our experiments, GLEU score correlates quite well with the BLEU -metric on a corpus level but does not have its drawbacks for our per -sentence reward objective. -""" - -_KWARGS_DESCRIPTION = """\ -Computes corpus-level Google BLEU (GLEU) score of translated segments against one or more references. -Instead of averaging the sentence level GLEU scores (i.e. macro-average precision), Wu et al. (2016) sum up the matching -tokens and the max of hypothesis and reference tokens for each sentence, then compute using the aggregate values. - -Args: - predictions (list of str): list of translations to score. - Each translation should be tokenized into a list of tokens. - references (list of list of str): list of lists of references for each translation. - Each reference should be tokenized into a list of tokens. - min_len (int): The minimum order of n-gram this function should extract. Defaults to 1. - max_len (int): The maximum order of n-gram this function should extract. Defaults to 4. - -Returns: - 'google_bleu': google_bleu score - -Examples: - Example 1: - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', - ... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] - >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> google_bleu = datasets.load_metric("google_bleu") - >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references) - >>> print(round(results["google_bleu"], 2)) - 0.44 - - Example 2: - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', - ... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] - >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] - >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', - ... 'heed', 'the', 'cat', 'commands'] - >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'cat'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> google_bleu = datasets.load_metric("google_bleu") - >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references) - >>> print(round(results["google_bleu"], 2)) - 0.61 - - Example 3: - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', - ... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] - >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] - >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', - ... 'heed', 'the', 'cat', 'commands'] - >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'cat'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> google_bleu = datasets.load_metric("google_bleu") - >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references, min_len=2) - >>> print(round(results["google_bleu"], 2)) - 0.53 - - Example 4: - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'always', - ... 'disobeys', 'the', 'commands', 'of', 'the', 'cat'] - >>> ref1a = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'rubber', 'duck', 'forces', 'never', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'cat'] - >>> ref1b = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'rubber', 'duck', 'will', 'never', - ... 'heed', 'the', 'cat', 'commands'] - >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'rubber', 'duck', 'army', 'never', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'cat'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> google_bleu = datasets.load_metric("google_bleu") - >>> results = google_bleu.compute(predictions=hypotheses,references=list_of_references, min_len=2, max_len=6) - >>> print(round(results["google_bleu"], 2)) - 0.4 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class GoogleBleu(datasets.Metric): - def _info(self) -> MetricInfo: - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("string", id="token"), id="sequence"), - "references": datasets.Sequence( - datasets.Sequence(datasets.Value("string", id="token"), id="sequence"), id="references" - ), - } - ), - ) - - def _compute( - self, - predictions: List[List[List[str]]], - references: List[List[str]], - min_len: int = 1, - max_len: int = 4, - ) -> Dict[str, float]: - return { - "google_bleu": gleu_score.corpus_gleu( - list_of_references=references, hypotheses=predictions, min_len=min_len, max_len=max_len - ) - } diff --git a/metrics/indic_glue/README.md b/metrics/indic_glue/README.md deleted file mode 100644 index e3c768d5c71..00000000000 --- a/metrics/indic_glue/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Metric Card for IndicGLUE - -## Metric description -This metric is used to compute the evaluation metric for the [IndicGLUE dataset](https://huggingface.co/datasets/indic_glue). - -IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide variety of tasks and covers 11 major Indian languages - Assamese (`as`), Bengali (`bn`), Gujarati (`gu`), Hindi (`hi`), Kannada (`kn`), Malayalam (`ml`), Marathi(`mr`), Oriya(`or`), Panjabi (`pa`), Tamil(`ta`) and Telugu (`te`). - -## How to use - -There are two steps: (1) loading the IndicGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. - -1. **Loading the relevant IndicGLUE metric** : the subsets of IndicGLUE are the following: `wnli`, `copa`, `sna`, `csqa`, `wstp`, `inltkh`, `bbca`, `cvit-mkb-clsr`, `iitp-mr`, `iitp-pr`, `actsa-sc`, `md`, and`wiki-ner`. - -More information about the different subsets of the Indic GLUE dataset can be found on the [IndicGLUE dataset page](https://indicnlp.ai4bharat.org/indic-glue/). - -2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation for all subsets of the dataset except for `cvit-mkb-clsr`, where each prediction and reference is a vector of floats. - -```python -from datasets import load_metric -indic_glue_metric = load_metric('indic_glue', 'wnli') -references = [0, 1] -predictions = [0, 1] -results = indic_glue_metric.compute(predictions=predictions, references=references) -``` - -## Output values - -The output of the metric depends on the IndicGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: - -`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). - -`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -`precision@10`: the fraction of the true examples among the top 10 predicted examples, with a range between 0 and 1 (see [precision](https://huggingface.co/metrics/precision) for more information). - -The `cvit-mkb-clsr` subset returns `precision@10`, the `wiki-ner` subset returns `accuracy` and `f1`, and all other subsets of Indic GLUE return only accuracy. - -### Values from popular papers - -The [original IndicGlue paper](https://aclanthology.org/2020.findings-emnlp.445.pdf) reported an average accuracy of 0.766 on the dataset, which varies depending on the subset selected. - -## Examples - -Maximal values for the WNLI subset (which outputs `accuracy`): - -```python -from datasets import load_metric -indic_glue_metric = load_metric('indic_glue', 'wnli') -references = [0, 1] -predictions = [0, 1] -results = indic_glue_metric.compute(predictions=predictions, references=references) -print(results) -{'accuracy': 1.0} -``` - -Minimal values for the Wiki-NER subset (which outputs `accuracy` and `f1`): - -```python ->>> from datasets import load_metric ->>> indic_glue_metric = load_metric('indic_glue', 'wiki-ner') ->>> references = [0, 1] ->>> predictions = [1,0] ->>> results = indic_glue_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'accuracy': 1.0, 'f1': 1.0} -``` - -Partial match for the CVIT-Mann Ki Baat subset (which outputs `precision@10`) - -```python ->>> from datasets import load_metric ->>> indic_glue_metric = load_metric('indic_glue', 'cvit-mkb-clsr') ->>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] ->>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] ->>> results = indic_glue_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'precision@10': 1.0} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as the [IndicGLUE dataset](https://huggingface.co/datasets/glue). - -## Citation - -```bibtex - @inproceedings{kakwani2020indicnlpsuite, - title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, - author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, - year={2020}, - booktitle={Findings of EMNLP}, -} -``` - -## Further References -- [IndicNLP website](https://indicnlp.ai4bharat.org/home/) -- diff --git a/metrics/indic_glue/indic_glue.py b/metrics/indic_glue/indic_glue.py deleted file mode 100644 index a3af970efe9..00000000000 --- a/metrics/indic_glue/indic_glue.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""IndicGLUE benchmark metric.""" - -import numpy as np -from scipy.spatial.distance import cdist -from sklearn.metrics import f1_score - -import datasets - - -_CITATION = """\ - @inproceedings{kakwani2020indicnlpsuite, - title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, - author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, - year={2020}, - booktitle={Findings of EMNLP}, -} -""" - -_DESCRIPTION = """\ - IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide - variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te. -""" - -_KWARGS_DESCRIPTION = """ -Compute IndicGLUE evaluation metric associated to each IndicGLUE dataset. -Args: - predictions: list of predictions to score (as int64), - except for 'cvit-mkb-clsr' where each prediction is a vector (of float32). - references: list of ground truth labels corresponding to the predictions (as int64), - except for 'cvit-mkb-clsr' where each reference is a vector (of float32). -Returns: depending on the IndicGLUE subset, one or several of: - "accuracy": Accuracy - "f1": F1 score - "precision": Precision@10 -Examples: - - >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wnli') # 'wnli' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"] - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = indic_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0} - - >>> indic_glue_metric = datasets.load_metric('indic_glue', 'wiki-ner') - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = indic_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0, 'f1': 1.0} - - >>> indic_glue_metric = datasets.load_metric('indic_glue', 'cvit-mkb-clsr') - >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] - >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] - >>> results = indic_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'precision@10': 1.0} - -""" - - -def simple_accuracy(preds, labels): - return float((preds == labels).mean()) - - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = float(f1_score(y_true=labels, y_pred=preds)) - return { - "accuracy": acc, - "f1": f1, - } - - -def precision_at_10(en_sentvecs, in_sentvecs): - en_sentvecs = np.array(en_sentvecs) - in_sentvecs = np.array(in_sentvecs) - n = en_sentvecs.shape[0] - - # mean centering - en_sentvecs = en_sentvecs - np.mean(en_sentvecs, axis=0) - in_sentvecs = in_sentvecs - np.mean(in_sentvecs, axis=0) - - sim = cdist(en_sentvecs, in_sentvecs, "cosine") - actual = np.array(range(n)) - preds = sim.argsort(axis=1)[:, :10] - matches = np.any(preds == actual[:, None], axis=1) - return float(matches.mean()) - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class IndicGlue(datasets.Metric): - def _info(self): - if self.config_name not in [ - "wnli", - "copa", - "sna", - "csqa", - "wstp", - "inltkh", - "bbca", - "cvit-mkb-clsr", - "iitp-mr", - "iitp-pr", - "actsa-sc", - "md", - "wiki-ner", - ]: - raise KeyError( - "You should supply a configuration name selected in " - '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' - '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' - '"wiki-ner"]' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("int64") - if self.config_name != "cvit-mkb-clsr" - else datasets.Sequence(datasets.Value("float32")), - "references": datasets.Value("int64") - if self.config_name != "cvit-mkb-clsr" - else datasets.Sequence(datasets.Value("float32")), - } - ), - codebase_urls=[], - reference_urls=[], - format="numpy" if self.config_name != "cvit-mkb-clsr" else None, - ) - - def _compute(self, predictions, references): - if self.config_name == "cvit-mkb-clsr": - return {"precision@10": precision_at_10(predictions, references)} - elif self.config_name in ["wiki-ner"]: - return acc_and_f1(predictions, references) - elif self.config_name in [ - "wnli", - "copa", - "sna", - "csqa", - "wstp", - "inltkh", - "bbca", - "iitp-mr", - "iitp-pr", - "actsa-sc", - "md", - ]: - return {"accuracy": simple_accuracy(predictions, references)} - else: - raise KeyError( - "You should supply a configuration name selected in " - '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' - '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' - '"wiki-ner"]' - ) diff --git a/metrics/mae/README.md b/metrics/mae/README.md deleted file mode 100644 index 0765e42ab94..00000000000 --- a/metrics/mae/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Metric Card for MAE - - -## Metric Description - -Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual numeric values: -![image](https://user-images.githubusercontent.com/14205986/165824243-e1078dfd-489d-456c-a0da-cbaa28726220.png) - - -## How to Use - -At minimum, this metric requires predictions and references as inputs. - -```python ->>> mae_metric = datasets.load_metric("mae") ->>> predictions = [2.5, 0.0, 2, 8] ->>> references = [3, -0.5, 2, 7] ->>> results = mae_metric.compute(predictions=predictions, references=references) -``` - -### Inputs - -Mandatory inputs: -- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. -- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. - -Optional arguments: -- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. -- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. - - `raw_values` returns a full set of errors in case of multioutput input. - - `uniform_average` means that the errors of all outputs are averaged with uniform weight. - - the array-like value defines weights used to average errors. - -### Output Values -This metric outputs a dictionary, containing the mean absolute error score, which is of type: -- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. -- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. - -Each MAE `float` value ranges from `0.0` to `1.0`, with the best value being 0.0. - -Output Example(s): -```python -{'mae': 0.5} -``` - -If `multioutput="raw_values"`: -```python -{'mae': array([0.5, 1. ])} -``` - -#### Values from Popular Papers - - -### Examples - -Example with the `uniform_average` config: -```python ->>> from datasets import load_metric ->>> mae_metric = load_metric("mae") ->>> predictions = [2.5, 0.0, 2, 8] ->>> references = [3, -0.5, 2, 7] ->>> results = mae_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'mae': 0.5} -``` - -Example with multi-dimensional lists, and the `raw_values` config: -```python ->>> from datasets import load_metric ->>> mae_metric = datasets.load_metric("mae", "multilist") ->>> predictions = [[0.5, 1], [-1, 1], [7, -6]] ->>> references = [[0, 2], [-1, 2], [8, -5]] ->>> results = mae_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'mae': 0.75} ->>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') ->>> print(results) -{'mae': array([0.5, 1. ])} -``` - -## Limitations and Bias -One limitation of MAE is that the relative size of the error is not always obvious, meaning that it can be difficult to tell a big error from a smaller one -- metrics such as Mean Absolute Percentage Error (MAPE) have been proposed to calculate MAE in percentage terms. - -Also, since it calculates the mean, MAE may underestimate the impact of big, but infrequent, errors -- metrics such as the Root Mean Square Error (RMSE) compensate for this by taking the root of error values. - -## Citation(s) -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - -```bibtex -@article{willmott2005advantages, - title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, - author={Willmott, Cort J and Matsuura, Kenji}, - journal={Climate research}, - volume={30}, - number={1}, - pages={79--82}, - year={2005} -} -``` - -## Further References -- [Mean Absolute Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_error) diff --git a/metrics/mae/mae.py b/metrics/mae/mae.py deleted file mode 100644 index 0c25c69cd1c..00000000000 --- a/metrics/mae/mae.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""MAE - Mean Absolute Error Metric""" - -from sklearn.metrics import mean_absolute_error - -import datasets - - -_CITATION = """\ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - -_DESCRIPTION = """\ -Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual -values. -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) - Estimated target values. - references: array-like of shape (n_samples,) or (n_samples, n_outputs) - Ground truth (correct) target values. - sample_weight: array-like of shape (n_samples,), default=None - Sample weights. - multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" - Defines aggregating of multiple output values. Array-like value defines weights used to average errors. - - "raw_values" : Returns a full set of errors in case of multioutput input. - - "uniform_average" : Errors of all outputs are averaged with uniform weight. - -Returns: - mae : mean absolute error. - If multioutput is "raw_values", then mean absolute error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. - MAE output is non-negative floating point. The best value is 0.0. -Examples: - - >>> mae_metric = datasets.load_metric("mae") - >>> predictions = [2.5, 0.0, 2, 8] - >>> references = [3, -0.5, 2, 7] - >>> results = mae_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'mae': 0.5} - - If you're using multi-dimensional lists, then set the config as follows : - - >>> mae_metric = datasets.load_metric("mae", "multilist") - >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] - >>> references = [[0, 2], [-1, 2], [8, -5]] - >>> results = mae_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'mae': 0.75} - >>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') - >>> print(results) - {'mae': array([0.5, 1. ])} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Mae(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features(self._get_feature_types()), - reference_urls=[ - "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html" - ], - ) - - def _get_feature_types(self): - if self.config_name == "multilist": - return { - "predictions": datasets.Sequence(datasets.Value("float")), - "references": datasets.Sequence(datasets.Value("float")), - } - else: - return { - "predictions": datasets.Value("float"), - "references": datasets.Value("float"), - } - - def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): - mae_score = mean_absolute_error(references, predictions, sample_weight=sample_weight, multioutput=multioutput) - - return {"mae": mae_score} diff --git a/metrics/mahalanobis/README.md b/metrics/mahalanobis/README.md deleted file mode 100644 index a25d116defa..00000000000 --- a/metrics/mahalanobis/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Metric Card for Mahalanobis Distance - -## Metric Description -Mahalonobis distance is the distance between a point and a distribution (as opposed to the distance between two points), making it the multivariate equivalent of the Euclidean distance. - -It is often used in multivariate anomaly detection, classification on highly imbalanced datasets and one-class classification. - -## How to Use -At minimum, this metric requires two `list`s of datapoints: - -```python ->>> mahalanobis_metric = datasets.load_metric("mahalanobis") ->>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) -``` - -### Inputs -- `X` (`list`): data points to be compared with the `reference_distribution`. -- `reference_distribution` (`list`): data points from the reference distribution that we want to compare to. - -### Output Values -`mahalanobis` (`array`): the Mahalonobis distance for each data point in `X`. - -```python ->>> print(results) -{'mahalanobis': array([0.5])} -``` - -#### Values from Popular Papers -*N/A* - -### Example - -```python ->>> mahalanobis_metric = datasets.load_metric("mahalanobis") ->>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) ->>> print(results) -{'mahalanobis': array([0.5])} -``` - -## Limitations and Bias - -The Mahalanobis distance is only able to capture linear relationships between the variables, which means it cannot capture all types of outliers. Mahalanobis distance also fails to faithfully represent data that is highly skewed or multimodal. - -## Citation -```bibtex -@inproceedings{mahalanobis1936generalized, - title={On the generalized distance in statistics}, - author={Mahalanobis, Prasanta Chandra}, - year={1936}, - organization={National Institute of Science of India} -} -``` - -```bibtex -@article{de2000mahalanobis, - title={The Mahalanobis distance}, - author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, - journal={Chemometrics and intelligent laboratory systems}, - volume={50}, - number={1}, - pages={1--18}, - year={2000}, - publisher={Elsevier} -} -``` - -## Further References --[Wikipedia -- Mahalanobis Distance](https://en.wikipedia.org/wiki/Mahalanobis_distance) - --[Machine Learning Plus -- Mahalanobis Distance](https://www.machinelearningplus.com/statistics/mahalanobis-distance/) diff --git a/metrics/mahalanobis/mahalanobis.py b/metrics/mahalanobis/mahalanobis.py deleted file mode 100644 index b5d58ab7f9c..00000000000 --- a/metrics/mahalanobis/mahalanobis.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Mahalanobis metric.""" - -import numpy as np - -import datasets - - -_DESCRIPTION = """ -Compute the Mahalanobis Distance - -Mahalonobis distance is the distance between a point and a distribution. -And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance. -It was introduced by Prof. P. C. Mahalanobis in 1936 -and has been used in various statistical applications ever since -[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/] -""" - -_CITATION = """\ -@article{de2000mahalanobis, - title={The mahalanobis distance}, - author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, - journal={Chemometrics and intelligent laboratory systems}, - volume={50}, - number={1}, - pages={1--18}, - year={2000}, - publisher={Elsevier} -} -""" - -_KWARGS_DESCRIPTION = """ -Args: - X: List of datapoints to be compared with the `reference_distribution`. - reference_distribution: List of datapoints from the reference distribution we want to compare to. -Returns: - mahalanobis: The Mahalonobis distance for each datapoint in `X`. -Examples: - - >>> mahalanobis_metric = datasets.load_metric("mahalanobis") - >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) - >>> print(results) - {'mahalanobis': array([0.5])} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Mahalanobis(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "X": datasets.Sequence(datasets.Value("float", id="sequence"), id="X"), - } - ), - ) - - def _compute(self, X, reference_distribution): - # convert to numpy arrays - X = np.array(X) - reference_distribution = np.array(reference_distribution) - - # Assert that arrays are 2D - if len(X.shape) != 2: - raise ValueError("Expected `X` to be a 2D vector") - if len(reference_distribution.shape) != 2: - raise ValueError("Expected `reference_distribution` to be a 2D vector") - if reference_distribution.shape[0] < 2: - raise ValueError( - "Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension" - ) - - # Get mahalanobis distance for each prediction - X_minus_mu = X - np.mean(reference_distribution) - cov = np.cov(reference_distribution.T) - try: - inv_covmat = np.linalg.inv(cov) - except np.linalg.LinAlgError: - inv_covmat = np.linalg.pinv(cov) - left_term = np.dot(X_minus_mu, inv_covmat) - mahal_dist = np.dot(left_term, X_minus_mu.T).diagonal() - - return {"mahalanobis": mahal_dist} diff --git a/metrics/matthews_correlation/README.md b/metrics/matthews_correlation/README.md deleted file mode 100644 index f4c5a643ae8..00000000000 --- a/metrics/matthews_correlation/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Metric Card for Matthews Correlation Coefficient - -## Metric Description -The Matthews correlation coefficient is used in machine learning as a -measure of the quality of binary and multiclass classifications. It takes -into account true and false positives and negatives and is generally -regarded as a balanced measure which can be used even if the classes are of -very different sizes. The MCC is in essence a correlation coefficient value -between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 -an average random prediction and -1 an inverse prediction. The statistic -is also known as the phi coefficient. [source: Wikipedia] - -## How to Use -At minimum, this metric requires a list of predictions and a list of references: -```python ->>> matthews_metric = datasets.load_metric("matthews_correlation") ->>> results = matthews_metric.compute(references=[0, 1], predictions=[0, 1]) ->>> print(results) -{'matthews_correlation': 1.0} -``` - -### Inputs -- **`predictions`** (`list` of `int`s): Predicted class labels. -- **`references`** (`list` of `int`s): Ground truth labels. -- **`sample_weight`** (`list` of `int`s, `float`s, or `bool`s): Sample weights. Defaults to `None`. - -### Output Values -- **`matthews_correlation`** (`float`): Matthews correlation coefficient. - -The metric output takes the following form: -```python -{'matthews_correlation': 0.54} -``` - -This metric can be any value from -1 to +1, inclusive. - -#### Values from Popular Papers - - -### Examples -A basic example with only predictions and references as inputs: -```python ->>> matthews_metric = datasets.load_metric("matthews_correlation") ->>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], -... predictions=[1, 2, 2, 0, 3, 3]) ->>> print(results) -{'matthews_correlation': 0.5384615384615384} -``` - -The same example as above, but also including sample weights: -```python ->>> matthews_metric = datasets.load_metric("matthews_correlation") ->>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], -... predictions=[1, 2, 2, 0, 3, 3], -... sample_weight=[0.5, 3, 1, 1, 1, 2]) ->>> print(results) -{'matthews_correlation': 0.09782608695652174} -``` - -The same example as above, with sample weights that cause a negative correlation: -```python ->>> matthews_metric = datasets.load_metric("matthews_correlation") ->>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], -... predictions=[1, 2, 2, 0, 3, 3], -... sample_weight=[0.5, 1, 0, 0, 0, 1]) ->>> print(results) -{'matthews_correlation': -0.25} -``` - -## Limitations and Bias -*Note any limitations or biases that the metric has.* - - -## Citation -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - -## Further References - -- This Hugging Face implementation uses [this scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html) \ No newline at end of file diff --git a/metrics/matthews_correlation/matthews_correlation.py b/metrics/matthews_correlation/matthews_correlation.py deleted file mode 100644 index 1c7f6bf8dfb..00000000000 --- a/metrics/matthews_correlation/matthews_correlation.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Matthews Correlation metric.""" - -from sklearn.metrics import matthews_corrcoef - -import datasets - - -_DESCRIPTION = """ -Compute the Matthews correlation coefficient (MCC) - -The Matthews correlation coefficient is used in machine learning as a -measure of the quality of binary and multiclass classifications. It takes -into account true and false positives and negatives and is generally -regarded as a balanced measure which can be used even if the classes are of -very different sizes. The MCC is in essence a correlation coefficient value -between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 -an average random prediction and -1 an inverse prediction. The statistic -is also known as the phi coefficient. [source: Wikipedia] -""" - -_KWARGS_DESCRIPTION = """ -Args: - predictions (list of int): Predicted labels, as returned by a model. - references (list of int): Ground truth labels. - sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`. -Returns: - matthews_correlation (dict containing float): Matthews correlation. -Examples: - Example 1, a basic example with only predictions and references as inputs: - >>> matthews_metric = datasets.load_metric("matthews_correlation") - >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], - ... predictions=[1, 2, 2, 0, 3, 3]) - >>> print(round(results['matthews_correlation'], 2)) - 0.54 - - Example 2, the same example as above, but also including sample weights: - >>> matthews_metric = datasets.load_metric("matthews_correlation") - >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], - ... predictions=[1, 2, 2, 0, 3, 3], - ... sample_weight=[0.5, 3, 1, 1, 1, 2]) - >>> print(round(results['matthews_correlation'], 2)) - 0.1 - - Example 3, the same example as above, but with sample weights that cause a negative correlation: - >>> matthews_metric = datasets.load_metric("matthews_correlation") - >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], - ... predictions=[1, 2, 2, 0, 3, 3], - ... sample_weight=[0.5, 1, 0, 0, 0, 1]) - >>> print(round(results['matthews_correlation'], 2)) - -0.25 -""" - -_CITATION = """\ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class MatthewsCorrelation(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("int32"), - "references": datasets.Value("int32"), - } - ), - reference_urls=[ - "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html" - ], - ) - - def _compute(self, predictions, references, sample_weight=None): - return { - "matthews_correlation": float(matthews_corrcoef(references, predictions, sample_weight=sample_weight)), - } diff --git a/metrics/mauve/README.md b/metrics/mauve/README.md deleted file mode 100644 index d9f5ce4f530..00000000000 --- a/metrics/mauve/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Metric Card for MAUVE - -## Metric description - -MAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure. It summarizes both Type I and Type II errors measured softly using [Kullback–Leibler (KL) divergences](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence). - -This metric is a wrapper around the [official implementation](https://github.com/krishnap25/mauve) of MAUVE. - -For more details, consult the [MAUVE paper](https://arxiv.org/abs/2102.01454). - - -## How to use - -The metric takes two lists of strings of tokens separated by spaces: one representing `predictions` (i.e. the text generated by the model) and the second representing `references` (a reference text for each prediction): - -```python -from datasets import load_metric -mauve = load_metric('mauve') -predictions = ["hello world", "goodnight moon"] -references = ["hello world", "goodnight moon"] -mauve_results = mauve.compute(predictions=predictions, references=references) -``` - -It also has several optional arguments: - -`num_buckets`: the size of the histogram to quantize P and Q. Options: `auto` (default) or an integer. - -`pca_max_data`: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. The default is `-1`. - -`kmeans_explained_var`: amount of variance of the data to keep in dimensionality reduction by PCA. The default is `0.9`. - -`kmeans_num_redo`: number of times to redo k-means clustering (the best objective is kept). The default is `5`. - -`kmeans_max_iter`: maximum number of k-means iterations. The default is `500`. - -`featurize_model_name`: name of the model from which features are obtained, from one of the following: `gpt2`, `gpt2-medium`, `gpt2-large`, `gpt2-xl`. The default is `gpt2-large`. - -`device_id`: Device for featurization. Supply a GPU id (e.g. `0` or `3`) to use GPU. If no GPU with this id is found, the metric will use CPU. - -`max_text_length`: maximum number of tokens to consider. The default is `1024`. - -`divergence_curve_discretization_size` Number of points to consider on the divergence curve. The default is `25`. - -`mauve_scaling_factor`: Hyperparameter for scaling. The default is `5`. - -`verbose`: If `True` (default), running the metric will print running time updates. - -`seed`: random seed to initialize k-means cluster assignments, randomly assigned by default. - - - -## Output values - -This metric outputs a dictionary with 5 key-value pairs: - -`mauve`: MAUVE score, which ranges between 0 and 1. **Larger** values indicate that P and Q are closer. - -`frontier_integral`: Frontier Integral, which ranges between 0 and 1. **Smaller** values indicate that P and Q are closer. - -`divergence_curve`: a numpy.ndarray of shape (m, 2); plot it with `matplotlib` to view the divergence curve. - -`p_hist`: a discrete distribution, which is a quantized version of the text distribution `p_text`. - -`q_hist`: same as above, but with `q_text`. - - -### Values from popular papers - -The [original MAUVE paper](https://arxiv.org/abs/2102.01454) reported values ranging from 0.88 to 0.94 for open-ended text generation using a text completion task in the web text domain. The authors found that bigger models resulted in higher MAUVE scores, and that MAUVE is correlated with human judgments. - - -## Examples - -Perfect match between prediction and reference: - -```python -from datasets import load_metric -mauve = load_metric('mauve') -predictions = ["hello world", "goodnight moon"] -references = ["hello world", "goodnight moon"] -mauve_results = mauve.compute(predictions=predictions, references=references) -print(mauve_results.mauve) -1.0 -``` - -Partial match between prediction and reference: - -```python -from datasets import load_metric -mauve = load_metric('mauve') -predictions = ["hello world", "goodnight moon"] -references = ["hello there", "general kenobi"] -mauve_results = mauve.compute(predictions=predictions, references=references) -print(mauve_results.mauve) -0.27811372536724027 -``` - -## Limitations and bias - -The [original MAUVE paper](https://arxiv.org/abs/2102.01454) did not analyze the inductive biases present in different embedding models, but related work has shown different kinds of biases exist in many popular generative language models including GPT-2 (see [Kirk et al., 2021](https://arxiv.org/pdf/2102.04130.pdf), [Abid et al., 2021](https://arxiv.org/abs/2101.05783)). The extent to which these biases can impact the MAUVE score has not been quantified. - -Also, calculating the MAUVE metric involves downloading the model from which features are obtained -- the default model, `gpt2-large`, takes over 3GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `gpt` is 523MB. - - -## Citation - -```bibtex -@inproceedings{pillutla-etal:mauve:neurips2021, - title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}, - author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, - booktitle = {NeurIPS}, - year = {2021} -} -``` - -## Further References -- [Official MAUVE implementation](https://github.com/krishnap25/mauve) -- [Hugging Face Tasks - Text Generation](https://huggingface.co/tasks/text-generation) diff --git a/metrics/mauve/mauve.py b/metrics/mauve/mauve.py deleted file mode 100644 index 5a7d43839f8..00000000000 --- a/metrics/mauve/mauve.py +++ /dev/null @@ -1,149 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""MAUVE metric from https://github.com/krishnap25/mauve.""" - -import faiss # noqa: F401 # Here to have a nice missing dependency error message early on -import numpy # noqa: F401 # Here to have a nice missing dependency error message early on -import requests # noqa: F401 # Here to have a nice missing dependency error message early on -import sklearn # noqa: F401 # Here to have a nice missing dependency error message early on -import tqdm # noqa: F401 # Here to have a nice missing dependency error message early on -from mauve import compute_mauve # From: mauve-text - -import datasets - - -_CITATION = """\ -@inproceedings{pillutla-etal:mauve:neurips2021, - title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}, - author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, - booktitle = {NeurIPS}, - year = {2021} -} - -""" - -_DESCRIPTION = """\ -MAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure. - -MAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences. - -For details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021). - -This metrics is a wrapper around the official implementation of MAUVE: -https://github.com/krishnap25/mauve -""" - -_KWARGS_DESCRIPTION = """ -Calculates MAUVE scores between two lists of generated text and reference text. -Args: - predictions: list of generated text to score. Each predictions - should be a string with tokens separated by spaces. - references: list of reference for each prediction. Each - reference should be a string with tokens separated by spaces. -Optional Args: - num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer - pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1 - kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9 - kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5 - kmeans_max_iter: maximum number of k-means iterations. Default 500 - featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl']. - device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU - max_text_length: maximum number of tokens to consider. Default 1024 - divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25 - mauve_scaling_factor: "c" from the paper. Default 5. - verbose: If True (default), print running time updates - seed: random seed to initialize k-means cluster assignments. -Returns: - mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer, - frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer, - divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve, - p_hist: a discrete distribution, which is a quantized version of the text distribution p_text, - q_hist: same as above, but with q_text. -Examples: - - >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest - >>> import datasets - >>> mauve = datasets.load_metric('mauve') - >>> predictions = ["hello there", "general kenobi"] - >>> references = ["hello there", "general kenobi"] - >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP - >>> print(out.mauve) # doctest: +SKIP - 1.0 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Mauve(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/krishnap25/mauve", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/krishnap25/mauve"], - reference_urls=[ - "https://arxiv.org/abs/2102.01454", - "https://github.com/krishnap25/mauve", - ], - ) - - def _compute( - self, - predictions, - references, - p_features=None, - q_features=None, - p_tokens=None, - q_tokens=None, - num_buckets="auto", - pca_max_data=-1, - kmeans_explained_var=0.9, - kmeans_num_redo=5, - kmeans_max_iter=500, - featurize_model_name="gpt2-large", - device_id=-1, - max_text_length=1024, - divergence_curve_discretization_size=25, - mauve_scaling_factor=5, - verbose=True, - seed=25, - ): - out = compute_mauve( - p_text=predictions, - q_text=references, - p_features=p_features, - q_features=q_features, - p_tokens=p_tokens, - q_tokens=q_tokens, - num_buckets=num_buckets, - pca_max_data=pca_max_data, - kmeans_explained_var=kmeans_explained_var, - kmeans_num_redo=kmeans_num_redo, - kmeans_max_iter=kmeans_max_iter, - featurize_model_name=featurize_model_name, - device_id=device_id, - max_text_length=max_text_length, - divergence_curve_discretization_size=divergence_curve_discretization_size, - mauve_scaling_factor=mauve_scaling_factor, - verbose=verbose, - seed=seed, - ) - return out diff --git a/metrics/mean_iou/README.md b/metrics/mean_iou/README.md deleted file mode 100644 index b1abe2c0d4e..00000000000 --- a/metrics/mean_iou/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Metric Card for Mean IoU - - -## Metric Description - -IoU (Intersection over Union) is the area of overlap between the predicted segmentation and the ground truth divided by the area of union between the predicted segmentation and the ground truth. - -For binary (two classes) or multi-class segmentation, the *mean IoU* of the image is calculated by taking the IoU of each class and averaging them. - -## How to Use - -The Mean IoU metric takes two numeric arrays as input corresponding to the predicted and ground truth segmentations: -```python ->>> import numpy as np ->>> mean_iou = datasets.load_metric("mean_iou") ->>> predicted = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) ->>> ground_truth = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) ->>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255) -``` - -### Inputs -**Mandatory inputs** -- `predictions` (`List[ndarray]`): List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. -- `references` (`List[ndarray]`): List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. -- `num_labels` (`int`): Number of classes (categories). -- `ignore_index` (`int`): Index that will be ignored during evaluation. - -**Optional inputs** -- `nan_to_num` (`int`): If specified, NaN values will be replaced by the number defined by the user. -- `label_map` (`dict`): If specified, dictionary mapping old label indices to new label indices. -- `reduce_labels` (`bool`): Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. The default value is `False`. - -### Output Values -The metric returns a dictionary with the following elements: -- `mean_iou` (`float`): Mean Intersection-over-Union (IoU averaged over all categories). -- `mean_accuracy` (`float`): Mean accuracy (averaged over all categories). -- `overall_accuracy` (`float`): Overall accuracy on all images. -- `per_category_accuracy` (`ndarray` of shape `(num_labels,)`): Per category accuracy. -- `per_category_iou` (`ndarray` of shape `(num_labels,)`): Per category IoU. - -The values of all of the scores reported range from from `0.0` (minimum) and `1.0` (maximum). - -Output Example: -```python -{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} -``` - -#### Values from Popular Papers - -The [leaderboard for the CityScapes dataset](https://paperswithcode.com/sota/semantic-segmentation-on-cityscapes) reports a Mean IOU ranging from 64 to 84; that of [ADE20k](https://paperswithcode.com/sota/semantic-segmentation-on-ade20k) ranges from 30 to a peak of 59.9, indicating that the dataset is more difficult for current approaches (as of 2022). - - -### Examples - -```python ->>> from datasets import load_metric ->>> import numpy as np ->>> mean_iou = load_metric("mean_iou") ->>> # suppose one has 3 different segmentation maps predicted ->>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) ->>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) ->>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) ->>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) ->>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) ->>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) ->>> predictions = [predicted_1, predicted_2, predicted_3] ->>> references = [actual_1, actual_2, actual_3] ->>> results = mean_iou.compute(predictions=predictions, references=references, num_labels=10, ignore_index=255, reduce_labels=False) ->>> print(results) # doctest: +NORMALIZE_WHITESPACE -{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} -``` - - -## Limitations and Bias -Mean IOU is an average metric, so it will not show you where model predictions differ from the ground truth (i.e. if there are particular regions or classes that the model does poorly on). Further error analysis is needed to gather actional insights that can be used to inform model improvements. - -## Citation(s) -```bibtex -@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, -author = {{MMSegmentation Contributors}}, -license = {Apache-2.0}, -month = {7}, -title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, -url = {https://github.com/open-mmlab/mmsegmentation}, -year = {2020} -}" -``` - - -## Further References -- [Wikipedia article - Jaccard Index](https://en.wikipedia.org/wiki/Jaccard_index) diff --git a/metrics/mean_iou/mean_iou.py b/metrics/mean_iou/mean_iou.py deleted file mode 100644 index 330f6e2000d..00000000000 --- a/metrics/mean_iou/mean_iou.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Mean IoU (Intersection-over-Union) metric.""" - -from typing import Dict, Optional - -import numpy as np - -import datasets - - -_DESCRIPTION = """ -IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union -between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation, -the mean IoU of the image is calculated by taking the IoU of each class and averaging them. -""" - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`List[ndarray]`): - List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. - references (`List[ndarray]`): - List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. - num_labels (`int`): - Number of classes (categories). - ignore_index (`int`): - Index that will be ignored during evaluation. - nan_to_num (`int`, *optional*): - If specified, NaN values will be replaced by the number defined by the user. - label_map (`dict`, *optional*): - If specified, dictionary mapping old label indices to new label indices. - reduce_labels (`bool`, *optional*, defaults to `False`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, - and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. - -Returns: - `Dict[str, float | ndarray]` comprising various elements: - - *mean_iou* (`float`): - Mean Intersection-over-Union (IoU averaged over all categories). - - *mean_accuracy* (`float`): - Mean accuracy (averaged over all categories). - - *overall_accuracy* (`float`): - Overall accuracy on all images. - - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): - Per category accuracy. - - *per_category_iou* (`ndarray` of shape `(num_labels,)`): - Per category IoU. - -Examples: - - >>> import numpy as np - - >>> mean_iou = datasets.load_metric("mean_iou") - - >>> # suppose one has 3 different segmentation maps predicted - >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) - >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) - - >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) - >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) - - >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) - >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) - - >>> predicted = [predicted_1, predicted_2, predicted_3] - >>> ground_truth = [actual_1, actual_2, actual_3] - - >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False) - >>> print(results) # doctest: +NORMALIZE_WHITESPACE - {'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} -""" - -_CITATION = """\ -@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, -author = {{MMSegmentation Contributors}}, -license = {Apache-2.0}, -month = {7}, -title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, -url = {https://github.com/open-mmlab/mmsegmentation}, -year = {2020} -}""" - - -def intersect_and_union( - pred_label, - label, - num_labels, - ignore_index: bool, - label_map: Optional[Dict[int, int]] = None, - reduce_labels: bool = False, -): - """Calculate intersection and Union. - - Args: - pred_label (`ndarray`): - Prediction segmentation map of shape (height, width). - label (`ndarray`): - Ground truth segmentation map of shape (height, width). - num_labels (`int`): - Number of categories. - ignore_index (`int`): - Index that will be ignored during evaluation. - label_map (`dict`, *optional*): - Mapping old labels to new labels. The parameter will work only when label is str. - reduce_labels (`bool`, *optional*, defaults to `False`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, - and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. - - Returns: - area_intersect (`ndarray`): - The intersection of prediction and ground truth histogram on all classes. - area_union (`ndarray`): - The union of prediction and ground truth histogram on all classes. - area_pred_label (`ndarray`): - The prediction histogram on all classes. - area_label (`ndarray`): - The ground truth histogram on all classes. - """ - if label_map is not None: - for old_id, new_id in label_map.items(): - label[label == old_id] = new_id - - # turn into Numpy arrays - pred_label = np.array(pred_label) - label = np.array(label) - - if reduce_labels: - label[label == 0] = 255 - label = label - 1 - label[label == 254] = 255 - - mask = label != ignore_index - mask = np.not_equal(label, ignore_index) - pred_label = pred_label[mask] - label = np.array(label)[mask] - - intersect = pred_label[pred_label == label] - - area_intersect = np.histogram(intersect, bins=num_labels, range=(0, num_labels - 1))[0] - area_pred_label = np.histogram(pred_label, bins=num_labels, range=(0, num_labels - 1))[0] - area_label = np.histogram(label, bins=num_labels, range=(0, num_labels - 1))[0] - - area_union = area_pred_label + area_label - area_intersect - - return area_intersect, area_union, area_pred_label, area_label - - -def total_intersect_and_union( - results, - gt_seg_maps, - num_labels, - ignore_index: bool, - label_map: Optional[Dict[int, int]] = None, - reduce_labels: bool = False, -): - """Calculate Total Intersection and Union, by calculating `intersect_and_union` for each (predicted, ground truth) pair. - - Args: - results (`ndarray`): - List of prediction segmentation maps, each of shape (height, width). - gt_seg_maps (`ndarray`): - List of ground truth segmentation maps, each of shape (height, width). - num_labels (`int`): - Number of categories. - ignore_index (`int`): - Index that will be ignored during evaluation. - label_map (`dict`, *optional*): - Mapping old labels to new labels. The parameter will work only when label is str. - reduce_labels (`bool`, *optional*, defaults to `False`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, - and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. - - Returns: - total_area_intersect (`ndarray`): - The intersection of prediction and ground truth histogram on all classes. - total_area_union (`ndarray`): - The union of prediction and ground truth histogram on all classes. - total_area_pred_label (`ndarray`): - The prediction histogram on all classes. - total_area_label (`ndarray`): - The ground truth histogram on all classes. - """ - total_area_intersect = np.zeros((num_labels,), dtype=np.float64) - total_area_union = np.zeros((num_labels,), dtype=np.float64) - total_area_pred_label = np.zeros((num_labels,), dtype=np.float64) - total_area_label = np.zeros((num_labels,), dtype=np.float64) - for result, gt_seg_map in zip(results, gt_seg_maps): - area_intersect, area_union, area_pred_label, area_label = intersect_and_union( - result, gt_seg_map, num_labels, ignore_index, label_map, reduce_labels - ) - total_area_intersect += area_intersect - total_area_union += area_union - total_area_pred_label += area_pred_label - total_area_label += area_label - return total_area_intersect, total_area_union, total_area_pred_label, total_area_label - - -def mean_iou( - results, - gt_seg_maps, - num_labels, - ignore_index: bool, - nan_to_num: Optional[int] = None, - label_map: Optional[Dict[int, int]] = None, - reduce_labels: bool = False, -): - """Calculate Mean Intersection and Union (mIoU). - - Args: - results (`ndarray`): - List of prediction segmentation maps, each of shape (height, width). - gt_seg_maps (`ndarray`): - List of ground truth segmentation maps, each of shape (height, width). - num_labels (`int`): - Number of categories. - ignore_index (`int`): - Index that will be ignored during evaluation. - nan_to_num (`int`, *optional*): - If specified, NaN values will be replaced by the number defined by the user. - label_map (`dict`, *optional*): - Mapping old labels to new labels. The parameter will work only when label is str. - reduce_labels (`bool`, *optional*, defaults to `False`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, - and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. - - Returns: - `Dict[str, float | ndarray]` comprising various elements: - - *mean_iou* (`float`): - Mean Intersection-over-Union (IoU averaged over all categories). - - *mean_accuracy* (`float`): - Mean accuracy (averaged over all categories). - - *overall_accuracy* (`float`): - Overall accuracy on all images. - - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): - Per category accuracy. - - *per_category_iou* (`ndarray` of shape `(num_labels,)`): - Per category IoU. - """ - total_area_intersect, total_area_union, total_area_pred_label, total_area_label = total_intersect_and_union( - results, gt_seg_maps, num_labels, ignore_index, label_map, reduce_labels - ) - - # compute metrics - metrics = {} - - all_acc = total_area_intersect.sum() / total_area_label.sum() - iou = total_area_intersect / total_area_union - acc = total_area_intersect / total_area_label - - metrics["mean_iou"] = np.nanmean(iou) - metrics["mean_accuracy"] = np.nanmean(acc) - metrics["overall_accuracy"] = all_acc - metrics["per_category_iou"] = iou - metrics["per_category_accuracy"] = acc - - if nan_to_num is not None: - metrics = {metric: np.nan_to_num(metric_value, nan=nan_to_num) for metric, metric_value in metrics.items()} - - return metrics - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class MeanIoU(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - # 1st Seq - height dim, 2nd - width dim - { - "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), - "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), - } - ), - reference_urls=[ - "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py" - ], - ) - - def _compute( - self, - predictions, - references, - num_labels: int, - ignore_index: bool, - nan_to_num: Optional[int] = None, - label_map: Optional[Dict[int, int]] = None, - reduce_labels: bool = False, - ): - iou_result = mean_iou( - results=predictions, - gt_seg_maps=references, - num_labels=num_labels, - ignore_index=ignore_index, - nan_to_num=nan_to_num, - label_map=label_map, - reduce_labels=reduce_labels, - ) - return iou_result diff --git a/metrics/meteor/README.md b/metrics/meteor/README.md deleted file mode 100644 index ba54a215aea..00000000000 --- a/metrics/meteor/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Metric Card for METEOR - -## Metric description - -METEOR (Metric for Evaluation of Translation with Explicit ORdering) is a machine translation evaluation metric, which is calculated based on the harmonic mean of precision and recall, with recall weighted more than precision. - -METEOR is based on a generalized concept of unigram matching between the machine-produced translation and human-produced reference translations. Unigrams can be matched based on their surface forms, stemmed forms, and meanings. Once all generalized unigram matches between the two strings have been found, METEOR computes a score for this matching using a combination of unigram-precision, unigram-recall, and a measure of fragmentation that is designed to directly capture how well-ordered the matched words in the machine translation are in relation to the reference. - - -## How to use - -METEOR has two mandatory arguments: - -`predictions`: a list of predictions to score. Each prediction should be a string with tokens separated by spaces. - -`references`: a list of references for each prediction. Each reference should be a string with tokens separated by spaces. - -It also has several optional parameters: - -`alpha`: Parameter for controlling relative weights of precision and recall. The default value is `0.9`. - -`beta`: Parameter for controlling shape of penalty as a function of fragmentation. The default value is `3`. - -`gamma`: The relative weight assigned to fragmentation penalty. The default is `0.5`. - -Refer to the [METEOR paper](https://aclanthology.org/W05-0909.pdf) for more information about parameter values and ranges. - -```python ->>> from datasets import load_metric ->>> meteor = load_metric('meteor') ->>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] ->>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] ->>> results = meteor.compute(predictions=predictions, references=references) -``` - -## Output values - -The metric outputs a dictionary containing the METEOR score. Its values range from 0 to 1. - - -### Values from popular papers -The [METEOR paper](https://aclanthology.org/W05-0909.pdf) does not report METEOR score values for different models, but it does report that METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic data and 0.331 on the Chinese data. - - -## Examples - -Maximal values : - -```python ->>> from datasets import load_metric ->>> meteor = load_metric('meteor') ->>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] ->>> references = ["It is a guide to action which ensures that the military always obeys the commands of the party"] ->>> results = meteor.compute(predictions=predictions, references=references) ->>> print(round(results['meteor'], 2)) -1.0 -``` - -Minimal values: - -```python ->>> from datasets import load_metric ->>> meteor = load_metric('meteor') ->>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] ->>> references = ["Hello world"] ->>> results = meteor.compute(predictions=predictions, references=references) ->>> print(round(results['meteor'], 2)) -0.0 -``` - -Partial match: - -```python ->>> from datasets import load_metric ->>> meteor = load_metric('meteor') ->>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] ->>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] ->>> results = meteor.compute(predictions=predictions, references=references) ->>> print(round(results['meteor'], 2)) -0.69 -``` - -## Limitations and bias - -While the correlation between METEOR and human judgments was measured for Chinese and Arabic and found to be significant, further experimentation is needed to check its correlation for other languages. - -Furthermore, while the alignment and matching done in METEOR is based on unigrams, using multiple word entities (e.g. bigrams) could contribute to improving its accuracy -- this has been proposed in [more recent publications](https://www.cs.cmu.edu/~alavie/METEOR/pdf/meteor-naacl-2010.pdf) on the subject. - - -## Citation - -```bibtex -@inproceedings{banarjee2005, - title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, - author = {Banerjee, Satanjeev and Lavie, Alon}, - booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, - month = jun, - year = {2005}, - address = {Ann Arbor, Michigan}, - publisher = {Association for Computational Linguistics}, - url = {https://www.aclweb.org/anthology/W05-0909}, - pages = {65--72}, -} -``` - -## Further References -- [METEOR -- Wikipedia](https://en.wikipedia.org/wiki/METEOR) -- [METEOR score -- NLTK](https://www.nltk.org/_modules/nltk/translate/meteor_score.html) - diff --git a/metrics/meteor/meteor.py b/metrics/meteor/meteor.py deleted file mode 100644 index 1ae3a08c232..00000000000 --- a/metrics/meteor/meteor.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""METEOR metric.""" - -import importlib.metadata - -import numpy as np -from nltk.translate import meteor_score -from packaging import version - -import datasets - - -NLTK_VERSION = version.parse(importlib.metadata.version("nltk")) -if NLTK_VERSION >= version.Version("3.6.4"): - from nltk import word_tokenize - - -_CITATION = """\ -@inproceedings{banarjee2005, - title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, - author = {Banerjee, Satanjeev and Lavie, Alon}, - booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, - month = jun, - year = {2005}, - address = {Ann Arbor, Michigan}, - publisher = {Association for Computational Linguistics}, - url = {https://www.aclweb.org/anthology/W05-0909}, - pages = {65--72}, -} -""" - -_DESCRIPTION = """\ -METEOR, an automatic metric for machine translation evaluation -that is based on a generalized concept of unigram matching between the -machine-produced translation and human-produced reference translations. -Unigrams can be matched based on their surface forms, stemmed forms, -and meanings; furthermore, METEOR can be easily extended to include more -advanced matching strategies. Once all generalized unigram matches -between the two strings have been found, METEOR computes a score for -this matching using a combination of unigram-precision, unigram-recall, and -a measure of fragmentation that is designed to directly capture how -well-ordered the matched words in the machine translation are in relation -to the reference. - -METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic -data and 0.331 on the Chinese data. This is shown to be an improvement on -using simply unigram-precision, unigram-recall and their harmonic F1 -combination. -""" - -_KWARGS_DESCRIPTION = """ -Computes METEOR score of translated segments against one or more references. -Args: - predictions: list of predictions to score. Each prediction - should be a string with tokens separated by spaces. - references: list of reference for each prediction. Each - reference should be a string with tokens separated by spaces. - alpha: Parameter for controlling relative weights of precision and recall. default: 0.9 - beta: Parameter for controlling shape of penalty as a function of fragmentation. default: 3 - gamma: Relative weight assigned to fragmentation penalty. default: 0.5 -Returns: - 'meteor': meteor score. -Examples: - - >>> meteor = datasets.load_metric('meteor') - >>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] - >>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] - >>> results = meteor.compute(predictions=predictions, references=references) - >>> print(round(results["meteor"], 4)) - 0.6944 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Meteor(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py"], - reference_urls=[ - "https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score", - "https://en.wikipedia.org/wiki/METEOR", - ], - ) - - def _download_and_prepare(self, dl_manager): - import nltk - - nltk.download("wordnet") - if NLTK_VERSION >= version.Version("3.6.5"): - nltk.download("punkt") - if NLTK_VERSION >= version.Version("3.6.6"): - nltk.download("omw-1.4") - - def _compute(self, predictions, references, alpha=0.9, beta=3, gamma=0.5): - if NLTK_VERSION >= version.Version("3.6.5"): - scores = [ - meteor_score.single_meteor_score( - word_tokenize(ref), word_tokenize(pred), alpha=alpha, beta=beta, gamma=gamma - ) - for ref, pred in zip(references, predictions) - ] - else: - scores = [ - meteor_score.single_meteor_score(ref, pred, alpha=alpha, beta=beta, gamma=gamma) - for ref, pred in zip(references, predictions) - ] - - return {"meteor": np.mean(scores)} diff --git a/metrics/mse/README.md b/metrics/mse/README.md deleted file mode 100644 index 421b1b8e601..00000000000 --- a/metrics/mse/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# Metric Card for MSE - - -## Metric Description - -Mean Squared Error(MSE) represents the average of the squares of errors -- i.e. the average squared difference between the estimated values and the actual values. - -![image](https://user-images.githubusercontent.com/14205986/165999302-eba3702d-81e3-4363-9c0e-d3bfceb7ec5a.png) - -## How to Use - -At minimum, this metric requires predictions and references as inputs. - -```python ->>> mse_metric = datasets.load_metric("mse") ->>> predictions = [2.5, 0.0, 2, 8] ->>> references = [3, -0.5, 2, 7] ->>> results = mse_metric.compute(predictions=predictions, references=references) -``` - -### Inputs - -Mandatory inputs: -- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. -- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. - -Optional arguments: -- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. -- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. - - `raw_values` returns a full set of errors in case of multioutput input. - - `uniform_average` means that the errors of all outputs are averaged with uniform weight. - - the array-like value defines weights used to average errors. -- `squared` (`bool`): If `True` returns MSE value, if `False` returns RMSE (Root Mean Squared Error). The default value is `True`. - - -### Output Values -This metric outputs a dictionary, containing the mean squared error score, which is of type: -- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. -- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. - -Each MSE `float` value ranges from `0.0` to `1.0`, with the best value being `0.0`. - -Output Example(s): -```python -{'mse': 0.5} -``` - -If `multioutput="raw_values"`: -```python -{'mse': array([0.41666667, 1. ])} -``` - -#### Values from Popular Papers - - -### Examples - -Example with the `uniform_average` config: -```python ->>> from datasets import load_metric ->>> mse_metric = load_metric("mse") ->>> predictions = [2.5, 0.0, 2, 8] ->>> references = [3, -0.5, 2, 7] ->>> results = mse_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'mse': 0.375} -``` - -Example with `squared = True`, which returns the RMSE: -```python ->>> from datasets import load_metric ->>> mse_metric = load_metric("mse") ->>> predictions = [2.5, 0.0, 2, 8] ->>> references = [3, -0.5, 2, 7] ->>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) ->>> print(rmse_result) -{'mse': 0.6123724356957945} -``` - -Example with multi-dimensional lists, and the `raw_values` config: -```python ->>> from datasets import load_metric ->>> mse_metric = load_metric("mse", "multilist") ->>> predictions = [[0.5, 1], [-1, 1], [7, -6]] ->>> references = [[0, 2], [-1, 2], [8, -5]] ->>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') ->>> print(results) -{'mse': array([0.41666667, 1. ])} -""" -``` - -## Limitations and Bias -MSE has the disadvantage of heavily weighting outliers -- given that it squares them, this results in large errors weighing more heavily than small ones. It can be used alongside [MAE](https://huggingface.co/metrics/mae), which is complementary given that it does not square the errors. - -## Citation(s) -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - -```bibtex -@article{willmott2005advantages, - title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, - author={Willmott, Cort J and Matsuura, Kenji}, - journal={Climate research}, - volume={30}, - number={1}, - pages={79--82}, - year={2005} -} -``` - -## Further References -- [Mean Squared Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_squared_error) diff --git a/metrics/mse/mse.py b/metrics/mse/mse.py deleted file mode 100644 index d1b8d002b03..00000000000 --- a/metrics/mse/mse.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""MSE - Mean Squared Error Metric""" - -from sklearn.metrics import mean_squared_error - -import datasets - - -_CITATION = """\ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - -_DESCRIPTION = """\ -Mean Squared Error(MSE) is the average of the square of difference between the predicted -and actual values. -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) - Estimated target values. - references: array-like of shape (n_samples,) or (n_samples, n_outputs) - Ground truth (correct) target values. - sample_weight: array-like of shape (n_samples,), default=None - Sample weights. - multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" - Defines aggregating of multiple output values. Array-like value defines weights used to average errors. - - "raw_values" : Returns a full set of errors in case of multioutput input. - - "uniform_average" : Errors of all outputs are averaged with uniform weight. - - squared : bool, default=True - If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value. - -Returns: - mse : mean squared error. -Examples: - - >>> mse_metric = datasets.load_metric("mse") - >>> predictions = [2.5, 0.0, 2, 8] - >>> references = [3, -0.5, 2, 7] - >>> results = mse_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'mse': 0.375} - >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) - >>> print(rmse_result) - {'mse': 0.6123724356957945} - - If you're using multi-dimensional lists, then set the config as follows : - - >>> mse_metric = datasets.load_metric("mse", "multilist") - >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] - >>> references = [[0, 2], [-1, 2], [8, -5]] - >>> results = mse_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'mse': 0.7083333333333334} - >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') - >>> print(results) # doctest: +NORMALIZE_WHITESPACE - {'mse': array([0.41666667, 1. ])} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Mse(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features(self._get_feature_types()), - reference_urls=[ - "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html" - ], - ) - - def _get_feature_types(self): - if self.config_name == "multilist": - return { - "predictions": datasets.Sequence(datasets.Value("float")), - "references": datasets.Sequence(datasets.Value("float")), - } - else: - return { - "predictions": datasets.Value("float"), - "references": datasets.Value("float"), - } - - def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average", squared=True): - mse = mean_squared_error( - references, predictions, sample_weight=sample_weight, multioutput=multioutput, squared=squared - ) - - return {"mse": mse} diff --git a/metrics/pearsonr/README.md b/metrics/pearsonr/README.md deleted file mode 100644 index 48f259ee586..00000000000 --- a/metrics/pearsonr/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Metric Card for Pearson Correlation Coefficient (pearsonr) - - -## Metric Description - -Pearson correlation coefficient and p-value for testing non-correlation. -The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. -The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. - - -## How to Use - -This metric takes a list of predictions and a list of references as input - -```python ->>> pearsonr_metric = datasets.load_metric("pearsonr") ->>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) ->>> print(round(results['pearsonr']), 2) -['-0.74'] -``` - - -### Inputs -- **predictions** (`list` of `int`): Predicted class labels, as returned by a model. -- **references** (`list` of `int`): Ground truth labels. -- **return_pvalue** (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. - - -### Output Values -- **pearsonr**(`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. -- **p-value**(`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. - -Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. - -Output Example(s): -```python -{'pearsonr': -0.7} -``` -```python -{'p-value': 0.15} -``` - -#### Values from Popular Papers - -### Examples - -Example 1-A simple example using only predictions and references. -```python ->>> pearsonr_metric = datasets.load_metric("pearsonr") ->>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) ->>> print(round(results['pearsonr'], 2)) --0.74 -``` - -Example 2-The same as Example 1, but that also returns the `p-value`. -```python ->>> pearsonr_metric = datasets.load_metric("pearsonr") ->>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) ->>> print(sorted(list(results.keys()))) -['p-value', 'pearsonr'] ->>> print(round(results['pearsonr'], 2)) --0.74 ->>> print(round(results['p-value'], 2)) -0.15 -``` - - -## Limitations and Bias - -As stated above, the calculation of the p-value relies on the assumption that each data set is normally distributed. This is not always the case, so verifying the true distribution of datasets is recommended. - - -## Citation(s) -```bibtex -@article{2020SciPy-NMeth, -author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and - Haberland, Matt and Reddy, Tyler and Cournapeau, David and - Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and - Bright, Jonathan and {van der Walt}, St{\'e}fan J. and - Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and - Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and - Kern, Robert and Larson, Eric and Carey, C J and - Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and - {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and - Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and - Harris, Charles R. and Archibald, Anne M. and - Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and - {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, -title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific - Computing in Python}}, -journal = {Nature Methods}, -year = {2020}, -volume = {17}, -pages = {261--272}, -adsurl = {https://rdcu.be/b08Wh}, -doi = {10.1038/s41592-019-0686-2}, -} -``` - - -## Further References diff --git a/metrics/pearsonr/pearsonr.py b/metrics/pearsonr/pearsonr.py deleted file mode 100644 index 34c31691f69..00000000000 --- a/metrics/pearsonr/pearsonr.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Pearson correlation coefficient metric.""" - -from scipy.stats import pearsonr - -import datasets - - -_DESCRIPTION = """ -Pearson correlation coefficient and p-value for testing non-correlation. -The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. -The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`list` of `int`): Predicted class labels, as returned by a model. - references (`list` of `int`): Ground truth labels. - return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. - -Returns: - pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. - p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. - -Examples: - - Example 1-A simple example using only predictions and references. - >>> pearsonr_metric = datasets.load_metric("pearsonr") - >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) - >>> print(round(results['pearsonr'], 2)) - -0.74 - - Example 2-The same as Example 1, but that also returns the `p-value`. - >>> pearsonr_metric = datasets.load_metric("pearsonr") - >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) - >>> print(sorted(list(results.keys()))) - ['p-value', 'pearsonr'] - >>> print(round(results['pearsonr'], 2)) - -0.74 - >>> print(round(results['p-value'], 2)) - 0.15 -""" - - -_CITATION = """ -@article{2020SciPy-NMeth, -author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and - Haberland, Matt and Reddy, Tyler and Cournapeau, David and - Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and - Bright, Jonathan and {van der Walt}, St{\'e}fan J. and - Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and - Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and - Kern, Robert and Larson, Eric and Carey, C J and - Polat, Ilhan and Feng, Yu and Moore, Eric W. and - {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and - Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and - Harris, Charles R. and Archibald, Anne M. and - Ribeiro, Antonio H. and Pedregosa, Fabian and - {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, -title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific - Computing in Python}}, -journal = {Nature Methods}, -year = {2020}, -volume = {17}, -pages = {261--272}, -adsurl = {https://rdcu.be/b08Wh}, -doi = {10.1038/s41592-019-0686-2}, -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Pearsonr(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("float"), - "references": datasets.Value("float"), - } - ), - reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"], - ) - - def _compute(self, predictions, references, return_pvalue=False): - if return_pvalue: - results = pearsonr(references, predictions) - return {"pearsonr": results[0], "p-value": results[1]} - else: - return {"pearsonr": float(pearsonr(references, predictions)[0])} diff --git a/metrics/perplexity/README.md b/metrics/perplexity/README.md deleted file mode 100644 index efa5216304c..00000000000 --- a/metrics/perplexity/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Metric Card for Perplexity - -## Metric Description -Given a model and an input text sequence, perplexity measures how likely the model is to generate the input text sequence. This can be used in two main ways: -1. to evaluate how well the model has learned the distribution of the text it was trained on - - In this case, the model input should be the trained model to be evaluated, and the input texts should be the text that the model was trained on. -2. to evaluate how well a selection of text matches the distribution of text that the input model was trained on - - In this case, the model input should be a trained model, and the input texts should be the text to be evaluated. - -## Intended Uses -Any language generation task. - -## How to Use - -The metric takes a list of text as input, as well as the name of the model used to compute the metric: - -```python -from datasets import load_metric -perplexity = load_metric("perplexity") -results = perplexity.compute(input_texts=input_texts, model_id='gpt2') -``` - -### Inputs -- **model_id** (str): model used for calculating Perplexity. NOTE: Perplexity can only be calculated for causal language models. - - This includes models such as gpt2, causal variations of bert, causal versions of t5, and more (the full list can be found in the AutoModelForCausalLM documentation here: https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) -- **input_texts** (list of str): input text, each separate text snippet is one list entry. -- **batch_size** (int): the batch size to run texts through the model. Defaults to 16. -- **add_start_token** (bool): whether to add the start token to the texts, so the perplexity can include the probability of the first word. Defaults to True. -- **device** (str): device to run on, defaults to 'cuda' when available - -### Output Values -This metric outputs a dictionary with the perplexity scores for the text input in the list, and the average perplexity. -If one of the input texts is longer than the max input length of the model, then it is truncated to the max length for the perplexity computation. - -``` -{'perplexities': [8.182524681091309, 33.42122268676758, 27.012239456176758], 'mean_perplexity': 22.871995608011883} -``` - -This metric's range is 0 and up. A lower score is better. - -#### Values from Popular Papers - - -### Examples -Calculating perplexity on input_texts defined here: -```python -perplexity = datasets.load_metric("perplexity") -input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] -results = perplexity.compute(model_id='gpt2', - add_start_token=False, - input_texts=input_texts) -print(list(results.keys())) ->>>['perplexities', 'mean_perplexity'] -print(round(results["mean_perplexity"], 2)) ->>>78.22 -print(round(results["perplexities"][0], 2)) ->>>11.11 -``` -Calculating perplexity on input_texts loaded in from a dataset: -```python -perplexity = datasets.load_metric("perplexity") -input_texts = datasets.load_dataset("wikitext", - "wikitext-2-raw-v1", - split="test")["text"][:50] -input_texts = [s for s in input_texts if s!=''] -results = perplexity.compute(model_id='gpt2', - input_texts=input_texts) -print(list(results.keys())) ->>>['perplexities', 'mean_perplexity'] -print(round(results["mean_perplexity"], 2)) ->>>60.35 -print(round(results["perplexities"][0], 2)) ->>>81.12 -``` - -## Limitations and Bias -Note that the output value is based heavily on what text the model was trained on. This means that perplexity scores are not comparable between models or datasets. - - -## Citation - -```bibtex -@article{jelinek1977perplexity, -title={Perplexityβ€”a measure of the difficulty of speech recognition tasks}, -author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K}, -journal={The Journal of the Acoustical Society of America}, -volume={62}, -number={S1}, -pages={S63--S63}, -year={1977}, -publisher={Acoustical Society of America} -} -``` - -## Further References -- [Hugging Face Perplexity Blog Post](https://huggingface.co/docs/transformers/perplexity) diff --git a/metrics/perplexity/perplexity.py b/metrics/perplexity/perplexity.py deleted file mode 100644 index 389ef8149e2..00000000000 --- a/metrics/perplexity/perplexity.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Perplexity Metric.""" - -import numpy as np -import torch -from torch.nn import CrossEntropyLoss -from transformers import AutoModelForCausalLM, AutoTokenizer - -import datasets -from datasets import logging - - -_CITATION = """\ - -""" - -_DESCRIPTION = """ -Perplexity (PPL) is one of the most common metrics for evaluating language models. -It is defined as the exponentiated average negative log-likelihood of a sequence. - -For more information, see https://huggingface.co/docs/transformers/perplexity -""" - -_KWARGS_DESCRIPTION = """ -Args: - model_id (str): model used for calculating Perplexity - NOTE: Perplexity can only be calculated for causal language models. - This includes models such as gpt2, causal variations of bert, - causal versions of t5, and more (the full list can be found - in the AutoModelForCausalLM documentation here: - https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) - - input_texts (list of str): input text, each separate text snippet - is one list entry. - batch_size (int): the batch size to run texts through the model. Defaults to 16. - add_start_token (bool): whether to add the start token to the texts, - so the perplexity can include the probability of the first word. Defaults to True. - device (str): device to run on, defaults to 'cuda' when available -Returns: - perplexity: dictionary containing the perplexity scores for the texts - in the input list, as well as the mean perplexity. If one of the input texts is - longer than the max input length of the model, then it is truncated to the - max length for the perplexity computation. -Examples: - Example 1: - >>> perplexity = datasets.load_metric("perplexity") - >>> input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] - >>> results = perplexity.compute(model_id='gpt2', - ... add_start_token=False, - ... input_texts=input_texts) # doctest:+ELLIPSIS - >>> print(list(results.keys())) - ['perplexities', 'mean_perplexity'] - >>> print(round(results["mean_perplexity"], 2)) - 78.22 - >>> print(round(results["perplexities"][0], 2)) - 11.11 - - Example 2: - >>> perplexity = datasets.load_metric("perplexity") - >>> input_texts = datasets.load_dataset("wikitext", - ... "wikitext-2-raw-v1", - ... split="test")["text"][:50] - >>> input_texts = [s for s in input_texts if s!=''] - >>> results = perplexity.compute(model_id='gpt2', - ... input_texts=input_texts) # doctest:+ELLIPSIS - >>> print(list(results.keys())) - ['perplexities', 'mean_perplexity'] - >>> print(round(results["mean_perplexity"], 2)) - 60.35 - >>> print(round(results["perplexities"][0], 2)) - 81.12 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Perplexity(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "input_texts": datasets.Value("string"), - } - ), - reference_urls=["https://huggingface.co/docs/transformers/perplexity"], - ) - - def _compute(self, input_texts, model_id, batch_size: int = 16, add_start_token: bool = True, device=None): - if device is not None: - assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu." - if device == "gpu": - device = "cuda" - else: - device = "cuda" if torch.cuda.is_available() else "cpu" - - model = AutoModelForCausalLM.from_pretrained(model_id) - model = model.to(device) - - tokenizer = AutoTokenizer.from_pretrained(model_id) - - # if batch_size > 1 (which generally leads to padding being required), and - # if there is not an already assigned pad_token, assign an existing - # special token to also be the padding token - if tokenizer.pad_token is None and batch_size > 1: - existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) - # check that the model already has at least one special token defined - assert ( - len(existing_special_tokens) > 0 - ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." - # assign one of the special tokens to also be the pad token - tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]}) - - if add_start_token: - # leave room for token to be added: - assert ( - tokenizer.bos_token is not None - ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False" - max_tokenized_len = model.config.max_length - 1 - else: - max_tokenized_len = model.config.max_length - - encodings = tokenizer( - input_texts, - add_special_tokens=False, - padding=True, - truncation=True, - max_length=max_tokenized_len, - return_tensors="pt", - return_attention_mask=True, - ).to(device) - - encoded_texts = encodings["input_ids"] - attn_masks = encodings["attention_mask"] - - # check that each input is long enough: - if add_start_token: - assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long." - else: - assert torch.all( - torch.ge(attn_masks.sum(1), 2) - ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings." - - ppls = [] - loss_fct = CrossEntropyLoss(reduction="none") - - for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)): - end_index = min(start_index + batch_size, len(encoded_texts)) - encoded_batch = encoded_texts[start_index:end_index] - attn_mask = attn_masks[start_index:end_index] - - if add_start_token: - bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device) - encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1) - attn_mask = torch.cat( - [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1 - ) - - labels = encoded_batch - - with torch.no_grad(): - out_logits = model(encoded_batch, attention_mask=attn_mask).logits - - shift_logits = out_logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - shift_attention_mask_batch = attn_mask[..., 1:].contiguous() - - perplexity_batch = torch.exp2( - (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1) - / shift_attention_mask_batch.sum(1) - ) - - ppls += perplexity_batch.tolist() - - return {"perplexities": ppls, "mean_perplexity": np.mean(ppls)} diff --git a/metrics/precision/README.md b/metrics/precision/README.md deleted file mode 100644 index 14b59982d58..00000000000 --- a/metrics/precision/README.md +++ /dev/null @@ -1,124 +0,0 @@ -# Metric Card for Precision - - -## Metric Description - -Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: -Precision = TP / (TP + FP) -where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). - - -## How to Use - -At minimum, precision takes as input a list of predicted labels, `predictions`, and a list of output labels, `references`. - -```python ->>> precision_metric = datasets.load_metric("precision") ->>> results = precision_metric.compute(references=[0, 1], predictions=[0, 1]) ->>> print(results) -{'precision': 1.0} -``` - - -### Inputs -- **predictions** (`list` of `int`): Predicted class labels. -- **references** (`list` of `int`): Actual class labels. -- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. -- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. -- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. - - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. - - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. - - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). -- **sample_weight** (`list` of `float`): Sample weights Defaults to None. -- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . - - 0: Returns 0 when there is a zero division. - - 1: Returns 1 when there is a zero division. - - 'warn': Raises warnings and then returns 0 when there is a zero division. - - -### Output Values -- **precision**(`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. - -Output Example(s): -```python -{'precision': 0.2222222222222222} -``` -```python -{'precision': array([0.66666667, 0.0, 0.0])} -``` - - - - -#### Values from Popular Papers - - -### Examples - -Example 1-A simple binary example -```python ->>> precision_metric = datasets.load_metric("precision") ->>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) ->>> print(results) -{'precision': 0.5} -``` - -Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. -```python ->>> precision_metric = datasets.load_metric("precision") ->>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) ->>> print(round(results['precision'], 2)) -0.67 -``` - -Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. -```python ->>> precision_metric = datasets.load_metric("precision") ->>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) ->>> print(results) -{'precision': 0.23529411764705882} -``` - -Example 4-A multiclass example, with different values for the `average` input. -```python ->>> predictions = [0, 2, 1, 0, 0, 1] ->>> references = [0, 1, 2, 0, 1, 2] ->>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') ->>> print(results) -{'precision': 0.2222222222222222} ->>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') ->>> print(results) -{'precision': 0.3333333333333333} ->>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') ->>> print(results) -{'precision': 0.2222222222222222} ->>> results = precision_metric.compute(predictions=predictions, references=references, average=None) ->>> print([round(res, 2) for res in results['precision']]) -[0.67, 0.0, 0.0] -``` - - -## Limitations and Bias - -[Precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall) are complementary and can be used to measure different aspects of model performance -- using both of them (or an averaged measure like [F1 score](https://huggingface.co/metrics/F1) to better represent different aspects of performance. See [Wikipedia](https://en.wikipedia.org/wiki/Precision_and_recall) for more information. - -## Citation(s) -```bibtex -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -``` - - -## Further References -- [Wikipedia -- Precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) diff --git a/metrics/precision/precision.py b/metrics/precision/precision.py deleted file mode 100644 index 95d0b898ad6..00000000000 --- a/metrics/precision/precision.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Precision metric.""" - -from sklearn.metrics import precision_score - -import datasets - - -_DESCRIPTION = """ -Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: -Precision = TP / (TP + FP) -where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). -""" - - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`list` of `int`): Predicted class labels. - references (`list` of `int`): Actual class labels. - labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. - pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. - average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. - - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. - - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. - - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). - sample_weight (`list` of `float`): Sample weights Defaults to None. - zero_division (`int` or `string`): Sets the value to return when there is a zero division. Defaults to 'warn'. - - - 0: Returns 0 when there is a zero division. - - 1: Returns 1 when there is a zero division. - - 'warn': Raises warnings and then returns 0 when there is a zero division. - -Returns: - precision (`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. - -Examples: - - Example 1-A simple binary example - >>> precision_metric = datasets.load_metric("precision") - >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) - >>> print(results) - {'precision': 0.5} - - Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. - >>> precision_metric = datasets.load_metric("precision") - >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) - >>> print(round(results['precision'], 2)) - 0.67 - - Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. - >>> precision_metric = datasets.load_metric("precision") - >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) - >>> print(results) - {'precision': 0.23529411764705882} - - Example 4-A multiclass example, with different values for the `average` input. - >>> predictions = [0, 2, 1, 0, 0, 1] - >>> references = [0, 1, 2, 0, 1, 2] - >>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') - >>> print(results) - {'precision': 0.2222222222222222} - >>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') - >>> print(results) - {'precision': 0.3333333333333333} - >>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') - >>> print(results) - {'precision': 0.2222222222222222} - >>> results = precision_metric.compute(predictions=predictions, references=references, average=None) - >>> print([round(res, 2) for res in results['precision']]) - [0.67, 0.0, 0.0] -""" - - -_CITATION = """ -@article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Precision(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("int32")), - "references": datasets.Sequence(datasets.Value("int32")), - } - if self.config_name == "multilabel" - else { - "predictions": datasets.Value("int32"), - "references": datasets.Value("int32"), - } - ), - reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html"], - ) - - def _compute( - self, - predictions, - references, - labels=None, - pos_label=1, - average="binary", - sample_weight=None, - zero_division="warn", - ): - score = precision_score( - references, - predictions, - labels=labels, - pos_label=pos_label, - average=average, - sample_weight=sample_weight, - zero_division=zero_division, - ) - return {"precision": float(score) if score.size == 1 else score} diff --git a/metrics/recall/README.md b/metrics/recall/README.md deleted file mode 100644 index cae1459ab93..00000000000 --- a/metrics/recall/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Metric Card for Recall - - -## Metric Description - -Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: -Recall = TP / (TP + FN) -Where TP is the number of true positives and FN is the number of false negatives. - - -## How to Use - -At minimum, this metric takes as input two `list`s, each containing `int`s: predictions and references. - -```python ->>> recall_metric = datasets.load_metric('recall') ->>> results = recall_metric.compute(references=[0, 1], predictions=[0, 1]) ->>> print(results) -["{'recall': 1.0}"] -``` - - -### Inputs -- **predictions** (`list` of `int`): The predicted labels. -- **references** (`list` of `int`): The ground truth labels. -- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. -- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. -- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. - - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. - - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. - - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). -- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. -- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . - - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. - - `0`: If there is a zero division, the return value is `0`. - - `1`: If there is a zero division, the return value is `1`. - - -### Output Values -- **recall**(`float`, or `array` of `float`, for multiclass targets): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. - -Output Example(s): -```python -{'recall': 1.0} -``` -```python -{'recall': array([1., 0., 0.])} -``` - -This metric outputs a dictionary with one entry, `'recall'`. - - -#### Values from Popular Papers - - -### Examples - -Example 1-A simple example with some errors -```python ->>> recall_metric = datasets.load_metric('recall') ->>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) ->>> print(results) -{'recall': 0.6666666666666666} -``` - -Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. -```python ->>> recall_metric = datasets.load_metric('recall') ->>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) ->>> print(results) -{'recall': 0.5} -``` - -Example 3-The same example as Example 1, but with `sample_weight` included. -```python ->>> recall_metric = datasets.load_metric('recall') ->>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] ->>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) ->>> print(results) -{'recall': 0.55} -``` - -Example 4-A multiclass example, using different averages. -```python ->>> recall_metric = datasets.load_metric('recall') ->>> predictions = [0, 2, 1, 0, 0, 1] ->>> references = [0, 1, 2, 0, 1, 2] ->>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') ->>> print(results) -{'recall': 0.3333333333333333} ->>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') ->>> print(results) -{'recall': 0.3333333333333333} ->>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') ->>> print(results) -{'recall': 0.3333333333333333} ->>> results = recall_metric.compute(predictions=predictions, references=references, average=None) ->>> print(results) -{'recall': array([1., 0., 0.])} -``` - - -## Limitations and Bias - - -## Citation(s) -```bibtex -@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} -``` - - -## Further References diff --git a/metrics/recall/recall.py b/metrics/recall/recall.py deleted file mode 100644 index 3ba11498a77..00000000000 --- a/metrics/recall/recall.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Recall metric.""" - -from sklearn.metrics import recall_score - -import datasets - - -_DESCRIPTION = """ -Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: -Recall = TP / (TP + FN) -Where TP is the true positives and FN is the false negatives. -""" - - -_KWARGS_DESCRIPTION = """ -Args: -- **predictions** (`list` of `int`): The predicted labels. -- **references** (`list` of `int`): The ground truth labels. -- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. -- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. -- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. - - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. - - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. - - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. - - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). -- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. -- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . - - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. - - `0`: If there is a zero division, the return value is `0`. - - `1`: If there is a zero division, the return value is `1`. - -Returns: -- **recall** (`float`, or `array` of `float`): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. - -Examples: - - Example 1-A simple example with some errors - >>> recall_metric = datasets.load_metric('recall') - >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) - >>> print(results) - {'recall': 0.6666666666666666} - - Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. - >>> recall_metric = datasets.load_metric('recall') - >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) - >>> print(results) - {'recall': 0.5} - - Example 3-The same example as Example 1, but with `sample_weight` included. - >>> recall_metric = datasets.load_metric('recall') - >>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] - >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) - >>> print(results) - {'recall': 0.55} - - Example 4-A multiclass example, using different averages. - >>> recall_metric = datasets.load_metric('recall') - >>> predictions = [0, 2, 1, 0, 0, 1] - >>> references = [0, 1, 2, 0, 1, 2] - >>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') - >>> print(results) - {'recall': 0.3333333333333333} - >>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') - >>> print(results) - {'recall': 0.3333333333333333} - >>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') - >>> print(results) - {'recall': 0.3333333333333333} - >>> results = recall_metric.compute(predictions=predictions, references=references, average=None) - >>> print(results) - {'recall': array([1., 0., 0.])} -""" - - -_CITATION = """ -@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Recall(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("int32")), - "references": datasets.Sequence(datasets.Value("int32")), - } - if self.config_name == "multilabel" - else { - "predictions": datasets.Value("int32"), - "references": datasets.Value("int32"), - } - ), - reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html"], - ) - - def _compute( - self, - predictions, - references, - labels=None, - pos_label=1, - average="binary", - sample_weight=None, - zero_division="warn", - ): - score = recall_score( - references, - predictions, - labels=labels, - pos_label=pos_label, - average=average, - sample_weight=sample_weight, - zero_division=zero_division, - ) - return {"recall": float(score) if score.size == 1 else score} diff --git a/metrics/roc_auc/README.md b/metrics/roc_auc/README.md deleted file mode 100644 index 0b1e65ef67f..00000000000 --- a/metrics/roc_auc/README.md +++ /dev/null @@ -1,183 +0,0 @@ -# Metric Card for ROC AUC - - -## Metric Description -This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. - -This metric has three separate use cases: -- **binary**: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. -- **multiclass**: The case in which there can be more than two different label classes, but each example still gets only one label. -- **multilabel**: The case in which there can be more than two different label classes, and each example can have more than one label. - - -## How to Use -At minimum, this metric requires references and prediction scores: -```python ->>> roc_auc_score = datasets.load_metric("roc_auc") ->>> refs = [1, 0, 1, 1, 0, 0] ->>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] ->>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) ->>> print(round(results['roc_auc'], 2)) -0.78 -``` - -The default implementation of this metric is the **binary** implementation. If employing the **multiclass** or **multilabel** use cases, the keyword `"multiclass"` or `"multilabel"` must be specified when loading the metric: -- In the **multiclass** case, the metric is loaded with: - ```python - >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass") - ``` -- In the **multilabel** case, the metric is loaded with: - ```python - >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel") - ``` - -See the [Examples Section Below](#examples_section) for more extensive examples. - - -### Inputs -- **`references`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different inputs based on use case: - - binary: expects an array-like of shape (n_samples,) - - multiclass: expects an array-like of shape (n_samples,) - - multilabel: expects an array-like of shape (n_samples, n_classes) -- **`prediction_scores`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: - - binary: expects an array-like of shape (n_samples,) - - multiclass: expects an array-like of shape (n_samples, n_classes). The probability estimates must sum to 1 across the possible classes. - - multilabel: expects an array-like of shape (n_samples, n_classes) -- **`average`** (`str`): Type of average, and is ignored in the binary use case. Defaults to `'macro'`. Options are: - - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. - - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). - - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. - - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. -- **`sample_weight`** (array-like of shape (n_samples,)): Sample weights. Defaults to None. -- **`max_fpr`** (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. -- **`multi_class`** (`str`): Only used for multiclass targets, in which case it is required. Determines the type of configuration to use. Options are: - - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. - - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. -- **`labels`** (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in `prediction_scores` is used. Defaults to `None`. - -### Output Values -This metric returns a dict containing the `roc_auc` score. The score is a `float`, unless it is the multilabel case with `average=None`, in which case the score is a numpy `array` with entries of type `float`. - -The output therefore generally takes the following format: -```python -{'roc_auc': 0.778} -``` - -In contrast, though, the output takes the following format in the multilabel case when `average=None`: -```python -{'roc_auc': array([0.83333333, 0.375, 0.94444444])} -``` - -ROC AUC scores can take on any value between `0` and `1`, inclusive. - -#### Values from Popular Papers - - -### Examples -Example 1, the **binary** use case: -```python ->>> roc_auc_score = datasets.load_metric("roc_auc") ->>> refs = [1, 0, 1, 1, 0, 0] ->>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] ->>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) ->>> print(round(results['roc_auc'], 2)) -0.78 -``` - -Example 2, the **multiclass** use case: -```python ->>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass") ->>> refs = [1, 0, 1, 2, 2, 0] ->>> pred_scores = [[0.3, 0.5, 0.2], -... [0.7, 0.2, 0.1], -... [0.005, 0.99, 0.005], -... [0.2, 0.3, 0.5], -... [0.1, 0.1, 0.8], -... [0.1, 0.7, 0.2]] ->>> results = roc_auc_score.compute(references=refs, -... prediction_scores=pred_scores, -... multi_class='ovr') ->>> print(round(results['roc_auc'], 2)) -0.85 -``` - -Example 3, the **multilabel** use case: -```python ->>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel") ->>> refs = [[1, 1, 0], -... [1, 1, 0], -... [0, 1, 0], -... [0, 0, 1], -... [0, 1, 1], -... [1, 0, 1]] ->>> pred_scores = [[0.3, 0.5, 0.2], -... [0.7, 0.2, 0.1], -... [0.005, 0.99, 0.005], -... [0.2, 0.3, 0.5], -... [0.1, 0.1, 0.8], -... [0.1, 0.7, 0.2]] ->>> results = roc_auc_score.compute(references=refs, -... prediction_scores=pred_scores, -... average=None) ->>> print([round(res, 2) for res in results['roc_auc']) -[0.83, 0.38, 0.94] -``` - - -## Limitations and Bias - - -## Citation -```bibtex -@article{doi:10.1177/0272989X8900900307, -author = {Donna Katzman McClish}, -title ={Analyzing a Portion of the ROC Curve}, -journal = {Medical Decision Making}, -volume = {9}, -number = {3}, -pages = {190-195}, -year = {1989}, -doi = {10.1177/0272989X8900900307}, - note ={PMID: 2668680}, -URL = {https://doi.org/10.1177/0272989X8900900307}, -eprint = {https://doi.org/10.1177/0272989X8900900307} -} -``` - -```bibtex -@article{10.1023/A:1010920819831, -author = {Hand, David J. and Till, Robert J.}, -title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, -year = {2001}, -issue_date = {November 2001}, -publisher = {Kluwer Academic Publishers}, -address = {USA}, -volume = {45}, -number = {2}, -issn = {0885-6125}, -url = {https://doi.org/10.1023/A:1010920819831}, -doi = {10.1023/A:1010920819831}, -journal = {Mach. Learn.}, -month = {oct}, -pages = {171–186}, -numpages = {16}, -keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} -} -``` - -```bibtex -@article{scikit-learn, -title={Scikit-learn: Machine Learning in {P}ython}, -author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, -journal={Journal of Machine Learning Research}, -volume={12}, -pages={2825--2830}, -year={2011} -} -``` - -## Further References -This implementation is a wrapper around the [Scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html). Much of the documentation here was adapted from their existing documentation, as well. - -The [Guide to ROC and AUC](https://youtu.be/iCZJfO-7C5Q) video from the channel Data Science Bits is also very informative. diff --git a/metrics/roc_auc/roc_auc.py b/metrics/roc_auc/roc_auc.py deleted file mode 100644 index 8acf2575fa3..00000000000 --- a/metrics/roc_auc/roc_auc.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Accuracy metric.""" - -from sklearn.metrics import roc_auc_score - -import datasets - - -_DESCRIPTION = """ -This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. - -This metric has three separate use cases: - - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. - - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label. - - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label. -""" - -_KWARGS_DESCRIPTION = """ -Args: -- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case: - - binary: expects an array-like of shape (n_samples,) - - multiclass: expects an array-like of shape (n_samples,) - - multilabel: expects an array-like of shape (n_samples, n_classes) -- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: - - binary: expects an array-like of shape (n_samples,) - - multiclass: expects an array-like of shape (n_samples, n_classes) - - multilabel: expects an array-like of shape (n_samples, n_classes) -- average (`str`): Type of average, and is ignored in the binary use case. Defaults to 'macro'. Options are: - - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. - - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. - - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). - - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. - - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. -- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None. -- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. -- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are: - - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. - - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. -- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in - `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in - `prediction_scores` is used. Defaults to `None`. -Returns: - roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average='None'`. Otherwise, returns `float`. -Examples: - Example 1: - >>> roc_auc_score = datasets.load_metric("roc_auc") - >>> refs = [1, 0, 1, 1, 0, 0] - >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] - >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) - >>> print(round(results['roc_auc'], 2)) - 0.78 - - Example 2: - >>> roc_auc_score = datasets.load_metric("roc_auc", "multiclass") - >>> refs = [1, 0, 1, 2, 2, 0] - >>> pred_scores = [[0.3, 0.5, 0.2], - ... [0.7, 0.2, 0.1], - ... [0.005, 0.99, 0.005], - ... [0.2, 0.3, 0.5], - ... [0.1, 0.1, 0.8], - ... [0.1, 0.7, 0.2]] - >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class='ovr') - >>> print(round(results['roc_auc'], 2)) - 0.85 - - Example 3: - >>> roc_auc_score = datasets.load_metric("roc_auc", "multilabel") - >>> refs = [[1, 1, 0], - ... [1, 1, 0], - ... [0, 1, 0], - ... [0, 0, 1], - ... [0, 1, 1], - ... [1, 0, 1]] - >>> pred_scores = [[0.3, 0.5, 0.2], - ... [0.7, 0.2, 0.1], - ... [0.005, 0.99, 0.005], - ... [0.2, 0.3, 0.5], - ... [0.1, 0.1, 0.8], - ... [0.1, 0.7, 0.2]] - >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None) - >>> print([round(res, 2) for res in results['roc_auc']]) - [0.83, 0.38, 0.94] -""" - -_CITATION = """\ -@article{doi:10.1177/0272989X8900900307, -author = {Donna Katzman McClish}, -title ={Analyzing a Portion of the ROC Curve}, -journal = {Medical Decision Making}, -volume = {9}, -number = {3}, -pages = {190-195}, -year = {1989}, -doi = {10.1177/0272989X8900900307}, - note ={PMID: 2668680}, -URL = {https://doi.org/10.1177/0272989X8900900307}, -eprint = {https://doi.org/10.1177/0272989X8900900307} -} - - -@article{10.1023/A:1010920819831, -author = {Hand, David J. and Till, Robert J.}, -title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, -year = {2001}, -issue_date = {November 2001}, -publisher = {Kluwer Academic Publishers}, -address = {USA}, -volume = {45}, -number = {2}, -issn = {0885-6125}, -url = {https://doi.org/10.1023/A:1010920819831}, -doi = {10.1023/A:1010920819831}, -journal = {Mach. Learn.}, -month = {oct}, -pages = {171–186}, -numpages = {16}, -keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} -} - - -@article{scikit-learn, -title={Scikit-learn: Machine Learning in {P}ython}, -author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, -journal={Journal of Machine Learning Research}, -volume={12}, -pages={2825--2830}, -year={2011} -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class ROCAUC(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "prediction_scores": datasets.Sequence(datasets.Value("float")), - "references": datasets.Value("int32"), - } - if self.config_name == "multiclass" - else { - "references": datasets.Sequence(datasets.Value("int32")), - "prediction_scores": datasets.Sequence(datasets.Value("float")), - } - if self.config_name == "multilabel" - else { - "references": datasets.Value("int32"), - "prediction_scores": datasets.Value("float"), - } - ), - reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html"], - ) - - def _compute( - self, - references, - prediction_scores, - average="macro", - sample_weight=None, - max_fpr=None, - multi_class="raise", - labels=None, - ): - return { - "roc_auc": roc_auc_score( - references, - prediction_scores, - average=average, - sample_weight=sample_weight, - max_fpr=max_fpr, - multi_class=multi_class, - labels=labels, - ) - } diff --git a/metrics/rouge/README.md b/metrics/rouge/README.md deleted file mode 100644 index 3bfb569cdf4..00000000000 --- a/metrics/rouge/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# Metric Card for ROUGE - -## Metric Description -ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. - -Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. - -This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) - -## How to Use -At minimum, this metric takes as input a list of predictions and a list of references: -```python ->>> rouge = datasets.load_metric('rouge') ->>> predictions = ["hello there", "general kenobi"] ->>> references = ["hello there", "general kenobi"] ->>> results = rouge.compute(predictions=predictions, -... references=references) ->>> print(list(results.keys())) -['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] ->>> print(results["rouge1"]) -AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0)) ->>> print(results["rouge1"].mid.fmeasure) -1.0 -``` - -### Inputs -- **predictions** (`list`): list of predictions to score. Each prediction - should be a string with tokens separated by spaces. -- **references** (`list`): list of reference for each prediction. Each - reference should be a string with tokens separated by spaces. -- **rouge_types** (`list`): A list of rouge types to calculate. Defaults to `['rouge1', 'rouge2', 'rougeL', 'rougeLsum']`. - - Valid rouge types: - - `"rouge1"`: unigram (1-gram) based scoring - - `"rouge2"`: bigram (2-gram) based scoring - - `"rougeL"`: Longest common subsequence based scoring. - - `"rougeLSum"`: splits text using `"\n"` - - See [here](https://github.com/huggingface/datasets/issues/617) for more information -- **use_aggregator** (`boolean`): If True, returns aggregates. Defaults to `True`. -- **use_stemmer** (`boolean`): If `True`, uses Porter stemmer to strip word suffixes. Defaults to `False`. - -### Output Values -The output is a dictionary with one entry for each rouge type in the input list `rouge_types`. If `use_aggregator=False`, each dictionary entry is a list of Score objects, with one score for each sentence. Each Score object includes the `precision`, `recall`, and `fmeasure`. E.g. if `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=False`, the output is: - -```python -{'rouge1': [Score(precision=1.0, recall=0.5, fmeasure=0.6666666666666666), Score(precision=1.0, recall=1.0, fmeasure=1.0)], 'rouge2': [Score(precision=0.0, recall=0.0, fmeasure=0.0), Score(precision=1.0, recall=1.0, fmeasure=1.0)]} -``` - -If `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=True`, the output is of the following format: -```python -{'rouge1': AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0)), 'rouge2': AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0))} -``` - -The `precision`, `recall`, and `fmeasure` values all have a range of 0 to 1. - - -#### Values from Popular Papers - - -### Examples -An example without aggregation: -```python ->>> rouge = datasets.load_metric('rouge') ->>> predictions = ["hello goodbye", "ankh morpork"] ->>> references = ["goodbye", "general kenobi"] ->>> results = rouge.compute(predictions=predictions, -... references=references) ->>> print(list(results.keys())) -['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] ->>> print(results["rouge1"]) -[Score(precision=0.5, recall=0.5, fmeasure=0.5), Score(precision=0.0, recall=0.0, fmeasure=0.0)] -``` - -The same example, but with aggregation: -```python ->>> rouge = datasets.load_metric('rouge') ->>> predictions = ["hello goodbye", "ankh morpork"] ->>> references = ["goodbye", "general kenobi"] ->>> results = rouge.compute(predictions=predictions, -... references=references, -... use_aggregator=True) ->>> print(list(results.keys())) -['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] ->>> print(results["rouge1"]) -AggregateScore(low=Score(precision=0.0, recall=0.0, fmeasure=0.0), mid=Score(precision=0.25, recall=0.25, fmeasure=0.25), high=Score(precision=0.5, recall=0.5, fmeasure=0.5)) -``` - -The same example, but only calculating `rouge_1`: -```python ->>> rouge = datasets.load_metric('rouge') ->>> predictions = ["hello goodbye", "ankh morpork"] ->>> references = ["goodbye", "general kenobi"] ->>> results = rouge.compute(predictions=predictions, -... references=references, -... rouge_types=['rouge_1'], -... use_aggregator=True) ->>> print(list(results.keys())) -['rouge1'] ->>> print(results["rouge1"]) -AggregateScore(low=Score(precision=0.0, recall=0.0, fmeasure=0.0), mid=Score(precision=0.25, recall=0.25, fmeasure=0.25), high=Score(precision=0.5, recall=0.5, fmeasure=0.5)) -``` - -## Limitations and Bias -See [Schluter (2017)](https://aclanthology.org/E17-2007/) for an in-depth discussion of many of ROUGE's limits. - -## Citation -```bibtex -@inproceedings{lin-2004-rouge, - title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", - author = "Lin, Chin-Yew", - booktitle = "Text Summarization Branches Out", - month = jul, - year = "2004", - address = "Barcelona, Spain", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W04-1013", - pages = "74--81", -} -``` - -## Further References -- This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) \ No newline at end of file diff --git a/metrics/rouge/rouge.py b/metrics/rouge/rouge.py deleted file mode 100644 index 0d821647c25..00000000000 --- a/metrics/rouge/rouge.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""ROUGE metric from Google Research github repo.""" - -# The dependencies in https://github.com/google-research/google-research/blob/master/rouge/requirements.txt -import absl # noqa: F401 # Here to have a nice missing dependency error message early on -import nltk # noqa: F401 # Here to have a nice missing dependency error message early on -import numpy # noqa: F401 # Here to have a nice missing dependency error message early on -import six # noqa: F401 # Here to have a nice missing dependency error message early on -from rouge_score import rouge_scorer, scoring - -import datasets - - -_CITATION = """\ -@inproceedings{lin-2004-rouge, - title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", - author = "Lin, Chin-Yew", - booktitle = "Text Summarization Branches Out", - month = jul, - year = "2004", - address = "Barcelona, Spain", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W04-1013", - pages = "74--81", -} -""" - -_DESCRIPTION = """\ -ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for -evaluating automatic summarization and machine translation software in natural language processing. -The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. - -Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. - -This metrics is a wrapper around Google Research reimplementation of ROUGE: -https://github.com/google-research/google-research/tree/master/rouge -""" - -_KWARGS_DESCRIPTION = """ -Calculates average rouge scores for a list of hypotheses and references -Args: - predictions: list of predictions to score. Each prediction - should be a string with tokens separated by spaces. - references: list of reference for each prediction. Each - reference should be a string with tokens separated by spaces. - rouge_types: A list of rouge types to calculate. - Valid names: - `"rouge{n}"` (e.g. `"rouge1"`, `"rouge2"`) where: {n} is the n-gram based scoring, - `"rougeL"`: Longest common subsequence based scoring. - `"rougeLSum"`: rougeLsum splits text using `"\n"`. - See details in https://github.com/huggingface/datasets/issues/617 - use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes. - use_aggregator: Return aggregates if this is set to True -Returns: - rouge1: rouge_1 (precision, recall, f1), - rouge2: rouge_2 (precision, recall, f1), - rougeL: rouge_l (precision, recall, f1), - rougeLsum: rouge_lsum (precision, recall, f1) -Examples: - - >>> rouge = datasets.load_metric('rouge') - >>> predictions = ["hello there", "general kenobi"] - >>> references = ["hello there", "general kenobi"] - >>> results = rouge.compute(predictions=predictions, references=references) - >>> print(list(results.keys())) - ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] - >>> print(results["rouge1"]) - AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0)) - >>> print(results["rouge1"].mid.fmeasure) - 1.0 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Rouge(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"], - reference_urls=[ - "https://en.wikipedia.org/wiki/ROUGE_(metric)", - "https://github.com/google-research/google-research/tree/master/rouge", - ], - ) - - def _compute(self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False): - if rouge_types is None: - rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"] - - scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer) - if use_aggregator: - aggregator = scoring.BootstrapAggregator() - else: - scores = [] - - for ref, pred in zip(references, predictions): - score = scorer.score(ref, pred) - if use_aggregator: - aggregator.add_scores(score) - else: - scores.append(score) - - if use_aggregator: - result = aggregator.aggregate() - else: - result = {} - for key in scores[0]: - result[key] = [score[key] for score in scores] - - return result diff --git a/metrics/sacrebleu/README.md b/metrics/sacrebleu/README.md deleted file mode 100644 index e2ea7fb24b7..00000000000 --- a/metrics/sacrebleu/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Metric Card for SacreBLEU - - -## Metric Description -SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official Workshop on Machine Translation (WMT) scores but works with plain text. It also knows all the standard test sets and handles downloading, processing, and tokenization. - -See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. - -## How to Use -This metric takes a set of predictions and a set of references as input, along with various optional parameters. - - -```python ->>> predictions = ["hello there general kenobi", "foo bar foobar"] ->>> references = [["hello there general kenobi", "hello there !"], -... ["foo bar foobar", "foo bar foobar"]] ->>> sacrebleu = datasets.load_metric("sacrebleu") ->>> results = sacrebleu.compute(predictions=predictions, -... references=references) ->>> print(list(results.keys())) -['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] ->>> print(round(results["score"], 1)) -100.0 -``` - -### Inputs -- **`predictions`** (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. -- **`references`** (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). -- **`smooth_method`** (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: - - `'none'`: no smoothing - - `'floor'`: increment zero counts - - `'add-k'`: increment num/denom by k for n>1 - - `'exp'`: exponential decay -- **`smooth_value`** (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). -- **`tokenize`** (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: - - `'none'`: No tokenization. - - `'zh'`: Chinese tokenization. - - `'13a'`: mimics the `mteval-v13a` script from Moses. - - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses - - `'char'`: Language-agnostic character-level tokenization. - - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). -- **`lowercase`** (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. -- **`force`** (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. -- **`use_effective_order`** (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. - -### Output Values -- `score`: BLEU score -- `counts`: Counts -- `totals`: Totals -- `precisions`: Precisions -- `bp`: Brevity penalty -- `sys_len`: predictions length -- `ref_len`: reference length - -The output is in the following format: -```python -{'score': 39.76353643835252, 'counts': [6, 4, 2, 1], 'totals': [10, 8, 6, 4], 'precisions': [60.0, 50.0, 33.333333333333336, 25.0], 'bp': 1.0, 'sys_len': 10, 'ref_len': 7} -``` -The score can take any value between `0.0` and `100.0`, inclusive. - -#### Values from Popular Papers - - -### Examples - -```python ->>> predictions = ["hello there general kenobi", -... "on our way to ankh morpork"] ->>> references = [["hello there general kenobi", "hello there !"], -... ["goodbye ankh morpork", "ankh morpork"]] ->>> sacrebleu = datasets.load_metric("sacrebleu") ->>> results = sacrebleu.compute(predictions=predictions, -... references=references) ->>> print(list(results.keys())) -['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] ->>> print(round(results["score"], 1)) -39.8 -``` - -## Limitations and Bias -Because what this metric calculates is BLEU scores, it has the same limitations as that metric, except that sacreBLEU is more easily reproducible. - -## Citation -```bibtex -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -``` - -## Further References -- See the [sacreBLEU README.md file](https://github.com/mjpost/sacreBLEU) for more information. \ No newline at end of file diff --git a/metrics/sacrebleu/sacrebleu.py b/metrics/sacrebleu/sacrebleu.py deleted file mode 100644 index b5117b0871f..00000000000 --- a/metrics/sacrebleu/sacrebleu.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""SACREBLEU metric.""" - -import sacrebleu as scb -from packaging import version - -import datasets - - -_CITATION = """\ -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -""" - -_DESCRIPTION = """\ -SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. -Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text. -It also knows all the standard test sets and handles downloading, processing, and tokenization for you. - -See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. -""" - -_KWARGS_DESCRIPTION = """ -Produces BLEU scores along with its sufficient statistics -from a source against one or more references. - -Args: - predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. - references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). - smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: - - `'none'`: no smoothing - - `'floor'`: increment zero counts - - `'add-k'`: increment num/denom by k for n>1 - - `'exp'`: exponential decay - smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). - tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: - - `'none'`: No tokenization. - - `'zh'`: Chinese tokenization. - - `'13a'`: mimics the `mteval-v13a` script from Moses. - - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses - - `'char'`: Language-agnostic character-level tokenization. - - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). - lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. - force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. - use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. - -Returns: - 'score': BLEU score, - 'counts': Counts, - 'totals': Totals, - 'precisions': Precisions, - 'bp': Brevity penalty, - 'sys_len': predictions length, - 'ref_len': reference length, - -Examples: - - Example 1: - >>> predictions = ["hello there general kenobi", "foo bar foobar"] - >>> references = [["hello there general kenobi", "hello there !"], ["foo bar foobar", "foo bar foobar"]] - >>> sacrebleu = datasets.load_metric("sacrebleu") - >>> results = sacrebleu.compute(predictions=predictions, references=references) - >>> print(list(results.keys())) - ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] - >>> print(round(results["score"], 1)) - 100.0 - - Example 2: - >>> predictions = ["hello there general kenobi", - ... "on our way to ankh morpork"] - >>> references = [["hello there general kenobi", "hello there !"], - ... ["goodbye ankh morpork", "ankh morpork"]] - >>> sacrebleu = datasets.load_metric("sacrebleu") - >>> results = sacrebleu.compute(predictions=predictions, - ... references=references) - >>> print(list(results.keys())) - ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] - >>> print(round(results["score"], 1)) - 39.8 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Sacrebleu(datasets.Metric): - def _info(self): - if version.parse(scb.__version__) < version.parse("1.4.12"): - raise ImportWarning( - "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" - 'You can install it with `pip install "sacrebleu>=1.4.12"`.' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/mjpost/sacreBLEU", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=["https://github.com/mjpost/sacreBLEU"], - reference_urls=[ - "https://github.com/mjpost/sacreBLEU", - "https://en.wikipedia.org/wiki/BLEU", - "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", - ], - ) - - def _compute( - self, - predictions, - references, - smooth_method="exp", - smooth_value=None, - force=False, - lowercase=False, - tokenize=None, - use_effective_order=False, - ): - references_per_prediction = len(references[0]) - if any(len(refs) != references_per_prediction for refs in references): - raise ValueError("Sacrebleu requires the same number of references for each prediction") - transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] - output = scb.corpus_bleu( - predictions, - transformed_references, - smooth_method=smooth_method, - smooth_value=smooth_value, - force=force, - lowercase=lowercase, - use_effective_order=use_effective_order, - **({"tokenize": tokenize} if tokenize else {}), - ) - output_dict = { - "score": output.score, - "counts": output.counts, - "totals": output.totals, - "precisions": output.precisions, - "bp": output.bp, - "sys_len": output.sys_len, - "ref_len": output.ref_len, - } - return output_dict diff --git a/metrics/sari/README.md b/metrics/sari/README.md deleted file mode 100644 index d5292b5b779..00000000000 --- a/metrics/sari/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# Metric Card for SARI - - -## Metric description -SARI (***s**ystem output **a**gainst **r**eferences and against the **i**nput sentence*) is a metric used for evaluating automatic text simplification systems. - -The metric compares the predicted simplified sentences against the reference and the source sentences. It explicitly measures the goodness of words that are added, deleted and kept by the system. - -SARI can be computed as: - -`sari = ( F1_add + F1_keep + P_del) / 3` - -where - -`F1_add` is the n-gram F1 score for add operations - -`F1_keep` is the n-gram F1 score for keep operations - -`P_del` is the n-gram precision score for delete operations - -The number of n grams, `n`, is equal to 4, as in the original paper. - -This implementation is adapted from [Tensorflow's tensor2tensor implementation](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py). -It has two differences with the [original GitHub implementation](https://github.com/cocoxu/simplification/blob/master/SARI.py): - -1) It defines 0/0=1 instead of 0 to give higher scores for predictions that match a target exactly. -2) It fixes an [alleged bug](https://github.com/cocoxu/simplification/issues/6) in the keep score computation. - - - -## How to use - -The metric takes 3 inputs: sources (a list of source sentence strings), predictions (a list of predicted sentence strings) and references (a list of lists of reference sentence strings) - -```python -from datasets import load_metric -sari = load_metric("sari") -sources=["About 95 species are currently accepted."] -predictions=["About 95 you now get in."] -references=[["About 95 species are currently known.","About 95 species are now accepted.","95 species are now accepted."]] -sari_score = sari.compute(sources=sources, predictions=predictions, references=references) -``` -## Output values - -This metric outputs a dictionary with the SARI score: - -``` -print(sari_score) -{'sari': 26.953601953601954} -``` - -The range of values for the SARI score is between 0 and 100 -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. - -### Values from popular papers - -The [original paper that proposes the SARI metric](https://aclanthology.org/Q16-1029.pdf) reports scores ranging from 26 to 43 for different simplification systems and different datasets. They also find that the metric ranks all of the simplification systems and human references in the same order as the human assessment used as a comparison, and that it correlates reasonably with human judgments. - -More recent SARI scores for text simplification can be found on leaderboards for datasets such as [TurkCorpus](https://paperswithcode.com/sota/text-simplification-on-turkcorpus) and [Newsela](https://paperswithcode.com/sota/text-simplification-on-newsela). - -## Examples - -Perfect match between prediction and reference: - -```python -from datasets import load_metric -sari = load_metric("sari") -sources=["About 95 species are currently accepted ."] -predictions=["About 95 species are currently accepted ."] -references=[["About 95 species are currently accepted ."]] -sari_score = sari.compute(sources=sources, predictions=predictions, references=references) -print(sari_score) -{'sari': 100.0} -``` - -Partial match between prediction and reference: - -```python -from datasets import load_metric -sari = load_metric("sari") -sources=["About 95 species are currently accepted ."] -predictions=["About 95 you now get in ."] -references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] -sari_score = sari.compute(sources=sources, predictions=predictions, references=references) -print(sari_score) -{'sari': 26.953601953601954} -``` - -## Limitations and bias - -SARI is a valuable measure for comparing different text simplification systems as well as one that can assist the iterative development of a system. - -However, while the [original paper presenting SARI](https://aclanthology.org/Q16-1029.pdf) states that it captures "the notion of grammaticality and meaning preservation", this is a difficult claim to empirically validate. - -## Citation - -```bibtex -@inproceedings{xu-etal-2016-optimizing, -title = {Optimizing Statistical Machine Translation for Text Simplification}, -authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, -journal = {Transactions of the Association for Computational Linguistics}, -volume = {4}, -year={2016}, -url = {https://www.aclweb.org/anthology/Q16-1029}, -pages = {401--415}, -} -``` - -## Further References - -- [NLP Progress -- Text Simplification](http://nlpprogress.com/english/simplification.html) -- [Hugging Face Hub -- Text Simplification Models](https://huggingface.co/datasets?filter=task_ids:text-simplification) diff --git a/metrics/sari/sari.py b/metrics/sari/sari.py deleted file mode 100644 index c492fc047d0..00000000000 --- a/metrics/sari/sari.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""SARI metric.""" - -from collections import Counter - -import sacrebleu -import sacremoses -from packaging import version - -import datasets - - -_CITATION = """\ -@inproceedings{xu-etal-2016-optimizing, -title = {Optimizing Statistical Machine Translation for Text Simplification}, -authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, -journal = {Transactions of the Association for Computational Linguistics}, -volume = {4}, -year={2016}, -url = {https://www.aclweb.org/anthology/Q16-1029}, -pages = {401--415}, -} -""" - -_DESCRIPTION = """\ -SARI is a metric used for evaluating automatic text simplification systems. -The metric compares the predicted simplified sentences against the reference -and the source sentences. It explicitly measures the goodness of words that are -added, deleted and kept by the system. -Sari = (F1_add + F1_keep + P_del) / 3 -where -F1_add: n-gram F1 score for add operation -F1_keep: n-gram F1 score for keep operation -P_del: n-gram precision score for delete operation -n = 4, as in the original paper. - -This implementation is adapted from Tensorflow's tensor2tensor implementation [3]. -It has two differences with the original GitHub [1] implementation: - (1) Defines 0/0=1 instead of 0 to give higher scores for predictions that match - a target exactly. - (2) Fixes an alleged bug [2] in the keep score computation. -[1] https://github.com/cocoxu/simplification/blob/master/SARI.py - (commit 0210f15) -[2] https://github.com/cocoxu/simplification/issues/6 -[3] https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py -""" - - -_KWARGS_DESCRIPTION = """ -Calculates sari score (between 0 and 100) given a list of source and predicted -sentences, and a list of lists of reference sentences. -Args: - sources: list of source sentences where each sentence should be a string. - predictions: list of predicted sentences where each sentence should be a string. - references: list of lists of reference sentences where each sentence should be a string. -Returns: - sari: sari score -Examples: - >>> sources=["About 95 species are currently accepted ."] - >>> predictions=["About 95 you now get in ."] - >>> references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] - >>> sari = datasets.load_metric("sari") - >>> results = sari.compute(sources=sources, predictions=predictions, references=references) - >>> print(results) - {'sari': 26.953601953601954} -""" - - -def SARIngram(sgrams, cgrams, rgramslist, numref): - rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] - rgramcounter = Counter(rgramsall) - - sgramcounter = Counter(sgrams) - sgramcounter_rep = Counter() - for sgram, scount in sgramcounter.items(): - sgramcounter_rep[sgram] = scount * numref - - cgramcounter = Counter(cgrams) - cgramcounter_rep = Counter() - for cgram, ccount in cgramcounter.items(): - cgramcounter_rep[cgram] = ccount * numref - - # KEEP - keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep - keepgramcountergood_rep = keepgramcounter_rep & rgramcounter - keepgramcounterall_rep = sgramcounter_rep & rgramcounter - - keeptmpscore1 = 0 - keeptmpscore2 = 0 - for keepgram in keepgramcountergood_rep: - keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] - # Fix an alleged bug [2] in the keep score computation. - # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] - keeptmpscore2 += keepgramcountergood_rep[keepgram] - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - keepscore_precision = 1 - keepscore_recall = 1 - if len(keepgramcounter_rep) > 0: - keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) - if len(keepgramcounterall_rep) > 0: - # Fix an alleged bug [2] in the keep score computation. - # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) - keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) - keepscore = 0 - if keepscore_precision > 0 or keepscore_recall > 0: - keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) - - # DELETION - delgramcounter_rep = sgramcounter_rep - cgramcounter_rep - delgramcountergood_rep = delgramcounter_rep - rgramcounter - delgramcounterall_rep = sgramcounter_rep - rgramcounter - deltmpscore1 = 0 - deltmpscore2 = 0 - for delgram in delgramcountergood_rep: - deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] - deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - delscore_precision = 1 - if len(delgramcounter_rep) > 0: - delscore_precision = deltmpscore1 / len(delgramcounter_rep) - - # ADDITION - addgramcounter = set(cgramcounter) - set(sgramcounter) - addgramcountergood = set(addgramcounter) & set(rgramcounter) - addgramcounterall = set(rgramcounter) - set(sgramcounter) - - addtmpscore = 0 - for addgram in addgramcountergood: - addtmpscore += 1 - - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - addscore_precision = 1 - addscore_recall = 1 - if len(addgramcounter) > 0: - addscore_precision = addtmpscore / len(addgramcounter) - if len(addgramcounterall) > 0: - addscore_recall = addtmpscore / len(addgramcounterall) - addscore = 0 - if addscore_precision > 0 or addscore_recall > 0: - addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) - - return (keepscore, delscore_precision, addscore) - - -def SARIsent(ssent, csent, rsents): - numref = len(rsents) - - s1grams = ssent.split(" ") - c1grams = csent.split(" ") - s2grams = [] - c2grams = [] - s3grams = [] - c3grams = [] - s4grams = [] - c4grams = [] - - r1gramslist = [] - r2gramslist = [] - r3gramslist = [] - r4gramslist = [] - for rsent in rsents: - r1grams = rsent.split(" ") - r2grams = [] - r3grams = [] - r4grams = [] - r1gramslist.append(r1grams) - for i in range(0, len(r1grams) - 1): - if i < len(r1grams) - 1: - r2gram = r1grams[i] + " " + r1grams[i + 1] - r2grams.append(r2gram) - if i < len(r1grams) - 2: - r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] - r3grams.append(r3gram) - if i < len(r1grams) - 3: - r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] - r4grams.append(r4gram) - r2gramslist.append(r2grams) - r3gramslist.append(r3grams) - r4gramslist.append(r4grams) - - for i in range(0, len(s1grams) - 1): - if i < len(s1grams) - 1: - s2gram = s1grams[i] + " " + s1grams[i + 1] - s2grams.append(s2gram) - if i < len(s1grams) - 2: - s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] - s3grams.append(s3gram) - if i < len(s1grams) - 3: - s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] - s4grams.append(s4gram) - - for i in range(0, len(c1grams) - 1): - if i < len(c1grams) - 1: - c2gram = c1grams[i] + " " + c1grams[i + 1] - c2grams.append(c2gram) - if i < len(c1grams) - 2: - c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] - c3grams.append(c3gram) - if i < len(c1grams) - 3: - c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] - c4grams.append(c4gram) - - (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) - (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) - (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) - (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) - avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 - avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 - avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 - finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 - return finalscore - - -def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): - # Normalization is requried for the ASSET dataset (one of the primary - # datasets in sentence simplification) to allow using space - # to split the sentence. Even though Wiki-Auto and TURK datasets, - # do not require normalization, we do it for consistency. - # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. - # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 - - if lowercase: - sentence = sentence.lower() - - if tokenizer in ["13a", "intl"]: - if version.parse(sacrebleu.__version__).major >= 2: - normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) - else: - normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) - elif tokenizer == "moses": - normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) - elif tokenizer == "penn": - normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) - else: - normalized_sent = sentence - - if not return_str: - normalized_sent = normalized_sent.split() - - return normalized_sent - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Sari(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "sources": datasets.Value("string", id="sequence"), - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=[ - "https://github.com/cocoxu/simplification/blob/master/SARI.py", - "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", - ], - reference_urls=["https://www.aclweb.org/anthology/Q16-1029.pdf"], - ) - - def _compute(self, sources, predictions, references): - if not (len(sources) == len(predictions) == len(references)): - raise ValueError("Sources length must match predictions and references lengths.") - sari_score = 0 - for src, pred, refs in zip(sources, predictions, references): - sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) - sari_score = sari_score / len(predictions) - return {"sari": 100 * sari_score} diff --git a/metrics/seqeval/README.md b/metrics/seqeval/README.md deleted file mode 100644 index 00026821879..00000000000 --- a/metrics/seqeval/README.md +++ /dev/null @@ -1,139 +0,0 @@ -# Metric Card for seqeval - -## Metric description - -seqeval is a Python framework for sequence labeling evaluation. seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. - - -## How to use - -Seqeval produces labelling scores along with its sufficient statistics from a source against one or more references. - -It takes two mandatory arguments: - -`predictions`: a list of lists of predicted labels, i.e. estimated targets as returned by a tagger. - -`references`: a list of lists of reference labels, i.e. the ground truth/target values. - -It can also take several optional arguments: - -`suffix` (boolean): `True` if the IOB tag is a suffix (after type) instead of a prefix (before type), `False` otherwise. The default value is `False`, i.e. the IOB tag is a prefix (before type). - -`scheme`: the target tagging scheme, which can be one of [`IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `BILOU`]. The default value is `None`. - -`mode`: whether to count correct entity labels with incorrect I/B tags as true positives or not. If you want to only count exact matches, pass `mode="strict"` and a specific `scheme` value. The default is `None`. - -`sample_weight`: An array-like of shape (n_samples,) that provides weights for individual samples. The default is `None`. - -`zero_division`: Which value to substitute as a metric value when encountering zero division. Should be one of [`0`,`1`,`"warn"`]. `"warn"` acts as `0`, but the warning is raised. - - -```python ->>> from datasets import load_metric ->>> seqeval = load_metric('seqeval') ->>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> results = seqeval.compute(predictions=predictions, references=references) -``` - -## Output values - -This metric returns a dictionary with a summary of scores for overall and per type: - -Overall: - -`accuracy`: the average [accuracy](https://huggingface.co/metrics/accuracy), on a scale between 0.0 and 1.0. - -`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. - -`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. - -`f1`: the average [F1 score](https://huggingface.co/metrics/f1), which is the harmonic mean of the precision and recall. It also has a scale of 0.0 to 1.0. - -Per type (e.g. `MISC`, `PER`, `LOC`,...): - -`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. - -`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. - -`f1`: the average [F1 score](https://huggingface.co/metrics/f1), on a scale between 0.0 and 1.0. - - -### Values from popular papers -The 1995 "Text Chunking using Transformation-Based Learning" [paper](https://aclanthology.org/W95-0107) reported a baseline recall of 81.9% and a precision of 78.2% using non Deep Learning-based methods. - -More recently, seqeval continues being used for reporting performance on tasks such as [named entity detection](https://www.mdpi.com/2306-5729/6/8/84/htm) and [information extraction](https://ieeexplore.ieee.org/abstract/document/9697942/). - - -## Examples - -Maximal values (full match) : - -```python ->>> from datasets import load_metric ->>> seqeval = load_metric('seqeval') ->>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> references = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> results = seqeval.compute(predictions=predictions, references=references) ->>> print(results) -{'MISC': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 1.0, 'overall_recall': 1.0, 'overall_f1': 1.0, 'overall_accuracy': 1.0} - -``` - -Minimal values (no match): - -```python ->>> from datasets import load_metric ->>> seqeval = load_metric('seqeval') ->>> predictions = [['O', 'B-MISC', 'I-MISC'], ['B-PER', 'I-PER', 'O']] ->>> references = [['B-MISC', 'O', 'O'], ['I-PER', '0', 'I-PER']] ->>> results = seqeval.compute(predictions=predictions, references=references) ->>> print(results) -{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 2}, '_': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'overall_precision': 0.0, 'overall_recall': 0.0, 'overall_f1': 0.0, 'overall_accuracy': 0.0} -``` - -Partial match: - -```python ->>> from datasets import load_metric ->>> seqeval = load_metric('seqeval') ->>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] ->>> results = seqeval.compute(predictions=predictions, references=references) ->>> print(results) -{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 0.5, 'overall_recall': 0.5, 'overall_f1': 0.5, 'overall_accuracy': 0.8} -``` - -## Limitations and bias - -seqeval supports following IOB formats (short for inside, outside, beginning) : `IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `IOBES` (only in strict mode) and `BILOU` (only in strict mode). - -For more information about IOB formats, refer to the [Wikipedia page](https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging)) and the description of the [CoNLL-2000 shared task](https://aclanthology.org/W02-2024). - - -## Citation - -```bibtex -@inproceedings{ramshaw-marcus-1995-text, - title = "Text Chunking using Transformation-Based Learning", - author = "Ramshaw, Lance and - Marcus, Mitch", - booktitle = "Third Workshop on Very Large Corpora", - year = "1995", - url = "https://www.aclweb.org/anthology/W95-0107", -} -``` - -```bibtex -@misc{seqeval, - title={{seqeval}: A Python framework for sequence labeling evaluation}, - url={https://github.com/chakki-works/seqeval}, - note={Software available from https://github.com/chakki-works/seqeval}, - author={Hiroki Nakayama}, - year={2018}, -} -``` - -## Further References -- [README for seqeval at GitHub](https://github.com/chakki-works/seqeval) -- [CoNLL-2000 shared task](https://www.clips.uantwerpen.be/conll2002/ner/bin/conlleval.txt) diff --git a/metrics/seqeval/seqeval.py b/metrics/seqeval/seqeval.py deleted file mode 100644 index e6b4f1b7603..00000000000 --- a/metrics/seqeval/seqeval.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""seqeval metric.""" - -import importlib -from typing import List, Optional, Union - -from seqeval.metrics import accuracy_score, classification_report - -import datasets - - -_CITATION = """\ -@inproceedings{ramshaw-marcus-1995-text, - title = "Text Chunking using Transformation-Based Learning", - author = "Ramshaw, Lance and - Marcus, Mitch", - booktitle = "Third Workshop on Very Large Corpora", - year = "1995", - url = "https://www.aclweb.org/anthology/W95-0107", -} -@misc{seqeval, - title={{seqeval}: A Python framework for sequence labeling evaluation}, - url={https://github.com/chakki-works/seqeval}, - note={Software available from https://github.com/chakki-works/seqeval}, - author={Hiroki Nakayama}, - year={2018}, -} -""" - -_DESCRIPTION = """\ -seqeval is a Python framework for sequence labeling evaluation. -seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. - -This is well-tested by using the Perl script conlleval, which can be used for -measuring the performance of a system that has processed the CoNLL-2000 shared task data. - -seqeval supports following formats: -IOB1 -IOB2 -IOE1 -IOE2 -IOBES - -See the [README.md] file at https://github.com/chakki-works/seqeval for more information. -""" - -_KWARGS_DESCRIPTION = """ -Produces labelling scores along with its sufficient statistics -from a source against one or more references. - -Args: - predictions: List of List of predicted labels (Estimated targets as returned by a tagger) - references: List of List of reference labels (Ground truth (correct) target values) - suffix: True if the IOB prefix is after type, False otherwise. default: False - scheme: Specify target tagging scheme. Should be one of ["IOB1", "IOB2", "IOE1", "IOE2", "IOBES", "BILOU"]. - default: None - mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not. - If you want to only count exact matches, pass mode="strict". default: None. - sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None - zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1, - "warn". "warn" acts as 0, but the warning is raised. - -Returns: - 'scores': dict. Summary of the scores for overall and per type - Overall: - 'accuracy': accuracy, - 'precision': precision, - 'recall': recall, - 'f1': F1 score, also known as balanced F-score or F-measure, - Per type: - 'precision': precision, - 'recall': recall, - 'f1': F1 score, also known as balanced F-score or F-measure -Examples: - - >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] - >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] - >>> seqeval = datasets.load_metric("seqeval") - >>> results = seqeval.compute(predictions=predictions, references=references) - >>> print(list(results.keys())) - ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy'] - >>> print(results["overall_f1"]) - 0.5 - >>> print(results["PER"]["f1"]) - 1.0 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Seqeval(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="https://github.com/chakki-works/seqeval", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), - } - ), - codebase_urls=["https://github.com/chakki-works/seqeval"], - reference_urls=["https://github.com/chakki-works/seqeval"], - ) - - def _compute( - self, - predictions, - references, - suffix: bool = False, - scheme: Optional[str] = None, - mode: Optional[str] = None, - sample_weight: Optional[List[int]] = None, - zero_division: Union[str, int] = "warn", - ): - if scheme is not None: - try: - scheme_module = importlib.import_module("seqeval.scheme") - scheme = getattr(scheme_module, scheme) - except AttributeError: - raise ValueError(f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}") - report = classification_report( - y_true=references, - y_pred=predictions, - suffix=suffix, - output_dict=True, - scheme=scheme, - mode=mode, - sample_weight=sample_weight, - zero_division=zero_division, - ) - report.pop("macro avg") - report.pop("weighted avg") - overall_score = report.pop("micro avg") - - scores = { - type_name: { - "precision": score["precision"], - "recall": score["recall"], - "f1": score["f1-score"], - "number": score["support"], - } - for type_name, score in report.items() - } - scores["overall_precision"] = overall_score["precision"] - scores["overall_recall"] = overall_score["recall"] - scores["overall_f1"] = overall_score["f1-score"] - scores["overall_accuracy"] = accuracy_score(y_true=references, y_pred=predictions) - - return scores diff --git a/metrics/spearmanr/README.md b/metrics/spearmanr/README.md deleted file mode 100644 index 007fb5d60c9..00000000000 --- a/metrics/spearmanr/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Metric Card for Spearman Correlation Coefficient Metric (spearmanr) - - -## Metric Description -The Spearman rank-order correlation coefficient is a measure of the -relationship between two datasets. Like other correlation coefficients, -this one varies between -1 and +1 with 0 implying no correlation. -Positive correlations imply that as data in dataset x increases, so -does data in dataset y. Negative correlations imply that as x increases, -y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. - -Unlike the Pearson correlation, the Spearman correlation does not -assume that both datasets are normally distributed. - -The p-value roughly indicates the probability of an uncorrelated system -producing datasets that have a Spearman correlation at least as extreme -as the one computed from these datasets. The p-values are not entirely -reliable but are probably reasonable for datasets larger than 500 or so. - - -## How to Use -At minimum, this metric only requires a `list` of predictions and a `list` of references: - -```python ->>> spearmanr_metric = datasets.load_metric("spearmanr") ->>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) ->>> print(results) -{'spearmanr': -0.7} -``` - -### Inputs -- **`predictions`** (`list` of `float`): Predicted labels, as returned by a model. -- **`references`** (`list` of `float`): Ground truth labels. -- **`return_pvalue`** (`bool`): If `True`, returns the p-value. If `False`, returns - only the spearmanr score. Defaults to `False`. - -### Output Values -- **`spearmanr`** (`float`): Spearman correlation coefficient. -- **`p-value`** (`float`): p-value. **Note**: is only returned - if `return_pvalue=True` is input. - -If `return_pvalue=False`, the output is a `dict` with one value, as below: -```python -{'spearmanr': -0.7} -``` - -Otherwise, if `return_pvalue=True`, the output is a `dict` containing a the `spearmanr` value as well as the corresponding `pvalue`: -```python -{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873} -``` - -Spearman rank-order correlations can take on any value from `-1` to `1`, inclusive. - -The p-values can take on any value from `0` to `1`, inclusive. - -#### Values from Popular Papers - - -### Examples -A basic example: -```python ->>> spearmanr_metric = datasets.load_metric("spearmanr") ->>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) ->>> print(results) -{'spearmanr': -0.7} -``` - -The same example, but that also returns the pvalue: -```python ->>> spearmanr_metric = datasets.load_metric("spearmanr") ->>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4], return_pvalue=True) ->>> print(results) -{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873 ->>> print(results['spearmanr']) --0.7 ->>> print(results['spearmanr_pvalue']) -0.1881204043741873 -``` - -## Limitations and Bias - - -## Citation -```bibtex -@book{kokoska2000crc, - title={CRC standard probability and statistics tables and formulae}, - author={Kokoska, Stephen and Zwillinger, Daniel}, - year={2000}, - publisher={Crc Press} -} -@article{2020SciPy-NMeth, - author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and - Haberland, Matt and Reddy, Tyler and Cournapeau, David and - Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and - Bright, Jonathan and {van der Walt}, St{\'e}fan J. and - Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and - Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and - Kern, Robert and Larson, Eric and Carey, C J and - Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and - {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and - Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and - Harris, Charles R. and Archibald, Anne M. and - Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and - {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, - title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific - Computing in Python}}, - journal = {Nature Methods}, - year = {2020}, - volume = {17}, - pages = {261--272}, - adsurl = {https://rdcu.be/b08Wh}, - doi = {10.1038/s41592-019-0686-2}, -} -``` - -## Further References -*Add any useful further references.* diff --git a/metrics/spearmanr/spearmanr.py b/metrics/spearmanr/spearmanr.py deleted file mode 100644 index bc5d4155101..00000000000 --- a/metrics/spearmanr/spearmanr.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Spearman correlation coefficient metric.""" - -from scipy.stats import spearmanr - -import datasets - - -_DESCRIPTION = """ -The Spearman rank-order correlation coefficient is a measure of the -relationship between two datasets. Like other correlation coefficients, -this one varies between -1 and +1 with 0 implying no correlation. -Positive correlations imply that as data in dataset x increases, so -does data in dataset y. Negative correlations imply that as x increases, -y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. - -Unlike the Pearson correlation, the Spearman correlation does not -assume that both datasets are normally distributed. - -The p-value roughly indicates the probability of an uncorrelated system -producing datasets that have a Spearman correlation at least as extreme -as the one computed from these datasets. The p-values are not entirely -reliable but are probably reasonable for datasets larger than 500 or so. -""" - -_KWARGS_DESCRIPTION = """ -Args: - predictions (`List[float]`): Predicted labels, as returned by a model. - references (`List[float]`): Ground truth labels. - return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns - only the spearmanr score. Defaults to `False`. -Returns: - spearmanr (`float`): Spearman correlation coefficient. - p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input. -Examples: - Example 1: - >>> spearmanr_metric = datasets.load_metric("spearmanr") - >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) - >>> print(results) - {'spearmanr': -0.7} - - Example 2: - >>> spearmanr_metric = datasets.load_metric("spearmanr") - >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], - ... predictions=[10, 9, 2.5, 6, 4], - ... return_pvalue=True) - >>> print(results['spearmanr']) - -0.7 - >>> print(round(results['spearmanr_pvalue'], 2)) - 0.19 -""" - -_CITATION = r"""\ -@book{kokoska2000crc, - title={CRC standard probability and statistics tables and formulae}, - author={Kokoska, Stephen and Zwillinger, Daniel}, - year={2000}, - publisher={Crc Press} -} -@article{2020SciPy-NMeth, - author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and - Haberland, Matt and Reddy, Tyler and Cournapeau, David and - Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and - Bright, Jonathan and {van der Walt}, St{\'e}fan J. and - Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and - Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and - Kern, Robert and Larson, Eric and Carey, C J and - Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and - {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and - Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and - Harris, Charles R. and Archibald, Anne M. and - Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and - {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, - title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific - Computing in Python}}, - journal = {Nature Methods}, - year = {2020}, - volume = {17}, - pages = {261--272}, - adsurl = {https://rdcu.be/b08Wh}, - doi = {10.1038/s41592-019-0686-2}, -} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Spearmanr(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("float"), - "references": datasets.Value("float"), - } - ), - reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"], - ) - - def _compute(self, predictions, references, return_pvalue=False): - results = spearmanr(references, predictions) - if return_pvalue: - return {"spearmanr": results[0], "spearmanr_pvalue": results[1]} - else: - return {"spearmanr": results[0]} diff --git a/metrics/squad/README.md b/metrics/squad/README.md deleted file mode 100644 index 2d5a5d78a8e..00000000000 --- a/metrics/squad/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Metric Card for SQuAD - -## Metric description -This metric wraps the official scoring script for version 1 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad). - -SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. - -## How to use - -The metric takes two files or two lists of question-answers dictionaries as inputs : one with the predictions of the model and the other with the references to be compared to: - -```python -from datasets import load_metric -squad_metric = load_metric("squad") -results = squad_metric.compute(predictions=predictions, references=references) -``` -## Output values - -This metric outputs a dictionary with two values: the average exact match score and the average [F1 score](https://huggingface.co/metrics/f1). - -``` -{'exact_match': 100.0, 'f1': 100.0} -``` - -The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. - -The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -### Values from popular papers -The [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an F1 score of 51.0% and an Exact Match score of 40.0%. They also report that human performance on the dataset represents an F1 score of 90.5% and an Exact Match score of 80.3%. - -For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). - -## Examples - -Maximal values for both exact match and F1 (perfect match): - -```python -from datasets import load_metric -squad_metric = load_metric("squad") -predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] -results = squad_metric.compute(predictions=predictions, references=references) -results -{'exact_match': 100.0, 'f1': 100.0} -``` - -Minimal values for both exact match and F1 (no match): - -```python -from datasets import load_metric -squad_metric = load_metric("squad") -predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22'}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] -results = squad_metric.compute(predictions=predictions, references=references) -results -{'exact_match': 0.0, 'f1': 0.0} -``` - -Partial match (2 out of 3 answers correct) : - -```python -from datasets import load_metric -squad_metric = load_metric("squad") -predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b'}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1'}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['BeyoncΓ© and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] -results = squad_metric.compute(predictions=predictions, references=references) -results -{'exact_match': 66.66666666666667, 'f1': 66.66666666666667} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as [SQuAD v.1 dataset](https://huggingface.co/datasets/squad). - -The SQuAD dataset does contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. - - -## Citation - - @inproceedings{Rajpurkar2016SQuAD10, - title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, - author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, - booktitle={EMNLP}, - year={2016} - } - -## Further References - -- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) -- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/metrics/squad/evaluate.py b/metrics/squad/evaluate.py deleted file mode 100644 index 7ab5977dd10..00000000000 --- a/metrics/squad/evaluate.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Official evaluation script for v1.1 of the SQuAD dataset.""" - -import argparse -import json -import re -import string -import sys -from collections import Counter - - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - return re.sub(r"\b(a|an|the)\b", " ", text) - - def white_space_fix(text): - return " ".join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def f1_score(prediction, ground_truth): - prediction_tokens = normalize_answer(prediction).split() - ground_truth_tokens = normalize_answer(ground_truth).split() - common = Counter(prediction_tokens) & Counter(ground_truth_tokens) - num_same = sum(common.values()) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(prediction_tokens) - recall = 1.0 * num_same / len(ground_truth_tokens) - f1 = (2 * precision * recall) / (precision + recall) - return f1 - - -def exact_match_score(prediction, ground_truth): - return normalize_answer(prediction) == normalize_answer(ground_truth) - - -def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): - scores_for_ground_truths = [] - for ground_truth in ground_truths: - score = metric_fn(prediction, ground_truth) - scores_for_ground_truths.append(score) - return max(scores_for_ground_truths) - - -def evaluate(dataset, predictions): - f1 = exact_match = total = 0 - for article in dataset: - for paragraph in article["paragraphs"]: - for qa in paragraph["qas"]: - total += 1 - if qa["id"] not in predictions: - message = "Unanswered question " + qa["id"] + " will receive score 0." - print(message, file=sys.stderr) - continue - ground_truths = [x["text"] for x in qa["answers"]] - prediction = predictions[qa["id"]] - exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) - f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) - - exact_match = 100.0 * exact_match / total - f1 = 100.0 * f1 / total - - return {"exact_match": exact_match, "f1": f1} - - -if __name__ == "__main__": - expected_version = "1.1" - parser = argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version) - parser.add_argument("dataset_file", help="Dataset file") - parser.add_argument("prediction_file", help="Prediction File") - args = parser.parse_args() - with open(args.dataset_file) as dataset_file: - dataset_json = json.load(dataset_file) - if dataset_json["version"] != expected_version: - print( - "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"], - file=sys.stderr, - ) - dataset = dataset_json["data"] - with open(args.prediction_file) as prediction_file: - predictions = json.load(prediction_file) - print(json.dumps(evaluate(dataset, predictions))) diff --git a/metrics/squad/squad.py b/metrics/squad/squad.py deleted file mode 100644 index 3f977d27c8a..00000000000 --- a/metrics/squad/squad.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""SQuAD metric.""" - -import datasets - -from .evaluate import evaluate - - -_CITATION = """\ -@inproceedings{Rajpurkar2016SQuAD10, - title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, - author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, - booktitle={EMNLP}, - year={2016} -} -""" - -_DESCRIPTION = """ -This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD). - -Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by -crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, -from the corresponding reading passage, or the question might be unanswerable. -""" - -_KWARGS_DESCRIPTION = """ -Computes SQuAD scores (F1 and EM). -Args: - predictions: List of question-answers dictionaries with the following key-values: - - 'id': id of the question-answer pair as given in the references (see below) - - 'prediction_text': the text of the answer - references: List of question-answers dictionaries with the following key-values: - - 'id': id of the question-answer pair (see above), - - 'answers': a Dict in the SQuAD dataset format - { - 'text': list of possible texts for the answer, as a list of strings - 'answer_start': list of start positions for the answer, as a list of ints - } - Note that answer_start values are not taken into account to compute the metric. -Returns: - 'exact_match': Exact match (the normalized answer exactly match the gold answer) - 'f1': The F-score of predicted tokens versus the gold answer -Examples: - - >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] - >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] - >>> squad_metric = datasets.load_metric("squad") - >>> results = squad_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'exact_match': 100.0, 'f1': 100.0} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Squad(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")}, - "references": { - "id": datasets.Value("string"), - "answers": datasets.features.Sequence( - { - "text": datasets.Value("string"), - "answer_start": datasets.Value("int32"), - } - ), - }, - } - ), - codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - ) - - def _compute(self, predictions, references): - pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} - dataset = [ - { - "paragraphs": [ - { - "qas": [ - { - "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], - "id": ref["id"], - } - for ref in references - ] - } - ] - } - ] - score = evaluate(dataset=dataset, predictions=pred_dict) - return score diff --git a/metrics/squad_v2/README.md b/metrics/squad_v2/README.md deleted file mode 100644 index fdd97e37790..00000000000 --- a/metrics/squad_v2/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Metric Card for SQuAD v2 - -## Metric description -This metric wraps the official scoring script for version 2 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad_v2). - -SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. - -SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -## How to use - -The metric takes two files or two lists - one representing model predictions and the other the references to compare them to. - -*Predictions* : List of triple for question-answers to score with the following key-value pairs: -* `'id'`: the question-answer identification field of the question and answer pair -* `'prediction_text'` : the text of the answer -* `'no_answer_probability'` : the probability that the question has no answer - -*References*: List of question-answers dictionaries with the following key-value pairs: -* `'id'`: id of the question-answer pair (see above), -* `'answers'`: a list of Dict {'text': text of the answer as a string} -* `'no_answer_threshold'`: the probability threshold to decide that a question has no answer. - -```python -from datasets import load_metric -squad_metric = load_metric("squad_v2") -results = squad_metric.compute(predictions=predictions, references=references) -``` -## Output values - -This metric outputs a dictionary with 13 values: -* `'exact'`: Exact match (the normalized answer exactly match the gold answer) (see the `exact_match` metric (forthcoming)) -* `'f1'`: The average F1-score of predicted tokens versus the gold answer (see the [F1 score](https://huggingface.co/metrics/f1) metric) -* `'total'`: Number of scores considered -* `'HasAns_exact'`: Exact match (the normalized answer exactly match the gold answer) -* `'HasAns_f1'`: The F-score of predicted tokens versus the gold answer -* `'HasAns_total'`: How many of the questions have answers -* `'NoAns_exact'`: Exact match (the normalized answer exactly match the gold answer) -* `'NoAns_f1'`: The F-score of predicted tokens versus the gold answer -* `'NoAns_total'`: How many of the questions have no answers -* `'best_exact'` : Best exact match (with varying threshold) -* `'best_exact_thresh'`: No-answer probability threshold associated to the best exact match -* `'best_f1'`: Best F1 score (with varying threshold) -* `'best_f1_thresh'`: No-answer probability threshold associated to the best F1 - - -The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. - -The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -The range of `total` depends on the length of predictions/references: its minimal value is 0, and maximal value is the total number of questions in the predictions and references. - -### Values from popular papers -The [SQuAD v2 paper](https://arxiv.org/pdf/1806.03822.pdf) reported an F1 score of 66.3% and an Exact Match score of 63.4%. -They also report that human performance on the dataset represents an F1 score of 89.5% and an Exact Match score of 86.9%. - -For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). - -## Examples - -Maximal values for both exact match and F1 (perfect match): - -```python -from datasets import load_metric -squad_v2_ metric = load_metric("squad_v2") -predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] -results = squad_v2_metric.compute(predictions=predictions, references=references) -results -{'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} -``` - -Minimal values for both exact match and F1 (no match): - -```python -from datasets import load_metric -squad_metric = load_metric("squad_v2") -predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] -results = squad_v2_metric.compute(predictions=predictions, references=references) -results -{'exact': 0.0, 'f1': 0.0, 'total': 1, 'HasAns_exact': 0.0, 'HasAns_f1': 0.0, 'HasAns_total': 1, 'best_exact': 0.0, 'best_exact_thresh': 0.0, 'best_f1': 0.0, 'best_f1_thresh': 0.0} -``` - -Partial match (2 out of 3 answers correct) : - -```python -from datasets import load_metric -squad_metric = load_metric("squad_v2") -predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b', 'no_answer_probability': 0.}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1', 'no_answer_probability': 0.}] -references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['BeyoncΓ© and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] -results = squad_v2_metric.compute(predictions=predictions, references=references) -results -{'exact': 66.66666666666667, 'f1': 66.66666666666667, 'total': 3, 'HasAns_exact': 66.66666666666667, 'HasAns_f1': 66.66666666666667, 'HasAns_total': 3, 'best_exact': 66.66666666666667, 'best_exact_thresh': 0.0, 'best_f1': 66.66666666666667, 'best_f1_thresh': 0.0} -``` - -## Limitations and bias -This metric works only with the datasets in the same format as the [SQuAD v.2 dataset](https://huggingface.co/datasets/squad_v2). - -The SQuAD datasets do contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. - - -## Citation - -```bibtex -@inproceedings{Rajpurkar2018SQuAD2, -title={Know What You Don't Know: Unanswerable Questions for SQuAD}, -author={Pranav Rajpurkar and Jian Zhang and Percy Liang}, -booktitle={ACL 2018}, -year={2018} -} -``` - -## Further References - -- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) -- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/metrics/squad_v2/evaluate.py b/metrics/squad_v2/evaluate.py deleted file mode 100644 index 2f8f2e11053..00000000000 --- a/metrics/squad_v2/evaluate.py +++ /dev/null @@ -1,324 +0,0 @@ -"""Official evaluation script for SQuAD version 2.0. - -In addition to basic functionality, we also compute additional statistics and -plot precision-recall curves if an additional na_prob.json file is provided. -This file is expected to map question ID's to the model's predicted probability -that a question is unanswerable. -""" - -import argparse -import collections -import json -import os -import re -import string -import sys - -import numpy as np - - -ARTICLES_REGEX = re.compile(r"\b(a|an|the)\b", re.UNICODE) - -OPTS = None - - -def parse_args(): - parser = argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.") - parser.add_argument("data_file", metavar="data.json", help="Input data JSON file.") - parser.add_argument("pred_file", metavar="pred.json", help="Model predictions.") - parser.add_argument( - "--out-file", "-o", metavar="eval.json", help="Write accuracy metrics to file (default is stdout)." - ) - parser.add_argument( - "--na-prob-file", "-n", metavar="na_prob.json", help="Model estimates of probability of no answer." - ) - parser.add_argument( - "--na-prob-thresh", - "-t", - type=float, - default=1.0, - help='Predict "" if no-answer probability exceeds this (default = 1.0).', - ) - parser.add_argument( - "--out-image-dir", "-p", metavar="out_images", default=None, help="Save precision-recall curves to directory." - ) - parser.add_argument("--verbose", "-v", action="store_true") - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - return parser.parse_args() - - -def make_qid_to_has_ans(dataset): - qid_to_has_ans = {} - for article in dataset: - for p in article["paragraphs"]: - for qa in p["qas"]: - qid_to_has_ans[qa["id"]] = bool(qa["answers"]["text"]) - return qid_to_has_ans - - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - return ARTICLES_REGEX.sub(" ", text) - - def white_space_fix(text): - return " ".join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def get_tokens(s): - if not s: - return [] - return normalize_answer(s).split() - - -def compute_exact(a_gold, a_pred): - return int(normalize_answer(a_gold) == normalize_answer(a_pred)) - - -def compute_f1(a_gold, a_pred): - gold_toks = get_tokens(a_gold) - pred_toks = get_tokens(a_pred) - common = collections.Counter(gold_toks) & collections.Counter(pred_toks) - num_same = sum(common.values()) - if len(gold_toks) == 0 or len(pred_toks) == 0: - # If either is no-answer, then F1 is 1 if they agree, 0 otherwise - return int(gold_toks == pred_toks) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(pred_toks) - recall = 1.0 * num_same / len(gold_toks) - f1 = (2 * precision * recall) / (precision + recall) - return f1 - - -def get_raw_scores(dataset, preds): - exact_scores = {} - f1_scores = {} - for article in dataset: - for p in article["paragraphs"]: - for qa in p["qas"]: - qid = qa["id"] - gold_answers = [t for t in qa["answers"]["text"] if normalize_answer(t)] - if not gold_answers: - # For unanswerable questions, only correct answer is empty string - gold_answers = [""] - if qid not in preds: - print(f"Missing prediction for {qid}") - continue - a_pred = preds[qid] - # Take max over all gold answers - exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) - f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) - return exact_scores, f1_scores - - -def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): - new_scores = {} - for qid, s in scores.items(): - pred_na = na_probs[qid] > na_prob_thresh - if pred_na: - new_scores[qid] = float(not qid_to_has_ans[qid]) - else: - new_scores[qid] = s - return new_scores - - -def make_eval_dict(exact_scores, f1_scores, qid_list=None): - if not qid_list: - total = len(exact_scores) - return collections.OrderedDict( - [ - ("exact", 100.0 * sum(exact_scores.values()) / total), - ("f1", 100.0 * sum(f1_scores.values()) / total), - ("total", total), - ] - ) - else: - total = len(qid_list) - return collections.OrderedDict( - [ - ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total), - ("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total), - ("total", total), - ] - ) - - -def merge_eval(main_eval, new_eval, prefix): - for k in new_eval: - main_eval[f"{prefix}_{k}"] = new_eval[k] - - -def plot_pr_curve(precisions, recalls, out_image, title): - plt.step(recalls, precisions, color="b", alpha=0.2, where="post") - plt.fill_between(recalls, precisions, step="post", alpha=0.2, color="b") - plt.xlabel("Recall") - plt.ylabel("Precision") - plt.xlim([0.0, 1.05]) - plt.ylim([0.0, 1.05]) - plt.title(title) - plt.savefig(out_image) - plt.clf() - - -def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, out_image=None, title=None): - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - true_pos = 0.0 - cur_p = 1.0 - cur_r = 0.0 - precisions = [1.0] - recalls = [0.0] - avg_prec = 0.0 - for i, qid in enumerate(qid_list): - if qid_to_has_ans[qid]: - true_pos += scores[qid] - cur_p = true_pos / float(i + 1) - cur_r = true_pos / float(num_true_pos) - if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]: - # i.e., if we can put a threshold after this point - avg_prec += cur_p * (cur_r - recalls[-1]) - precisions.append(cur_p) - recalls.append(cur_r) - if out_image: - plot_pr_curve(precisions, recalls, out_image, title) - return {"ap": 100.0 * avg_prec} - - -def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, out_image_dir): - if out_image_dir and not os.path.exists(out_image_dir): - os.makedirs(out_image_dir) - num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) - if num_true_pos == 0: - return - pr_exact = make_precision_recall_eval( - exact_raw, - na_probs, - num_true_pos, - qid_to_has_ans, - out_image=os.path.join(out_image_dir, "pr_exact.png"), - title="Precision-Recall curve for Exact Match score", - ) - pr_f1 = make_precision_recall_eval( - f1_raw, - na_probs, - num_true_pos, - qid_to_has_ans, - out_image=os.path.join(out_image_dir, "pr_f1.png"), - title="Precision-Recall curve for F1 score", - ) - oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} - pr_oracle = make_precision_recall_eval( - oracle_scores, - na_probs, - num_true_pos, - qid_to_has_ans, - out_image=os.path.join(out_image_dir, "pr_oracle.png"), - title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)", - ) - merge_eval(main_eval, pr_exact, "pr_exact") - merge_eval(main_eval, pr_f1, "pr_f1") - merge_eval(main_eval, pr_oracle, "pr_oracle") - - -def histogram_na_prob(na_probs, qid_list, image_dir, name): - if not qid_list: - return - x = [na_probs[k] for k in qid_list] - weights = np.ones_like(x) / float(len(x)) - plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0)) - plt.xlabel("Model probability of no-answer") - plt.ylabel("Proportion of dataset") - plt.title(f"Histogram of no-answer probability: {name}") - plt.savefig(os.path.join(image_dir, f"na_prob_hist_{name}.png")) - plt.clf() - - -def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): - num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) - cur_score = num_no_ans - best_score = cur_score - best_thresh = 0.0 - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - for i, qid in enumerate(qid_list): - if qid not in scores: - continue - if qid_to_has_ans[qid]: - diff = scores[qid] - else: - if preds[qid]: - diff = -1 - else: - diff = 0 - cur_score += diff - if cur_score > best_score: - best_score = cur_score - best_thresh = na_probs[qid] - return 100.0 * best_score / len(scores), best_thresh - - -def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): - best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) - best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) - main_eval["best_exact"] = best_exact - main_eval["best_exact_thresh"] = exact_thresh - main_eval["best_f1"] = best_f1 - main_eval["best_f1_thresh"] = f1_thresh - - -def main(): - with open(OPTS.data_file) as f: - dataset_json = json.load(f) - dataset = dataset_json["data"] - with open(OPTS.pred_file) as f: - preds = json.load(f) - if OPTS.na_prob_file: - with open(OPTS.na_prob_file) as f: - na_probs = json.load(f) - else: - na_probs = {k: 0.0 for k in preds} - qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(dataset, preds) - exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) - f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) - out_eval = make_eval_dict(exact_thresh, f1_thresh) - if has_ans_qids: - has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) - merge_eval(out_eval, has_ans_eval, "HasAns") - if no_ans_qids: - no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) - merge_eval(out_eval, no_ans_eval, "NoAns") - if OPTS.na_prob_file: - find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans) - if OPTS.na_prob_file and OPTS.out_image_dir: - run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, OPTS.out_image_dir) - histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, "hasAns") - histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, "noAns") - if OPTS.out_file: - with open(OPTS.out_file, "w") as f: - json.dump(out_eval, f) - else: - print(json.dumps(out_eval, indent=2)) - - -if __name__ == "__main__": - OPTS = parse_args() - if OPTS.out_image_dir: - import matplotlib - - matplotlib.use("Agg") - import matplotlib.pyplot as plt - main() diff --git a/metrics/squad_v2/squad_v2.py b/metrics/squad_v2/squad_v2.py deleted file mode 100644 index f6c77f63553..00000000000 --- a/metrics/squad_v2/squad_v2.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""SQuAD v2 metric.""" - -import datasets - -from .evaluate import ( - apply_no_ans_threshold, - find_all_best_thresh, - get_raw_scores, - make_eval_dict, - make_qid_to_has_ans, - merge_eval, -) - - -_CITATION = """\ -@inproceedings{Rajpurkar2016SQuAD10, - title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, - author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, - booktitle={EMNLP}, - year={2016} -} -""" - -_DESCRIPTION = """ -This metric wrap the official scoring script for version 2 of the Stanford Question -Answering Dataset (SQuAD). - -Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by -crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, -from the corresponding reading passage, or the question might be unanswerable. - -SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions -written adversarially by crowdworkers to look similar to answerable ones. -To do well on SQuAD2.0, systems must not only answer questions when possible, but also -determine when no answer is supported by the paragraph and abstain from answering. -""" - -_KWARGS_DESCRIPTION = """ -Computes SQuAD v2 scores (F1 and EM). -Args: - predictions: List of triple for question-answers to score with the following elements: - - the question-answer 'id' field as given in the references (see below) - - the text of the answer - - the probability that the question has no answer - references: List of question-answers dictionaries with the following key-values: - - 'id': id of the question-answer pair (see above), - - 'answers': a list of Dict {'text': text of the answer as a string} - no_answer_threshold: float - Probability threshold to decide that a question has no answer. -Returns: - 'exact': Exact match (the normalized answer exactly match the gold answer) - 'f1': The F-score of predicted tokens versus the gold answer - 'total': Number of score considered - 'HasAns_exact': Exact match (the normalized answer exactly match the gold answer) - 'HasAns_f1': The F-score of predicted tokens versus the gold answer - 'HasAns_total': Number of score considered - 'NoAns_exact': Exact match (the normalized answer exactly match the gold answer) - 'NoAns_f1': The F-score of predicted tokens versus the gold answer - 'NoAns_total': Number of score considered - 'best_exact': Best exact match (with varying threshold) - 'best_exact_thresh': No-answer probability threshold associated to the best exact match - 'best_f1': Best F1 (with varying threshold) - 'best_f1_thresh': No-answer probability threshold associated to the best F1 -Examples: - - >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] - >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] - >>> squad_v2_metric = datasets.load_metric("squad_v2") - >>> results = squad_v2_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class SquadV2(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": { - "id": datasets.Value("string"), - "prediction_text": datasets.Value("string"), - "no_answer_probability": datasets.Value("float32"), - }, - "references": { - "id": datasets.Value("string"), - "answers": datasets.features.Sequence( - {"text": datasets.Value("string"), "answer_start": datasets.Value("int32")} - ), - }, - } - ), - codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], - ) - - def _compute(self, predictions, references, no_answer_threshold=1.0): - no_answer_probabilities = {p["id"]: p["no_answer_probability"] for p in predictions} - dataset = [{"paragraphs": [{"qas": references}]}] - predictions = {p["id"]: p["prediction_text"] for p in predictions} - - qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - - exact_raw, f1_raw = get_raw_scores(dataset, predictions) - exact_thresh = apply_no_ans_threshold(exact_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) - f1_thresh = apply_no_ans_threshold(f1_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) - out_eval = make_eval_dict(exact_thresh, f1_thresh) - - if has_ans_qids: - has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) - merge_eval(out_eval, has_ans_eval, "HasAns") - if no_ans_qids: - no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) - merge_eval(out_eval, no_ans_eval, "NoAns") - find_all_best_thresh(out_eval, predictions, exact_raw, f1_raw, no_answer_probabilities, qid_to_has_ans) - return dict(out_eval) diff --git a/metrics/super_glue/README.md b/metrics/super_glue/README.md deleted file mode 100644 index d1e2b5b8409..00000000000 --- a/metrics/super_glue/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Metric Card for SuperGLUE - -## Metric description -This metric is used to compute the SuperGLUE evaluation metric associated to each of the subsets of the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). - -SuperGLUE is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. - - -## How to use - -There are two steps: (1) loading the SuperGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. - -1. **Loading the relevant SuperGLUE metric** : the subsets of SuperGLUE are the following: `boolq`, `cb`, `copa`, `multirc`, `record`, `rte`, `wic`, `wsc`, `wsc.fixed`, `axb`, `axg`. - -More information about the different subsets of the SuperGLUE dataset can be found on the [SuperGLUE dataset page](https://huggingface.co/datasets/super_glue) and on the [official dataset website](https://super.gluebenchmark.com/). - -2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one list of reference labels. The structure of both inputs depends on the SuperGlUE subset being used: - -Format of `predictions`: -- for `record`: list of question-answer dictionaries with the following keys: - - `idx`: index of the question as specified by the dataset - - `prediction_text`: the predicted answer text -- for `multirc`: list of question-answer dictionaries with the following keys: - - `idx`: index of the question-answer pair as specified by the dataset - - `prediction`: the predicted answer label -- otherwise: list of predicted labels - -Format of `references`: -- for `record`: list of question-answers dictionaries with the following keys: - - `idx`: index of the question as specified by the dataset - - `answers`: list of possible answers -- otherwise: list of reference labels - -```python -from datasets import load_metric -super_glue_metric = load_metric('super_glue', 'copa') -predictions = [0, 1] -references = [0, 1] -results = super_glue_metric.compute(predictions=predictions, references=references) -``` -## Output values - -The output of the metric depends on the SuperGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: - -`exact_match`: A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. (See [Exact Match](https://huggingface.co/metrics/exact_match) for more information). - -`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. - -`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. - -### Values from popular papers -The [original SuperGLUE paper](https://arxiv.org/pdf/1905.00537.pdf) reported average scores ranging from 47 to 71.5%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). - -For more recent model performance, see the [dataset leaderboard](https://super.gluebenchmark.com/leaderboard). - -## Examples - -Maximal values for the COPA subset (which outputs `accuracy`): - -```python -from datasets import load_metric -super_glue_metric = load_metric('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] -predictions = [0, 1] -references = [0, 1] -results = super_glue_metric.compute(predictions=predictions, references=references) -print(results) -{'accuracy': 1.0} -``` - -Minimal values for the MultiRC subset (which outputs `pearson` and `spearmanr`): - -```python -from datasets import load_metric -super_glue_metric = load_metric('super_glue', 'multirc') -predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] -references = [1,0] -results = super_glue_metric.compute(predictions=predictions, references=references) -print(results) -{'exact_match': 0.0, 'f1_m': 0.0, 'f1_a': 0.0} -``` - -Partial match for the COLA subset (which outputs `matthews_correlation`) - -```python -from datasets import load_metric -super_glue_metric = load_metric('super_glue', 'axb') -references = [0, 1] -predictions = [1,1] -results = super_glue_metric.compute(predictions=predictions, references=references) -print(results) -{'matthews_correlation': 0.0} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). - -The dataset also includes Winogender, a subset of the dataset that is designed to measure gender bias in coreference resolution systems. However, as noted in the SuperGLUE paper, this subset has its limitations: *"It offers only positive predictive value: A poor bias score is clear evidence that a model exhibits gender bias, but a good score does not mean that the model is unbiased.[...] Also, Winogender does not cover all forms of social bias, or even all forms of gender. For instance, the version of the data used here offers no coverage of gender-neutral they or non-binary pronouns." - -## Citation - -```bibtex -@article{wang2019superglue, - title={Super{GLUE}: A Stickier Benchmark for General-Purpose Language Understanding Systems}, - author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, - journal={arXiv preprint arXiv:1905.00537}, - year={2019} -} -``` - -## Further References - -- [SuperGLUE benchmark homepage](https://super.gluebenchmark.com/) diff --git a/metrics/super_glue/record_evaluation.py b/metrics/super_glue/record_evaluation.py deleted file mode 100644 index 69ba8b969b8..00000000000 --- a/metrics/super_glue/record_evaluation.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Official evaluation script for ReCoRD v1.0. -(Some functions are adopted from the SQuAD evaluation script.) -""" - -import argparse -import json -import re -import string -import sys -from collections import Counter - - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - return re.sub(r"\b(a|an|the)\b", " ", text) - - def white_space_fix(text): - return " ".join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def f1_score(prediction, ground_truth): - prediction_tokens = normalize_answer(prediction).split() - ground_truth_tokens = normalize_answer(ground_truth).split() - common = Counter(prediction_tokens) & Counter(ground_truth_tokens) - num_same = sum(common.values()) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(prediction_tokens) - recall = 1.0 * num_same / len(ground_truth_tokens) - f1 = (2 * precision * recall) / (precision + recall) - return f1 - - -def exact_match_score(prediction, ground_truth): - return normalize_answer(prediction) == normalize_answer(ground_truth) - - -def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): - scores_for_ground_truths = [] - for ground_truth in ground_truths: - score = metric_fn(prediction, ground_truth) - scores_for_ground_truths.append(score) - return max(scores_for_ground_truths) - - -def evaluate(dataset, predictions): - f1 = exact_match = total = 0 - correct_ids = [] - for passage in dataset: - for qa in passage["qas"]: - total += 1 - if qa["id"] not in predictions: - message = f'Unanswered question {qa["id"]} will receive score 0.' - print(message, file=sys.stderr) - continue - - ground_truths = [x["text"] for x in qa["answers"]] - prediction = predictions[qa["id"]] - - _exact_match = metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) - if int(_exact_match) == 1: - correct_ids.append(qa["id"]) - exact_match += _exact_match - - f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) - - exact_match = exact_match / total - f1 = f1 / total - - return {"exact_match": exact_match, "f1": f1}, correct_ids - - -if __name__ == "__main__": - expected_version = "1.0" - parser = argparse.ArgumentParser("Official evaluation script for ReCoRD v1.0.") - parser.add_argument("data_file", help="The dataset file in JSON format.") - parser.add_argument("pred_file", help="The model prediction file in JSON format.") - parser.add_argument("--output_correct_ids", action="store_true", help="Output the correctly answered query IDs.") - args = parser.parse_args() - - with open(args.data_file) as data_file: - dataset_json = json.load(data_file) - if dataset_json["version"] != expected_version: - print( - f'Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}', - file=sys.stderr, - ) - dataset = dataset_json["data"] - - with open(args.pred_file) as pred_file: - predictions = json.load(pred_file) - - metrics, correct_ids = evaluate(dataset, predictions) - - if args.output_correct_ids: - print(f"Output {len(correct_ids)} correctly answered question IDs.") - with open("correct_ids.json", "w") as f: - json.dump(correct_ids, f) diff --git a/metrics/super_glue/super_glue.py b/metrics/super_glue/super_glue.py deleted file mode 100644 index aeb6eb56ca3..00000000000 --- a/metrics/super_glue/super_glue.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The SuperGLUE benchmark metric.""" - -from sklearn.metrics import f1_score, matthews_corrcoef - -import datasets - -from .record_evaluation import evaluate as evaluate_record - - -_CITATION = """\ -@article{wang2019superglue, - title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems}, - author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, - journal={arXiv preprint arXiv:1905.00537}, - year={2019} -} -""" - -_DESCRIPTION = """\ -SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after -GLUE with a new set of more difficult language understanding tasks, improved -resources, and a new public leaderboard. -""" - -_KWARGS_DESCRIPTION = """ -Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset. -Args: - predictions: list of predictions to score. Depending on the SuperGlUE subset: - - for 'record': list of question-answer dictionaries with the following keys: - - 'idx': index of the question as specified by the dataset - - 'prediction_text': the predicted answer text - - for 'multirc': list of question-answer dictionaries with the following keys: - - 'idx': index of the question-answer pair as specified by the dataset - - 'prediction': the predicted answer label - - otherwise: list of predicted labels - references: list of reference labels. Depending on the SuperGLUE subset: - - for 'record': list of question-answers dictionaries with the following keys: - - 'idx': index of the question as specified by the dataset - - 'answers': list of possible answers - - otherwise: list of reference labels -Returns: depending on the SuperGLUE subset: - - for 'record': - - 'exact_match': Exact match between answer and gold answer - - 'f1': F1 score - - for 'multirc': - - 'exact_match': Exact match between answer and gold answer - - 'f1_m': Per-question macro-F1 score - - 'f1_a': Average F1 score over all answers - - for 'axb': - 'matthews_correlation': Matthew Correlation - - for 'cb': - - 'accuracy': Accuracy - - 'f1': F1 score - - for all others: - - 'accuracy': Accuracy -Examples: - - >>> super_glue_metric = datasets.load_metric('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] - >>> predictions = [0, 1] - >>> references = [0, 1] - >>> results = super_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0} - - >>> super_glue_metric = datasets.load_metric('super_glue', 'cb') - >>> predictions = [0, 1] - >>> references = [0, 1] - >>> results = super_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0, 'f1': 1.0} - - >>> super_glue_metric = datasets.load_metric('super_glue', 'record') - >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}] - >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}] - >>> results = super_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'exact_match': 1.0, 'f1': 1.0} - - >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc') - >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] - >>> references = [0, 1] - >>> results = super_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0} - - >>> super_glue_metric = datasets.load_metric('super_glue', 'axb') - >>> references = [0, 1] - >>> predictions = [0, 1] - >>> results = super_glue_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'matthews_correlation': 1.0} -""" - - -def simple_accuracy(preds, labels): - return float((preds == labels).mean()) - - -def acc_and_f1(preds, labels, f1_avg="binary"): - acc = simple_accuracy(preds, labels) - f1 = float(f1_score(y_true=labels, y_pred=preds, average=f1_avg)) - return { - "accuracy": acc, - "f1": f1, - } - - -def evaluate_multirc(ids_preds, labels): - """ - Computes F1 score and Exact Match for MultiRC predictions. - """ - question_map = {} - for id_pred, label in zip(ids_preds, labels): - question_id = f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}' - pred = id_pred["prediction"] - if question_id in question_map: - question_map[question_id].append((pred, label)) - else: - question_map[question_id] = [(pred, label)] - f1s, ems = [], [] - for question, preds_labels in question_map.items(): - question_preds, question_labels = zip(*preds_labels) - f1 = f1_score(y_true=question_labels, y_pred=question_preds, average="macro") - f1s.append(f1) - em = int(sum(pred == label for pred, label in preds_labels) == len(preds_labels)) - ems.append(em) - f1_m = float(sum(f1s) / len(f1s)) - em = sum(ems) / len(ems) - f1_a = float(f1_score(y_true=labels, y_pred=[id_pred["prediction"] for id_pred in ids_preds])) - return {"exact_match": em, "f1_m": f1_m, "f1_a": f1_a} - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class SuperGlue(datasets.Metric): - def _info(self): - if self.config_name not in [ - "boolq", - "cb", - "copa", - "multirc", - "record", - "rte", - "wic", - "wsc", - "wsc.fixed", - "axb", - "axg", - ]: - raise KeyError( - "You should supply a configuration name selected in " - '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features(self._get_feature_types()), - codebase_urls=[], - reference_urls=[], - format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None, - ) - - def _get_feature_types(self): - if self.config_name == "record": - return { - "predictions": { - "idx": { - "passage": datasets.Value("int64"), - "query": datasets.Value("int64"), - }, - "prediction_text": datasets.Value("string"), - }, - "references": { - "idx": { - "passage": datasets.Value("int64"), - "query": datasets.Value("int64"), - }, - "answers": datasets.Sequence(datasets.Value("string")), - }, - } - elif self.config_name == "multirc": - return { - "predictions": { - "idx": { - "answer": datasets.Value("int64"), - "paragraph": datasets.Value("int64"), - "question": datasets.Value("int64"), - }, - "prediction": datasets.Value("int64"), - }, - "references": datasets.Value("int64"), - } - else: - return { - "predictions": datasets.Value("int64"), - "references": datasets.Value("int64"), - } - - def _compute(self, predictions, references): - if self.config_name == "axb": - return {"matthews_correlation": matthews_corrcoef(references, predictions)} - elif self.config_name == "cb": - return acc_and_f1(predictions, references, f1_avg="macro") - elif self.config_name == "record": - dataset = [ - { - "qas": [ - {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]} - for ref in references - ] - } - ] - predictions = {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions} - return evaluate_record(dataset, predictions)[0] - elif self.config_name == "multirc": - return evaluate_multirc(predictions, references) - elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]: - return {"accuracy": simple_accuracy(predictions, references)} - else: - raise KeyError( - "You should supply a configuration name selected in " - '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' - ) diff --git a/metrics/ter/README.md b/metrics/ter/README.md deleted file mode 100644 index ae44bbce782..00000000000 --- a/metrics/ter/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# Metric Card for TER - -## Metric Description -TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a hypothesis requires to match a reference translation. We use the implementation that is already present in [sacrebleu](https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the [TERCOM implementation](https://github.com/jhclark/tercom). - -The implementation here is slightly different from sacrebleu in terms of the required input format. The length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See [this github issue](https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534). - -See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. - - -## How to Use -This metric takes, at minimum, predicted sentences and reference sentences: -```python ->>> predictions = ["does this sentence match??", -... "what about this sentence?", -... "What did the TER metric user say to the developer?"] ->>> references = [["does this sentence match", "does this sentence match!?!"], -... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], -... ["Your jokes are...", "...TERrible"]] ->>> ter = datasets.load_metric("ter") ->>> results = ter.compute(predictions=predictions, -... references=references, -... case_sensitive=True) ->>> print(results) -{'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} -``` - -### Inputs -This metric takes the following as input: -- **`predictions`** (`list` of `str`): The system stream (a sequence of segments). -- **`references`** (`list` of `list` of `str`): A list of one or more reference streams (each a sequence of segments). -- **`normalized`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. -- **`ignore_punct`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. -- **`support_zh_ja_chars`** (`boolean`): If `True`, tokenization/normalization supports processing of Chinese characters, as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. Only applies if `normalized = True`. Defaults to `False`. -- **`case_sensitive`** (`boolean`): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. - -### Output Values -This metric returns the following: -- **`score`** (`float`): TER score (num_edits / sum_ref_lengths * 100) -- **`num_edits`** (`int`): The cumulative number of edits -- **`ref_length`** (`float`): The cumulative average reference length - -The output takes the following form: -```python -{'score': ter_score, 'num_edits': num_edits, 'ref_length': ref_length} -``` - -The metric can take on any value `0` and above. `0` is a perfect score, meaning the predictions exactly match the references and no edits were necessary. Higher scores are worse. Scores above 100 mean that the cumulative number of edits, `num_edits`, is higher than the cumulative length of the references, `ref_length`. - -#### Values from Popular Papers - - -### Examples -Basic example with only predictions and references as inputs: -```python ->>> predictions = ["does this sentence match??", -... "what about this sentence?"] ->>> references = [["does this sentence match", "does this sentence match!?!"], -... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] ->>> ter = datasets.load_metric("ter") ->>> results = ter.compute(predictions=predictions, -... references=references, -... case_sensitive=True) ->>> print(results) -{'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} -``` - -Example with `normalization = True`: -```python ->>> predictions = ["does this sentence match??", -... "what about this sentence?"] ->>> references = [["does this sentence match", "does this sentence match!?!"], -... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] ->>> ter = datasets.load_metric("ter") ->>> results = ter.compute(predictions=predictions, -... references=references, -... normalized=True, -... case_sensitive=True) ->>> print(results) -{'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} -``` - -Example ignoring punctuation and capitalization, and everything matches: -```python ->>> predictions = ["does this sentence match??", -... "what about this sentence?"] ->>> references = [["does this sentence match", "does this sentence match!?!"], -... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] ->>> ter = datasets.load_metric("ter") ->>> results = ter.compute(predictions=predictions, -... references=references, -... ignore_punct=True, -... case_sensitive=False) ->>> print(results) -{'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} -``` - -Example ignoring punctuation and capitalization, but with an extra (incorrect) sample: -```python ->>> predictions = ["does this sentence match??", -... "what about this sentence?", -... "What did the TER metric user say to the developer?"] ->>> references = [["does this sentence match", "does this sentence match!?!"], -... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], -... ["Your jokes are...", "...TERrible"]] ->>> ter = datasets.load_metric("ter") ->>> results = ter.compute(predictions=predictions, -... references=references, -... ignore_punct=True, -... case_sensitive=False) ->>> print(results) -{'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} -``` - - -## Limitations and Bias - - -## Citation -```bibtex -@inproceedings{snover-etal-2006-study, - title = "A Study of Translation Edit Rate with Targeted Human Annotation", - author = "Snover, Matthew and - Dorr, Bonnie and - Schwartz, Rich and - Micciulla, Linnea and - Makhoul, John", - booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", - month = aug # " 8-12", - year = "2006", - address = "Cambridge, Massachusetts, USA", - publisher = "Association for Machine Translation in the Americas", - url = "https://aclanthology.org/2006.amta-papers.25", - pages = "223--231", -} -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -``` - -## Further References -- See [the sacreBLEU github repo](https://github.com/mjpost/sacreBLEU#ter) for more information. diff --git a/metrics/ter/ter.py b/metrics/ter/ter.py deleted file mode 100644 index dff3f17ffb3..00000000000 --- a/metrics/ter/ter.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""TER metric as available in sacrebleu.""" - -import sacrebleu as scb -from packaging import version -from sacrebleu import TER - -import datasets - - -_CITATION = """\ -@inproceedings{snover-etal-2006-study, - title = "A Study of Translation Edit Rate with Targeted Human Annotation", - author = "Snover, Matthew and - Dorr, Bonnie and - Schwartz, Rich and - Micciulla, Linnea and - Makhoul, John", - booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", - month = aug # " 8-12", - year = "2006", - address = "Cambridge, Massachusetts, USA", - publisher = "Association for Machine Translation in the Americas", - url = "https://aclanthology.org/2006.amta-papers.25", - pages = "223--231", -} -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -""" - -_DESCRIPTION = """\ -TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a -hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu -(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found -here: https://github.com/jhclark/tercom. - -The implementation here is slightly different from sacrebleu in terms of the required input format. The length of -the references and hypotheses lists need to be the same, so you may need to transpose your references compared to -sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 - -See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. -""" - -_KWARGS_DESCRIPTION = """ -Produces TER scores alongside the number of edits and reference length. - -Args: - predictions (list of str): The system stream (a sequence of segments). - references (list of list of str): A list of one or more reference streams (each a sequence of segments). - normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. - ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. - support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters, - as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. - Only applies if `normalized = True`. Defaults to `False`. - case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. - -Returns: - 'score' (float): TER score (num_edits / sum_ref_lengths * 100) - 'num_edits' (int): The cumulative number of edits - 'ref_length' (float): The cumulative average reference length - -Examples: - Example 1: - >>> predictions = ["does this sentence match??", - ... "what about this sentence?", - ... "What did the TER metric user say to the developer?"] - >>> references = [["does this sentence match", "does this sentence match!?!"], - ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], - ... ["Your jokes are...", "...TERrible"]] - >>> ter = datasets.load_metric("ter") - >>> results = ter.compute(predictions=predictions, - ... references=references, - ... case_sensitive=True) - >>> print(results) - {'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} - - Example 2: - >>> predictions = ["does this sentence match??", - ... "what about this sentence?"] - >>> references = [["does this sentence match", "does this sentence match!?!"], - ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] - >>> ter = datasets.load_metric("ter") - >>> results = ter.compute(predictions=predictions, - ... references=references, - ... case_sensitive=True) - >>> print(results) - {'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} - - Example 3: - >>> predictions = ["does this sentence match??", - ... "what about this sentence?"] - >>> references = [["does this sentence match", "does this sentence match!?!"], - ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] - >>> ter = datasets.load_metric("ter") - >>> results = ter.compute(predictions=predictions, - ... references=references, - ... normalized=True, - ... case_sensitive=True) - >>> print(results) - {'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} - - Example 4: - >>> predictions = ["does this sentence match??", - ... "what about this sentence?"] - >>> references = [["does this sentence match", "does this sentence match!?!"], - ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] - >>> ter = datasets.load_metric("ter") - >>> results = ter.compute(predictions=predictions, - ... references=references, - ... ignore_punct=True, - ... case_sensitive=False) - >>> print(results) - {'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} - - Example 5: - >>> predictions = ["does this sentence match??", - ... "what about this sentence?", - ... "What did the TER metric user say to the developer?"] - >>> references = [["does this sentence match", "does this sentence match!?!"], - ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], - ... ["Your jokes are...", "...TERrible"]] - >>> ter = datasets.load_metric("ter") - >>> results = ter.compute(predictions=predictions, - ... references=references, - ... ignore_punct=True, - ... case_sensitive=False) - >>> print(results) - {'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Ter(datasets.Metric): - def _info(self): - if version.parse(scb.__version__) < version.parse("1.4.12"): - raise ImportWarning( - "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" - 'You can install it with `pip install "sacrebleu>=1.4.12"`.' - ) - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - homepage="http://www.cs.umd.edu/~snover/tercom/", - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=["https://github.com/mjpost/sacreBLEU#ter"], - reference_urls=[ - "https://github.com/jhclark/tercom", - ], - ) - - def _compute( - self, - predictions, - references, - normalized: bool = False, - ignore_punct: bool = False, - support_zh_ja_chars: bool = False, - case_sensitive: bool = False, - ): - references_per_prediction = len(references[0]) - if any(len(refs) != references_per_prediction for refs in references): - raise ValueError("Sacrebleu requires the same number of references for each prediction") - transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] - - sb_ter = TER( - normalized=normalized, - no_punct=ignore_punct, - asian_support=support_zh_ja_chars, - case_sensitive=case_sensitive, - ) - output = sb_ter.corpus_score(predictions, transformed_references) - - return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length} diff --git a/metrics/wer/README.md b/metrics/wer/README.md deleted file mode 100644 index ae71505378b..00000000000 --- a/metrics/wer/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# Metric Card for WER - -## Metric description -Word error rate (WER) is a common metric of the performance of an automatic speech recognition (ASR) system. - -The general difficulty of measuring the performance of ASR systems lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance), working at the word level. - -This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between [perplexity](https://huggingface.co/metrics/perplexity) and word error rate (see [this article](https://www.cs.cmu.edu/~roni/papers/eval-metrics-bntuw-9802.pdf) for further information). - -Word error rate can then be computed as: - -`WER = (S + D + I) / N = (S + D + I) / (S + D + C)` - -where - -`S` is the number of substitutions, - -`D` is the number of deletions, - -`I` is the number of insertions, - -`C` is the number of correct words, - -`N` is the number of words in the reference (`N=S+D+C`). - - -## How to use - -The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). - - -```python -from datasets import load_metric -wer = load_metric("wer") -wer_score = wer.compute(predictions=predictions, references=references) -``` -## Output values - -This metric outputs a float representing the word error rate. - -``` -print(wer_score) -0.5 -``` - -This value indicates the average number of errors per reference word. - -The **lower** the value, the **better** the performance of the ASR system, with a WER of 0 being a perfect score. - -### Values from popular papers - -This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. - -For example, datasets such as [LibriSpeech](https://huggingface.co/datasets/librispeech_asr) report a WER in the 1.8-3.3 range, whereas ASR models evaluated on [Timit](https://huggingface.co/datasets/timit_asr) report a WER in the 8.3-20.4 range. -See the leaderboards for [LibriSpeech](https://paperswithcode.com/sota/speech-recognition-on-librispeech-test-clean) and [Timit](https://paperswithcode.com/sota/speech-recognition-on-timit) for the most recent values. - -## Examples - -Perfect match between prediction and reference: - -```python -from datasets import load_metric -wer = load_metric("wer") -predictions = ["hello world", "good night moon"] -references = ["hello world", "good night moon"] -wer_score = wer.compute(predictions=predictions, references=references) -print(wer_score) -0.0 -``` - -Partial match between prediction and reference: - -```python -from datasets import load_metric -wer = load_metric("wer") -predictions = ["this is the prediction", "there is an other sample"] -references = ["this is the reference", "there is another one"] -wer_score = wer.compute(predictions=predictions, references=references) -print(wer_score) -0.5 -``` - -No match between prediction and reference: - -```python -from datasets import load_metric -wer = load_metric("wer") -predictions = ["hello world", "good night moon"] -references = ["hi everyone", "have a great day"] -wer_score = wer.compute(predictions=predictions, references=references) -print(wer_score) -1.0 -``` - -## Limitations and bias - -WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. - -## Citation - -```bibtex -@inproceedings{woodard1982, -author = {Woodard, J.P. and Nelson, J.T., -year = {1982}, -journal = αΊ„orkshop on standardisation for speech I/O technology, Naval Air Development Center, Warminster, PA}, -title = {An information theoretic measure of speech recognition performance} -} -``` - -```bibtex -@inproceedings{morris2004, -author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, -year = {2004}, -month = {01}, -pages = {}, -title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} -} -``` - -## Further References - -- [Word Error Rate -- Wikipedia](https://en.wikipedia.org/wiki/Word_error_rate) -- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/metrics/wer/wer.py b/metrics/wer/wer.py deleted file mode 100644 index f79d7d5e6b4..00000000000 --- a/metrics/wer/wer.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2021 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Word Error Ratio (WER) metric.""" - -from jiwer import compute_measures - -import datasets - - -_CITATION = """\ -@inproceedings{inproceedings, - author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, - year = {2004}, - month = {01}, - pages = {}, - title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} -} -""" - -_DESCRIPTION = """\ -Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. - -The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. - -This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate. - -Word error rate can then be computed as: - -WER = (S + D + I) / N = (S + D + I) / (S + D + C) - -where - -S is the number of substitutions, -D is the number of deletions, -I is the number of insertions, -C is the number of correct words, -N is the number of words in the reference (N=S+D+C). - -This value indicates the average number of errors per reference word. The lower the value, the better the -performance of the ASR system with a WER of 0 being a perfect score. -""" - -_KWARGS_DESCRIPTION = """ -Compute WER score of transcribed segments against references. - -Args: - references: List of references for each speech input. - predictions: List of transcriptions to score. - concatenate_texts (bool, default=False): Whether to concatenate all input texts or compute WER iteratively. - -Returns: - (float): the word error rate - -Examples: - - >>> predictions = ["this is the prediction", "there is an other sample"] - >>> references = ["this is the reference", "there is another one"] - >>> wer = datasets.load_metric("wer") - >>> wer_score = wer.compute(predictions=predictions, references=references) - >>> print(wer_score) - 0.5 -""" - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class WER(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Value("string", id="sequence"), - } - ), - codebase_urls=["https://github.com/jitsi/jiwer/"], - reference_urls=[ - "https://en.wikipedia.org/wiki/Word_error_rate", - ], - ) - - def _compute(self, predictions=None, references=None, concatenate_texts=False): - if concatenate_texts: - return compute_measures(references, predictions)["wer"] - else: - incorrect = 0 - total = 0 - for prediction, reference in zip(predictions, references): - measures = compute_measures(reference, prediction) - incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] - total += measures["substitutions"] + measures["deletions"] + measures["hits"] - return incorrect / total diff --git a/metrics/wiki_split/README.md b/metrics/wiki_split/README.md deleted file mode 100644 index 1a9c934640a..00000000000 --- a/metrics/wiki_split/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Metric Card for WikiSplit - -## Metric description - -WikiSplit is the combination of three metrics: [SARI](https://huggingface.co/metrics/sari), [exact match](https://huggingface.co/metrics/exact_match) and [SacreBLEU](https://huggingface.co/metrics/sacrebleu). - -It can be used to evaluate the quality of sentence splitting approaches, which require rewriting a long sentence into two or more coherent short sentences, e.g. based on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). - -## How to use - -The WIKI_SPLIT metric takes three inputs: - -`sources`: a list of source sentences, where each sentence should be a string. - -`predictions`: a list of predicted sentences, where each sentence should be a string. - -`references`: a list of lists of reference sentences, where each sentence should be a string. - -```python ->>> from datasets import load_metric ->>> wiki_split = load_metric("wiki_split") ->>> sources = ["About 95 species are currently accepted ."] ->>> predictions = ["About 95 you now get in ."] ->>> references= [["About 95 species are currently known ."]] ->>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) -``` -## Output values - -This metric outputs a dictionary containing three scores: - -`sari`: the [SARI](https://huggingface.co/metrics/sari) score, whose range is between `0.0` and `100.0` -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. - -`sacrebleu`: the [SacreBLEU](https://huggingface.co/metrics/sacrebleu) score, which can take any value between `0.0` and `100.0`, inclusive. - -`exact`: the [exact match](https://huggingface.co/metrics/exact_match) score, which represents the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. It ranges from `0.0` to `100`, inclusive. Here, `0.0` means no prediction/reference pairs were matches, while `100.0` means they all were. - -```python ->>> print(results) -{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} -``` - -### Values from popular papers - -This metric was initially used by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf) to evaluate the performance of different split-and-rephrase approaches on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). They reported a SARI score of 63.5, a SacreBLEU score of 77.2, and an EXACT_MATCH score of 16.3. - -## Examples - -Perfect match between prediction and reference: - -```python ->>> from datasets import load_metric ->>> wiki_split = load_metric("wiki_split") ->>> sources = ["About 95 species are currently accepted ."] ->>> predictions = ["About 95 species are currently accepted ."] ->>> references= [["About 95 species are currently accepted ."]] ->>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) ->>> print(results) -{'sari': 100.0, 'sacrebleu': 100.00000000000004, 'exact': 100.0 -``` - -Partial match between prediction and reference: - -```python ->>> from datasets import load_metric ->>> wiki_split = load_metric("wiki_split") ->>> sources = ["About 95 species are currently accepted ."] ->>> predictions = ["About 95 you now get in ."] ->>> references= [["About 95 species are currently known ."]] ->>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) ->>> print(results) -{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} -``` - -No match between prediction and reference: - -```python ->>> from datasets import load_metric ->>> wiki_split = load_metric("wiki_split") ->>> sources = ["About 95 species are currently accepted ."] ->>> predictions = ["Hello world ."] ->>> references= [["About 95 species are currently known ."]] ->>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) ->>> print(results) -{'sari': 14.047619047619046, 'sacrebleu': 0.0, 'exact': 0.0} -``` -## Limitations and bias - -This metric is not the official metric to evaluate models on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). It was initially proposed by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf), whereas the [original paper introducing the WikiSplit dataset (2018)](https://aclanthology.org/D18-1080.pdf) uses different metrics to evaluate performance, such as corpus-level [BLEU](https://huggingface.co/metrics/bleu) and sentence-level BLEU. - -## Citation - -```bibtex -@article{rothe2020leveraging, - title={Leveraging pre-trained checkpoints for sequence generation tasks}, - author={Rothe, Sascha and Narayan, Shashi and Severyn, Aliaksei}, - journal={Transactions of the Association for Computational Linguistics}, - volume={8}, - pages={264--280}, - year={2020}, - publisher={MIT Press} -} -``` - -## Further References - -- [WikiSplit dataset](https://huggingface.co/datasets/wiki_split) -- [WikiSplit paper (Botha et al., 2018)](https://aclanthology.org/D18-1080.pdf) diff --git a/metrics/wiki_split/wiki_split.py b/metrics/wiki_split/wiki_split.py deleted file mode 100644 index a912372cb36..00000000000 --- a/metrics/wiki_split/wiki_split.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""WIKI_SPLIT metric.""" - -import re -import string -from collections import Counter - -import sacrebleu -import sacremoses -from packaging import version - -import datasets - - -_CITATION = """ -@inproceedings{xu-etal-2016-optimizing, - title = {Optimizing Statistical Machine Translation for Text Simplification}, - authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, - journal = {Transactions of the Association for Computational Linguistics}, - volume = {4}, - year={2016}, - url = {https://www.aclweb.org/anthology/Q16-1029}, - pages = {401--415 -}, -@inproceedings{post-2018-call, - title = "A Call for Clarity in Reporting {BLEU} Scores", - author = "Post, Matt", - booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", - month = oct, - year = "2018", - address = "Belgium, Brussels", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/W18-6319", - pages = "186--191", -} -""" - -_DESCRIPTION = """\ -WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU -It can be used to evaluate the quality of machine-generated texts. -""" - - -_KWARGS_DESCRIPTION = """ -Calculates sari score (between 0 and 100) given a list of source and predicted -sentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score. -Args: - sources: list of source sentences where each sentence should be a string. - predictions: list of predicted sentences where each sentence should be a string. - references: list of lists of reference sentences where each sentence should be a string. -Returns: - sari: sari score - sacrebleu: sacrebleu score - exact: exact score - -Examples: - >>> sources=["About 95 species are currently accepted ."] - >>> predictions=["About 95 you now get in ."] - >>> references=[["About 95 species are currently known ."]] - >>> wiki_split = datasets.load_metric("wiki_split") - >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) - >>> print(results) - {'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} -""" - - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) - return re.sub(regex, " ", text) - - def white_space_fix(text): - return " ".join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def compute_exact(a_gold, a_pred): - return int(normalize_answer(a_gold) == normalize_answer(a_pred)) - - -def compute_em(predictions, references): - scores = [any(compute_exact(ref, pred) for ref in refs) for pred, refs in zip(predictions, references)] - return (sum(scores) / len(scores)) * 100 - - -def SARIngram(sgrams, cgrams, rgramslist, numref): - rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] - rgramcounter = Counter(rgramsall) - - sgramcounter = Counter(sgrams) - sgramcounter_rep = Counter() - for sgram, scount in sgramcounter.items(): - sgramcounter_rep[sgram] = scount * numref - - cgramcounter = Counter(cgrams) - cgramcounter_rep = Counter() - for cgram, ccount in cgramcounter.items(): - cgramcounter_rep[cgram] = ccount * numref - - # KEEP - keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep - keepgramcountergood_rep = keepgramcounter_rep & rgramcounter - keepgramcounterall_rep = sgramcounter_rep & rgramcounter - - keeptmpscore1 = 0 - keeptmpscore2 = 0 - for keepgram in keepgramcountergood_rep: - keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] - # Fix an alleged bug [2] in the keep score computation. - # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] - keeptmpscore2 += keepgramcountergood_rep[keepgram] - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - keepscore_precision = 1 - keepscore_recall = 1 - if len(keepgramcounter_rep) > 0: - keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) - if len(keepgramcounterall_rep) > 0: - # Fix an alleged bug [2] in the keep score computation. - # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) - keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) - keepscore = 0 - if keepscore_precision > 0 or keepscore_recall > 0: - keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) - - # DELETION - delgramcounter_rep = sgramcounter_rep - cgramcounter_rep - delgramcountergood_rep = delgramcounter_rep - rgramcounter - delgramcounterall_rep = sgramcounter_rep - rgramcounter - deltmpscore1 = 0 - deltmpscore2 = 0 - for delgram in delgramcountergood_rep: - deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] - deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - delscore_precision = 1 - if len(delgramcounter_rep) > 0: - delscore_precision = deltmpscore1 / len(delgramcounter_rep) - - # ADDITION - addgramcounter = set(cgramcounter) - set(sgramcounter) - addgramcountergood = set(addgramcounter) & set(rgramcounter) - addgramcounterall = set(rgramcounter) - set(sgramcounter) - - addtmpscore = 0 - for addgram in addgramcountergood: - addtmpscore += 1 - - # Define 0/0=1 instead of 0 to give higher scores for predictions that match - # a target exactly. - addscore_precision = 1 - addscore_recall = 1 - if len(addgramcounter) > 0: - addscore_precision = addtmpscore / len(addgramcounter) - if len(addgramcounterall) > 0: - addscore_recall = addtmpscore / len(addgramcounterall) - addscore = 0 - if addscore_precision > 0 or addscore_recall > 0: - addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) - - return (keepscore, delscore_precision, addscore) - - -def SARIsent(ssent, csent, rsents): - numref = len(rsents) - - s1grams = ssent.split(" ") - c1grams = csent.split(" ") - s2grams = [] - c2grams = [] - s3grams = [] - c3grams = [] - s4grams = [] - c4grams = [] - - r1gramslist = [] - r2gramslist = [] - r3gramslist = [] - r4gramslist = [] - for rsent in rsents: - r1grams = rsent.split(" ") - r2grams = [] - r3grams = [] - r4grams = [] - r1gramslist.append(r1grams) - for i in range(0, len(r1grams) - 1): - if i < len(r1grams) - 1: - r2gram = r1grams[i] + " " + r1grams[i + 1] - r2grams.append(r2gram) - if i < len(r1grams) - 2: - r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] - r3grams.append(r3gram) - if i < len(r1grams) - 3: - r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] - r4grams.append(r4gram) - r2gramslist.append(r2grams) - r3gramslist.append(r3grams) - r4gramslist.append(r4grams) - - for i in range(0, len(s1grams) - 1): - if i < len(s1grams) - 1: - s2gram = s1grams[i] + " " + s1grams[i + 1] - s2grams.append(s2gram) - if i < len(s1grams) - 2: - s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] - s3grams.append(s3gram) - if i < len(s1grams) - 3: - s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] - s4grams.append(s4gram) - - for i in range(0, len(c1grams) - 1): - if i < len(c1grams) - 1: - c2gram = c1grams[i] + " " + c1grams[i + 1] - c2grams.append(c2gram) - if i < len(c1grams) - 2: - c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] - c3grams.append(c3gram) - if i < len(c1grams) - 3: - c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] - c4grams.append(c4gram) - - (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) - (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) - (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) - (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) - avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 - avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 - avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 - finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 - return finalscore - - -def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): - # Normalization is requried for the ASSET dataset (one of the primary - # datasets in sentence simplification) to allow using space - # to split the sentence. Even though Wiki-Auto and TURK datasets, - # do not require normalization, we do it for consistency. - # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. - # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 - - if lowercase: - sentence = sentence.lower() - - if tokenizer in ["13a", "intl"]: - if version.parse(sacrebleu.__version__).major >= 2: - normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) - else: - normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) - elif tokenizer == "moses": - normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) - elif tokenizer == "penn": - normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) - else: - normalized_sent = sentence - - if not return_str: - normalized_sent = normalized_sent.split() - - return normalized_sent - - -def compute_sari(sources, predictions, references): - if not (len(sources) == len(predictions) == len(references)): - raise ValueError("Sources length must match predictions and references lengths.") - sari_score = 0 - for src, pred, refs in zip(sources, predictions, references): - sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) - sari_score = sari_score / len(predictions) - return 100 * sari_score - - -def compute_sacrebleu( - predictions, - references, - smooth_method="exp", - smooth_value=None, - force=False, - lowercase=False, - use_effective_order=False, -): - references_per_prediction = len(references[0]) - if any(len(refs) != references_per_prediction for refs in references): - raise ValueError("Sacrebleu requires the same number of references for each prediction") - transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] - output = sacrebleu.corpus_bleu( - predictions, - transformed_references, - smooth_method=smooth_method, - smooth_value=smooth_value, - force=force, - lowercase=lowercase, - use_effective_order=use_effective_order, - ) - return output.score - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class WikiSplit(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("string", id="sequence"), - "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), - } - ), - codebase_urls=[ - "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py", - "https://github.com/cocoxu/simplification/blob/master/SARI.py", - "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", - "https://github.com/mjpost/sacreBLEU", - ], - reference_urls=[ - "https://www.aclweb.org/anthology/Q16-1029.pdf", - "https://github.com/mjpost/sacreBLEU", - "https://en.wikipedia.org/wiki/BLEU", - "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", - ], - ) - - def _compute(self, sources, predictions, references): - result = {} - result.update({"sari": compute_sari(sources=sources, predictions=predictions, references=references)}) - result.update({"sacrebleu": compute_sacrebleu(predictions=predictions, references=references)}) - result.update({"exact": compute_em(predictions=predictions, references=references)}) - return result diff --git a/metrics/xnli/README.md b/metrics/xnli/README.md deleted file mode 100644 index 889804f77e4..00000000000 --- a/metrics/xnli/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Metric Card for XNLI - -## Metric description - -The XNLI metric allows to evaluate a model's score on the [XNLI dataset](https://huggingface.co/datasets/xnli), which is a subset of a few thousand examples from the [MNLI dataset](https://huggingface.co/datasets/glue/viewer/mnli) that have been translated into a 14 different languages, some of which are relatively low resource such as Swahili and Urdu. - -As with MNLI, the task is to predict textual entailment (does sentence A imply/contradict/neither sentence B) and is a classification task (given two sentences, predict one of three labels). - -## How to use - -The XNLI metric is computed based on the `predictions` (a list of predicted labels) and the `references` (a list of ground truth labels). - -```python -from datasets import load_metric -xnli_metric = load_metric("xnli") -predictions = [0, 1] -references = [0, 1] -results = xnli_metric.compute(predictions=predictions, references=references) -``` - -## Output values - -The output of the XNLI metric is simply the `accuracy`, i.e. the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). - -### Values from popular papers -The [original XNLI paper](https://arxiv.org/pdf/1809.05053.pdf) reported accuracies ranging from 59.3 (for `ur`) to 73.7 (for `en`) for the BiLSTM-max model. - -For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/xnli). - -## Examples - -Maximal values: - -```python ->>> from datasets import load_metric ->>> xnli_metric = load_metric("xnli") ->>> predictions = [0, 1] ->>> references = [0, 1] ->>> results = xnli_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'accuracy': 1.0} -``` - -Minimal values: - -```python ->>> from datasets import load_metric ->>> xnli_metric = load_metric("xnli") ->>> predictions = [1, 0] ->>> references = [0, 1] ->>> results = xnli_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'accuracy': 0.0} -``` - -Partial match: - -```python ->>> from datasets import load_metric ->>> xnli_metric = load_metric("xnli") ->>> predictions = [1, 0, 1] ->>> references = [1, 0, 0] ->>> results = xnli_metric.compute(predictions=predictions, references=references) ->>> print(results) -{'accuracy': 0.6666666666666666} -``` - -## Limitations and bias - -While accuracy alone does give a certain indication of performance, it can be supplemented by error analysis and a better understanding of the model's mistakes on each of the categories represented in the dataset, especially if they are unbalanced. - -While the XNLI dataset is multilingual and represents a diversity of languages, in reality, cross-lingual sentence understanding goes beyond translation, given that there are many cultural differences that have an impact on human sentiment annotations. Since the XNLI dataset was obtained by translation based on English sentences, it does not capture these cultural differences. - - - -## Citation - -```bibtex -@InProceedings{conneau2018xnli, - author = "Conneau, Alexis - and Rinott, Ruty - and Lample, Guillaume - and Williams, Adina - and Bowman, Samuel R. - and Schwenk, Holger - and Stoyanov, Veselin", - title = "XNLI: Evaluating Cross-lingual Sentence Representations", - booktitle = "Proceedings of the 2018 Conference on Empirical Methods - in Natural Language Processing", - year = "2018", - publisher = "Association for Computational Linguistics", - location = "Brussels, Belgium", -} -``` - -## Further References - -- [XNI Dataset GitHub](https://github.com/facebookresearch/XNLI) -- [HuggingFace Tasks -- Text Classification](https://huggingface.co/tasks/text-classification) diff --git a/metrics/xnli/xnli.py b/metrics/xnli/xnli.py deleted file mode 100644 index 9b06eeea0de..00000000000 --- a/metrics/xnli/xnli.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""XNLI benchmark metric.""" - -import datasets - - -_CITATION = """\ -@InProceedings{conneau2018xnli, - author = "Conneau, Alexis - and Rinott, Ruty - and Lample, Guillaume - and Williams, Adina - and Bowman, Samuel R. - and Schwenk, Holger - and Stoyanov, Veselin", - title = "XNLI: Evaluating Cross-lingual Sentence Representations", - booktitle = "Proceedings of the 2018 Conference on Empirical Methods - in Natural Language Processing", - year = "2018", - publisher = "Association for Computational Linguistics", - location = "Brussels, Belgium", -} -""" - -_DESCRIPTION = """\ -XNLI is a subset of a few thousand examples from MNLI which has been translated -into a 14 different languages (some low-ish resource). As with MNLI, the goal is -to predict textual entailment (does sentence A imply/contradict/neither sentence -B) and is a classification task (given two sentences, predict one of three -labels). -""" - -_KWARGS_DESCRIPTION = """ -Computes XNLI score which is just simple accuracy. -Args: - predictions: Predicted labels. - references: Ground truth labels. -Returns: - 'accuracy': accuracy -Examples: - - >>> predictions = [0, 1] - >>> references = [0, 1] - >>> xnli_metric = datasets.load_metric("xnli") - >>> results = xnli_metric.compute(predictions=predictions, references=references) - >>> print(results) - {'accuracy': 1.0} -""" - - -def simple_accuracy(preds, labels): - return (preds == labels).mean() - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class Xnli(datasets.Metric): - def _info(self): - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - { - "predictions": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), - "references": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), - } - ), - codebase_urls=[], - reference_urls=[], - format="numpy", - ) - - def _compute(self, predictions, references): - return {"accuracy": simple_accuracy(predictions, references)} diff --git a/metrics/xtreme_s/README.md b/metrics/xtreme_s/README.md deleted file mode 100644 index 80c5239aaff..00000000000 --- a/metrics/xtreme_s/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# Metric Card for XTREME-S - - -## Metric Description - -The XTREME-S metric aims to evaluate model performance on the Cross-lingual TRansfer Evaluation of Multilingual Encoders for Speech (XTREME-S) benchmark. - -This benchmark was designed to evaluate speech representations across languages, tasks, domains and data regimes. It covers 102 languages from 10+ language families, 3 different domains and 4 task families: speech recognition, translation, classification and retrieval. - -## How to Use - -There are two steps: (1) loading the XTREME-S metric relevant to the subset of the benchmark being used for evaluation; and (2) calculating the metric. - -1. **Loading the relevant XTREME-S metric** : the subsets of XTREME-S are the following: `mls`, `voxpopuli`, `covost2`, `fleurs-asr`, `fleurs-lang_id`, `minds14` and `babel`. More information about the different subsets can be found on the [XTREME-S benchmark page](https://huggingface.co/datasets/google/xtreme_s). - - -```python ->>> from datasets import load_metric ->>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls') -``` - -2. **Calculating the metric**: the metric takes two inputs : - -- `predictions`: a list of predictions to score, with each prediction a `str`. - -- `references`: a list of lists of references for each translation, with each reference a `str`. - -```python ->>> references = ["it is sunny here", "paper and pen are essentials"] ->>> predictions = ["it's sunny", "paper pen are essential"] ->>> results = xtreme_s_metric.compute(predictions=predictions, references=references) -``` - -It also has two optional arguments: - -- `bleu_kwargs`: a `dict` of keywords to be passed when computing the `bleu` metric for the `covost2` subset. Keywords can be one of `smooth_method`, `smooth_value`, `force`, `lowercase`, `tokenize`, `use_effective_order`. - -- `wer_kwargs`: optional dict of keywords to be passed when computing `wer` and `cer`, which are computed for the `mls`, `fleurs-asr`, `voxpopuli`, and `babel` subsets. Keywords are `concatenate_texts`. - -## Output values - -The output of the metric depends on the XTREME-S subset chosen, consisting of a dictionary that contains one or several of the following metrics: - -- `accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). This is returned for the `fleurs-lang_id` and `minds14` subsets. - -- `f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. It is returned for the `minds14` subset. - -- `wer`: Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. The lower the value, the better the performance of the ASR system, with a WER of 0 being a perfect score (see [WER score](https://huggingface.co/metrics/wer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. - -- `cer`: Character error rate (CER) is similar to WER, but operates on character instead of word. The lower the CER value, the better the performance of the ASR system, with a CER of 0 being a perfect score (see [CER score](https://huggingface.co/metrics/cer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. - -- `bleu`: the BLEU score, calculated according to the SacreBLEU metric approach. It can take any value between 0.0 and 100.0, inclusive, with higher values being better (see [SacreBLEU](https://huggingface.co/metrics/sacrebleu) for more details). This is returned for the `covost2` subset. - - -### Values from popular papers -The [original XTREME-S paper](https://arxiv.org/pdf/2203.10752.pdf) reported average WERs ranging from 9.2 to 14.6, a BLEU score of 20.6, an accuracy of 73.3 and F1 score of 86.9, depending on the subsets of the dataset tested on. - -## Examples - -For the `mls` subset (which outputs `wer` and `cer`): - -```python ->>> from datasets import load_metric ->>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls') ->>> references = ["it is sunny here", "paper and pen are essentials"] ->>> predictions = ["it's sunny", "paper pen are essential"] ->>> results = xtreme_s_metric.compute(predictions=predictions, references=references) ->>> print({k: round(v, 2) for k, v in results.items()}) -{'wer': 0.56, 'cer': 0.27} -``` - -For the `covost2` subset (which outputs `bleu`): - -```python ->>> from datasets import load_metric ->>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'covost2') ->>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] ->>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] ->>> results = xtreme_s_metric.compute(predictions=predictions, references=references) ->>> print({k: round(v, 2) for k, v in results.items()}) -{'bleu': 31.65} -``` - -For the `fleurs-lang_id` subset (which outputs `accuracy`): - -```python ->>> from datasets import load_metric ->>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'fleurs-lang_id') ->>> references = [0, 1, 0, 0, 1] ->>> predictions = [0, 1, 1, 0, 0] ->>> results = xtreme_s_metric.compute(predictions=predictions, references=references) ->>> print({k: round(v, 2) for k, v in results.items()}) -{'accuracy': 0.6} - ``` - -For the `minds14` subset (which outputs `f1` and `accuracy`): - -```python ->>> from datasets import load_metric ->>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'minds14') ->>> references = [0, 1, 0, 0, 1] ->>> predictions = [0, 1, 1, 0, 0] ->>> results = xtreme_s_metric.compute(predictions=predictions, references=references) ->>> print({k: round(v, 2) for k, v in results.items()}) -{'f1': 0.58, 'accuracy': 0.6} -``` - -## Limitations and bias -This metric works only with datasets that have the same format as the [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s). - -While the XTREME-S dataset is meant to represent a variety of languages and tasks, it has inherent biases: it is missing many languages that are important and under-represented in NLP datasets. - -It also has a particular focus on read-speech because common evaluation benchmarks like CoVoST-2 or LibriSpeech evaluate on this type of speech, which results in a mismatch between performance obtained in a read-speech setting and a more noisy setting (in production or live deployment, for instance). - -## Citation - -```bibtex -@article{conneau2022xtreme, - title={XTREME-S: Evaluating Cross-lingual Speech Representations}, - author={Conneau, Alexis and Bapna, Ankur and Zhang, Yu and Ma, Min and von Platen, Patrick and Lozhkov, Anton and Cherry, Colin and Jia, Ye and Rivera, Clara and Kale, Mihir and others}, - journal={arXiv preprint arXiv:2203.10752}, - year={2022} -} -``` - -## Further References - -- [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s) -- [XTREME-S github repository](https://github.com/google-research/xtreme) diff --git a/metrics/xtreme_s/xtreme_s.py b/metrics/xtreme_s/xtreme_s.py deleted file mode 100644 index bab49c1a4b0..00000000000 --- a/metrics/xtreme_s/xtreme_s.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2022 The HuggingFace Datasets Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""XTREME-S benchmark metric.""" - -from typing import List - -from packaging import version -from sklearn.metrics import f1_score - -import datasets -from datasets.config import PY_VERSION - - -if PY_VERSION < version.parse("3.8"): - import importlib_metadata -else: - import importlib.metadata as importlib_metadata - - -# TODO(Patrick/Anton) -_CITATION = """\ -""" - -_DESCRIPTION = """\ - XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages. - XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval. -""" - -_KWARGS_DESCRIPTION = """ -Compute XTREME-S evaluation metric associated to each XTREME-S dataset. -Args: - predictions: list of predictions to score. - Each translation should be tokenized into a list of tokens. - references: list of lists of references for each translation. - Each reference should be tokenized into a list of tokens. - bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'. - Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase', - 'tokenize', 'use_effective_order'. - wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'. - Keywords include 'concatenate_texts'. -Returns: depending on the XTREME-S task, one or several of: - "accuracy": Accuracy - for 'fleurs-lang_id', 'minds14' - "f1": F1 score - for 'minds14' - "wer": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' - "cer": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' - "bleu": BLEU score according to the `sacrebleu` metric - for 'covost2' -Examples: - - >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls') # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel' - >>> references = ["it is sunny here", "paper and pen are essentials"] - >>> predictions = ["it's sunny", "paper pen are essential"] - >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) - >>> print({k: round(v, 2) for k, v in results.items()}) - {'wer': 0.56, 'cer': 0.27} - - >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'covost2') - >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] - >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] - >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) - >>> print({k: round(v, 2) for k, v in results.items()}) - {'bleu': 31.65} - - >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'fleurs-lang_id') - >>> references = [0, 1, 0, 0, 1] - >>> predictions = [0, 1, 1, 0, 0] - >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) - >>> print({k: round(v, 2) for k, v in results.items()}) - {'accuracy': 0.6} - - >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'minds14') - >>> references = [0, 1, 0, 0, 1] - >>> predictions = [0, 1, 1, 0, 0] - >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) - >>> print({k: round(v, 2) for k, v in results.items()}) - {'f1': 0.58, 'accuracy': 0.6} -""" - -_CONFIG_NAMES = ["fleurs-asr", "mls", "voxpopuli", "babel", "covost2", "fleurs-lang_id", "minds14"] -SENTENCE_DELIMITER = "" - -try: - from jiwer import transforms as tr - - _jiwer_available = True -except ImportError: - _jiwer_available = False - -if _jiwer_available and version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): - - class SentencesToListOfCharacters(tr.AbstractTransform): - def __init__(self, sentence_delimiter: str = " "): - self.sentence_delimiter = sentence_delimiter - - def process_string(self, s: str): - return list(s) - - def process_list(self, inp: List[str]): - chars = [] - for sent_idx, sentence in enumerate(inp): - chars.extend(self.process_string(sentence)) - if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: - chars.append(self.sentence_delimiter) - return chars - - cer_transform = tr.Compose( - [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] - ) -elif _jiwer_available: - cer_transform = tr.Compose( - [ - tr.RemoveMultipleSpaces(), - tr.Strip(), - tr.ReduceToSingleSentence(SENTENCE_DELIMITER), - tr.ReduceToListOfListOfChars(), - ] - ) -else: - cer_transform = None - - -def simple_accuracy(preds, labels): - return float((preds == labels).mean()) - - -def f1_and_simple_accuracy(preds, labels): - return { - "f1": float(f1_score(y_true=labels, y_pred=preds, average="macro")), - "accuracy": simple_accuracy(preds, labels), - } - - -def bleu( - preds, - labels, - smooth_method="exp", - smooth_value=None, - force=False, - lowercase=False, - tokenize=None, - use_effective_order=False, -): - # xtreme-s can only have one label - labels = [[label] for label in labels] - preds = list(preds) - try: - import sacrebleu as scb - except ImportError: - raise ValueError( - "sacrebleu has to be installed in order to apply the bleu metric for covost2." - "You can install it via `pip install sacrebleu`." - ) - - if version.parse(scb.__version__) < version.parse("1.4.12"): - raise ImportWarning( - "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" - 'You can install it with `pip install "sacrebleu>=1.4.12"`.' - ) - - references_per_prediction = len(labels[0]) - if any(len(refs) != references_per_prediction for refs in labels): - raise ValueError("Sacrebleu requires the same number of references for each prediction") - transformed_references = [[refs[i] for refs in labels] for i in range(references_per_prediction)] - output = scb.corpus_bleu( - preds, - transformed_references, - smooth_method=smooth_method, - smooth_value=smooth_value, - force=force, - lowercase=lowercase, - use_effective_order=use_effective_order, - **({"tokenize": tokenize} if tokenize else {}), - ) - return {"bleu": output.score} - - -def wer_and_cer(preds, labels, concatenate_texts, config_name): - try: - from jiwer import compute_measures - except ImportError: - raise ValueError( - f"jiwer has to be installed in order to apply the wer metric for {config_name}." - "You can install it via `pip install jiwer`." - ) - - if concatenate_texts: - wer = compute_measures(labels, preds)["wer"] - - cer = compute_measures(labels, preds, truth_transform=cer_transform, hypothesis_transform=cer_transform)["wer"] - return {"wer": wer, "cer": cer} - else: - - def compute_score(preds, labels, score_type="wer"): - incorrect = 0 - total = 0 - for prediction, reference in zip(preds, labels): - if score_type == "wer": - measures = compute_measures(reference, prediction) - elif score_type == "cer": - measures = compute_measures( - reference, prediction, truth_transform=cer_transform, hypothesis_transform=cer_transform - ) - incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] - total += measures["substitutions"] + measures["deletions"] + measures["hits"] - return incorrect / total - - return {"wer": compute_score(preds, labels, "wer"), "cer": compute_score(preds, labels, "cer")} - - -@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) -class XtremeS(datasets.Metric): - def _info(self): - if self.config_name not in _CONFIG_NAMES: - raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") - - pred_type = "int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string" - - return datasets.MetricInfo( - description=_DESCRIPTION, - citation=_CITATION, - inputs_description=_KWARGS_DESCRIPTION, - features=datasets.Features( - {"predictions": datasets.Value(pred_type), "references": datasets.Value(pred_type)} - ), - codebase_urls=[], - reference_urls=[], - format="numpy", - ) - - def _compute(self, predictions, references, bleu_kwargs=None, wer_kwargs=None): - bleu_kwargs = bleu_kwargs if bleu_kwargs is not None else {} - wer_kwargs = wer_kwargs if wer_kwargs is not None else {} - - if self.config_name == "fleurs-lang_id": - return {"accuracy": simple_accuracy(predictions, references)} - elif self.config_name == "minds14": - return f1_and_simple_accuracy(predictions, references) - elif self.config_name == "covost2": - smooth_method = bleu_kwargs.pop("smooth_method", "exp") - smooth_value = bleu_kwargs.pop("smooth_value", None) - force = bleu_kwargs.pop("force", False) - lowercase = bleu_kwargs.pop("lowercase", False) - tokenize = bleu_kwargs.pop("tokenize", None) - use_effective_order = bleu_kwargs.pop("use_effective_order", False) - return bleu( - preds=predictions, - labels=references, - smooth_method=smooth_method, - smooth_value=smooth_value, - force=force, - lowercase=lowercase, - tokenize=tokenize, - use_effective_order=use_effective_order, - ) - elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]: - concatenate_texts = wer_kwargs.pop("concatenate_texts", False) - return wer_and_cer(predictions, references, concatenate_texts, self.config_name) - else: - raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") diff --git a/pyproject.toml b/pyproject.toml index 53328fa65cc..511a9e0d744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,9 @@ known-first-party = ["datasets"] [tool.pytest.ini_options] # Test fails if a FutureWarning is thrown by `huggingface_hub` -# Temporarily disabled because transformers 4.41.1 calls deprecated code from `huggingface_hub` that causes FutureWarning -# filterwarnings = [ -# "error::FutureWarning:huggingface_hub*", -# ] +filterwarnings = [ + "error::FutureWarning:huggingface_hub*", +] markers = [ "unit: unit test", "integration: integration test", diff --git a/setup.py b/setup.py index 66d8080115f..ae99e1dbec8 100644 --- a/setup.py +++ b/setup.py @@ -185,33 +185,6 @@ ] -METRICS_TESTS_REQUIRE = [ - # metrics dependencies - "accelerate", # for frugalscore (calls transformers' Trainer) - "bert_score>=0.3.6", - "jiwer", - "langdetect", - "mauve-text", - "nltk", - "rouge_score", - "sacrebleu", - "sacremoses", - "scikit-learn", - "scipy", - "sentencepiece", # for bleurt - "seqeval", - "spacy>=3.0.0", - "tldextract", - # to speed up pip backtracking - "toml>=0.10.1", - "typer<0.5.0", # pinned to work with Spacy==3.4.3 on Windows: see https://github.com/tiangolo/typer/issues/427 - "requests_file>=1.5.1", - "tldextract>=3.1.0", - "texttable>=1.6.3", - "Werkzeug>=1.0.1", - "six~=1.15.0", -] - TESTS_REQUIRE.extend(VISION_REQUIRE) TESTS_REQUIRE.extend(AUDIO_REQUIRE) @@ -239,7 +212,6 @@ "streaming": [], # for backward compatibility "dev": TESTS_REQUIRE + QUALITY_REQUIRE + DOCS_REQUIRE, "tests": TESTS_REQUIRE, - "metrics-tests": METRICS_TESTS_REQUIRE, "quality": QUALITY_REQUIRE, "benchmarks": BENCHMARKS_REQUIRE, "docs": DOCS_REQUIRE, @@ -279,6 +251,6 @@ "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering :: Artificial Intelligence", ], - keywords="datasets machine learning datasets metrics", + keywords="datasets machine learning datasets", zip_safe=False, # Required for mypy to find the py.typed file ) diff --git a/src/datasets/__init__.py b/src/datasets/__init__.py index 4045f76aadd..111f930c564 100644 --- a/src/datasets/__init__.py +++ b/src/datasets/__init__.py @@ -22,19 +22,16 @@ from .download import * from .features import * from .fingerprint import disable_caching, enable_caching, is_caching_enabled -from .info import DatasetInfo, MetricInfo +from .info import DatasetInfo from .inspect import ( get_dataset_config_info, get_dataset_config_names, get_dataset_default_config_name, get_dataset_infos, get_dataset_split_names, - inspect_metric, - list_metrics, ) from .iterable_dataset import IterableDataset -from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric -from .metric import Metric +from .load import load_dataset, load_dataset_builder, load_from_disk from .splits import ( NamedSplit, NamedSplitAll, diff --git a/src/datasets/config.py b/src/datasets/config.py index b4bc21b90d2..e2de170bcba 100644 --- a/src/datasets/config.py +++ b/src/datasets/config.py @@ -17,11 +17,6 @@ CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets" REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}" -# Metrics -S3_METRICS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics" -CLOUDFRONT_METRICS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/metric" -REPO_METRICS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}" - # Hub HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}" @@ -150,9 +145,6 @@ DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets") HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE)) -DEFAULT_HF_METRICS_CACHE = os.path.join(HF_CACHE_HOME, "metrics") -HF_METRICS_CACHE = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE)) - DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) @@ -229,7 +221,6 @@ DATASET_INFO_FILENAME = "dataset_info.json" DATASETDICT_INFOS_FILENAME = "dataset_infos.json" LICENSE_FILENAME = "LICENSE" -METRIC_INFO_FILENAME = "metric_info.json" DATASETDICT_JSON_FILENAME = "dataset_dict.json" METADATA_CONFIGS_FIELD = "configs" REPOCARD_FILENAME = "README.md" diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py index 0973a1fed28..df8eeffd306 100644 --- a/src/datasets/features/features.py +++ b/src/datasets/features/features.py @@ -841,7 +841,7 @@ def __array__(self, dtype=None): https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray """ - if dtype == object: + if dtype == np.dtype(object): out = np.empty(len(self._data), dtype=object) for i in range(len(self._data)): out[i] = self._data[i] diff --git a/src/datasets/info.py b/src/datasets/info.py index 066c30a8a1e..204f2f6c1cc 100644 --- a/src/datasets/info.py +++ b/src/datasets/info.py @@ -13,7 +13,7 @@ # limitations under the License. # Lint as: python3 -"""DatasetInfo and MetricInfo record information we know about a dataset and a metric. +"""DatasetInfo record information we know about a dataset. This includes things that we know about the dataset statically, i.e.: - description @@ -42,7 +42,7 @@ from huggingface_hub import DatasetCard, DatasetCardData from . import config -from .features import Features, Value +from .features import Features from .splits import SplitDict from .tasks import TaskTemplate, task_template_from_dict from .utils import Version @@ -465,88 +465,3 @@ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: dataset_info_yaml_dict.pop("config_name", None) dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict} dataset_card_data["dataset_info"].append(dataset_info_yaml_dict) - - -@dataclass -class MetricInfo: - """Information about a metric. - - `MetricInfo` documents a metric, including its name, version, and features. - See the constructor arguments and properties for a full list. - - Note: Not all fields are known on construction and may be updated later. - """ - - # Set in the dataset scripts - description: str - citation: str - features: Features - inputs_description: str = dataclasses.field(default_factory=str) - homepage: str = dataclasses.field(default_factory=str) - license: str = dataclasses.field(default_factory=str) - codebase_urls: List[str] = dataclasses.field(default_factory=list) - reference_urls: List[str] = dataclasses.field(default_factory=list) - streamable: bool = False - format: Optional[str] = None - - # Set later by the builder - metric_name: Optional[str] = None - config_name: Optional[str] = None - experiment_id: Optional[str] = None - - def __post_init__(self): - if self.format is not None: - for key, value in self.features.items(): - if not isinstance(value, Value): - raise ValueError( - f"When using 'numpy' format, all features should be a `datasets.Value` feature. " - f"Here {key} is an instance of {value.__class__.__name__}" - ) - - def write_to_directory(self, metric_info_dir, pretty_print=False): - """Write `MetricInfo` as JSON to `metric_info_dir`. - Also save the license separately in LICENCE. - If `pretty_print` is True, the JSON will be pretty-printed with the indent level of 4. - - Example: - - ```py - >>> from datasets import load_metric - >>> metric = load_metric("accuracy") - >>> metric.info.write_to_directory("/path/to/directory/") - ``` - """ - with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), "w", encoding="utf-8") as f: - json.dump(asdict(self), f, indent=4 if pretty_print else None) - - if self.license: - with open(os.path.join(metric_info_dir, config.LICENSE_FILENAME), "w", encoding="utf-8") as f: - f.write(self.license) - - @classmethod - def from_directory(cls, metric_info_dir) -> "MetricInfo": - """Create MetricInfo from the JSON file in `metric_info_dir`. - - Args: - metric_info_dir: `str` The directory containing the metadata file. This - should be the root directory of a specific dataset version. - - Example: - - ```py - >>> from datasets import MetricInfo - >>> metric_info = MetricInfo.from_directory("/path/to/directory/") - ``` - """ - logger.info(f"Loading Metric info from {metric_info_dir}") - if not metric_info_dir: - raise ValueError("Calling MetricInfo.from_directory() with undefined metric_info_dir.") - - with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), encoding="utf-8") as f: - metric_info_dict = json.load(f) - return cls.from_dict(metric_info_dict) - - @classmethod - def from_dict(cls, metric_info_dict: dict) -> "MetricInfo": - field_names = {f.name for f in dataclasses.fields(cls)} - return cls(**{k: v for k, v in metric_info_dict.items() if k in field_names}) diff --git a/src/datasets/inspect.py b/src/datasets/inspect.py index 30ae8440f41..a383ef5b4b6 100644 --- a/src/datasets/inspect.py +++ b/src/datasets/inspect.py @@ -15,14 +15,9 @@ # Lint as: python3 """List and inspect datasets.""" -import inspect import os -import shutil -from pathlib import PurePath from typing import Dict, List, Mapping, Optional, Sequence, Union -import huggingface_hub - from .download.download_config import DownloadConfig from .download.download_manager import DownloadMode from .download.streaming_download_manager import StreamingDownloadManager @@ -30,12 +25,8 @@ from .load import ( dataset_module_factory, get_dataset_builder_class, - import_main_class, load_dataset_builder, - metric_module_factory, ) -from .utils.deprecation_utils import deprecated -from .utils.file_utils import relative_to_absolute_path from .utils.logging import get_logger from .utils.version import Version @@ -47,89 +38,6 @@ class SplitsNotFoundError(ValueError): pass -@deprecated( - "Use 'evaluate.list_evaluation_modules' instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate" -) -def list_metrics(with_community_metrics=True, with_details=False): - """List all the metrics script available on the Hugging Face Hub. - - - - Use `evaluate.list_evaluation_modules` instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate - - - - Args: - with_community_metrics (:obj:`bool`, optional, default ``True``): Include the community provided metrics. - with_details (:obj:`bool`, optional, default ``False``): Return the full details on the metrics instead of only the short name. - - Example: - - ```py - >>> from datasets import list_metrics - >>> list_metrics() - ['accuracy', - 'bertscore', - 'bleu', - 'bleurt', - 'cer', - 'chrf', - ... - ] - ``` - """ - metrics = huggingface_hub.list_metrics() - if not with_community_metrics: - metrics = [metric for metric in metrics if "/" not in metric.id] - if not with_details: - metrics = [metric.id for metric in metrics] - return metrics - - -@deprecated( - "Use 'evaluate.inspect_evaluation_module' instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate" -) -def inspect_metric(path: str, local_path: str, download_config: Optional[DownloadConfig] = None, **download_kwargs): - r""" - Allow inspection/modification of a metric script by copying it on local drive at local_path. - - - - Use `evaluate.inspect_evaluation_module` instead, from the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - - Args: - path (``str``): path to the dataset processing script with the dataset builder. Can be either: - - - a local path to processing script or the directory containing the script (if the script has the same name as the directory), - e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'`` - - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``) - e.g. ``'squad'``, ``'glue'`` or ``'openai/webtext'`` - local_path (``str``): path to the local folder to copy the datset script to. - download_config (Optional ``datasets.DownloadConfig``): specific download configuration parameters. - **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. - """ - metric_module = metric_module_factory(path, download_config=download_config, **download_kwargs) - metric_cls = import_main_class(metric_module.module_path, dataset=False) - module_source_path = inspect.getsourcefile(metric_cls) - module_source_dirpath = os.path.dirname(module_source_path) - for dirpath, dirnames, filenames in os.walk(module_source_dirpath): - dst_dirpath = os.path.join(local_path, os.path.relpath(dirpath, module_source_dirpath)) - os.makedirs(dst_dirpath, exist_ok=True) - # skipping hidden directories; prune the search - dirnames[:] = [dirname for dirname in dirnames if not dirname.startswith((".", "__"))] - for filename in filenames: - shutil.copy2(os.path.join(dirpath, filename), os.path.join(dst_dirpath, filename)) - shutil.copystat(dirpath, dst_dirpath) - local_path = relative_to_absolute_path(local_path) - print( - f"The processing scripts for metric {path} can be inspected at {local_path}. " - f"The main class is in {module_source_dirpath}. " - f'You can modify this processing scripts and use it with `datasets.load_metric("{PurePath(local_path).as_posix()}")`.' - ) - - def get_dataset_infos( path: str, data_files: Optional[Union[Dict, List, str]] = None, @@ -227,7 +135,7 @@ def get_dataset_config_names( Download/generate mode. dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`): Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`. - By default the datasets and metrics are stored inside the `datasets_modules` module. + By default the datasets are stored inside the `datasets_modules` module. data_files (`Union[Dict, List, str]`, *optional*): Defining the data_files of the dataset configuration. **download_kwargs (additional keyword arguments): @@ -299,7 +207,7 @@ def get_dataset_default_config_name( Download/generate mode. dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`): Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`. - By default the datasets and metrics are stored inside the `datasets_modules` module. + By default the datasets are stored inside the `datasets_modules` module. data_files (`Union[Dict, List, str]`, *optional*): Defining the data_files of the dataset configuration. **download_kwargs (additional keyword arguments): diff --git a/src/datasets/load.py b/src/datasets/load.py index 2bd242c573a..a593233a2c2 100644 --- a/src/datasets/load.py +++ b/src/datasets/load.py @@ -62,7 +62,6 @@ from .fingerprint import Hasher from .info import DatasetInfo, DatasetInfosDict from .iterable_dataset import IterableDataset -from .metric import Metric from .naming import camelcase_to_snakecase, snakecase_to_camelcase from .packaged_modules import ( _EXTENSION_TO_MODULE, @@ -73,13 +72,11 @@ ) from .splits import Split from .utils import _dataset_viewer -from .utils.deprecation_utils import deprecated from .utils.file_utils import ( OfflineModeIsEnabled, _raise_if_offline_mode_is_enabled, cached_path, head_hf_s3, - hf_github_url, init_hf_modules, is_relative_path, relative_to_absolute_path, @@ -146,7 +143,7 @@ def init_dynamic_modules( ): """ Create a module with name `name` in which you can add dynamic modules - such as metrics or datasets. The module can be imported using its name. + such as datasets. The module can be imported using its name. The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can be overridden by specifying a path to another directory in `hf_modules_cache`. """ @@ -159,22 +156,13 @@ def init_dynamic_modules( return dynamic_modules_path -def import_main_class(module_path, dataset=True) -> Optional[Union[Type[DatasetBuilder], Type[Metric]]]: - """Import a module at module_path and return its main class: - - a DatasetBuilder if dataset is True - - a Metric if dataset is False - """ +def import_main_class(module_path) -> Optional[Type[DatasetBuilder]]: + """Import a module at module_path and return its main class: a DatasetBuilder""" module = importlib.import_module(module_path) - - if dataset: - main_cls_type = DatasetBuilder - else: - main_cls_type = Metric - # Find the main class in our imported module module_main_cls = None for name, obj in module.__dict__.items(): - if inspect.isclass(obj) and issubclass(obj, main_cls_type): + if inspect.isclass(obj) and issubclass(obj, DatasetBuilder): if inspect.isabstract(obj): continue module_main_cls = obj @@ -283,11 +271,11 @@ def files_to_hash(file_paths: List[str]) -> str: return _hash_python_lines(lines) -def increase_load_count(name: str, resource_type: str): - """Update the download count of a dataset or metric.""" +def increase_load_count(name: str): + """Update the download count of a dataset.""" if not config.HF_HUB_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: try: - head_hf_s3(name, filename=name + ".py", dataset=(resource_type == "dataset")) + head_hf_s3(name, filename=name + ".py") except Exception: pass @@ -382,14 +370,14 @@ def _copy_script_and_other_resources_in_importable_dir( Return: importable_file: path to an importable module with importlib.import_module """ - # Define a directory with a unique name in our dataset or metric folder - # path is: ./datasets|metrics/dataset|metric_name/hash_from_code/script.py - # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + # Define a directory with a unique name in our dataset folder + # path is: ./datasets/dataset_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset processing file together importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) importable_file = os.path.join(importable_subdirectory, name + ".py") # Prevent parallel disk operations with lock_importable_file(importable_file): - # Create main dataset/metrics folder if needed + # Create main dataset folder if needed if download_mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(importable_directory_path): shutil.rmtree(importable_directory_path) os.makedirs(importable_directory_path, exist_ok=True) @@ -698,215 +686,11 @@ class DatasetModule: importable_file_path: Optional[str] = None -@dataclass -class MetricModule: - module_path: str - hash: str - - class _DatasetModuleFactory: def get_module(self) -> DatasetModule: raise NotImplementedError -class _MetricModuleFactory: - def get_module(self) -> MetricModule: - raise NotImplementedError - - -class GithubMetricModuleFactory(_MetricModuleFactory): - """Get the module of a metric. The metric script is downloaded from GitHub. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - """ - - @deprecated("Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate") - def __init__( - self, - name: str, - revision: Optional[Union[str, Version]] = None, - download_config: Optional[DownloadConfig] = None, - download_mode: Optional[Union[DownloadMode, str]] = None, - dynamic_modules_path: Optional[str] = None, - trust_remote_code: Optional[str] = None, - ): - self.name = name - self.revision = revision - self.download_config = download_config.copy() if download_config else DownloadConfig() - if self.download_config.max_retries < 3: - self.download_config.max_retries = 3 - self.download_mode = download_mode - self.dynamic_modules_path = dynamic_modules_path - self.trust_remote_code = trust_remote_code - assert self.name.count("/") == 0 - increase_load_count(name, resource_type="metric") - - def download_loading_script(self, revision: Optional[str]) -> str: - file_path = hf_github_url(path=self.name, name=self.name + ".py", revision=revision, dataset=False) - download_config = self.download_config.copy() - if download_config.download_desc is None: - download_config.download_desc = "Downloading builder script" - return cached_path(file_path, download_config=download_config) - - def get_module(self) -> MetricModule: - if config.HF_DATASETS_TRUST_REMOTE_CODE and self.trust_remote_code is None: - _loading_script_url = hf_github_url( - path=self.name, name=self.name + ".py", revision=self.revision, dataset=False - ) - warnings.warn( - f"The repository for {self.name} contains custom code which must be executed to correctly " - f"load the metric. You can inspect the repository content at {_loading_script_url}\n" - f"You can avoid this message in future by passing the argument `trust_remote_code=True`.\n" - f"Passing `trust_remote_code=True` will be mandatory to load this metric from the next major release of `datasets`.", - FutureWarning, - ) - # get script and other files - revision = self.revision - try: - local_path = self.download_loading_script(revision) - revision = self.revision - except FileNotFoundError: - if revision is not None: - raise - else: - revision = "main" - local_path = self.download_loading_script(revision) - logger.warning( - f"Couldn't find a directory or a metric named '{self.name}' in this version. " - f"It was picked from the main branch on github instead." - ) - imports = get_imports(local_path) - local_imports = _download_additional_modules( - name=self.name, - base_path=hf_github_url(path=self.name, name="", revision=revision, dataset=False), - imports=imports, - download_config=self.download_config, - ) - # copy the script and the files in an importable directory - dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() - hash = files_to_hash([local_path] + [loc[1] for loc in local_imports]) - importable_file_path = _get_importable_file_path( - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - ) - if not os.path.exists(importable_file_path): - trust_remote_code = resolve_trust_remote_code(self.trust_remote_code, self.name) - if trust_remote_code: - _create_importable_file( - local_path=local_path, - local_imports=local_imports, - additional_files=[], - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - download_mode=self.download_mode, - ) - else: - raise ValueError( - f"Loading {self.name} requires you to execute the dataset script in that" - " repo on your local machine. Make sure you have read the code there to avoid malicious use, then" - " set the option `trust_remote_code=True` to remove this error." - ) - module_path, hash = _load_importable_file( - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - ) - # make the new module to be noticed by the import system - importlib.invalidate_caches() - return MetricModule(module_path, hash) - - -class LocalMetricModuleFactory(_MetricModuleFactory): - """Get the module of a local metric. The metric script is loaded from a local script. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - """ - - @deprecated("Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate") - def __init__( - self, - path: str, - download_config: Optional[DownloadConfig] = None, - download_mode: Optional[Union[DownloadMode, str]] = None, - dynamic_modules_path: Optional[str] = None, - trust_remote_code: Optional[str] = None, - ): - self.path = path - self.name = Path(path).stem - self.download_config = download_config or DownloadConfig() - self.download_mode = download_mode - self.dynamic_modules_path = dynamic_modules_path - self.trust_remote_code = trust_remote_code - - def get_module(self) -> MetricModule: - if config.HF_DATASETS_TRUST_REMOTE_CODE and self.trust_remote_code is None: - warnings.warn( - f"The repository for {self.name} contains custom code which must be executed to correctly " - f"load the metric. You can inspect the repository content at {self.path}\n" - f"You can avoid this message in future by passing the argument `trust_remote_code=True`.\n" - f"Passing `trust_remote_code=True` will be mandatory to load this metric from the next major release of `datasets`.", - FutureWarning, - ) - # get script and other files - imports = get_imports(self.path) - local_imports = _download_additional_modules( - name=self.name, - base_path=str(Path(self.path).parent), - imports=imports, - download_config=self.download_config, - ) - # copy the script and the files in an importable directory - dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() - hash = files_to_hash([self.path] + [loc[1] for loc in local_imports]) - importable_file_path = _get_importable_file_path( - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - ) - if not os.path.exists(importable_file_path): - trust_remote_code = resolve_trust_remote_code(self.trust_remote_code, self.name) - if trust_remote_code: - _create_importable_file( - local_path=self.path, - local_imports=local_imports, - additional_files=[], - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - download_mode=self.download_mode, - ) - else: - raise ValueError( - f"Loading {self.name} requires you to execute the dataset script in that" - " repo on your local machine. Make sure you have read the code there to avoid malicious use, then" - " set the option `trust_remote_code=True` to remove this error." - ) - module_path, hash = _load_importable_file( - dynamic_modules_path=dynamic_modules_path, - module_namespace="metrics", - subdirectory_name=hash, - name=self.name, - ) - # make the new module to be noticed by the import system - importlib.invalidate_caches() - return MetricModule(module_path, hash) - - class LocalDatasetModuleFactoryWithScript(_DatasetModuleFactory): """Get the module of a local dataset. The dataset script is loaded from a local script.""" @@ -1130,7 +914,7 @@ def __init__( self.data_dir = data_dir self.download_config = download_config self.download_mode = download_mode - increase_load_count(name, resource_type="dataset") + increase_load_count(name) def get_module(self) -> DatasetModule: base_path = Path(self.data_dir or "").expanduser().resolve().as_posix() @@ -1193,7 +977,7 @@ def __init__( self.data_dir = data_dir self.download_config = download_config or DownloadConfig() self.download_mode = download_mode - increase_load_count(name, resource_type="dataset") + increase_load_count(name) def get_module(self) -> DatasetModule: hfh_dataset_info = HfApi(config.HF_ENDPOINT).dataset_info( @@ -1378,7 +1162,7 @@ def __init__( self.name = name self.revision = revision self.download_config = download_config or DownloadConfig() - increase_load_count(name, resource_type="dataset") + increase_load_count(name) def get_module(self) -> DatasetModule: exported_parquet_files = _dataset_viewer.get_exported_parquet_files( @@ -1450,7 +1234,7 @@ def __init__( self.download_mode = download_mode self.dynamic_modules_path = dynamic_modules_path self.trust_remote_code = trust_remote_code - increase_load_count(name, resource_type="dataset") + increase_load_count(name) def download_loading_script(self) -> str: file_path = hf_dataset_url(self.name, self.name.split("/")[-1] + ".py", revision=self.revision) @@ -1643,55 +1427,6 @@ def _get_modification_time(module_hash): raise FileNotFoundError(f"Dataset {self.name} is not cached in {self.cache_dir}") -class CachedMetricModuleFactory(_MetricModuleFactory): - """ - Get the module of a metric that has been loaded once already and cached. - The script that is loaded from the cache is the most recent one with a matching name. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - """ - - @deprecated("Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate") - def __init__( - self, - name: str, - dynamic_modules_path: Optional[str] = None, - ): - self.name = name - self.dynamic_modules_path = dynamic_modules_path - assert self.name.count("/") == 0 - - def get_module(self) -> MetricModule: - dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() - importable_directory_path = os.path.join(dynamic_modules_path, "metrics", self.name) - hashes = ( - [h for h in os.listdir(importable_directory_path) if len(h) == 64] - if os.path.isdir(importable_directory_path) - else None - ) - if not hashes: - raise FileNotFoundError(f"Metric {self.name} is not cached in {dynamic_modules_path}") - # get most recent - - def _get_modification_time(module_hash): - return (Path(importable_directory_path) / module_hash / (self.name + ".py")).stat().st_mtime - - hash = sorted(hashes, key=_get_modification_time)[-1] - logger.warning( - f"Using the latest cached version of the module from {os.path.join(importable_directory_path, hash)} " - f"(last modified on {time.ctime(_get_modification_time(hash))}) since it " - f"couldn't be found locally at {self.name}, or remotely on the Hugging Face Hub." - ) - # make the new module to be noticed by the import system - module_path = ".".join([os.path.basename(dynamic_modules_path), "metrics", self.name, hash, self.name]) - importlib.invalidate_caches() - return MetricModule(module_path, hash) - - def dataset_module_factory( path: str, revision: Optional[Union[str, Version]] = None, @@ -1741,7 +1476,7 @@ def dataset_module_factory( download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. - By default, the datasets and metrics are stored inside the `datasets_modules` module. + By default, the datasets are stored inside the `datasets_modules` module. data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified, in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`. data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration. @@ -1919,221 +1654,6 @@ def dataset_module_factory( ) -@deprecated("Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate") -def metric_module_factory( - path: str, - revision: Optional[Union[str, Version]] = None, - download_config: Optional[DownloadConfig] = None, - download_mode: Optional[Union[DownloadMode, str]] = None, - dynamic_modules_path: Optional[str] = None, - trust_remote_code: Optional[bool] = None, - **download_kwargs, -) -> MetricModule: - """ - Download/extract/cache a metric module. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - - Metrics codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). - - Args: - - path (str): Path or name of the metric script. - - - if ``path`` is a local metric script or a directory containing a local metric script (if the script has the same name as the directory): - -> load the module from the metric script - e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'``. - - if ``path`` is a metric on the Hugging Face Hub (ex: `glue`, `squad`) - -> load the module from the metric script in the GitHub repository at huggingface/datasets - e.g. ``'accuracy'`` or ``'rouge'``. - - revision (Optional ``Union[str, datasets.Version]``): - If specified, the module will be loaded from the datasets repository at this version. - By default: - - it is set to the local version of the lib. - - it will also try to load it from the main branch if it's not available at the local version of the lib. - Specifying a version that is different from your local version of the lib might cause compatibility issues. - download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. - download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. - dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): - Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. - By default, the datasets and metrics are stored inside the `datasets_modules` module. - trust_remote_code (`bool`, defaults to `False`): - Whether or not to allow for datasets defined on the Hub using a dataset script. This option - should only be set to `True` for repositories you trust and in which you have read the code, as it will - execute code present on the Hub on your local machine. - - - - - - `trust_remote_code` defaults to `False` if not specified. - - - - **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override - the attributes in download_config if supplied. - - Returns: - MetricModule - """ - with warnings.catch_warnings(): - # Ignore equivalent warnings to the one already issued - warnings.filterwarnings("ignore", message=".*https://huggingface.co/docs/evaluate$", category=FutureWarning) - - if download_config is None: - download_config = DownloadConfig(**download_kwargs) - download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) - download_config.extract_compressed_file = True - download_config.force_extract = True - - filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] - if not filename.endswith(".py"): - filename = filename + ".py" - combined_path = os.path.join(path, filename) - # Try locally - if path.endswith(filename): - if os.path.isfile(path): - return LocalMetricModuleFactory( - path, - download_mode=download_mode, - dynamic_modules_path=dynamic_modules_path, - trust_remote_code=trust_remote_code, - ).get_module() - else: - raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}") - elif os.path.isfile(combined_path): - return LocalMetricModuleFactory( - combined_path, - download_mode=download_mode, - dynamic_modules_path=dynamic_modules_path, - trust_remote_code=trust_remote_code, - ).get_module() - elif is_relative_path(path) and path.count("/") == 0: - try: - return GithubMetricModuleFactory( - path, - revision=revision, - download_config=download_config, - download_mode=download_mode, - dynamic_modules_path=dynamic_modules_path, - trust_remote_code=trust_remote_code, - ).get_module() - except Exception as e1: # noqa all the attempts failed, before raising the error we should check if the module is already cached. - try: - return CachedMetricModuleFactory(path, dynamic_modules_path=dynamic_modules_path).get_module() - except Exception: # noqa if it's not in the cache, then it doesn't exist. - if not isinstance(e1, FileNotFoundError): - raise e1 from None - raise FileNotFoundError( - f"Couldn't find a metric script at {relative_to_absolute_path(combined_path)}. " - f"Metric '{path}' doesn't exist on the Hugging Face Hub either." - ) from None - else: - raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(combined_path)}.") - - -@deprecated("Use 'evaluate.load' instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate") -def load_metric( - path: str, - config_name: Optional[str] = None, - process_id: int = 0, - num_process: int = 1, - cache_dir: Optional[str] = None, - experiment_id: Optional[str] = None, - keep_in_memory: bool = False, - download_config: Optional[DownloadConfig] = None, - download_mode: Optional[Union[DownloadMode, str]] = None, - revision: Optional[Union[str, Version]] = None, - trust_remote_code: Optional[bool] = None, - **metric_init_kwargs, -) -> Metric: - """Load a `datasets.Metric`. - - - - Use `evaluate.load` instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate - - - - Args: - - path (``str``): - path to the metric processing script with the metric builder. Can be either: - - a local path to processing script or the directory containing the script (if the script has the same name as the directory), - e.g. ``'./metrics/rouge'`` or ``'./metrics/rogue/rouge.py'`` - - a metric identifier on the HuggingFace datasets repo (list all available metrics with ``datasets.list_metrics()``) - e.g. ``'rouge'`` or ``'bleu'`` - config_name (:obj:`str`, optional): selecting a configuration for the metric (e.g. the GLUE metric has a configuration for each subset) - process_id (:obj:`int`, optional): for distributed evaluation: id of the process - num_process (:obj:`int`, optional): for distributed evaluation: total number of processes - cache_dir (Optional str): path to store the temporary predictions and references (default to `~/.cache/huggingface/metrics/`) - experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system. - This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). - keep_in_memory (bool): Whether to store the temporary results in memory (defaults to False) - download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters. - download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. - revision (Optional ``Union[str, datasets.Version]``): if specified, the module will be loaded from the datasets repository - at this version. By default, it is set to the local version of the lib. Specifying a version that is different from - your local version of the lib might cause compatibility issues. - trust_remote_code (`bool`, defaults to `False`): - Whether or not to allow for datasets defined on the Hub using a dataset script. This option - should only be set to `True` for repositories you trust and in which you have read the code, as it will - execute code present on the Hub on your local machine. - - - - - - `trust_remote_code` defaults to `False` if not specified. - - - - Returns: - `datasets.Metric` - - Example: - - ```py - >>> from datasets import load_metric - >>> accuracy = load_metric('accuracy') - >>> accuracy.compute(references=[1, 0], predictions=[1, 1]) - {'accuracy': 0.5} - ``` - """ - with warnings.catch_warnings(): - # Ignore equivalent warnings to the one already issued - warnings.filterwarnings("ignore", message=".*https://huggingface.co/docs/evaluate$", category=FutureWarning) - - download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) - metric_module = metric_module_factory( - path, - revision=revision, - download_config=download_config, - download_mode=download_mode, - trust_remote_code=trust_remote_code, - ).module_path - metric_cls = import_main_class(metric_module, dataset=False) - metric = metric_cls( - config_name=config_name, - process_id=process_id, - num_process=num_process, - cache_dir=cache_dir, - keep_in_memory=keep_in_memory, - experiment_id=experiment_id, - **metric_init_kwargs, - ) - - # Download and prepare resources for the metric - metric.download_and_prepare(download_config=download_config) - - return metric - - def load_dataset_builder( path: str, name: Optional[str] = None, diff --git a/src/datasets/metric.py b/src/datasets/metric.py deleted file mode 100644 index 187c5e5c925..00000000000 --- a/src/datasets/metric.py +++ /dev/null @@ -1,652 +0,0 @@ -# Copyright 2020 The HuggingFace Datasets Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Lint as: python3 -"""Metrics base class.""" - -import os -import types -import uuid -from typing import Any, Dict, List, Optional, Tuple, Union - -import numpy as np -import pyarrow as pa -from filelock import BaseFileLock, Timeout - -from . import config -from .arrow_dataset import Dataset -from .arrow_reader import ArrowReader -from .arrow_writer import ArrowWriter -from .download.download_config import DownloadConfig -from .download.download_manager import DownloadManager -from .features import Features -from .info import DatasetInfo, MetricInfo -from .naming import camelcase_to_snakecase -from .utils._filelock import FileLock -from .utils.deprecation_utils import deprecated -from .utils.logging import get_logger -from .utils.py_utils import copyfunc, temp_seed - - -logger = get_logger(__name__) - - -class FileFreeLock(BaseFileLock): - """Thread lock until a file **cannot** be locked""" - - def __init__(self, lock_file, *args, **kwargs): - self.filelock = FileLock(lock_file) - super().__init__(self.filelock.lock_file, *args, **kwargs) - - def _acquire(self): - try: - self.filelock.acquire(timeout=0.01, poll_intervall=0.02) # Try to lock once - except Timeout: - # We couldn't acquire the lock, the file is locked! - self._context.lock_file_fd = self.filelock.lock_file - else: - # We were able to acquire the lock, the file is not yet locked! - self.filelock.release() - self._context.lock_file_fd = None - - def _release(self): - self._context.lock_file_fd = None - - -# lists - summarize long lists similarly to NumPy -# arrays/tensors - let the frameworks control formatting -def summarize_if_long_list(obj): - if not type(obj) == list or len(obj) <= 6: # noqa: E721 - return f"{obj}" - - def format_chunk(chunk): - return ", ".join(repr(x) for x in chunk) - - return f"[{format_chunk(obj[:3])}, ..., {format_chunk(obj[-3:])}]" - - -class MetricInfoMixin: - """This base class exposes some attributes of MetricInfo - at the base level of the Metric for easy access. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - - """ - - def __init__(self, info: MetricInfo): - self._metric_info = info - - @property - def info(self): - """:class:`datasets.MetricInfo` object containing all the metadata in the metric.""" - return self._metric_info - - @property - def name(self) -> str: - return self._metric_info.metric_name - - @property - def experiment_id(self) -> Optional[str]: - return self._metric_info.experiment_id - - @property - def description(self) -> str: - return self._metric_info.description - - @property - def citation(self) -> str: - return self._metric_info.citation - - @property - def features(self) -> Features: - return self._metric_info.features - - @property - def inputs_description(self) -> str: - return self._metric_info.inputs_description - - @property - def homepage(self) -> Optional[str]: - return self._metric_info.homepage - - @property - def license(self) -> str: - return self._metric_info.license - - @property - def codebase_urls(self) -> Optional[List[str]]: - return self._metric_info.codebase_urls - - @property - def reference_urls(self) -> Optional[List[str]]: - return self._metric_info.reference_urls - - @property - def streamable(self) -> bool: - return self._metric_info.streamable - - @property - def format(self) -> Optional[str]: - return self._metric_info.format - - -class Metric(MetricInfoMixin): - """A Metric is the base class and common API for all metrics. - - - - Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate - - - - Args: - config_name (``str``): This is used to define a hash specific to a metrics computation script and prevents the metric's data - to be overridden when the metric loading script is modified. - keep_in_memory (:obj:`bool`): keep all predictions and references in memory. Not possible in distributed settings. - cache_dir (``str``): Path to a directory in which temporary prediction/references data will be stored. - The data directory should be located on a shared file-system in distributed setups. - num_process (``int``): specify the total number of nodes in a distributed settings. - This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). - process_id (``int``): specify the id of the current process in a distributed setup (between 0 and num_process-1) - This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). - seed (:obj:`int`, optional): If specified, this will temporarily set numpy's random seed when :func:`datasets.Metric.compute` is run. - experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system. - This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). - max_concurrent_cache_files (``int``): Max number of concurrent metrics cache files (default 10000). - timeout (``Union[int, float]``): Timeout in second for distributed setting synchronization. - """ - - @deprecated("Use the new library πŸ€— Evaluate instead: https://huggingface.co/docs/evaluate") - def __init__( - self, - config_name: Optional[str] = None, - keep_in_memory: bool = False, - cache_dir: Optional[str] = None, - num_process: int = 1, - process_id: int = 0, - seed: Optional[int] = None, - experiment_id: Optional[str] = None, - max_concurrent_cache_files: int = 10000, - timeout: Union[int, float] = 100, - **kwargs, - ): - # prepare info - self.config_name = config_name or "default" - info = self._info() - info.metric_name = camelcase_to_snakecase(self.__class__.__name__) - info.config_name = self.config_name - info.experiment_id = experiment_id or "default_experiment" - MetricInfoMixin.__init__(self, info) # For easy access on low level - - # Safety checks on num_process and process_id - if not isinstance(process_id, int) or process_id < 0: - raise ValueError("'process_id' should be a number greater than 0") - if not isinstance(num_process, int) or num_process <= process_id: - raise ValueError("'num_process' should be a number greater than process_id") - if keep_in_memory and num_process != 1: - raise ValueError("Using 'keep_in_memory' is not possible in distributed setting (num_process > 1).") - - self.num_process = num_process - self.process_id = process_id - self.max_concurrent_cache_files = max_concurrent_cache_files - - self.keep_in_memory = keep_in_memory - self._data_dir_root = os.path.expanduser(cache_dir or config.HF_METRICS_CACHE) - self.data_dir = self._build_data_dir() - if seed is None: - _, seed, pos, *_ = np.random.get_state() - self.seed: int = seed[pos] if pos < 624 else seed[0] - else: - self.seed: int = seed - self.timeout: Union[int, float] = timeout - - # Update 'compute' and 'add' docstring - # methods need to be copied otherwise it changes the docstrings of every instance - self.compute = types.MethodType(copyfunc(self.compute), self) - self.add_batch = types.MethodType(copyfunc(self.add_batch), self) - self.add = types.MethodType(copyfunc(self.add), self) - self.compute.__func__.__doc__ += self.info.inputs_description - self.add_batch.__func__.__doc__ += self.info.inputs_description - self.add.__func__.__doc__ += self.info.inputs_description - - # self.arrow_schema = pa.schema(field for field in self.info.features.type) - self.buf_writer = None - self.writer = None - self.writer_batch_size = None - self.data = None - - # This is the cache file we store our predictions/references in - # Keep it None for now so we can (cloud)pickle the object - self.cache_file_name = None - self.filelock = None - self.rendez_vous_lock = None - - # This is all the cache files on which we have a lock when we are in a distributed setting - self.file_paths = None - self.filelocks = None - - def __len__(self): - """Return the number of examples (predictions or predictions/references pair) - currently stored in the metric's cache. - """ - return 0 if self.writer is None else len(self.writer) - - def __repr__(self): - return ( - f'Metric(name: "{self.name}", features: {self.features}, ' - f'usage: """{self.inputs_description}""", ' - f"stored examples: {len(self)})" - ) - - def _build_data_dir(self): - """Path of this metric in cache_dir: - Will be: - self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ - If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. - """ - builder_data_dir = self._data_dir_root - builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name) - os.makedirs(builder_data_dir, exist_ok=True) - return builder_data_dir - - def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]: - """Create a new cache file. If the default cache file is used, we generated a new hash.""" - file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow") - filelock = None - for i in range(self.max_concurrent_cache_files): - filelock = FileLock(file_path + ".lock") - try: - filelock.acquire(timeout=timeout) - except Timeout: - # If we have reached the max number of attempts or we are not allow to find a free name (distributed setup) - # We raise an error - if self.num_process != 1: - raise ValueError( - f"Error in _create_cache_file: another metric instance is already using the local cache file at {file_path}. " - f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " - f"between distributed metric instances." - ) from None - if i == self.max_concurrent_cache_files - 1: - raise ValueError( - f"Cannot acquire lock, too many metric instance are operating concurrently on this file system." - f"You should set a larger value of max_concurrent_cache_files when creating the metric " - f"(current value is {self.max_concurrent_cache_files})." - ) from None - # In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name. - file_uuid = str(uuid.uuid4()) - file_path = os.path.join( - self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow" - ) - else: - break - - return file_path, filelock - - def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]: - """Get a lock on all the cache files in a distributed setup. - We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). - """ - if self.num_process == 1: - if self.cache_file_name is None: - raise ValueError( - "Metric cache file doesn't exist. Please make sure that you call `add` or `add_batch` " - "at least once before calling `compute`." - ) - file_paths = [self.cache_file_name] - else: - file_paths = [ - os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow") - for process_id in range(self.num_process) - ] - - # Let's acquire a lock on each process files to be sure they are finished writing - filelocks = [] - for process_id, file_path in enumerate(file_paths): - if process_id == 0: # process 0 already has its lock file - filelocks.append(self.filelock) - else: - filelock = FileLock(file_path + ".lock") - try: - filelock.acquire(timeout=self.timeout) - except Timeout: - raise ValueError( - f"Cannot acquire lock on cached file {file_path} for process {process_id}." - ) from None - else: - filelocks.append(filelock) - - return file_paths, filelocks - - def _check_all_processes_locks(self): - expected_lock_file_names = [ - os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock") - for process_id in range(self.num_process) - ] - for expected_lock_file_name in expected_lock_file_names: - nofilelock = FileFreeLock(expected_lock_file_name) - try: - nofilelock.acquire(timeout=self.timeout) - except Timeout: - raise ValueError( - f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." - ) from None - else: - nofilelock.release() - - def _check_rendez_vous(self): - expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock") - nofilelock = FileFreeLock(expected_lock_file_name) - try: - nofilelock.acquire(timeout=self.timeout) - except Timeout: - raise ValueError( - f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." - ) from None - else: - nofilelock.release() - lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") - rendez_vous_lock = FileLock(lock_file_name) - try: - rendez_vous_lock.acquire(timeout=self.timeout) - except Timeout: - raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.") from None - else: - rendez_vous_lock.release() - - def _finalize(self): - """Close all the writing process and load/gather the data - from all the nodes if main node or all_process is True. - """ - if self.writer is not None: - self.writer.finalize() - self.writer = None - # release the locks of the processes > 0 so that process 0 can lock them to read + delete the data - if self.filelock is not None and self.process_id > 0: - self.filelock.release() - - if self.keep_in_memory: - # Read the predictions and references - reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.features)) - self.data = Dataset.from_buffer(self.buf_writer.getvalue()) - - elif self.process_id == 0: - # Let's acquire a lock on each node files to be sure they are finished writing - file_paths, filelocks = self._get_all_cache_files() - - # Read the predictions and references - try: - reader = ArrowReader(path="", info=DatasetInfo(features=self.features)) - self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths])) - except FileNotFoundError: - raise ValueError( - "Error in finalize: another metric instance is already using the local cache file. " - "Please specify an experiment_id to avoid collision between distributed metric instances." - ) from None - - # Store file paths and locks and we will release/delete them after the computation. - self.file_paths = file_paths - self.filelocks = filelocks - - def compute(self, *, predictions=None, references=None, **kwargs) -> Optional[dict]: - """Compute the metrics. - - Usage of positional arguments is not allowed to prevent mistakes. - - Args: - predictions (list/array/tensor, optional): Predictions. - references (list/array/tensor, optional): References. - **kwargs (optional): Keyword arguments that will be forwarded to the metrics :meth:`_compute` - method (see details in the docstring). - - Return: - dict or None - - - Dictionary with the metrics if this metric is run on the main process (``process_id == 0``). - - None if the metric is not run on the main process (``process_id != 0``). - - Example: - - ```py - >>> from datasets import load_metric - >>> metric = load_metric("accuracy") - >>> accuracy = metric.compute(predictions=model_prediction, references=labels) - ``` - """ - all_kwargs = {"predictions": predictions, "references": references, **kwargs} - if predictions is None and references is None: - missing_kwargs = {k: None for k in self.features if k not in all_kwargs} - all_kwargs.update(missing_kwargs) - else: - missing_inputs = [k for k in self.features if k not in all_kwargs] - if missing_inputs: - raise ValueError( - f"Metric inputs are missing: {missing_inputs}. All required inputs are {list(self.features)}" - ) - inputs = {input_name: all_kwargs[input_name] for input_name in self.features} - compute_kwargs = {k: kwargs[k] for k in kwargs if k not in self.features} - - if any(v is not None for v in inputs.values()): - self.add_batch(**inputs) - self._finalize() - - self.cache_file_name = None - self.filelock = None - - if self.process_id == 0: - self.data.set_format(type=self.info.format) - - inputs = {input_name: self.data[input_name] for input_name in self.features} - with temp_seed(self.seed): - output = self._compute(**inputs, **compute_kwargs) - - if self.buf_writer is not None: - self.buf_writer = None - del self.data - self.data = None - else: - # Release locks and delete all the cache files. Process 0 is released last. - for filelock, file_path in reversed(list(zip(self.filelocks, self.file_paths))): - logger.info(f"Removing {file_path}") - del self.data - self.data = None - del self.writer - self.writer = None - os.remove(file_path) - filelock.release() - - return output - else: - return None - - def add_batch(self, *, predictions=None, references=None, **kwargs): - """Add a batch of predictions and references for the metric's stack. - - Args: - predictions (list/array/tensor, optional): Predictions. - references (list/array/tensor, optional): References. - - Example: - - ```py - >>> from datasets import load_metric - >>> metric = load_metric("accuracy") - >>> metric.add_batch(predictions=model_prediction, references=labels) - ``` - """ - bad_inputs = [input_name for input_name in kwargs if input_name not in self.features] - if bad_inputs: - raise ValueError(f"Bad inputs for metric: {bad_inputs}. All required inputs are {list(self.features)}") - batch = {"predictions": predictions, "references": references, **kwargs} - batch = {intput_name: batch[intput_name] for intput_name in self.features} - batch = self.info.features.encode_batch(batch) - if self.writer is None: - self._init_writer() - try: - self.writer.write_batch(batch) - except pa.ArrowInvalid: - if any(len(batch[c]) != len(next(iter(batch.values()))) for c in batch): - col0 = next(iter(batch)) - bad_col = [c for c in batch if len(batch[c]) != len(batch[col0])][0] - error_msg = ( - f"Mismatch in the number of {col0} ({len(batch[col0])}) and {bad_col} ({len(batch[bad_col])})" - ) - elif sorted(self.features) != ["references", "predictions"]: - error_msg = f"Metric inputs don't match the expected format.\n" f"Expected format: {self.features},\n" - error_msg_inputs = ",\n".join( - f"Input {input_name}: {summarize_if_long_list(batch[input_name])}" for input_name in self.features - ) - error_msg += error_msg_inputs - else: - error_msg = ( - f"Predictions and/or references don't match the expected format.\n" - f"Expected format: {self.features},\n" - f"Input predictions: {summarize_if_long_list(predictions)},\n" - f"Input references: {summarize_if_long_list(references)}" - ) - raise ValueError(error_msg) from None - - def add(self, *, prediction=None, reference=None, **kwargs): - """Add one prediction and reference for the metric's stack. - - Args: - prediction (list/array/tensor, optional): Predictions. - reference (list/array/tensor, optional): References. - - Example: - - ```py - >>> from datasets import load_metric - >>> metric = load_metric("accuracy") - >>> metric.add(predictions=model_predictions, references=labels) - ``` - """ - bad_inputs = [input_name for input_name in kwargs if input_name not in self.features] - if bad_inputs: - raise ValueError(f"Bad inputs for metric: {bad_inputs}. All required inputs are {list(self.features)}") - example = {"predictions": prediction, "references": reference, **kwargs} - example = {intput_name: example[intput_name] for intput_name in self.features} - example = self.info.features.encode_example(example) - if self.writer is None: - self._init_writer() - try: - self.writer.write(example) - except pa.ArrowInvalid: - error_msg = f"Metric inputs don't match the expected format.\n" f"Expected format: {self.features},\n" - error_msg_inputs = ",\n".join( - f"Input {input_name}: {summarize_if_long_list(example[input_name])}" for input_name in self.features - ) - error_msg += error_msg_inputs - raise ValueError(error_msg) from None - - def _init_writer(self, timeout=1): - if self.num_process > 1: - if self.process_id == 0: - file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") - self.rendez_vous_lock = FileLock(file_path) - try: - self.rendez_vous_lock.acquire(timeout=timeout) - except TimeoutError: - raise ValueError( - f"Error in _init_writer: another metric instance is already using the local cache file at {file_path}. " - f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " - f"between distributed metric instances." - ) from None - - if self.keep_in_memory: - self.buf_writer = pa.BufferOutputStream() - self.writer = ArrowWriter( - features=self.info.features, stream=self.buf_writer, writer_batch_size=self.writer_batch_size - ) - else: - self.buf_writer = None - - # Get cache file name and lock it - if self.cache_file_name is None or self.filelock is None: - cache_file_name, filelock = self._create_cache_file() # get ready - self.cache_file_name = cache_file_name - self.filelock = filelock - - self.writer = ArrowWriter( - features=self.info.features, path=self.cache_file_name, writer_batch_size=self.writer_batch_size - ) - # Setup rendez-vous here if - if self.num_process > 1: - if self.process_id == 0: - self._check_all_processes_locks() # wait for everyone to be ready - self.rendez_vous_lock.release() # let everyone go - else: - self._check_rendez_vous() # wait for master to be ready and to let everyone go - - def _info(self) -> MetricInfo: - """Construct the MetricInfo object. See `MetricInfo` for details. - - Warning: This function is only called once and the result is cached for all - following .info() calls. - - Returns: - info: (MetricInfo) The metrics information - """ - raise NotImplementedError - - def download_and_prepare( - self, - download_config: Optional[DownloadConfig] = None, - dl_manager: Optional[DownloadManager] = None, - ): - """Downloads and prepares dataset for reading. - - Args: - download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. - dl_manager (:class:`DownloadManager`, optional): Specific download manager to use. - """ - if dl_manager is None: - if download_config is None: - download_config = DownloadConfig() - download_config.cache_dir = os.path.join(self.data_dir, "downloads") - download_config.force_download = False - - dl_manager = DownloadManager( - dataset_name=self.name, download_config=download_config, data_dir=self.data_dir - ) - - self._download_and_prepare(dl_manager) - - def _download_and_prepare(self, dl_manager): - """Downloads and prepares resources for the metric. - - This is the internal implementation to overwrite called when user calls - `download_and_prepare`. It should download all required resources for the metric. - - Args: - dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. - """ - return None - - def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]: - """This method defines the common API for all the metrics in the library""" - raise NotImplementedError - - def __del__(self): - if hasattr(self, "filelock") and self.filelock is not None: - self.filelock.release() - if hasattr(self, "rendez_vous_lock") and self.rendez_vous_lock is not None: - self.rendez_vous_lock.release() - if hasattr(self, "writer"): # in case it was already deleted - del self.writer - if hasattr(self, "data"): # in case it was already deleted - del self.data diff --git a/src/datasets/packaged_modules/webdataset/webdataset.py b/src/datasets/packaged_modules/webdataset/webdataset.py index fc1567d4ac0..1b3ce7c9799 100644 --- a/src/datasets/packaged_modules/webdataset/webdataset.py +++ b/src/datasets/packaged_modules/webdataset/webdataset.py @@ -109,10 +109,18 @@ def _generate_examples(self, tar_paths, tar_iterators): audio_field_names = [ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio) ] + all_field_names = list(self.info.features.keys()) for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)): for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)): + for field_name in all_field_names: + if field_name not in example: + example[field_name] = None for field_name in image_field_names + audio_field_names: - example[field_name] = {"path": example["__key__"] + "." + field_name, "bytes": example[field_name]} + if example[field_name] is not None: + example[field_name] = { + "path": example["__key__"] + "." + field_name, + "bytes": example[field_name], + } yield f"{tar_idx}_{example_idx}", example diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py index 07949690dd5..d1d70419d69 100644 --- a/src/datasets/utils/file_utils.py +++ b/src/datasets/utils/file_utils.py @@ -4,6 +4,7 @@ Copyright by the AllenNLP authors. """ +import asyncio import copy import glob import io @@ -19,7 +20,6 @@ import urllib import xml.dom.minidom import zipfile -from asyncio import TimeoutError from contextlib import closing, contextmanager from functools import partial from io import BytesIO @@ -30,10 +30,10 @@ from urllib.parse import urljoin, urlparse from xml.etree import ElementTree as ET +import aiohttp.client_exceptions import fsspec import huggingface_hub import requests -from aiohttp.client_exceptions import ClientError from fsspec.core import strip_protocol, url_to_fs from fsspec.utils import can_be_local from huggingface_hub.utils import EntryNotFoundError, insecure_hashlib @@ -96,30 +96,22 @@ def relative_to_absolute_path(path: T) -> T: return Path(abs_path_str) if isinstance(path, Path) else abs_path_str -def hf_bucket_url(identifier: str, filename: str, use_cdn=False, dataset=True) -> str: - if dataset: - endpoint = config.CLOUDFRONT_DATASETS_DISTRIB_PREFIX if use_cdn else config.S3_DATASETS_BUCKET_PREFIX - else: - endpoint = config.CLOUDFRONT_METRICS_DISTRIB_PREFIX if use_cdn else config.S3_METRICS_BUCKET_PREFIX +def hf_bucket_url(identifier: str, filename: str, use_cdn=False) -> str: + endpoint = config.CLOUDFRONT_DATASETS_DISTRIB_PREFIX if use_cdn else config.S3_DATASETS_BUCKET_PREFIX return "/".join((endpoint, identifier, filename)) -def head_hf_s3( - identifier: str, filename: str, use_cdn=False, dataset=True, max_retries=0 -) -> Union[requests.Response, Exception]: +def head_hf_s3(identifier: str, filename: str, use_cdn=False, max_retries=0) -> Union[requests.Response, Exception]: return http_head( - hf_bucket_url(identifier=identifier, filename=filename, use_cdn=use_cdn, dataset=dataset), + hf_bucket_url(identifier=identifier, filename=filename, use_cdn=use_cdn), max_retries=max_retries, ) -def hf_github_url(path: str, name: str, dataset=True, revision: Optional[str] = None) -> str: +def hf_github_url(path: str, name: str, revision: Optional[str] = None) -> str: default_revision = "main" if version.parse(__version__).is_devrelease else __version__ revision = revision or default_revision - if dataset: - return config.REPO_DATASETS_URL.format(revision=revision, path=path, name=name) - else: - return config.REPO_METRICS_URL.format(revision=revision, path=path, name=name) + return config.REPO_DATASETS_URL.format(revision=revision, path=path, name=name) def url_or_path_join(base_name: str, *pathnames: str) -> str: @@ -1074,7 +1066,12 @@ def read_with_retries(*args, **kwargs): try: out = read(*args, **kwargs) break - except (ClientError, TimeoutError) as err: + except ( + aiohttp.client_exceptions.ClientError, + asyncio.TimeoutError, + requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError, + ) as err: disconnect_err = err logger.warning( f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]" diff --git a/src/datasets/utils/py_utils.py b/src/datasets/utils/py_utils.py index 97dd6fecf52..55602e26996 100644 --- a/src/datasets/utils/py_utils.py +++ b/src/datasets/utils/py_utils.py @@ -586,7 +586,7 @@ def _convert_github_url(url_path: str) -> Tuple[str, Optional[str]]: def lock_importable_file(importable_local_file: str) -> FileLock: # Check the directory with a unique name in our dataset folder # path is: ./datasets/dataset_name/hash_from_code/script.py - # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset processing file together importable_directory_path = str(Path(importable_local_file).resolve().parent.parent) lock_path = importable_directory_path + ".lock" return FileLock(lock_path) diff --git a/templates/metric_card_template.md b/templates/metric_card_template.md deleted file mode 100644 index bf31ec59ef7..00000000000 --- a/templates/metric_card_template.md +++ /dev/null @@ -1,37 +0,0 @@ -# Metric Card for *Current Metric* - -***Metric Card Instructions:*** *Copy this file into the relevant metric folder, then fill it out and save it as README.md. Feel free to take a look at existing metric cards if you'd like examples.* - -## Metric Description -*Give a brief overview of this metric.* - -## How to Use -*Give general statement of how to use the metric* - -*Provide simplest possible example for using the metric* - -### Inputs -*List all input arguments in the format below* -- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).* - -### Output Values -*Explain what this metric outputs (e.g. a single score, a list of scores)* - -*Give an example of what the metric output looks like.* - -*State the range of possible values that the metric's output can take, as well as what in that range is considered good. For example: "This metric can take on any value between 0 and 100, inclusive. Higher scores are better."* - -#### Values from Popular Papers -*Give examples, preferrably with links, to papers that have reported this metric, along with the values they have reported.* - -### Examples -*Give code examples of the metric being used. Try to include examples that clear up any potential ambiguity left from the metric description above. If possible, provide a range of examples that show both typical and atypical results, as well as examples where a variety of input parameters are passed.* - -## Limitations and Bias -*Note any known limitations or biases that the metric has, with links and references if possible.* - -## Citation -*Cite the source where this metric was introduced.* - -## Further References -*Add any useful further references.* diff --git a/tests/conftest.py b/tests/conftest.py index eeec9e7111c..3b59b6b7d53 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,10 +21,8 @@ def set_test_cache_config(tmp_path_factory, monkeypatch): # test_hf_cache_home = tmp_path_factory.mktemp("cache") # TODO: why a cache dir per test function does not work? test_hf_cache_home = tmp_path_factory.getbasetemp() / "cache" test_hf_datasets_cache = test_hf_cache_home / "datasets" - test_hf_metrics_cache = test_hf_cache_home / "metrics" test_hf_modules_cache = test_hf_cache_home / "modules" monkeypatch.setattr("datasets.config.HF_DATASETS_CACHE", str(test_hf_datasets_cache)) - monkeypatch.setattr("datasets.config.HF_METRICS_CACHE", str(test_hf_metrics_cache)) monkeypatch.setattr("datasets.config.HF_MODULES_CACHE", str(test_hf_modules_cache)) test_downloaded_datasets_path = test_hf_datasets_cache / "downloads" monkeypatch.setattr("datasets.config.DOWNLOADED_DATASETS_PATH", str(test_downloaded_datasets_path)) diff --git a/tests/features/test_array_xd.py b/tests/features/test_array_xd.py index 61a0fd7650e..8a50823b996 100644 --- a/tests/features/test_array_xd.py +++ b/tests/features/test_array_xd.py @@ -362,7 +362,7 @@ def test_table_to_pandas(dtype, dummy_value): features = datasets.Features({"foo": datasets.Array2D(dtype=dtype, shape=(2, 2))}) dataset = datasets.Dataset.from_dict({"foo": [[[dummy_value] * 2] * 2]}, features=features) df = dataset._data.to_pandas() - assert type(df.foo.dtype) == PandasArrayExtensionDtype + assert isinstance(df.foo.dtype, PandasArrayExtensionDtype) arr = df.foo.to_numpy() np.testing.assert_equal(arr, np.array([[[dummy_value] * 2] * 2], dtype=np.dtype(dtype))) diff --git a/tests/packaged_modules/test_webdataset.py b/tests/packaged_modules/test_webdataset.py index c2b06abc3d5..e122275cec9 100644 --- a/tests/packaged_modules/test_webdataset.py +++ b/tests/packaged_modules/test_webdataset.py @@ -128,6 +128,37 @@ def test_image_webdataset(image_wds_file): assert isinstance(decoded["jpg"], PIL.Image.Image) +@require_pil +def test_image_webdataset_missing_keys(image_wds_file): + import PIL.Image + + data_files = {"train": [image_wds_file]} + features = Features( + { + "__key__": Value("string"), + "__url__": Value("string"), + "json": {"caption": Value("string")}, + "jpg": Image(), + "jpeg": Image(), # additional field + "txt": Value("string"), # additional field + } + ) + webdataset = WebDataset(data_files=data_files, features=features) + split_generators = webdataset._split_generators(DownloadManager()) + assert webdataset.info.features == features + split_generator = split_generators[0] + assert split_generator.name == "train" + generator = webdataset._generate_examples(**split_generator.gen_kwargs) + _, example = next(iter(generator)) + encoded = webdataset.info.features.encode_example(example) + decoded = webdataset.info.features.decode_example(encoded) + assert isinstance(decoded["json"], dict) + assert isinstance(decoded["json"]["caption"], str) + assert isinstance(decoded["jpg"], PIL.Image.Image) + assert decoded["jpeg"] is None + assert decoded["txt"] is None + + @require_sndfile def test_audio_webdataset(audio_wds_file): data_files = {"train": [audio_wds_file]} diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 1b110660b0f..05870e30521 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -1,5 +1,3 @@ -import os - import pytest from datasets.inspect import ( @@ -8,23 +6,12 @@ get_dataset_default_config_name, get_dataset_infos, get_dataset_split_names, - inspect_metric, ) pytestmark = pytest.mark.integration -@pytest.mark.filterwarnings("ignore:inspect_metric is deprecated:FutureWarning") -@pytest.mark.filterwarnings("ignore:metric_module_factory is deprecated:FutureWarning") -@pytest.mark.parametrize("path", ["accuracy"]) -def test_inspect_metric(path, tmp_path): - inspect_metric(path, tmp_path, trust_remote_code=True) - script_name = path + ".py" - assert script_name in os.listdir(tmp_path) - assert "__pycache__" not in os.listdir(tmp_path) - - @pytest.mark.parametrize( "path, config_name, expected_splits", [ diff --git a/tests/test_load.py b/tests/test_load.py index 3bc272b4c60..e55f4027a92 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -29,14 +29,11 @@ from datasets.iterable_dataset import IterableDataset from datasets.load import ( CachedDatasetModuleFactory, - CachedMetricModuleFactory, - GithubMetricModuleFactory, HubDatasetModuleFactoryWithoutScript, HubDatasetModuleFactoryWithParquetExport, HubDatasetModuleFactoryWithScript, LocalDatasetModuleFactoryWithoutScript, LocalDatasetModuleFactoryWithScript, - LocalMetricModuleFactory, PackagedDatasetModuleFactory, infer_module_for_data_files_list, infer_module_for_data_files_list_in_archives, @@ -103,23 +100,6 @@ def _generate_examples(self, filepath, **kwargs): SAMPLE_DATASET_CAPITAL_LETTERS_IN_NAME = "hf-internal-testing/DatasetWithCapitalLetters" -METRIC_LOADING_SCRIPT_NAME = "__dummy_metric1__" - -METRIC_LOADING_SCRIPT_CODE = """ -import datasets -from datasets import MetricInfo, Features, Value - - -class __DummyMetric1__(datasets.Metric): - - def _info(self): - return MetricInfo(features=Features({"predictions": Value("int"), "references": Value("int")})) - - def _compute(self, predictions, references): - return {"__dummy_metric1__": sum(int(p == r) for p, r in zip(predictions, references))} -""" - - @pytest.fixture def data_dir(tmp_path): data_dir = tmp_path / "data_dir" @@ -327,17 +307,6 @@ def dataset_loading_script_dir_readonly(tmp_path): return dataset_loading_script_dir -@pytest.fixture -def metric_loading_script_dir(tmp_path): - script_name = METRIC_LOADING_SCRIPT_NAME - script_dir = tmp_path / script_name - script_dir.mkdir() - script_path = script_dir / f"{script_name}.py" - with open(script_path, "w") as f: - f.write(METRIC_LOADING_SCRIPT_CODE) - return str(script_dir) - - @pytest.mark.parametrize( "data_files, expected_module, expected_builder_kwargs", [ @@ -396,7 +365,6 @@ def inject_fixtures( data_dir_with_two_config_in_metadata, sub_data_dirs, dataset_loading_script_dir, - metric_loading_script_dir, ): self._jsonl_path = jsonl_path self._data_dir = data_dir @@ -407,7 +375,6 @@ def inject_fixtures( self._data_dir2 = sub_data_dirs[0] self._sub_data_dir = sub_data_dirs[1] self._dataset_loading_script_dir = dataset_loading_script_dir - self._metric_loading_script_dir = metric_loading_script_dir def setUp(self): self.hf_modules_cache = tempfile.mkdtemp() @@ -447,40 +414,6 @@ def test_HubDatasetModuleFactoryWithScript_with_hub_dataset(self): assert importlib.import_module(module_factory_result.module_path) is not None assert module_factory_result.builder_kwargs["base_path"].startswith(config.HF_ENDPOINT) - def test_GithubMetricModuleFactory_with_internal_import(self): - # "squad_v2" requires additional imports (internal) - factory = GithubMetricModuleFactory( - "squad_v2", - download_config=self.download_config, - dynamic_modules_path=self.dynamic_modules_path, - trust_remote_code=True, - ) - module_factory_result = factory.get_module() - assert importlib.import_module(module_factory_result.module_path) is not None - - @pytest.mark.filterwarnings("ignore:GithubMetricModuleFactory is deprecated:FutureWarning") - def test_GithubMetricModuleFactory_with_external_import(self): - # "bleu" requires additional imports (external from github) - factory = GithubMetricModuleFactory( - "bleu", - download_config=self.download_config, - dynamic_modules_path=self.dynamic_modules_path, - trust_remote_code=True, - ) - module_factory_result = factory.get_module() - assert importlib.import_module(module_factory_result.module_path) is not None - - def test_LocalMetricModuleFactory(self): - path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") - factory = LocalMetricModuleFactory( - path, - download_config=self.download_config, - dynamic_modules_path=self.dynamic_modules_path, - trust_remote_code=True, - ) - module_factory_result = factory.get_module() - assert importlib.import_module(module_factory_result.module_path) is not None - def test_LocalDatasetModuleFactoryWithScript(self): path = os.path.join(self._dataset_loading_script_dir, f"{DATASET_LOADING_SCRIPT_NAME}.py") factory = LocalDatasetModuleFactoryWithScript( @@ -910,38 +843,15 @@ def test_CachedDatasetModuleFactory_with_script(self): module_factory_result = factory.get_module() assert importlib.import_module(module_factory_result.module_path) is not None - @pytest.mark.filterwarnings("ignore:LocalMetricModuleFactory is deprecated:FutureWarning") - @pytest.mark.filterwarnings("ignore:CachedMetricModuleFactory is deprecated:FutureWarning") - def test_CachedMetricModuleFactory(self): - path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") - factory = LocalMetricModuleFactory( - path, - download_config=self.download_config, - dynamic_modules_path=self.dynamic_modules_path, - trust_remote_code=True, - ) - module_factory_result = factory.get_module() - for offline_mode in OfflineSimulationMode: - with offline(offline_mode): - factory = CachedMetricModuleFactory( - METRIC_LOADING_SCRIPT_NAME, - dynamic_modules_path=self.dynamic_modules_path, - ) - module_factory_result = factory.get_module() - assert importlib.import_module(module_factory_result.module_path) is not None - @pytest.mark.parametrize( "factory_class", [ CachedDatasetModuleFactory, - CachedMetricModuleFactory, - GithubMetricModuleFactory, HubDatasetModuleFactoryWithoutScript, HubDatasetModuleFactoryWithScript, LocalDatasetModuleFactoryWithoutScript, LocalDatasetModuleFactoryWithScript, - LocalMetricModuleFactory, PackagedDatasetModuleFactory, ], ) diff --git a/tests/test_metric.py b/tests/test_metric.py deleted file mode 100644 index 918fb2bbb37..00000000000 --- a/tests/test_metric.py +++ /dev/null @@ -1,582 +0,0 @@ -import os -import pickle -import tempfile -import time -from multiprocessing import Pool -from unittest import TestCase - -import pytest - -from datasets.features import Features, Sequence, Value -from datasets.metric import Metric, MetricInfo - -from .utils import require_tf, require_torch - - -class DummyMetric(Metric): - def _info(self): - return MetricInfo( - description="dummy metric for tests", - citation="insert citation here", - features=Features({"predictions": Value("int64"), "references": Value("int64")}), - ) - - def _compute(self, predictions, references): - return ( - { - "accuracy": sum(i == j for i, j in zip(predictions, references)) / len(predictions), - "set_equality": set(predictions) == set(references), - } - if predictions - else {} - ) - - @classmethod - def predictions_and_references(cls): - return ([1, 2, 3, 4], [1, 2, 4, 3]) - - @classmethod - def expected_results(cls): - return {"accuracy": 0.5, "set_equality": True} - - @classmethod - def other_predictions_and_references(cls): - return ([1, 3, 4, 5], [1, 2, 3, 4]) - - @classmethod - def other_expected_results(cls): - return {"accuracy": 0.25, "set_equality": False} - - @classmethod - def distributed_predictions_and_references(cls): - return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) - - @classmethod - def distributed_expected_results(cls): - return {"accuracy": 0.75, "set_equality": False} - - @classmethod - def separate_predictions_and_references(cls): - return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) - - @classmethod - def separate_expected_results(cls): - return [{"accuracy": 1.0, "set_equality": True}, {"accuracy": 0.5, "set_equality": False}] - - -def properly_del_metric(metric): - """properly delete a metric on windows if the process is killed during multiprocessing""" - if metric is not None: - if metric.filelock is not None: - metric.filelock.release() - if metric.rendez_vous_lock is not None: - metric.rendez_vous_lock.release() - del metric.writer - del metric.data - del metric - - -def metric_compute(arg): - """Thread worker function for distributed evaluation testing. - On base level to be pickable. - """ - metric = None - try: - num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg - metric = DummyMetric( - num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 - ) - time.sleep(wait) - results = metric.compute(predictions=preds, references=refs) - return results - finally: - properly_del_metric(metric) - - -def metric_add_batch_and_compute(arg): - """Thread worker function for distributed evaluation testing. - On base level to be pickable. - """ - metric = None - try: - num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg - metric = DummyMetric( - num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 - ) - metric.add_batch(predictions=preds, references=refs) - time.sleep(wait) - results = metric.compute() - return results - finally: - properly_del_metric(metric) - - -def metric_add_and_compute(arg): - """Thread worker function for distributed evaluation testing. - On base level to be pickable. - """ - metric = None - try: - num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg - metric = DummyMetric( - num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 - ) - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - time.sleep(wait) - results = metric.compute() - return results - finally: - properly_del_metric(metric) - - -@pytest.mark.filterwarnings("ignore:Metric is deprecated:FutureWarning") -class TestMetric(TestCase): - def test_dummy_metric(self): - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - - metric = DummyMetric(experiment_id="test_dummy_metric") - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - metric = DummyMetric(experiment_id="test_dummy_metric") - metric.add_batch(predictions=preds, references=refs) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(experiment_id="test_dummy_metric") - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - # With keep_in_memory - metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") - metric.add_batch(predictions=preds, references=refs) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") - self.assertDictEqual({}, metric.compute(predictions=[], references=[])) - del metric - - metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") - with self.assertRaisesRegex(ValueError, "Mismatch in the number"): - metric.add_batch(predictions=[1, 2, 3], references=[1, 2, 3, 4]) - del metric - - def test_metric_with_cache_dir(self): - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - - with tempfile.TemporaryDirectory() as tmp_dir: - metric = DummyMetric(experiment_id="test_dummy_metric", cache_dir=tmp_dir) - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - def test_concurrent_metrics(self): - preds, refs = DummyMetric.predictions_and_references() - other_preds, other_refs = DummyMetric.other_predictions_and_references() - expected_results = DummyMetric.expected_results() - other_expected_results = DummyMetric.other_expected_results() - - metric = DummyMetric(experiment_id="test_concurrent_metrics") - other_metric = DummyMetric( - experiment_id="test_concurrent_metrics", - ) - - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - self.assertDictEqual( - other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) - ) - del metric, other_metric - - metric = DummyMetric( - experiment_id="test_concurrent_metrics", - ) - other_metric = DummyMetric( - experiment_id="test_concurrent_metrics", - ) - metric.add_batch(predictions=preds, references=refs) - other_metric.add_batch(predictions=other_preds, references=other_refs) - self.assertDictEqual(expected_results, metric.compute()) - self.assertDictEqual(other_expected_results, other_metric.compute()) - - for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): - metric.add(prediction=pred, reference=ref) - other_metric.add(prediction=other_pred, reference=other_ref) - self.assertDictEqual(expected_results, metric.compute()) - self.assertDictEqual(other_expected_results, other_metric.compute()) - del metric, other_metric - - # With keep_in_memory - metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) - other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) - - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - self.assertDictEqual( - other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) - ) - - metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) - other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) - metric.add_batch(predictions=preds, references=refs) - other_metric.add_batch(predictions=other_preds, references=other_refs) - self.assertDictEqual(expected_results, metric.compute()) - self.assertDictEqual(other_expected_results, other_metric.compute()) - - for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): - metric.add(prediction=pred, reference=ref) - other_metric.add(prediction=other_pred, reference=other_ref) - self.assertDictEqual(expected_results, metric.compute()) - self.assertDictEqual(other_expected_results, other_metric.compute()) - del metric, other_metric - - def test_separate_experiments_in_parallel(self): - with tempfile.TemporaryDirectory() as tmp_dir: - (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.separate_predictions_and_references() - expected_results = DummyMetric.separate_expected_results() - - pool = Pool(processes=4) - - results = pool.map( - metric_compute, - [ - (1, 0, preds_0, refs_0, None, tmp_dir, 0), - (1, 0, preds_1, refs_1, None, tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results[0], results[0]) - self.assertDictEqual(expected_results[1], results[1]) - del results - - # more than one sec of waiting so that the second metric has to sample a new hashing name - results = pool.map( - metric_compute, - [ - (1, 0, preds_0, refs_0, None, tmp_dir, 2), - (1, 0, preds_1, refs_1, None, tmp_dir, 2), - ], - ) - self.assertDictEqual(expected_results[0], results[0]) - self.assertDictEqual(expected_results[1], results[1]) - del results - - results = pool.map( - metric_add_and_compute, - [ - (1, 0, preds_0, refs_0, None, tmp_dir, 0), - (1, 0, preds_1, refs_1, None, tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results[0], results[0]) - self.assertDictEqual(expected_results[1], results[1]) - del results - - results = pool.map( - metric_add_batch_and_compute, - [ - (1, 0, preds_0, refs_0, None, tmp_dir, 0), - (1, 0, preds_1, refs_1, None, tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results[0], results[0]) - self.assertDictEqual(expected_results[1], results[1]) - del results - - def test_distributed_metrics(self): - with tempfile.TemporaryDirectory() as tmp_dir: - (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.distributed_predictions_and_references() - expected_results = DummyMetric.distributed_expected_results() - - pool = Pool(processes=4) - - results = pool.map( - metric_compute, - [ - (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0), - (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0.5), - ], - ) - self.assertDictEqual(expected_results, results[0]) - self.assertIsNone(results[1]) - del results - - results = pool.map( - metric_compute, - [ - (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0.5), - (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results, results[0]) - self.assertIsNone(results[1]) - del results - - results = pool.map( - metric_add_and_compute, - [ - (2, 0, preds_0, refs_0, "test_distributed_metrics_1", tmp_dir, 0), - (2, 1, preds_1, refs_1, "test_distributed_metrics_1", tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results, results[0]) - self.assertIsNone(results[1]) - del results - - results = pool.map( - metric_add_batch_and_compute, - [ - (2, 0, preds_0, refs_0, "test_distributed_metrics_2", tmp_dir, 0), - (2, 1, preds_1, refs_1, "test_distributed_metrics_2", tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results, results[0]) - self.assertIsNone(results[1]) - del results - - # To use several distributed metrics on the same local file system, need to specify an experiment_id - try: - results = pool.map( - metric_add_and_compute, - [ - (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), - (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), - (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), - (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), - ], - ) - except ValueError: - # We are fine with either raising a ValueError or computing well the metric - # Being sure we raise the error would means making the dummy dataset bigger - # and the test longer... - pass - else: - self.assertDictEqual(expected_results, results[0]) - self.assertDictEqual(expected_results, results[2]) - self.assertIsNone(results[1]) - self.assertIsNone(results[3]) - del results - - results = pool.map( - metric_add_and_compute, - [ - (2, 0, preds_0, refs_0, "exp_0", tmp_dir, 0), - (2, 1, preds_1, refs_1, "exp_0", tmp_dir, 0), - (2, 0, preds_0, refs_0, "exp_1", tmp_dir, 0), - (2, 1, preds_1, refs_1, "exp_1", tmp_dir, 0), - ], - ) - self.assertDictEqual(expected_results, results[0]) - self.assertDictEqual(expected_results, results[2]) - self.assertIsNone(results[1]) - self.assertIsNone(results[3]) - del results - - # With keep_in_memory is not allowed - with self.assertRaises(ValueError): - DummyMetric( - experiment_id="test_distributed_metrics_4", - keep_in_memory=True, - num_process=2, - process_id=0, - cache_dir=tmp_dir, - ) - - def test_dummy_metric_pickle(self): - with tempfile.TemporaryDirectory() as tmp_dir: - tmp_file = os.path.join(tmp_dir, "metric.pt") - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - - metric = DummyMetric(experiment_id="test_dummy_metric_pickle") - - with open(tmp_file, "wb") as f: - pickle.dump(metric, f) - del metric - - with open(tmp_file, "rb") as f: - metric = pickle.load(f) - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - def test_input_numpy(self): - import numpy as np - - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - preds, refs = np.array(preds), np.array(refs) - - metric = DummyMetric(experiment_id="test_input_numpy") - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - metric = DummyMetric(experiment_id="test_input_numpy") - metric.add_batch(predictions=preds, references=refs) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(experiment_id="test_input_numpy") - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - @require_torch - def test_input_torch(self): - import torch - - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - preds, refs = torch.tensor(preds), torch.tensor(refs) - - metric = DummyMetric(experiment_id="test_input_torch") - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - metric = DummyMetric(experiment_id="test_input_torch") - metric.add_batch(predictions=preds, references=refs) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(experiment_id="test_input_torch") - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - @require_tf - def test_input_tf(self): - import tensorflow as tf - - preds, refs = DummyMetric.predictions_and_references() - expected_results = DummyMetric.expected_results() - preds, refs = tf.constant(preds), tf.constant(refs) - - metric = DummyMetric(experiment_id="test_input_tf") - self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) - del metric - - metric = DummyMetric(experiment_id="test_input_tf") - metric.add_batch(predictions=preds, references=refs) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - metric = DummyMetric(experiment_id="test_input_tf") - for pred, ref in zip(preds, refs): - metric.add(prediction=pred, reference=ref) - self.assertDictEqual(expected_results, metric.compute()) - del metric - - -class MetricWithMultiLabel(Metric): - def _info(self): - return MetricInfo( - description="dummy metric for tests", - citation="insert citation here", - features=Features( - {"predictions": Sequence(Value("int64")), "references": Sequence(Value("int64"))} - if self.config_name == "multilabel" - else {"predictions": Value("int64"), "references": Value("int64")} - ), - ) - - def _compute(self, predictions=None, references=None): - return ( - { - "accuracy": sum(i == j for i, j in zip(predictions, references)) / len(predictions), - } - if predictions - else {} - ) - - -@pytest.mark.parametrize( - "config_name, predictions, references, expected", - [ - (None, [1, 2, 3, 4], [1, 2, 4, 3], 0.5), # Multiclass: Value("int64") - ( - "multilabel", - [[1, 0], [1, 0], [1, 0], [1, 0]], - [[1, 0], [0, 1], [1, 1], [0, 0]], - 0.25, - ), # Multilabel: Sequence(Value("int64")) - ], -) -def test_metric_with_multilabel(config_name, predictions, references, expected, tmp_path): - cache_dir = tmp_path / "cache" - metric = MetricWithMultiLabel(config_name, cache_dir=cache_dir) - results = metric.compute(predictions=predictions, references=references) - assert results["accuracy"] == expected - - -def test_safety_checks_process_vars(): - with pytest.raises(ValueError): - _ = DummyMetric(process_id=-2) - - with pytest.raises(ValueError): - _ = DummyMetric(num_process=2, process_id=3) - - -class AccuracyWithNonStandardFeatureNames(Metric): - def _info(self): - return MetricInfo( - description="dummy metric for tests", - citation="insert citation here", - features=Features({"inputs": Value("int64"), "targets": Value("int64")}), - ) - - def _compute(self, inputs, targets): - return ( - { - "accuracy": sum(i == j for i, j in zip(inputs, targets)) / len(targets), - } - if targets - else {} - ) - - @classmethod - def inputs_and_targets(cls): - return ([1, 2, 3, 4], [1, 2, 4, 3]) - - @classmethod - def expected_results(cls): - return {"accuracy": 0.5} - - -def test_metric_with_non_standard_feature_names_add(tmp_path): - cache_dir = tmp_path / "cache" - inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() - metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) - for input, target in zip(inputs, targets): - metric.add(inputs=input, targets=target) - results = metric.compute() - assert results == AccuracyWithNonStandardFeatureNames.expected_results() - - -def test_metric_with_non_standard_feature_names_add_batch(tmp_path): - cache_dir = tmp_path / "cache" - inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() - metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) - metric.add_batch(inputs=inputs, targets=targets) - results = metric.compute() - assert results == AccuracyWithNonStandardFeatureNames.expected_results() - - -def test_metric_with_non_standard_feature_names_compute(tmp_path): - cache_dir = tmp_path / "cache" - inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() - metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) - results = metric.compute(inputs=inputs, targets=targets) - assert results == AccuracyWithNonStandardFeatureNames.expected_results() diff --git a/tests/test_metric_common.py b/tests/test_metric_common.py deleted file mode 100644 index 2a7407e7a43..00000000000 --- a/tests/test_metric_common.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright 2020 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import doctest -import glob -import importlib -import inspect -import os -import re -from contextlib import contextmanager -from functools import wraps -from unittest.mock import patch - -import numpy as np -import pytest -from absl.testing import parameterized - -import datasets -from datasets import load_metric - -from .utils import for_all_test_methods, local, slow - - -# mark all tests as integration -pytestmark = pytest.mark.integration - - -REQUIRE_FAIRSEQ = {"comet"} -_has_fairseq = importlib.util.find_spec("fairseq") is not None - -# code_eval is unsupported, and the other are not installed in the windows CI -UNSUPPORTED_ON_WINDOWS = {"code_eval", "bleurt", "competition_math", "coval"} -_on_windows = os.name == "nt" - -REQUIRE_TRANSFORMERS = {"bertscore", "frugalscore", "perplexity"} -_has_transformers = importlib.util.find_spec("transformers") is not None - - -def skip_if_metric_requires_fairseq(test_case): - @wraps(test_case) - def wrapper(self, metric_name): - if not _has_fairseq and metric_name in REQUIRE_FAIRSEQ: - self.skipTest('"test requires Fairseq"') - else: - test_case(self, metric_name) - - return wrapper - - -def skip_if_metric_requires_transformers(test_case): - @wraps(test_case) - def wrapper(self, metric_name): - if not _has_transformers and metric_name in REQUIRE_TRANSFORMERS: - self.skipTest('"test requires transformers"') - else: - test_case(self, metric_name) - - return wrapper - - -def skip_on_windows_if_not_windows_compatible(test_case): - @wraps(test_case) - def wrapper(self, metric_name): - if _on_windows and metric_name in UNSUPPORTED_ON_WINDOWS: - self.skipTest('"test not supported on Windows"') - else: - test_case(self, metric_name) - - return wrapper - - -def get_local_metric_names(): - metrics = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./metrics/*/")] - return [{"testcase_name": x, "metric_name": x} for x in metrics if x != "gleu"] # gleu is unfinished - - -@parameterized.named_parameters(get_local_metric_names()) -@for_all_test_methods( - skip_if_metric_requires_fairseq, skip_if_metric_requires_transformers, skip_on_windows_if_not_windows_compatible -) -@local -class LocalMetricTest(parameterized.TestCase): - INTENSIVE_CALLS_PATCHER = {} - metric_name = None - - @pytest.mark.filterwarnings("ignore:metric_module_factory is deprecated:FutureWarning") - @pytest.mark.filterwarnings("ignore:load_metric is deprecated:FutureWarning") - def test_load_metric(self, metric_name): - doctest.ELLIPSIS_MARKER = "[...]" - metric_module = importlib.import_module( - datasets.load.metric_module_factory( - os.path.join("metrics", metric_name), trust_remote_code=True - ).module_path - ) - metric = datasets.load.import_main_class(metric_module.__name__, dataset=False) - # check parameters - parameters = inspect.signature(metric._compute).parameters - self.assertTrue(all(p.kind != p.VAR_KEYWORD for p in parameters.values())) # no **kwargs - # run doctest - with self.patch_intensive_calls(metric_name, metric_module.__name__): - with self.use_local_metrics(): - try: - results = doctest.testmod(metric_module, verbose=True, raise_on_error=True) - except doctest.UnexpectedException as e: - raise e.exc_info[1] # raise the exception that doctest caught - self.assertEqual(results.failed, 0) - self.assertGreater(results.attempted, 1) - - @slow - def test_load_real_metric(self, metric_name): - doctest.ELLIPSIS_MARKER = "[...]" - metric_module = importlib.import_module( - datasets.load.metric_module_factory(os.path.join("metrics", metric_name)).module_path - ) - # run doctest - with self.use_local_metrics(): - results = doctest.testmod(metric_module, verbose=True, raise_on_error=True) - self.assertEqual(results.failed, 0) - self.assertGreater(results.attempted, 1) - - @contextmanager - def patch_intensive_calls(self, metric_name, module_name): - if metric_name in self.INTENSIVE_CALLS_PATCHER: - with self.INTENSIVE_CALLS_PATCHER[metric_name](module_name): - yield - else: - yield - - @contextmanager - def use_local_metrics(self): - def load_local_metric(metric_name, *args, **kwargs): - return load_metric(os.path.join("metrics", metric_name), *args, **kwargs) - - with patch("datasets.load_metric") as mock_load_metric: - mock_load_metric.side_effect = load_local_metric - yield - - @classmethod - def register_intensive_calls_patcher(cls, metric_name): - def wrapper(patcher): - patcher = contextmanager(patcher) - cls.INTENSIVE_CALLS_PATCHER[metric_name] = patcher - return patcher - - return wrapper - - -# Metrics intensive calls patchers -# -------------------------------- - - -@LocalMetricTest.register_intensive_calls_patcher("bleurt") -def patch_bleurt(module_name): - import tensorflow.compat.v1 as tf - from bleurt.score import Predictor - - tf.flags.DEFINE_string("sv", "", "") # handle pytest cli flags - - class MockedPredictor(Predictor): - def predict(self, input_dict): - assert len(input_dict["input_ids"]) == 2 - return np.array([1.03, 1.04]) - - # mock predict_fn which is supposed to do a forward pass with a bleurt model - with patch("bleurt.score._create_predictor") as mock_create_predictor: - mock_create_predictor.return_value = MockedPredictor() - yield - - -@LocalMetricTest.register_intensive_calls_patcher("bertscore") -def patch_bertscore(module_name): - import torch - - def bert_cos_score_idf(model, refs, *args, **kwargs): - return torch.tensor([[1.0, 1.0, 1.0]] * len(refs)) - - # mock get_model which is supposed to do download a bert model - # mock bert_cos_score_idf which is supposed to do a forward pass with a bert model - with patch("bert_score.scorer.get_model"), patch( - "bert_score.scorer.bert_cos_score_idf" - ) as mock_bert_cos_score_idf: - mock_bert_cos_score_idf.side_effect = bert_cos_score_idf - yield - - -@LocalMetricTest.register_intensive_calls_patcher("comet") -def patch_comet(module_name): - def load_from_checkpoint(model_path): - class Model: - def predict(self, data, *args, **kwargs): - assert len(data) == 2 - scores = [0.19, 0.92] - return scores, sum(scores) / len(scores) - - return Model() - - # mock load_from_checkpoint which is supposed to do download a bert model - # mock load_from_checkpoint which is supposed to do download a bert model - with patch("comet.download_model") as mock_download_model: - mock_download_model.return_value = None - with patch("comet.load_from_checkpoint") as mock_load_from_checkpoint: - mock_load_from_checkpoint.side_effect = load_from_checkpoint - yield - - -def test_seqeval_raises_when_incorrect_scheme(): - metric = load_metric(os.path.join("metrics", "seqeval"), trust_remote_code=True) - wrong_scheme = "ERROR" - error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}" - with pytest.raises(ValueError, match=re.escape(error_message)): - metric.compute(predictions=[], references=[], scheme=wrong_scheme) diff --git a/tests/test_warnings.py b/tests/test_warnings.py deleted file mode 100644 index 62028507ef4..00000000000 --- a/tests/test_warnings.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from datasets import inspect_metric, list_metrics, load_metric - - -@pytest.fixture -def mock_emitted_deprecation_warnings(monkeypatch): - monkeypatch.setattr("datasets.utils.deprecation_utils._emitted_deprecation_warnings", set()) - - -# Used by list_metrics -@pytest.fixture -def mock_hfh(monkeypatch): - class MetricMock: - def __init__(self, metric_id): - self.id = metric_id - - class HfhMock: - _metrics = [MetricMock(metric_id) for metric_id in ["accuracy", "mse", "precision", "codeparrot/apps_metric"]] - - def list_metrics(self): - return self._metrics - - monkeypatch.setattr("datasets.inspect.huggingface_hub", HfhMock()) - - -@pytest.mark.parametrize( - "func, args, kwargs", - [ - (load_metric, ("metrics/mse",), {"trust_remote_code": True}), - (list_metrics, (), {}), - (inspect_metric, ("metrics/mse", "tmp_path"), {"trust_remote_code": True}), - ], -) -def test_metric_deprecation_warning(func, args, kwargs, mock_emitted_deprecation_warnings, mock_hfh, tmp_path): - if "tmp_path" in args: - args = tuple(arg if arg != "tmp_path" else tmp_path for arg in args) - with pytest.warns(FutureWarning, match="https://huggingface.co/docs/evaluate"): - func(*args, **kwargs)