Add metrics/wer

.gitignore

@@ -87,6 +87,7 @@ target/
 
 # pyenv
 .python-version
+venv
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.

Makefile

@@ -1,4 +1,4 @@
-SOURCES := speechmatics/ tests/ examples/ setup.py
+SOURCES := speechmatics/ tests/ examples/ metrics/ setup.py
 VERSION ?= $(shell cat VERSION)
 
 .PHONY: all

metrics/README.md

@@ -0,0 +1,47 @@
+# SM Metrics
+
+We provide some additional tooling to help benchmark transcription and diarization performance.
+
+## Getting Started
+
+### CLI
+
+The `sm-metrics` binary is built after installing with PyPI or running `python3 setup.py` from the source code. To see the options from the command-line, use the following:
+``` bash
+sm-metrics -h
+```
+
+### Source Code
+
+When executing directly from the source code:
+```bash
+python3 -m metrics.cli -h
+```
+
+## What's Included?
+
+### Transcription Metrics
+
+This includes tools to:
+- Normalise transcripts
+- Calculate Word Error Rate and Character Error Rate
+- Calculate the number of substitutions, deletions and insertions for a given ASR transcript
+- Visualise the alignment and differences between a reference and ASR transcript
+
+### Diarization Metrics
+
+This includes tools to calculate a number of metrics used in benchmarking diarization, including:
+
+- Diarization Error Rate
+- Segmentation precision, recall and F1-Scores
+- Word Diarization Error Rate
+
+## Documentation
+
+More extensive information on the metrics themselves, as well as how to run them can be found on the READMEs.
+
+For diarization, we provide an additional PDF.
+
+## Support
+
+If you have any issues with this library or encounter any bugs then please get in touch with us at [email protected] or raise an issue for this repo.

metrics/cli.py

@@ -0,0 +1,36 @@
+"""Entrypoint for SM metrics"""
+import argparse
+
+import metrics.diarization.sm_diarization_metrics.cookbook as diarization_metrics
+import metrics.wer.__main__ as wer_metrics
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Your CLI description")
+
+    # Create subparsers
+    subparsers = parser.add_subparsers(
+        dest="mode", help="Metrics mode. Choose from 'wer' or 'diarization"
+    )
+    subparsers.required = True  # Make sure a subparser id always provided
+
+    wer_parser = subparsers.add_parser("wer", help="Entrypoint for WER metrics")
+    wer_metrics.get_wer_args(wer_parser)
+
+    diarization_parser = subparsers.add_parser(
+        "diarization", help="Entrypoint for diarization metrics"
+    )
+    diarization_metrics.get_diarization_args(diarization_parser)
+
+    args = parser.parse_args()
+
+    if args.mode == "wer":
+        wer_metrics.main(args)
+    elif args.mode == "diarization":
+        diarization_metrics.main(args)
+    else:
+        print("Unsupported mode. Please use 'wer' or 'diarization'")
+
+
+if __name__ == "__main__":
+    main()

metrics/diarization/Makefile

@@ -0,0 +1,13 @@
+clean_files := .deps .deps-dev build dist
+
+.PHONY: clean
+clean:
+	$(RM) -rf $(clean_files)
+
+.PHONY: wheel
+wheel:
+	(pip install wheel && python3 setup.py bdist_wheel)
+
+.PHONY: install
+install: 
+	pip install ./dist/*

metrics/diarization/README.md

@@ -0,0 +1,81 @@
+# SM Diarisation Metrics
+
+## Getting Started
+
+This project is Speechmatics' fork of https://github.com/pyannote/pyannote-metrics used to calculate various speaker diarization metrics from reference/hypothesis transcript pairs.
+
+### Run from PyPI
+
+```
+pip install speechmatics-python
+```
+
+This package has a CLI supporting ctm, lab, or V2 JSON format transcripts and can be run using:
+
+```bash
+sm-metrics diarization <reference file> <hypothesis file>
+```
+
+For further guidance run:
+
+```
+sm-metrics diarization -h
+```
+
+### Run from source code
+
+If you would prefer to clone the repo and run the source code, that can be done as follows.
+
+Clone the repository and install package:
+```bash
+git clone https://github.com/speechmatics/speechmatics-python.git && cd speechmatics-python && python setup.py install
+```
+
+And run directly:
+```
+python3 -m metrics.cli <reference file> <transcript_file>
+```
+
+
+
+## Permitted Formats
+
+### CTM
+
+Plain text file with the '.ctm' extension. Each line is of the form:
+```
+<file id> <speaker> <start time> <end time> <word> <confidence>
+```
+
+### LAB
+
+Plain text file with the '.lab' extension. Each line is of the form:
+```
+<start time> <end time> <speaker>
+```
+
+### JSON (Diarisation Reference format)
+
+JSON file of the form:
+
+```json
+[
+    {
+        "speaker_name": "Speaker 1",
+        "word": "Seems",
+        "start": 0.75,
+        "duration": 0.29
+    },
+]
+
+```
+
+### JSON (Speechmatics ASR Output)
+
+V2 JSON output of Speechmatics ASR can be directly used as a hypothesis for diarization metrics
+
+## Docs
+
+Further description of how to use the tool and the metrics available are in sm_diarization_metrics.pdf
+
+When using the PDF, be aware that it assumes you are running the source code directly from `./metrics/diarization`

metrics/diarization/requirements.txt

@@ -0,0 +1,4 @@
+pyannote.core
+pyannote.database
+docopt
+tabulate

metrics/diarization/setup.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""Package module."""
+
+import os
+
+from pip._internal.req import parse_requirements
+from setuptools import find_packages, setup
+
+requirements = parse_requirements("./requirements.txt", session=False)
+
+git_tag = os.environ.get("CI_COMMIT_TAG")
+if git_tag:
+    assert git_tag.startswith("diarization-metrics")
+version = git_tag.lstrip("diarization-metrics/") if git_tag else "0.0.3"
+
+
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+setup(
+    author="Speechmatics",
+    author_email="[email protected]",
+    description="Python module for evaluating speaker diarization.",
+    install_requires=[str(r.requirement) for r in requirements],
+    name="speechmatics_diarization_metrics",
+    license="Speechmatics Proprietary License",
+    packages=find_packages(exclude=("tests",)),
+    platforms=["linux"],
+    python_requires=">=3.5",
+    version=version,
+    long_description=read("README.md"),
+    long_description_content_type="text/markdown",
+)