Add QE APIs support in Language pages (#549)

* Add QE APIs to languages and fix languages nav bar * Exclude languages from nav bar and improve wording --------- Co-authored-by: Tovmas <[email protected]>
machinetranslate · Oct 21, 2023 · 1184139 · 1184139
1 parent a2188c5
commit 1184139
Show file tree

Hide file tree

Showing 229 changed files with 2,050 additions and 250 deletions.
diff --git a/_layouts/language.html b/_layouts/language.html
@@ -15,6 +15,9 @@ <h2>
 <p>
   Machine translation for the <strong>{{ page.title }}</strong> language is supported by <a href="#machine-translation">{{ page.supported_apis | size }} machine translation APIs</a>.
 </p>
+<p>
+  Quality estimation for the <strong>{{ page.title }}</strong> language is supported by <a href="#quality-estimation">{{ page.supported_qe_apis | size }} quality estimation APIs</a>.
+</p>
 
 <blockquote>
 
@@ -54,13 +57,13 @@ <h2>
 {{ content }}
 
 <section id="machine-translation">
-  <h2>API support</h2>
+  <h2>Machine translation</h2>
   <details>
     <summary>
       {% if page.supported_apis.size == 1 %}
-        <strong>1</strong> API supports {{ page.title }}.
+        <strong>1</strong> machine translation API supports {{ page.title }}.
       {% else %}
-        <strong>{{ page.supported_apis | size }}</strong> APIs support {{ page.title }}.
+        <strong>{{ page.supported_apis | size }}</strong> machine translation APIs support {{ page.title }}.
       {% endif %}
       <p class="preview hint">
         {{ page.supported_apis | slice: 0, 5 | map: 'name' | join: ', ' }}
@@ -78,3 +81,29 @@ <h2>API support</h2>
     </ul>
   </details>
 </section>
+
+<section id="quality-estimation">
+  <h2>Quality estimation</h2>
+  <details>
+    <summary>
+      {% if page.supported_qe_apis.size == 1 %}
+        <strong>1</strong> quality estimation API supports {{ page.title }}.
+      {% else %}
+        <strong>{{ page.supported_qe_apis | size }}</strong> quality estimation APIs support {{ page.title }}.
+      {% endif %}
+      <p class="preview hint">
+        {{ page.supported_qe_apis | slice: 0, 5 | map: 'name' | join: ', ' }}
+        {% if page.supported_qe_apis.size > 5 %}, …{% endif %}
+      </p>
+    </summary>
+    <ul>
+      {% for qe_api in page.supported_qe_apis %}
+        <li>
+          <a href="/{{ qe_api.slug }}">
+            {{ qe_api.name }}
+          </a>
+        </li>
+      {% endfor %}
+    </ul>
+  </details>
+</section>
diff --git a/generate.py b/generate.py
@@ -106,7 +106,7 @@ def slugify(name):
 def flatten(l):
   _ = []
   for item in l:
-    if type(item) is list:
+    if isinstance(item, list):
       _ += flatten(item)
     else:
       _.append(item)
@@ -122,22 +122,20 @@ def read_content(filepath):
     i += len('\n---\n')
     return page[i:].strip()
 
-SUPPORTED_LANGUAGE_BASE_CODES = {}
-for api in APIS:
-  api_id = api['id']
-
-  codes = flatten(api['languages'])
-
-
-  def normalize(code):
-    return normalize_language_code(code, api_id)
-
-  codes = map(normalize, codes)
-
-  codes = map(base_language_code, codes)
+def supported_language_base_codes(file_name):
+  SUPPORTED_LANGUAGE_BASE_CODES = {}
+  for api in file_name:
+    api_id = api['id']
+    codes = flatten(api['languages'])
 
+    def normalize(code):
+      return normalize_language_code(code, api_id)
+
+    codes = map(normalize, codes)
+    codes = map(base_language_code, codes)
 
-  SUPPORTED_LANGUAGE_BASE_CODES[api_id] = list(set(codes))
+    SUPPORTED_LANGUAGE_BASE_CODES[api_id] = list(set(codes))
+  return SUPPORTED_LANGUAGE_BASE_CODES
 
 
 ### Write language families
@@ -191,6 +189,8 @@ def normalize(code):
 
 
 ### Write languages
+API_SUPPORTED_LANGUAGE_BASE_CODES = supported_language_base_codes(APIS)
+QE_API_SUPPORTED_LANGUAGE_BASE_CODES = supported_language_base_codes(QUALITY_ESTIMATION)
 for language in LANGUAGES:
   code = language['codes'][0]
 
@@ -212,7 +212,7 @@ def normalize(code):
   # "Join"
   supported_apis = []
   for api in APIS:
-    codes = SUPPORTED_LANGUAGE_BASE_CODES[api['id']]
+    codes = API_SUPPORTED_LANGUAGE_BASE_CODES[api['id']]
     if code in codes:
       supported_apis.append({
         'id': api['id'],
@@ -222,19 +222,34 @@ def normalize(code):
 
   supported_apis.sort(key=lambda api: api['supported_language_count'])
 
+  supported_qe_apis =[]
+  for qe_api in QUALITY_ESTIMATION:
+    codes = QE_API_SUPPORTED_LANGUAGE_BASE_CODES[qe_api['id']]
+    if code in codes:
+      supported_qe_apis.append({
+        'slug': slugify(qe_api['name']),
+        'id': qe_api['id'],
+        'name': qe_api['name'],
+        'supported_language_count': len(codes)
+      })
+
+  supported_qe_apis.sort(key=lambda api: api['supported_language_count'])
+
+
   desc = f'Machine translation for { name }'
 
   frontmatter = {
     'autogenerated': True,
-    'nav_exclude': True,
     'nav_order': 1000 - len(supported_apis),
+    'nav_exclude': True,
     'parent': 'Languages',
     'layout': 'language',
     'title': name,
     'description': desc,
     **language,
     'family': family,
     'supported_apis': supported_apis,
+    'supported_qe_apis': supported_qe_apis,
     'seo': {
       'name': desc,
       'type': 'Language'
@@ -398,6 +413,7 @@ def normalize(code):
 
   print(f'[{ text }]({ link })')
 
+
 # Generate TMS files
 QE_APIS_BY_ID = {api['id']: api for api in QUALITY_ESTIMATION}
 APIS_BY_ID = {api['id']: api for api in APIS + AGGREGATORS}
@@ -492,7 +508,6 @@ def normalize(code):
 
 
 ### Generate Aggregators files
-
 for a in AGGREGATORS:
     a_id = a['id']
     a_name = a['name']
@@ -548,8 +563,8 @@ def normalize(code):
 { content }
 ''')
 
-### Generate Quality Estimation files
 
+### Generate Quality Estimation files
 for estimation in QUALITY_ESTIMATION:
 
   name = estimation['name']
@@ -664,7 +679,8 @@ def normalize(code):
     }
   }
 
-  slug = slugify(name.replace('-', '').replace('  ', ' '))
+  slug = slugify(name)
+
   filepath = f'quality-estimation/{ slug }.md'
 
   content = read_content(filepath)

diff --git a/languages/abkhaz.md b/languages/abkhaz.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 998
+nav_exclude: true
 parent: Languages
 layout: language
 title: Abkhaz
@@ -30,6 +30,7 @@ supported_apis:
 - id: alibaba
   name: Alibaba Translate
   supported_language_count: 212
+supported_qe_apis: []
 seo:
   name: Machine translation for Abkhaz
   type: Language

diff --git a/languages/acehnese.md b/languages/acehnese.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 997
+nav_exclude: true
 parent: Languages
 layout: language
 title: Acehnese
@@ -33,6 +33,7 @@ supported_apis:
 - id: niutrans
   name: Niutrans
   supported_language_count: 381
+supported_qe_apis: []
 seo:
   name: Machine translation for Acehnese
   type: Language

diff --git a/languages/acholi.md b/languages/acholi.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 998
+nav_exclude: true
 parent: Languages
 layout: language
 title: Acholi
@@ -31,6 +31,7 @@ supported_apis:
 - id: niutrans
   name: Niutrans
   supported_language_count: 381
+supported_qe_apis: []
 seo:
   name: Machine translation for Acholi
   type: Language

diff --git a/languages/afrikaans.md b/languages/afrikaans.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 982
+nav_exclude: true
 parent: Languages
 layout: language
 title: Afrikaans
@@ -84,6 +84,19 @@ supported_apis:
 - id: niutrans
   name: Niutrans
   supported_language_count: 381
+supported_qe_apis:
+- slug: phrase-mtqe
+  id: phrase-mtqe
+  name: Phrase MTQE
+  supported_language_count: 65
+- slug: demt-estimate
+  id: demt-estimate
+  name: DeMT Estimate
+  supported_language_count: 109
+- slug: modelfront
+  id: modelfront
+  name: ModelFront
+  supported_language_count: 126
 seo:
   name: Machine translation for Afrikaans
   type: Language

diff --git a/languages/akan.md b/languages/akan.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 996
+nav_exclude: true
 parent: Languages
 layout: language
 title: Akan
@@ -38,6 +38,7 @@ supported_apis:
 - id: alibaba
   name: Alibaba Translate
   supported_language_count: 212
+supported_qe_apis: []
 seo:
   name: Machine translation for Akan
   type: Language

diff --git a/languages/albanian.md b/languages/albanian.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 983
+nav_exclude: true
 parent: Languages
 layout: language
 title: Albanian
@@ -85,6 +85,19 @@ supported_apis:
 - id: niutrans
   name: Niutrans
   supported_language_count: 381
+supported_qe_apis:
+- slug: phrase-mtqe
+  id: phrase-mtqe
+  name: Phrase MTQE
+  supported_language_count: 65
+- slug: demt-estimate
+  id: demt-estimate
+  name: DeMT Estimate
+  supported_language_count: 109
+- slug: modelfront
+  id: modelfront
+  name: ModelFront
+  supported_language_count: 126
 seo:
   name: Machine translation for Albanian
   type: Language

diff --git a/languages/alemannic.md b/languages/alemannic.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 999
+nav_exclude: true
 parent: Languages
 layout: language
 title: Alemannic
@@ -42,6 +42,7 @@ supported_apis:
 - id: textshuttle
   name: Textshuttle
   supported_language_count: 21
+supported_qe_apis: []
 seo:
   name: Machine translation for Alemannic
   type: Language

diff --git a/languages/algerian-arabic.md b/languages/algerian-arabic.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 999
+nav_exclude: true
 parent: Languages
 layout: language
 title: Algerian Arabic
@@ -34,6 +34,7 @@ supported_apis:
 - id: baidu
   name: Baidu Translate
   supported_language_count: 197
+supported_qe_apis: []
 seo:
   name: Machine translation for Algerian Arabic
   type: Language

diff --git a/languages/amharic.md b/languages/amharic.md
@@ -1,7 +1,7 @@
 ---
 autogenerated: true
-nav_exclude: true
 nav_order: 987
+nav_exclude: true
 parent: Languages
 layout: language
 title: Amharic
@@ -66,6 +66,15 @@ supported_apis:
 - id: niutrans
   name: Niutrans
   supported_language_count: 381
+supported_qe_apis:
+- slug: demt-estimate
+  id: demt-estimate
+  name: DeMT Estimate
+  supported_language_count: 109
+- slug: modelfront
+  id: modelfront
+  name: ModelFront
+  supported_language_count: 126
 seo:
   name: Machine translation for Amharic
   type: Language