From 5b6ba18e77d54d338c8020c988fed12be869d54a Mon Sep 17 00:00:00 2001 From: "D. Ror" Date: Thu, 2 May 2024 12:18:32 -0400 Subject: [PATCH] Update mui-language-picker and font scripts (#3060) --- maintenance/Dockerfile | 4 +- .../mui_language_picker_font_map.json | 19 -- .../font_lists/mui_language_picker_fonts.txt | 188 ------------------ maintenance/scripts/get_fonts.py | 86 ++++---- .../scripts/mui_language_picker_fonts.txt | 152 ++++++++++++++ package-lock.json | 8 +- package.json | 2 +- scripts/get_fonts_dev.py | 39 +++- 8 files changed, 232 insertions(+), 266 deletions(-) delete mode 100644 maintenance/scripts/font_lists/mui_language_picker_font_map.json delete mode 100644 maintenance/scripts/font_lists/mui_language_picker_fonts.txt create mode 100644 maintenance/scripts/mui_language_picker_fonts.txt diff --git a/maintenance/Dockerfile b/maintenance/Dockerfile index 7e2302eaa7..b897aa2dbb 100644 --- a/maintenance/Dockerfile +++ b/maintenance/Dockerfile @@ -33,12 +33,10 @@ WORKDIR ${HOME} ENV PATH ${PATH}:${SCRIPT_DIR} -RUN mkdir -p ${SCRIPT_DIR}/font_lists - COPY --chown=user:user requirements.txt . RUN pip3 install -r requirements.txt COPY --chown=user:user scripts/*.py ${SCRIPT_DIR} COPY --chown=user:user scripts/*.sh ${SCRIPT_DIR} -COPY --chown=user:user scripts/font_lists/* ${SCRIPT_DIR}/font_lists +COPY --chown=user:user scripts/mui_language_picker_fonts.txt ${SCRIPT_DIR} diff --git a/maintenance/scripts/font_lists/mui_language_picker_font_map.json b/maintenance/scripts/font_lists/mui_language_picker_font_map.json deleted file mode 100644 index 257fc91086..0000000000 --- a/maintenance/scripts/font_lists/mui_language_picker_font_map.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "KNDABadami": "Badami", - "Galatia": "GalatiaSIL", - "Kyebogyi": "KyebogyiSIL", - "NamdhinggoSIL": "Namdhinggo", - "NotoSansCJKJP": "NotoSansJP", - "NotoSansCJKKR": "NotoSansKR", - "NotoSansCJKSC": "NotoSansSC", - "NotoSansCJKTC": "NotoSansTC", - "NotoSansLatin": "NotoSans", - "NotoSansTangut": "NotoSerifTangut", - "NotoSerifCJKJP": "NotoSerifJP", - "NotoSerifCJKKR": "NotoSerifKR", - "NotoSerifCJKSC": "NotoSerifSC", - "NotoSerifCJKTC": "NotoSerifTC", - "NotoSerifLatin": "NotoSerif", - "Scheherazade": "ScheherazadeNew", - "TAMLThiruValluvar": "ThiruValluvar" -} diff --git a/maintenance/scripts/font_lists/mui_language_picker_fonts.txt b/maintenance/scripts/font_lists/mui_language_picker_fonts.txt deleted file mode 100644 index 1a8e38ed63..0000000000 --- a/maintenance/scripts/font_lists/mui_language_picker_fonts.txt +++ /dev/null @@ -1,188 +0,0 @@ -Abyssinica SIL -Annapurna SIL -Awami Nastaliq -Charis SIL -Dai Banna SIL -DokChampa -Doulos SIL -Ebrima -Ezra SIL -Galatia -Gentium Plus -Harmattan -KNDA Badami -Kayahli -Khmer Mondulkiri -Kyebogyi -LisuTzimu -Mingzat -Namdhinggo SIL -Narnoor -Nokyung -Noto Sans CJK JP -Noto Sans CJK KR -Noto Sans CJK SC -Noto Sans CJK TC -Noto Serif CJK JP -Noto Serif CJK KR -Noto Serif CJK SC -Noto Serif CJK TC -NotoNastaliqUrdu -NotoSans -NotoSansAdlam -NotoSansAnatolianHieroglyphs -NotoSansArabic -NotoSansArmenian -NotoSansAvestan -NotoSansBalinese -NotoSansBamum -NotoSansBassaVah -NotoSansBatak -NotoSansBengali -NotoSansBhaiksuki -NotoSansBrahmi -NotoSansBuginese -NotoSansBuhid -NotoSansCanadianAboriginal -NotoSansCarian -NotoSansCaucasianAlbanian -NotoSansChakma -NotoSansCham -NotoSansCherokee -NotoSansCoptic -NotoSansCuneiform -NotoSansCypriot -NotoSansDeseret -NotoSansDevanagari -NotoSansDuployan -NotoSansEgyptianHieroglyphs -NotoSansElbasan -NotoSansEthiopic -NotoSansGeorgian -NotoSansGlagolitic -NotoSansGothic -NotoSansGrantha -NotoSansGujarati -NotoSansGurmukhi -NotoSansHanunoo -NotoSansHatran -NotoSansHebrew -NotoSansImperialAramaic -NotoSansInscriptionalPahlavi -NotoSansInscriptionalParthian -NotoSansJavanese -NotoSansKaithi -NotoSansKannada -NotoSansKayah Li -NotoSansKharoshthi -NotoSansKhmer -NotoSansKhojki -NotoSansKhudawadi -NotoSansLao -NotoSansLatin -NotoSansLeke -NotoSansLepcha -NotoSansLimbu -NotoSansLinearA -NotoSansLinearB -NotoSansLisu -NotoSansLycian -NotoSansLydian -NotoSansMahajani -NotoSansMalayalam -NotoSansMandaic -NotoSansManichaean -NotoSansMarchen -NotoSansMeeteiMayek -NotoSansMendeKikakui -NotoSansMeroitic -NotoSansMeroitic Cursive -NotoSansMiao -NotoSansModi -NotoSansMongolian -NotoSansMro -NotoSansMultani -NotoSansMyanmar -NotoSansNKo -NotoSansNabataean -NotoSansNandinagari -NotoSansNewTaiLue -NotoSansNewa -NotoSansOgham -NotoSansOl Chiki -NotoSansOld Permic -NotoSansOldHungarian -NotoSansOldItalic -NotoSansOldNorthArabian -NotoSansOldPersian -NotoSansOldSouthArabian -NotoSansOldTurkic -NotoSansOriya -NotoSansOsage -NotoSansOsmanya -NotoSansPahawhHmong -NotoSansPalmyrene -NotoSansPauCinHau -NotoSansPhagsPa -NotoSansPhoenician -NotoSansPsalterPahlavi -NotoSansRejang -NotoSansRunic -NotoSansSamaritan -NotoSansSaurashtra -NotoSansShuishu -NotoSansSiddham -NotoSansSinhala -NotoSansSora Sompeng -NotoSansSylotiNagri -NotoSansSyriac -NotoSansTagalog -NotoSansTagbanwa -NotoSansTaiLe -NotoSansTaiTham -NotoSansTaiViet -NotoSansTakri -NotoSansTamil -NotoSansTangut -NotoSansTelugu -NotoSansThaana -NotoSansThai -NotoSansTibetan -NotoSansTifinagh -NotoSansTirhuta -NotoSansUgaritic -NotoSansVai -NotoSansWarangCiti -NotoSansYi -NotoSerif -NotoSerifAhom -NotoSerifArmenian -NotoSerifBengali -NotoSerifDevanagari -NotoSerifEthiopic -NotoSerifGeorgian -NotoSerifGujarati -NotoSerifGurmukhi -NotoSerifHebrew -NotoSerifKannada -NotoSerifKhmer -NotoSerifLao -NotoSerifLatin -NotoSerifMalayalam -NotoSerifMyanmar -NotoSerifSinhala -NotoSerifTamil -NotoSerifTelugu -NotoSerifThai -NotoSerifTibetan -Nuosu SIL -Padauk -Saysettha MX -Scheherazade -ShiShan -Shimenkan -SimSun -Sophia Nubian -Surma -TAML ThiruValluvar -Tai Heritage Pro diff --git a/maintenance/scripts/get_fonts.py b/maintenance/scripts/get_fonts.py index f2f40056cd..2186be0cd1 100755 --- a/maintenance/scripts/get_fonts.py +++ b/maintenance/scripts/get_fonts.py @@ -19,10 +19,7 @@ scripts_dir = Path(__file__).resolve().parent file_name_fallback = "fallback.json" -font_lists_dir = scripts_dir / "font_lists" -mlp_font_list = font_lists_dir / "mui_language_picker_fonts.txt" -mlp_font_map = font_lists_dir / "mui_language_picker_font_map.json" -mlp_fonts_known_unavailable = ["NotoSansLeke", "NotoSansShuishu", "SimSun"] +mlp_font_list = scripts_dir / "mui_language_picker_fonts.txt" url_font_families_info = "https://github.com/silnrsi/fonts/raw/main/families.json" url_lang_tags_list = "https://ldml.api.sil.org/en?query=langtags" url_script_font_table = ( @@ -48,6 +45,13 @@ def parse_args() -> argparse.Namespace: metavar="LANG", help="List of language tags for which fonts should be downloaded.", ) + parser.add_argument( + "--scripts", + "-s", + nargs="*", + metavar="SCRIPT", + help="List of script tags for which fonts should be downloaded.", + ) parser.add_argument( "--url", "-u", @@ -142,10 +146,10 @@ def fetch_scripts_for_langs(langs: List[str]) -> List[str]: def fetch_fonts_for_scripts(scripts: List[str]) -> List[str]: """Given a list of script tags, look up the default fonts used with those scripts.""" - scripts = [script.capitalize() for script in scripts] + scripts = list({script.capitalize() for script in scripts}) - # Always have the Mui-Language-Picker default/safe fonts (except proprietary "SimSun"). - fonts = ["AnnapurnaSIL", "CharisSIL", "DoulosSIL", "NotoSans", "ScheherazadeNew"] + # Always have the Mui-Language-Picker default/safe fonts. + fonts = ["Annapurna SIL", "Charis SIL", "Noto Sans TC", "Scheherazade New"] logging.debug(f"Downloading script font table from {url_script_font_table}") req = requests.get(url_script_font_table) @@ -174,7 +178,7 @@ def fetch_fonts_for_scripts(scripts: List[str]) -> List[str]: if len(row) == 0 or row[0] not in scripts: continue for i in font_indices: - font = row[i].replace(" ", "") + font = row[i] if font != "" and font not in fonts: fonts.append(font) @@ -200,19 +204,25 @@ def main() -> None: logging.error(f"Invalid output directory: '{args.output}'") exit(1) + is_for_offline: bool = args.langs or args.scripts + with open(mlp_font_list, "r") as mlp_fonts_list: - # MLP use of spaces in fonts is inconsistent, so remove all spaces for simplicity. - fonts = [f.strip().replace(" ", "") for f in mlp_fonts_list.readlines()] + fonts = [f.strip() for f in mlp_fonts_list.readlines()] - if args.langs: - logging.info(f"Language tags: {', '.join(args.langs)}") + if is_for_offline: + scripts: List[str] = [] + if args.langs: + logging.info(f"Specified languages: {', '.join(args.langs)}") + scripts = fetch_scripts_for_langs(args.langs) + logging.info(f"Scripts for specified languages: {', '.join(scripts)}") - scripts = fetch_scripts_for_langs(args.langs) - logging.info(f"Scripts used for specified language tags: {', '.join(scripts)}") + if args.scripts: + logging.info(f"Specified scripts: {', '.join(args.scripts)}") + scripts.extend(args.scripts) script_fonts = fetch_fonts_for_scripts(scripts) logging.info( - f"Default fonts and fonts used for specified language tags: {', '.join(script_fonts)}" + f"Fonts (default and those for specified languages/scripts): {', '.join(script_fonts)}" ) if args.clean: @@ -227,30 +237,20 @@ def main() -> None: families = fetch_font_families_info() - with open(mlp_font_map, "r") as mlp_map_file: - mlp_map: dict[str, str] = json.load(mlp_map_file) - # Assumes no two keys map to the same value. - mlp_map_rev = {val: key for key, val in mlp_map.items()} - # Fonts for which the frontend will get css files from Google's font API. google_fallback: dict[str, str] = {} for font in fonts: logging.debug(f"Font: {font}") - font_id: str = font.lower() - if font in mlp_map.keys(): - font_id = mlp_map[font].lower() + font_id: str = font.replace(" ", "").lower() # Get font family info from font families info, using fallback font if necessary. while font_id != "" and font_id in families.keys(): font_info = families[font_id] family: str = font_info["family"] - from_google = ( - (not args.langs) - and "source" in font_info.keys() - and font_info["source"] == "Google" - ) - if check_font_info(font_info) or from_google: + source_is_google = "source" in font_info.keys() and font_info["source"] == "Google" + get_from_google = source_is_google and not is_for_offline + if check_font_info(font_info) or get_from_google: # Font available. break if "fallback" in font_info.keys(): @@ -261,22 +261,20 @@ def main() -> None: font_id = "" # When downloading, only download fonts used for scripts of the specified languages. - if args.langs and family.replace(" ", "") not in script_fonts: - logging.debug(f"Skipping font {font} as irrelevant for specified languages.") + if is_for_offline and family not in script_fonts: + logging.debug(f'Skipping font "{font}" as irrelevant for specified languages/scripts.') continue # Check if font was determined available. if font_id == "" or font_id not in families.keys(): - if font in mlp_fonts_known_unavailable: - logging.debug(f"Font {font} not available (but we knew that already)") - elif args.langs: - logging.warning(f"Font {font} not available for download") + if is_for_offline: + logging.warning(f'Font "{font}" not available for download') else: - logging.warning(f"Font {font} css info not available") + logging.warning(f'Font "{font}" css info not available') continue # When not downloading, prefer fetching css info from Google when available. - if from_google: + if get_from_google: google_fallback[font] = family logging.debug(f"Using Google fallback for {font}: {google_fallback[font]}") continue @@ -305,7 +303,7 @@ def main() -> None: logging.warning(f"{file_name}: No 'flourl' or 'url' for this file") continue - if args.langs: + if is_for_offline: # With the https://fonts.languagetechnology.org "flourl" urls, # urllib.request.urlretrieve() is denied (403), but requests.get() works. req = requests.get(src) @@ -326,17 +324,7 @@ def main() -> None: with open(css_file_path, "w") as css_file: css_file.writelines(css_lines) - # If the font corresponds to a different MPL font name, - # create a css file for that font name too. - if font in mlp_map_rev.keys(): - font = mlp_map_rev[font] - css_lines[2] = f" font-family: '{font}';\n" - css_file_path = args.output / f"{font}.css" - logging.debug(f"Writing {css_file_path}") - with open(css_file_path, "w") as css_file: - css_file.writelines(css_lines) - - if not args.langs: + if not is_for_offline: fallback_lines = ['{\n "google": {\n'] for key, val in google_fallback.items(): fallback_lines.append(f' "{key}": "{val}",\n') diff --git a/maintenance/scripts/mui_language_picker_fonts.txt b/maintenance/scripts/mui_language_picker_fonts.txt new file mode 100644 index 0000000000..5b9eaeb3c6 --- /dev/null +++ b/maintenance/scripts/mui_language_picker_fonts.txt @@ -0,0 +1,152 @@ +Abyssinica SIL +Annapurna SIL +BJCree UNI +Badami +Charis SIL +Dai Banna SIL +Dukor +Ezra SIL +Gentium Plus +Kay Pho Du +Khmer Mondulkiri +Lisu Bosa +Mingzat +Namdhinggo +Narnoor +Noto Sans Anatolian Hieroglyphs +Noto Sans Avestan +Noto Sans Balinese +Noto Sans Bamum +Noto Sans Bassa Vah +Noto Sans Batak +Noto Sans Bengali +Noto Sans Bhaiksuki +Noto Sans Brahmi +Noto Sans Buginese +Noto Sans Buhid +Noto Sans Carian +Noto Sans Caucasian Albanian +Noto Sans Chakma +Noto Sans Cham +Noto Sans Cherokee +Noto Sans Chorasmian +Noto Sans Coptic +Noto Sans Cuneiform +Noto Sans Cypriot +Noto Sans Deseret +Noto Sans Duployan +Noto Sans Egyptian Hieroglyphs +Noto Sans Elbasan +Noto Sans Elymaic +Noto Sans Georgian +Noto Sans Glagolitic +Noto Sans Gothic +Noto Sans Grantha +Noto Sans Gujarati +Noto Sans Gurmukhi +Noto Sans Hanifi Rohingya +Noto Sans Hanunoo +Noto Sans Hatran +Noto Sans Imperial Aramaic +Noto Sans Inscriptional Pahlavi +Noto Sans Inscriptional Parthian +Noto Sans JP +Noto Sans Javanese +Noto Sans KR +Noto Sans Kaithi +Noto Sans Kharoshthi +Noto Sans Khojki +Noto Sans Khudawadi +Noto Sans Linear A +Noto Sans Linear B +Noto Sans Lycian +Noto Sans Lydian +Noto Sans Mahajani +Noto Sans Mandaic +Noto Sans Manichaean +Noto Sans Marchen +Noto Sans Medefaidrin +Noto Sans Meetei Mayek +Noto Sans Mende Kikakui +Noto Sans Meroitic +Noto Sans Modi +Noto Sans Mongolian +Noto Sans Mro +Noto Sans Multani +Noto Sans NKo +Noto Sans Nabataean +Noto Sans Nag Mundari +Noto Sans Nandinagari +Noto Sans Newa +Noto Sans Ogham +Noto Sans Ol Chiki +Noto Sans Old Hungarian +Noto Sans Old Italic +Noto Sans Old North Arabian +Noto Sans Old Permic +Noto Sans Old Persian +Noto Sans Old Sogdian +Noto Sans Old South Arabian +Noto Sans Old Turkic +Noto Sans Oriya +Noto Sans Osage +Noto Sans Osmanya +Noto Sans Pahawh Hmong +Noto Sans Palmyrene +Noto Sans Pau Cin Hau +Noto Sans Phags Pa +Noto Sans Phoenician +Noto Sans Psalter Pahlavi +Noto Sans Rejang +Noto Sans Runic +Noto Sans SC +Noto Sans Samaritan +Noto Sans Saurashtra +Noto Sans Sharada +Noto Sans Shavian +Noto Sans Siddham +Noto Sans SignWriting +Noto Sans Sinhala +Noto Sans Sogdian +Noto Sans Sora Sompeng +Noto Sans Soyombo +Noto Sans Sundanese +Noto Sans Syriac +Noto Sans TC +Noto Sans Tagalog +Noto Sans Tai Le +Noto Sans Tai Tham +Noto Sans Takri +Noto Sans Tamil +Noto Sans Tangsa +Noto Sans Thaana +Noto Sans Thai +Noto Sans Tifinagh +Noto Sans Tirhuta +Noto Sans Ugaritic +Noto Sans Vithkuqi +Noto Sans Wancho +Noto Sans Warang Citi +Noto Sans Zanabazar Square +Noto Serif Ahom +Noto Serif Armenian +Noto Serif Dogra +Noto Serif JP +Noto Serif KR +Noto Serif Khitan Small Script +Noto Serif Makasar +Noto Serif Malayalam +Noto Serif NP Hmong +Noto Serif SC +Noto Serif Tangut +Noto Serif Telugu +Noto Serif Tibetan +Noto Serif Toto +Noto Serif Yezidi +Nuosu SIL +Padauk +Sapushan +Saysettha +Scheherazade New +Surma +Tai Heritage Pro diff --git a/package-lock.json b/package-lock.json index 641926ecb6..628d11847d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "make-dir": "^4.0.0", "material-react-table": "^2.9.2", "motion": "^10.16.2", - "mui-language-picker": "^1.2.8", + "mui-language-picker": "^2.1.5", "notistack": "^3.0.1", "nspell": "^2.1.5", "punycode": "^2.3.0", @@ -20040,9 +20040,9 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/mui-language-picker": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/mui-language-picker/-/mui-language-picker-1.2.8.tgz", - "integrity": "sha512-siNEsHI07YuckB2zm4QDFYQpzmgK6IPdJLdVM1zGTILwg+QfATO6DvJ4yaxJyOe92/dNHC5+kfHah+6ORSlstA==", + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/mui-language-picker/-/mui-language-picker-2.1.5.tgz", + "integrity": "sha512-vXd1+jwlluMHkCndbOOVrxBr00B72i3p3Pj2uX8WwqmCBkKIF81PXo3AdIDe+bQ48b/DeyvJsl9dDrovGQ1ODQ==", "dependencies": { "lodash": "^4.17.21" } diff --git a/package.json b/package.json index 5673ac6355..d1f953c9be 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "make-dir": "^4.0.0", "material-react-table": "^2.9.2", "motion": "^10.16.2", - "mui-language-picker": "^1.2.8", + "mui-language-picker": "^2.1.5", "notistack": "^3.0.1", "nspell": "^2.1.5", "punycode": "^2.3.0", diff --git a/scripts/get_fonts_dev.py b/scripts/get_fonts_dev.py index 4ec7287248..1c035b6460 100755 --- a/scripts/get_fonts_dev.py +++ b/scripts/get_fonts_dev.py @@ -1,14 +1,23 @@ #!/usr/bin/env python3 -"""Runs maintenance/scripts/get_fonts.py with dev arguments for -f and -o""" +""" +Runs maintenance/scripts/get_fonts.py with dev arguments for -f and -o. +Run this script with -U/--update whenever the mui-language-picker version is updated. +""" import argparse from pathlib import Path import platform +import re import subprocess project_dir = Path(__file__).resolve().parent.parent dev_output_dir = project_dir / "public" / "fonts" +maintenance_scripts_dir = project_dir / "maintenance" / "scripts" +mlp_font_list = maintenance_scripts_dir / "mui_language_picker_fonts.txt" +mlp_font_families = ( + project_dir / "node_modules" / "mui-language-picker" / "dist" / "data" / "scriptFontIndex.js" +) def parse_args() -> argparse.Namespace: @@ -26,6 +35,13 @@ def parse_args() -> argparse.Namespace: metavar="LANG", help="List of language tags for which fonts should be downloaded.", ) + parser.add_argument( + "--scripts", + "-s", + nargs="*", + metavar="SCRIPT", + help="List of script tags for which fonts should be downloaded.", + ) parser.add_argument( "--url", "-u", @@ -35,6 +51,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output", "-o", default=dev_output_dir, help="Output directory for font data." ) + parser.add_argument( + "--update", + "-U", + action="store_true", + help="Updates the list of fonts from mui-language-picker. (Must have run `npm i`.)", + ) parser.add_argument( "--verbose", "-v", @@ -49,11 +71,21 @@ def parse_args() -> argparse.Namespace: def main() -> None: args = parse_args() + if args.update: + # Font families are in the file as: \"family\":\"Font Family Name\" + family_pattern = re.compile(r'\\"family\\"\:\\"([^\\]+)\\"') + with open(mlp_font_families, "r") as families_file: + matches = re.findall(family_pattern, families_file.read()) + font_lines = [match + "\n" for match in set(matches)] + font_lines.sort() + with open(mlp_font_list, "w") as fonts_file: + fonts_file.writelines(font_lines) + args.output.mkdir(mode=0o755, parents=True, exist_ok=True) command = [ "python", - project_dir / "maintenance" / "scripts" / "get_fonts.py", + maintenance_scripts_dir / "get_fonts.py", "-o", args.output, ] @@ -65,6 +97,9 @@ def main() -> None: if args.langs: command.append("-l") command.extend(args.langs) + if args.scripts: + command.append("-s") + command.extend(args.scripts) if args.verbose: command.append("-v") subprocess.run(command, shell=(platform.system() == "Windows"), check=True, text=True)