Skip to content

Commit

Permalink
CLDR-17600 BRS 7 No2 Update language names
Browse files Browse the repository at this point in the history
  • Loading branch information
@macchiati
macchiati committed May 2, 2024
1 parent dd4a0bd commit 44e1ef8
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 362 deletions.
177 changes: 80 additions & 97 deletions common/supplemental/attributeValueValidity.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,105 +7,88 @@
<metadata>
<validity>
<!-- BCP 47 contains many more language codes than we are interested in maintaining as a part of the CLDR -->
<!-- This $language list contains ONLY those languages that are in the CLDR locales' IDs (after maximizing with LikelySubtags) -->
<!-- This $language list contains ONLY those languages that TC Locale's IDs (after maximizing with LikelySubtags) -->
<variable id='$language' type='choice'>
aa ab af agq ak am an ann apc ar arn as asa ast az
ba bal bas be bem bew bez bg bgc bgn bho blo blt bm bn bo br brx bs bss byn
ca cad cch ccp ce ceb cgg cho chr cic ckb co cs csw cu cv cy
da dav de dje doi dsb dua dv dyo dz
ebu ee el en eo es et eu ewo
fa ff fi fil fo fr frr fur fy
ga gaa gd gez gl gn gsw gu guz gv
ha haw he hi hnj hr hsb hu hy
ia id ie ig ii io is it iu
ja jbo jgo jmc jv
ka kab kaj kam kcg kde kea ken kgp khq ki kk kkj kl kln km kn ko kok kpe ks ksb ksf ksh ku kw kxv ky
la lag lb lg lij lkt lld lmo ln lo lrc lt ltg lu luo luy lv
mai mas mdf mer mfe mg mgh mgo mhn mi mic mk ml mn mni moh mr ms mt mua mus my myv mzn
naq nb nd nds ne nl nmg nn nnh no nqo nr nso nus nv ny nyn
oc om or os osa
pa pap pcm pis pl prg ps pt
qu quc
raj rhg rif rm rn ro rof ru rw rwk
sa sah saq sat sbp sc scn sd sdh se seh ses sg shi shn si sid sk skr sl sma smj smn sms sn so sq sr ss ssy st su sv sw syr szl
ta te teo tg th ti tig tk tn to tok tpi tr trv trw ts tt twq tyv tzm
ug uk ur uz
vai ve vec vi vmw vo vun
wa wae wal wbp wo
xh xnr xog
yav yi yo yrl yue
za zgh zh zu
af am ar as az
be bg bgc bho bn brx bs
ca ceb cs cv cy
da de doi
el en es et eu
fa fi fil fr
ga gd gl gu
ha he hi hr hu hy
id ig is it
ja jv
ka kk km kn ko kok ks ky
lo lt lv
mai mi mk ml mn mni mr ms my
ne nl nn no
or
pa pcm pl ps pt
raj ro ru
sa sat sd si sk sl so sq sr su sv sw
ta te tg th ti tk tr tt
uk ur uz
vi
wo
xh
yo yue
zh zu
</variable>
<!-- The following are exceptional cases.
In v44 this contains a copy of the previous $language list to make review of CLDR-16789 easier;
this should be rationalized later.
-->
<variable id='$languageExceptions' type='choice'>
af agq ak am ann apc ar as asa ast az
bas be bem bez bg bgc bho bm bn bo br brx bs
ca ccp ce ceb cgg chr ckb cs cu cv cy
da dav de dje doi dsb dua dyo dz
ebu ee el en eo es et eu ewo
fa ff fi fil fo fr frr fur fy
ga gd gl gsw gu guz gv
ha haw he hi hr hsb hu hy
ia id ig ii is it
ja jgo jmc jv
ka kab kam kde kea kgp khq ki kk kkj kl kln km kn ko kok ks ksb ksf ksh ku kw ky
lag lb lg lij lkt lmo ln lo lrc lt lu luo luy lv
mai mas mer mfe mg mgh mgo mi mk ml mn mni mr ms mt mua my mzn
naq nb nd nds ne nl nmg nn nnh no nus nyn
om or os
pa pap pcm pis pl prg ps pt
qu
raj rif rm rn ro rof
und
ru rw rwk
sa sah saq sat sbp sc sd se seh ses sg shi si sk sl smn sms sn so sq sr su sv sw
ta te teo tg th ti tk to tok tr tt twq tzm
ug uk ur uz
vai vec vi vo vun
wae wo
xh xog
yav yi yo yrl yue
zgh zh zu

mul root zxx
ab ace ada ady ain ale alt an anp arn arp ars atj av awa ay
ba ban bi bin bla bug byn
cay ch chk chm cho chp chy clc co crg crj crk crl crm crr csw
dak dar dgr dv dzg
efi eka
fj fon frc
gaa gez gil gn gor gwi
hai hax hil hmn ht hup hur hz
iba ibb ikt ilo inh io iu
jbo
kac kaj kbd kcg kfo kha kj kmb kpe kr krc krl kru kum kv kwk
la lad lez li lil lou loz lsm lua lun lus
mad mag mak mdf men mh mic min moe moh mos mus mwl myv
na nap new ng nia niu nog nqo nr nso nv ny
oc ojb ojc ojs ojw oka
pag pam pau pqm
rap rar rhg rup
sad sba scn sco shn slh sm snk srn ss st str suk swb syr
tce tem tet tgx tht tig tlh tli tn tpi trv ts ttm tum tvl ty tyv
udm umb
ve
wa wal war wuu
xal
ybb
zun zza
<!-- The following are non-TC locales that have reached at least Basic, plus specials mul root zxx und -->
<variable id='$languageNonTcGeqBasic' type='choice'>
ast
blo br
chr csw
dsb
eo
ff fo fy
hsb
ia ie
kea kgp ku kxv
lb lij lmo
mt
nds nqo
oc
prg
qu
rm
sah sc syr szl
to
ug
vec vmw
xnr
yrl
za
mul root zxx und
</variable>
<!-- The following are not in modern coverage: -->
<variable id='$oldLanguages' type='choice'>
aa
crs
gan
hak hsn
nan
quc
sma smj sms ssy
<!-- The following have not yet made it to Basic -->
<variable id='$languageNonTcLtBasic' type='choice'>
aa ab agq ak an ann apc arn asa
ba bal bas bem bew bez bgn blt bm bo bss byn
cad cch ccp ce cgg cho cic ckb co cu
dav dje dua dv dyo dz
ebu ee ewo
ff frr fur
gaa gez gn gsw guz gv
haw hnj
ii io iu
jbo jgo jmc
kab kaj kam kcg kde ken khq ki kkj kl kln kpe ksb ksf ksh kw
la lag lg lkt lld ln lrc ltg lu luo luy
mas mdf mer mfe mhn mg mgh mgo mic moh mua mus myv mzn
naq nb nd nmg nnh nr nso nus nv ny nyn om
os osa
pap pis
quc
rhg rif rn rof rw rwk
saq sbp scn sdh se seh ses sg shi shn sid skr sma smj smn sms sn ss ssy st
teo tig tn tok tpi trv trw ts twq tyv tzm
vai ve vo vun
wa wae wal wbp
xog
yav yi
zgh
</variable>
<variable id='$scriptNonUnicode' type='choice'>Afak Aran Blis Cirt Cyrs Egyd Egyh Geok Inds Jurc Kitl Kpel Latf Latg Loma Maya Moon
Nkgb Phlv Roro Sara Syre Syrj Syrn Teng Visp Wole
Expand Down Expand Up @@ -458,4 +441,4 @@
<attributeValues dtds='keyboard3' elements='version' attributes='number' type='TODO'></attributeValues>
</validity>
</metadata>
</supplementalData>
</supplementalData>
5 changes: 2 additions & 3 deletions common/supplemental/coverageLevels.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,7 @@ For terms of use, see http://www.unicode.org/copyright.html
<coverageVariable key="%language60_TD" value="(shu|dzg|kbl|mde|mua|sba)"/>
<!-- See CLDR-16673: All basic+ locales (per coverageLevels.txt) MUST have their language's name at least at level 80 (modern), except for those on exception list -->
<!-- Can use MinimizeRegex.java to "unpack" the compressed lists that are painful to edit -->
<coverageVariable key="%language80" value="(ace|ada|ady|ain|ale|alt|anp|arn|arp|ars|ast|atj|awa|ban|bho|bin|bla|brx|bug|byn|cay|ceb|chk|chm|cho|chp|chr|chy|clc|crg|crj|crk|crl|crm|crr|csw|dak|dar|dgr|doi|dsb|dzg|efi|eka|fil|fon|frc|gaa|gez|gil|gor|gwi|hai|hax|hil|hmn|hsb|hup|hur|iba|ibb|ikt|ilo|inh|jbo|kac|kaj|kbd|kcg|kea|kfo|kgp|kha|kmb|kok|kpe|krc|krl|kru|kum|kwk|lad|lez|lil|lou|loz|lsm|lua|lun|lus|mad|mag|mai|mak|mdf|men|mic|min|mni|moe|moh|mos|mul|mus|mwl|myv|nap|new|nia|niu|nog|nqo|nso|ojb|ojc|ojs|ojw|oka|pag|pam|pap|pau|pcm|pqm|rap|rar|rhg|rup|sad|sah|sat|sba|scn|sco|shn|slh|snk|srn|str|suk|swb|syr|tce|tem|tet|tgx|tht|tig|tlh|tli|tpi|trv|ttm|tum|tvl|tyv|udm|umb|wal|war|wuu|xal|ybb|yrl|yue|zun|zxx|zza|ab|af|am|an|ar|as|av|ay|az|ba|be|bg|bi|bn|br|bs|ca|ch|co|cs|cv|cy|da|de|dv|el|en|es|et|eu|fa|fi|fj|fo|fr|fy|ga|gd|gl|gn|gu|ha|he|hi|hr|ht|hu|hy|hz|ia|id|ig|io|is|it|iu|ja|jv|ka|kj|kk|km|kn|ko|kr|ks|ku|kv|ky|la|lb|li|lo|lt|lv|mh|mi|mk|ml|mn|mr|ms|mt|my|na|ne|ng|nl|nn|no|nr|nv|ny|oc|or|pa|pl|ps|pt|qu|rm|ro|ru|sa|sc|sd|si|sk|sl|sm|so|sq|sr|ss|st|su|sv|sw|ta|te|tg|th|ti|tk|tn|to|tr|ts|tt|ty|ug|uk|ur|uz|ve|vi|wa|wo|xh|yo|zh|zu)"/>
<coverageVariable key="%languagecomp" value="(gan|hak|hsn|nan)"/> <!-- not currently used, just for reference: the only valid language codes that are not in modern coverage -->
<coverageVariable key="%language80" value="(af|am|ar|as|az|be|bg|bgc|bho|bn|brx|bs|ca|ceb|cs|cv|cy|da|de|doi|el|en|es|et|eu|fa|fi|fil|fr|ga|gd|gl|gu|ha|he|hi|hr|hu|hy|id|ig|is|it|ja|jv|ka|kk|km|kn|ko|kok|ks|ky|lo|lt|lv|mai|mi|mk|ml|mn|mni|mr|ms|my|ne|nl|nn|no|or|pa|pcm|pl|ps|pt|raj|ro|ru|sa|sat|sd|si|sk|sl|so|sq|sr|su|sv|sw|ta|te|tg|th|ti|tk|tr|tt|uk|ur|uz|vi|wo|xh|yo|yue|zh|zu|ast|blo|br|chr|csw|dsb|eo|ff|fo|fy|hsb|ia|ie|kea|kgp|ku|kxv|lb|lij|lmo|mt|nds|nqo|oc|prg|qu|rm|sah|sc|syr|szl|to|ug|vec|vmw|xnr|yrl|za|mul|root|zxx|und)"/>
<coverageVariable key="%lbTypes80" value="(strict|normal|loose)"/>
<coverageVariable key="%lwTypes" value="(normal|breakall|keepall|phrase)"/>
<coverageVariable key="%m0Types80" value="(bgn|prprname|ungegn)"/>
Expand Down Expand Up @@ -1007,4 +1006,4 @@ For terms of use, see http://www.unicode.org/copyright.html
<pathMatch id="annotations1" match="annotations/annotation[@cp='%anyAttribute'][@type='%anyAttribute']"/>
<pathMatch id="annotations2" match="annotations/annotation[@cp='%anyAttribute']"/>
</coverageLevels>
</supplementalData>
</supplementalData>
37 changes: 30 additions & 7 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1348,6 +1348,8 @@ private void makeStuffSafe() {
}
}
CLDRScriptCodes = newScripts.build();
CLDRLanguageCodes = CldrUtility.protectCollection(CLDRLanguageCodes);
languageNonTcLtBasic = CldrUtility.protectCollection(languageNonTcLtBasic);
}

/**
Expand Down Expand Up @@ -2144,13 +2146,21 @@ private boolean handleMetadata(String level2, String value, XPathValue parts) {
String level3 = parts.getElement(3);
if (level3.equals("variable")) {
Map<String, String> attributes = parts.getAttributes(-1);
validityInfo.put(attributes.get("id"), Row.of(attributes.get("type"), value));
String idString = attributes.get("id");
if (("$language".equals(idString)
|| "$languageExceptions".equals(attributes.get("id")))
&& "choice".equals(attributes.get("type"))) {
String[] validCodeArray = value.trim().split("\\s+");
CLDRLanguageCodes.addAll(Arrays.asList(validCodeArray));
final String idString = attributes.get("id");
final String typeString = attributes.get("type");
validityInfo.put(idString, Row.of(typeString, value));
if ("choice".equals(typeString)) {
if ("$language".equals(idString)
|| "$languageNonTcGeqBasic".equals(idString)) {
String[] validCodeArray = value.trim().split("\\s+");
CLDRLanguageCodes.addAll(Arrays.asList(validCodeArray));
}
if ("$languageNonTcLtBasic".equals(idString)) { // not yet basic
String[] validCodeArray = value.trim().split("\\s+");
final List<String> asList = Arrays.asList(validCodeArray);
languageNonTcLtBasic.addAll(asList);
CLDRLanguageCodes.addAll(asList);
}
}
return true;
} else if (level3.equals("attributeValues")) {
Expand Down Expand Up @@ -2495,6 +2505,8 @@ public int parseIntegerOrNull(String attributeValue) {
public Map<CLDRLocale, CLDRLocale> baseToDefaultContent; // wo -> wo_Arab_SN
public Map<CLDRLocale, CLDRLocale> defaultContentToBase; // wo_Arab_SN -> wo
private Set<String> CLDRLanguageCodes = new TreeSet<>();
private Set<String> languageNonTcLtBasic = new TreeSet<>();

private Set<String> CLDRScriptCodes;

/**
Expand Down Expand Up @@ -4793,10 +4805,21 @@ public Map<String, R2<String, String>> getValidityInfo() {
return validityInfo;
}

/** TC languages or at Basic or better, or worse than Basic (grandfathered or Core). */
public Set<String> getCLDRLanguageCodes() {
return CLDRLanguageCodes;
}

/** TC languages or those at Basic or better. */
public Set<String> getLanguageTcOrBasic() {
return Sets.difference(CLDRLanguageCodes, languageNonTcLtBasic);
}

/** Non TC languages that are worse than Basic (grandfathered or Core) */
public Set<String> getLanguageNonTcLtBasic() {
return languageNonTcLtBasic;
}

public boolean isCLDRLanguageCode(String code) {
return CLDRLanguageCodes.contains(code);
}
Expand Down
Loading

0 comments on commit 44e1ef8

Please sign in to comment.