Skip to content

Commit

Permalink
CLDR-11155 Pages too big; move some paths to new People2 and Symbols3 (…
Browse files Browse the repository at this point in the history 
…#3506)

-New People2 (People & Body 2) starting with subgroup person-fantasy

-New Symbols3 starting with subgroup other-symbol

-New method Emoji.getPageId

-No more reference to obsolete Smileys & People

-Remove some dead code
  • Loading branch information
@btangmu
btangmu authored Feb 15, 2024
1 parent 54b4262 commit 0745094
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 55 deletions.
72 changes: 39 additions & 33 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/Emoji.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.Map.Entry;
import java.util.Set;
import org.unicode.cldr.draft.FileUtilities;
import org.unicode.cldr.util.PathHeader.PageId;

public class Emoji {
public static final String EMOJI_VARIANT = "\uFE0F";
Expand Down Expand Up @@ -74,34 +75,28 @@ public class Emoji {

static {
/*
# group: Smileys & People
# subgroup: face-positive
1F600 ; fully-qualified # 😀 grinning face
*/
* Example from emoji-test.txt:
* # group: Smileys & Emotion
* # subgroup: face-smiling
* 1F600 ; fully-qualified # 😀 grinning face
*/
Splitter semi = Splitter.on(CharMatcher.anyOf(";#")).trimResults();
String majorCategory = null;
String minorCategory = null;
long majorOrder = 0;
long minorOrder = 0;
// Multimap<Pair<Integer,Integer>,String> majorPlusMinorToEmoji = TreeMultimap.create();
for (String line : FileUtilities.in(Emoji.class, "data/emoji/emoji-test.txt")) {
if (line.startsWith("#")) {
line = line.substring(1).trim();
if (line.startsWith("group:")) {
majorCategory = line.substring("group:".length()).trim();
Long oldMajorOrder = majorToOrder.get(majorCategory);
if (oldMajorOrder == null) {
majorToOrder.put(majorCategory, majorOrder = majorToOrder.size());
} else {
majorOrder = oldMajorOrder;
majorToOrder.put(majorCategory, (long) majorToOrder.size());
}
} else if (line.startsWith("subgroup:")) {
minorCategory = line.substring("subgroup:".length()).trim();
Long oldMinorOrder = minorToOrder.get(minorCategory);
if (oldMinorOrder == null) {
minorToOrder.put(minorCategory, minorOrder = minorToOrder.size());
} else {
minorOrder = oldMinorOrder;
minorToOrder.put(minorCategory, (long) minorToOrder.size());
}
}
continue;
Expand All @@ -113,17 +108,7 @@ public class Emoji {
Iterator<String> it = semi.split(line).iterator();

String emojiHex = it.next();
if (emojiHex.contains("1F48F")) {
int debug = 0;
}

String original = Utility.fromHex(emojiHex, 4, " ");
if (original.contains("💏")) {
if (false) {
System.out.println(original + "\t" + Utility.hex(original));
}
}

String type = it.next();
if (type.startsWith("fully-qualified")) {
allRgi.add(original);
Expand Down Expand Up @@ -151,13 +136,8 @@ public class Emoji {
if (!emojiToOrder.containsKey(minimal)) {
putUnique(emojiToOrder, minimal, emojiToOrder.size() * 100L);
}
//
// majorPlusMinorToEmoji.put(Pair.of(majorOrder, minorOrder), minimal);

boolean singleton = CharSequences.getSingleCodePoint(minimal) != Integer.MAX_VALUE;
// if (!emojiToOrder.containsKey(minimal)) {
// emojiToOrder.put(minimal, emojiToOrder.size());
// }

// skip constructed values
if (minimal.contains(COMBINING_ENCLOSING_KEYCAP)
Expand All @@ -177,11 +157,6 @@ public class Emoji {
nonConstructed.add(minimal);
}
}
// for (Entry<Pair<Integer,Integer>, String> entry : majorPlusMinorToEmoji.entries())
// {
// String minimal = entry.getValue();
// emojiToOrder.put(minimal, emojiToOrder.size());
// }
emojiToMajorCategory.freeze();
emojiToMinorCategory.freeze();
nonConstructed.add(MODIFIERS); // needed for names
Expand Down Expand Up @@ -381,4 +356,35 @@ private static ImmutableSet<String> buildPaths(String suffix) {
}
return builder.build();
}

/**
* Return the PageId for the given emoji, making adjustments for pages that are united in
* emoji-test.txt but divided in Survey Tool, such as Symbols, Symbols2, and Symbols3
*
* @param emoji the emoji as a string
* @return the adjusted PageId
*/
public static PageId getPageId(String emoji) {
final String major = getMajorCategory(emoji);
final String minor = getMinorCategory(emoji);
final PageId pageId = PageId.forString(major);
switch (pageId) {
case People:
if (minorToOrder.get(minor) < minorToOrder.get("person-fantasy")) {
return PageId.People;
} else {
return PageId.People2;
}
case Symbols:
if (minorToOrder.get(minor) < minorToOrder.get("transport-sign")) {
return PageId.Symbols;
} else if (minorToOrder.get(minor) < minorToOrder.get("other-symbol")) {
return PageId.Symbols2;
} else {
return PageId.Symbols3;
}
default:
return pageId;
}
}
}
25 changes: 3 additions & 22 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/PathHeader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,12 +309,14 @@ public enum PageId {
// Symbols, Flags]
Smileys(SectionId.Characters, "Smileys & Emotion"),
People(SectionId.Characters, "People & Body"),
People2(SectionId.Characters, "People & Body 2"),
Animals_Nature(SectionId.Characters, "Animals & Nature"),
Food_Drink(SectionId.Characters, "Food & Drink"),
Travel_Places(SectionId.Characters, "Travel & Places"),
Activities(SectionId.Characters),
Objects(SectionId.Characters),
Symbols2(SectionId.Characters),
Symbols3(SectionId.Characters),
Flags(SectionId.Characters),
Component(SectionId.Characters),
Typography(SectionId.Characters),
Expand Down Expand Up @@ -2197,28 +2199,7 @@ public String transform(String source) {
new Transform<String, String>() {
@Override
public String transform(String source) {
String major = Emoji.getMajorCategory(source);
// check that result is reasonable by running through PageId.
switch (major) {
default:
PageId pageId2 = PageId.forString(major);
if (pageId2.getSectionId() != SectionId.Characters) {
if (pageId2 == PageId.Symbols) {
pageId2 = PageId.Symbols2;
}
}
return pageId2.toString();
case "Smileys & People":
String minorCat = Emoji.getMinorCategory(source);
if (minorCat.equals("skin-tone")
|| minorCat.equals("hair-style")) {
return PageId.Component.toString();
} else if (!minorCat.contains("face")) {
return PageId.People.toString();
} else {
return PageId.Smileys.toString();
}
}
return Emoji.getPageId(source).toString();
}
});
functionMap.put(
Expand Down

0 comments on commit 0745094

Please sign in to comment.