Merge remote-tracking branch 'la-vache/main' into modifier-capital-s

unicode-org · Jun 6, 2024 · f0043e2 · f0043e2
2 parents 538d9bb + 184d7e5
commit f0043e2
Show file tree

Hide file tree

Showing 26 changed files with 602 additions and 240 deletions.
diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml
@@ -275,6 +275,22 @@ jobs:
         with:
           repository: unicode-org/unicodetools
           path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
         run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
@@ -316,6 +332,10 @@ jobs:
       - name: Run command - UCA - collation validity log
         run: |
           cd unicodetools/mine/src
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then set +e
+          fi
           # invoke main() in class ...UCA.Main
           mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
           # check for output file
@@ -333,6 +353,22 @@ jobs:
         with:
           repository: unicode-org/unicodetools
           path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
         run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
@@ -372,6 +408,16 @@ jobs:
       - name: Run invariant tests
         run: |
           cd unicodetools/mine/src
-          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then ERROR="::notice"
+          else ERROR="::error"
+          fi
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS 2>&1 | sed "s/^::error/$ERROR/"
+          STATUS=${PIPESTATUS[0]}
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then exit 0
+          else exit $STATUS
+          fi
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/docs/build.md b/docs/build.md
@@ -550,7 +550,7 @@ We no longer post files to FTP folders, nor publish individual files without con
     *   org.unicode.text.UCD
         *   TestUnicodeInvariants.java
 1.  Run>Run As... Java Application\
-    Will create the following file of results:
+    Will create the following files of results:
     ```
     {Generated}/UnicodeTestResults.txt
     ```
@@ -623,6 +623,8 @@ We no longer post files to FTP folders, nor publish individual files without con
         and what are likely remedies
         (changing properties, adding to an exceptions list, changing the test case).
         Improve these comments as needed.
+1.  Additional tests for UTS #39 data are found in [unicodetools/src/main/resources/org/unicode/text/UCD/SecurityInvariantTest.txt](https://github.com/unicode-org/unicodetools/blob/main/unicodetools/src/main/resources/org/unicode/text/UCD/SecurityInvariantTest.txt).
+    1.  These are reported in `{Generated}/UnicodeTestResults-security.txt` when running `TestTestUnicodeInvariants`.
 
 ### Options
 

diff --git a/docs/pipeline.md b/docs/pipeline.md
@@ -49,7 +49,7 @@ Indic scripts only:
 - [ ] Commit
 
 ---
-- [ ] PropsList.txt — Add Other_Alphabetic, Diacritic, and Extender to satisfy invariants, or to taste
+- [ ] PropsList.txt — Add Other_Alphabetic, Other_Lowercase, Diacritic, and Extender to satisfy invariants, or to taste
 - [ ] Commit
 
 ---
@@ -67,8 +67,13 @@ PR preparation:
 - [ ] PR button — Set to DRAFT pull request
   - unless approved for the upcoming version
 - [ ] PR button — Press
-  - The **Check UCA data** CI check might fail; many character additions need separate handling there, 
-    but that is out of scope for the PAG work of preparing `data-for-new`. This will get resolved later.
+  - The **Check UCA data** and **Check security data invariants** CI checks are
+    suppressed; many character additions need separate handling there,
+    but that is out of scope for the PAG work of preparing `data-for-new`,
+    so reporting those failures could distract from real issues
+    in the UCD invariants.
+    UCA and security data issues are addressed later in the process,
+    before the start of β review.
 
 ## Scripts
 

diff --git a/docs/security.md b/docs/security.md
@@ -9,8 +9,8 @@ machine-generated, then tweaked. They have names like
 source/confusables-winFonts.txt. The main file is confusables-source.txt.
 
 ***There is fairly complex processing for the confusables, so carefully diff the
-results. Sometimes you may get an unexpected union of two equivalence sets. Look
-at Testing below for help.***
+results. Sometimes you may get an unexpected union of two equivalence sets.
+Look at Testing below for help.***
 
 Look at the following spreadsheets / bugs to see if there are any additional
 suggestions.
@@ -19,17 +19,38 @@ suggestions.
     Suggestions](https://docs.google.com/spreadsheet/ccc?key=0ArRWBHdd5mx-dHRXelRVbXRYSVp2QTNDdTBlV1I5X1E&usp=drive_web#gid=0)**
 *   **[Identifier Restriction
     Suggestions](https://docs.google.com/spreadsheet/ccc?key=0ArRWBHdd5mx-dEJJWkdzZzk4cDRYbEVLTmhraGN0Q3c&usp=drive_web#gid=0)**
-*   *[Unicode
-    Bugs](http://www.unicode.org/edcom/bugtrack/query?status=accepted&status=assigned&status=new&status=reopened&group=component&order=priority&col=id&col=summary&col=status&col=type&col=priority&col=milestone&col=component&owner=mark&report=10)
-    (under TR #36/39)*\
-    :construction: **TODO**: That Trac instance is gone.
-    Markus thinks we decided that there was nothing useful in it,
-    and deleted it without saving data. Check with Mark.
+*   *[Sample PRs](https://github.com/unicode-org/unicodetools/pull/841)
 
 If so, assess and add to unicodetools/data/security/{version}/data/source/confusables-source.txt — *if needed.*
-
 Then in the spreadsheets, move the "new stuff" line to the end.
 
+### File Format
+There is a brief description of the file format at the top. 
+Each line represents a mapping from a code point or set of code points to a sequence of one or more code points.
+
+For example:
+```
+0021 ;  01C3    # ( ! → ǃ) EXCLAMATION MARK → LATIN LETTER RETROFLEX CLICK
+```
+
+The ordering of characters doesn't matter. 
+So it doesn't matter whether you have the above line, or
+```
+01C3 ; 0021    # ( ǃ → !) LATIN LETTER RETROFLEX CLICK → EXCLAMATION MARK
+```
+It also doesn't matter if you have identical lines; the second one will be a NOOP.
+
+The mappings are used to generate equivalence classes.
+From each equivalence class, one representative member will be chosen,
+and in the resulting data file, all the other characters will map to that representative.
+Because of transitivity, the equivalence class will tend to be somewhat looser than expected.
+
+We've discussed possible future enhancements:
+- Have a second, narrower mapping that is more exact.
+- Allow for mappings from sequences to sequences (instead of just code points to sequences).
+- Provide for context, perhaps like the Transform rules. 
+  Eg [x { a } y → A](https://util.unicode.org/UnicodeJsps/list-unicodeset.jsp?a=%5B%3Aarabic_type%3A%5D&g=&i=)
+
 ## Before generating
 
 First, in CLDR, update the script metadata:
@@ -51,13 +72,10 @@ Run GenerateConfusables -c -b to generate the files. They will appear in two pla
 *   reformatted source, log
     *   $UNICODETOOLS_DIR/data/security/11.0.0/* *including log.txt*
 
-**Run TestSecurity to verify that the confusable mappings are idempotent!**
+The TestSecurity.java test is part of the unit test suite, run by a github CI.
+It verifies that the confusable mappings are idempotent.
 
-With the same VM arguments as the generator.
-Starting in 2021q3, TestSecurity needs to be run as a JUnit test.
-It is also now part of the unit test suite and run on GitHub CI.
-
-Copy the following from the output directory to the top level of the revision directory:
+Copy the following from the output directory to the top level of the revision directory, and check in.
 
 *   confusables.txt
 *   confusablesSummary.txt
@@ -66,6 +84,12 @@ Copy the following from the output directory to the top level of the revision di
 *   ReadMe.txt
 *   xidmodifications.txt
 
+### Review
+
+Review the mappings to make sure that there are no surprises.
+The biggest issue is if two equivalence classes are mistakenly joined. 
+For example, if you map b to d, then that will join the equivalence class for b with that of d.
+
 ### IdentifierStatus.txt & IdentifierType.txt
 
 Markus 2020-feb-07 for Unicode 13.0:

diff --git a/unicodetools/data/emoji/dev/emoji-test.txt b/unicodetools/data/emoji/dev/emoji-test.txt
@@ -1,5 +1,5 @@
 # emoji-test.txt
-# Date: 2024-05-01, 21:25:24 GMT
+# Date: 2024-06-04, 16:46:01 GMT
 # © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
@@ -1751,12 +1751,12 @@
 1F936 1F3FD                                            ; fully-qualified     # 🤶🏽 E3.0 Mrs. Claus: medium skin tone
 1F936 1F3FE                                            ; fully-qualified     # 🤶🏾 E3.0 Mrs. Claus: medium-dark skin tone
 1F936 1F3FF                                            ; fully-qualified     # 🤶🏿 E3.0 Mrs. Claus: dark skin tone
-1F9D1 200D 1F384                                       ; fully-qualified     # 🧑‍🎄 E13.0 mx claus
-1F9D1 1F3FB 200D 1F384                                 ; fully-qualified     # 🧑🏻‍🎄 E13.0 mx claus: light skin tone
-1F9D1 1F3FC 200D 1F384                                 ; fully-qualified     # 🧑🏼‍🎄 E13.0 mx claus: medium-light skin tone
-1F9D1 1F3FD 200D 1F384                                 ; fully-qualified     # 🧑🏽‍🎄 E13.0 mx claus: medium skin tone
-1F9D1 1F3FE 200D 1F384                                 ; fully-qualified     # 🧑🏾‍🎄 E13.0 mx claus: medium-dark skin tone
-1F9D1 1F3FF 200D 1F384                                 ; fully-qualified     # 🧑🏿‍🎄 E13.0 mx claus: dark skin tone
+1F9D1 200D 1F384                                       ; fully-qualified     # 🧑‍🎄 E13.0 Mx claus
+1F9D1 1F3FB 200D 1F384                                 ; fully-qualified     # 🧑🏻‍🎄 E13.0 Mx claus: light skin tone
+1F9D1 1F3FC 200D 1F384                                 ; fully-qualified     # 🧑🏼‍🎄 E13.0 Mx claus: medium-light skin tone
+1F9D1 1F3FD 200D 1F384                                 ; fully-qualified     # 🧑🏽‍🎄 E13.0 Mx claus: medium skin tone
+1F9D1 1F3FE 200D 1F384                                 ; fully-qualified     # 🧑🏾‍🎄 E13.0 Mx claus: medium-dark skin tone
+1F9D1 1F3FF 200D 1F384                                 ; fully-qualified     # 🧑🏿‍🎄 E13.0 Mx claus: dark skin tone
 1F9B8                                                  ; fully-qualified     # 🦸 E11.0 superhero
 1F9B8 1F3FB                                            ; fully-qualified     # 🦸🏻 E11.0 superhero: light skin tone
 1F9B8 1F3FC                                            ; fully-qualified     # 🦸🏼 E11.0 superhero: medium-light skin tone
@@ -3721,6 +3721,11 @@
 1F41A                                                  ; fully-qualified     # 🐚 E0.6 spiral shell
 1FAB8                                                  ; fully-qualified     # 🪸 E14.0 coral
 1FABC                                                  ; fully-qualified     # 🪼 E15.0 jellyfish
+1F980                                                  ; fully-qualified     # 🦀 E1.0 crab
+1F99E                                                  ; fully-qualified     # 🦞 E11.0 lobster
+1F990                                                  ; fully-qualified     # 🦐 E3.0 shrimp
+1F991                                                  ; fully-qualified     # 🦑 E3.0 squid
+1F9AA                                                  ; fully-qualified     # 🦪 E12.0 oyster
 
 # subgroup: animal-bug
 1F40C                                                  ; fully-qualified     # 🐌 E0.6 snail
@@ -3777,8 +3782,8 @@
 1F344                                                  ; fully-qualified     # 🍄 E0.6 mushroom
 1FABE                                                  ; fully-qualified     # 🪾 E16.0 leafless tree
 
-# Animals & Nature subtotal:		161
-# Animals & Nature subtotal:		161	w/o modifiers
+# Animals & Nature subtotal:		166
+# Animals & Nature subtotal:		166	w/o modifiers
 
 # group: Food & Drink
 
@@ -3881,13 +3886,6 @@
 1F960                                                  ; fully-qualified     # 🥠 E5.0 fortune cookie
 1F961                                                  ; fully-qualified     # 🥡 E5.0 takeout box
 
-# subgroup: food-marine
-1F980                                                  ; fully-qualified     # 🦀 E1.0 crab
-1F99E                                                  ; fully-qualified     # 🦞 E11.0 lobster
-1F990                                                  ; fully-qualified     # 🦐 E3.0 shrimp
-1F991                                                  ; fully-qualified     # 🦑 E3.0 squid
-1F9AA                                                  ; fully-qualified     # 🦪 E12.0 oyster
-
 # subgroup: food-sweet
 1F366                                                  ; fully-qualified     # 🍦 E0.6 soft ice cream
 1F367                                                  ; fully-qualified     # 🍧 E0.6 shaved ice
@@ -3936,8 +3934,8 @@
 1FAD9                                                  ; fully-qualified     # 🫙 E14.0 jar
 1F3FA                                                  ; fully-qualified     # 🏺 E1.0 amphora
 
-# Food & Drink subtotal:		138
-# Food & Drink subtotal:		138	w/o modifiers
+# Food & Drink subtotal:		133
+# Food & Drink subtotal:		133	w/o modifiers
 
 # group: Travel & Places