Merge remote-tracking branch 'la-vache/main' into 12-ipa-diacritics-1…

…1-above-and-1-below

.github/workflows/cli-build-instructions.yml

@@ -88,7 +88,7 @@ jobs:
       # change anything, which makes little sense; but that is the job of the
       # other job.
       - name: Run invariant tests
-        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -170,7 +170,7 @@ jobs:
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
-          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -Dtest=!TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -275,6 +275,22 @@ jobs:
         with:
           repository: unicode-org/unicodetools
           path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
         run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
@@ -316,9 +332,92 @@ jobs:
       - name: Run command - UCA - collation validity log
         run: |
           cd unicodetools/mine/src
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then set +e
+          fi
           # invoke main() in class ...UCA.Main
           mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
           # check for output file
           compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Out-of-source build.
+  security:
+    name: Check security data invariants
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Unicode Tools
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/unicodetools
+          path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
+      - name: Get the CLDR_REF from pom.xml
+        id: cldr_ref
+        run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
+      - name: Verify CLDR checkout ref
+        run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
+      - name: Cache CLDR repository
+        uses: actions/cache@v3
+        with:
+          path: cldr/mine/src
+          key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
+          restore-keys: |
+            cldr
+      - name: Check out CLDR
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/cldr
+          path: cldr/mine/src
+          ref: main
+          fetch-depth: 0
+      - name: Switch CLDR to CLDR_REF
+        run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      - name: Run invariant tests
+        run: |
+          cd unicodetools/mine/src
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then ERROR="::notice"
+          else ERROR="::error"
+          fi
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS 2>&1 | sed "s/^::error/$ERROR/"
+          STATUS=${PIPESTATUS[0]}
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then exit 0
+          else exit $STATUS
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

UnicodeJsps/pom.xml

@@ -29,7 +29,7 @@
 		<!-- icu -->
 		<dependency>
 			<groupId>com.ibm.icu</groupId>
-			<artifactId>icu4j-for-cldr</artifactId>
+			<artifactId>icu4j</artifactId>
 		</dependency>
 
 		<dependency>

UnicodeJsps/src/main/java/org/unicode/cldr/tool/TablePrinter.java

@@ -451,6 +451,7 @@ private int findIdentical(Comparable[][] sortedFlat, int rowIndex, int colIndex)
         }
         return sortedFlat.length - rowIndex;
     }
+
     // to-do: prevent overlap when it would cause information to be lost.
     private BitSet breaksSpans = new BitSet();
 

UnicodeJsps/src/main/java/org/unicode/jsp/Confusables.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.CollectionUtilities;
 import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.text.Normalizer;
@@ -41,7 +40,7 @@ public static UnicodeMap<String> getMap() {
         UnicodeMap<String> result = new UnicodeMap<String>();
         for (String s : equivalents) {
             Set<String> others = new TreeSet<String>(equivalents.getEquivalences(s));
-            String list = "\u2051" + CollectionUtilities.join(others, "\u2051") + "\u2051";
+            String list = "\u2051" + String.join("\u2051", others) + "\u2051";
             for (String other : others) {
                 result.put(other, list);
             }

UnicodeJsps/src/main/java/org/unicode/jsp/Globe.java

@@ -495,6 +495,7 @@ private static void loadSourceMap(String sname) {
         }
         changeImage(frame);
     }
+
     /**
      * @param frame
      */
@@ -704,12 +705,14 @@ public Shape getClip() {
             }
             return clip;
         }
+
         /**
          * @return
          */
         public boolean usesOrigin() {
             return false;
         }
+
         // must set before use
         Transform set(double srcW, double srcH, double dstW, double dstH) {
             this.srcW = srcW;
@@ -724,6 +727,7 @@ Transform set(double srcW, double srcH, double dstW, double dstH) {
             clip = null;
             return this;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.
         // cos(-angle) =  cos(angle)
@@ -750,6 +754,7 @@ public final boolean transform(double x, double y, DPoint retcoord) {
             retcoord.y = lat_dstH.map(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= dstW && retcoord.y >= 0 && retcoord.y <= dstH;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.  Since inverting
         // the transform is also the same as negating the angle, itransform
@@ -774,18 +779,21 @@ public final boolean itransform(double x, double y, DPoint retcoord) {
             retcoord.y = srcH_lat.back(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= srcW && retcoord.y >= 0 && retcoord.y <= srcH;
         }
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _transform(DPoint retcoord);
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _itransform(DPoint retcoord);
 
         protected abstract Shape _getClip();
+
         /**
          * @param style
          * @return
@@ -866,6 +874,7 @@ public void _transform(DPoint retcoord) {
         public void _itransform(DPoint retcoord) {
             // nothing
         }
+
         /* (non-Javadoc)
          * @see Globe.Transform#_getClip()
          */
@@ -979,6 +988,7 @@ public static class TransformEqualAreaEllipse extends Transform {
                 debugTemp = false;
             }
         }
+
         // Area of a spherical cap is 2 pi r^2 (1-sin(lat))
         // Area of a circular segment is r^2 ( acos(p) - p sqrt(1-p^2)), where p = dist to chord/r
         // Thus we get the itransform easily:
@@ -1016,6 +1026,7 @@ public void _itransform(DPoint retcoord) {
         protected Shape _getClip() {
             return new Ellipse2D.Double(0, 0, dstW, dstH);
         }
+
         /**
          * @param in -1..1
          * @return value in 0..PI
@@ -1573,6 +1584,7 @@ public void imageComplete(int status) {
             consumer.imageComplete(status);
         }
     }
+
     /*
      public static double convertDegreesToDecimal(double degrees, double minutes, double seconds, boolean NorthOrEast) {
          double result = (degrees + minutes / 60 + seconds / 3600);
@@ -1932,6 +1944,7 @@ void fixContents() {
                 lineContents[c.yStart].add(c);
             }
         }
+
         /**
          * @param c
          * @return
@@ -2066,6 +2079,7 @@ private static void drawDegrees(
     }
 
     private static DPoint drawLineP1 = new DPoint();
+
     /*
 
      private static void drawLine(Graphics2D graphics2D, Transform trans, double x1, double y1, double x2, double y2) {
@@ -2111,6 +2125,7 @@ private static void drawLine2(Graphics2D graphics2D, Transform trans, double x,
 
     abstract static class TTransform {
         double x, y;
+
         // t is 0..1
         abstract void transform(double t);
     }
@@ -2174,11 +2189,13 @@ static class LineDrawer {
         transient double startX, startY, startT;
         // transient double endX, endY, endT;
         TTransform ttransform;
+
         // int segments = 0;
         LineDrawer(Graphics2D graphics2D, TTransform ttransform) {
             this.graphics2D = graphics2D;
             this.ttransform = ttransform;
         }
+
         // t is 0..1
         void draw(double startT, double endT) {
             this.startT = startT;

UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.CollectionUtilities;
 import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
@@ -46,6 +45,7 @@ public enum ScriptSpecials {
     public static final int EXTRA_COUNT = 16; // should be enough, hard working as UTC is!
 
     public static final Map<String, Integer> extraScripts = new ConcurrentHashMap<>(EXTRA_COUNT);
+
     /** Extended scripts; note that they do not have stable numbers, and should not be persisted. */
     public static final int
             // HANT = UScript.CODE_LIMIT,
@@ -391,7 +391,7 @@ public static String getNames(
             for (int i = value.nextSetBit(0); i >= 0; i = value.nextSetBit(i + 1)) {
                 names.add(ScriptTester.getScriptName(i, choice));
             }
-            return CollectionUtilities.join(names, separator).toString();
+            return String.join(separator, names);
         }
     }
 
@@ -552,6 +552,7 @@ public ScriptTester get() {
             }
             return new ScriptTester(character_scripts);
         }
+
         /**
          * Add list of scripts that are acceptable in combination together.
          *

UnicodeJsps/src/main/java/org/unicode/jsp/Typology.java

@@ -34,6 +34,7 @@ public class Typology {
     public static Relation<String, String> labelToPaths =
             new Relation(new TreeMap(), TreeSet.class);
     public static Map<String, Map<String, UnicodeSet>> label_parent_uset = new TreeMap();
+
     // public static Relation<String, String> pathToList = new Relation(new TreeMap(),
     // TreeSet.class);
 

UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeUtilities.java

@@ -1348,6 +1348,7 @@ public static void getDifferences(
     };
 
     static Comparator<String> col = UnicodeSetUtilities.MAIN_COLLATOR;
+
     //    Collator.getInstance(ULocale.ROOT);
     //    static {
     //        ((RuleBasedCollator) col).setNumericCollation(true);
@@ -1958,6 +1959,7 @@ static Subheader getSubheader() {
     static String removals =
             new UnicodeSet("[\u1806[:di:]-[:cn:]]").complement().complement().toPattern(false);
     static Matcher rem = Pattern.compile(removals).matcher("");
+
     // TODO use UnicodeRegex
 
     //  static IdnaLabelTester getIdna2008Tester() {

UnicodeJsps/src/test/java/org/unicode/jsptest/TestBuilder.java

@@ -46,6 +46,7 @@ public void TestCollection() {
         Set<Foo> w = Builder.with(EnumSet.noneOf(Foo.class)).add(Foo.foo1).freeze();
         assertTrue("w", w.size() == 1 && w.contains(Foo.foo1));
     }
+
     /**
      *
      *

UnicodeJsps/src/test/java/org/unicode/jsptest/TestJsp.java

@@ -666,6 +666,7 @@ public void assertNoMatch(String message, String pattern, Object actual) {
                 false,
                 actual);
     }
+
     //         return handleAssert(expected == actual, message, stringFor(expected),
     // stringFor(actual), "==", false);