Merge remote-tracking branch 'la-vache/main' into arabic-letter-noon-…

…with-ring-above

.github/workflows/build-jsp.yml

@@ -52,22 +52,15 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
     # TODO: move this DOWN after JSPs once it works
-    - name: Generate Unicode data
-      # TODO: hard coded version
-      # TODO: symlink of security here?
+    - name: Regenerate the property cache files
       run: >
-        mkdir -pv $(pwd)/output/Generated/ &&
-        mvn -s .github/workflows/mvn-settings.xml -B compile exec:java -DskipTests=true
-        -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version ${CURRENT_UVERSION} build MakeUnicodeFiles"
-        -am -pl unicodetools -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr
-        -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=${CURRENT_UVERSION}
-        -DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - name: Run unicodetools tests
-      run: >
-        mvn -s .github/workflows/mvn-settings.xml -B test -am -pl unicodetools
-        -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr -T 1C -Dparallel=classes -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=${CURRENT_UVERSION} -DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
+        mkdir -pv $(pwd)/Generated/ &&
+        mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.jsp.RebuildPropertyCache"
+        -am -pl unicodetools
+        -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr
+        -DUNICODETOOLS_REPO_DIR=$(pwd)
+        -DUNICODETOOLS_GEN_DIR=$(pwd)/Generated &&
+        tar -cpz -f UnicodeJsps/target/generated.tgz ./Generated
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Package JSPs

.github/workflows/cli-build-instructions.yml

@@ -88,7 +88,7 @@ jobs:
       # change anything, which makes little sense; but that is the job of the
       # other job.
       - name: Run invariant tests
-        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -170,7 +170,7 @@ jobs:
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
-          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -Dtest=!TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -322,3 +322,56 @@ jobs:
           compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Out-of-source build.
+  security:
+    name: Check security data invariants
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Unicode Tools
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/unicodetools
+          path: unicodetools/mine/src
+      - name: Get the CLDR_REF from pom.xml
+        id: cldr_ref
+        run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
+      - name: Verify CLDR checkout ref
+        run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
+      - name: Cache CLDR repository
+        uses: actions/cache@v3
+        with:
+          path: cldr/mine/src
+          key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
+          restore-keys: |
+            cldr
+      - name: Check out CLDR
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/cldr
+          path: cldr/mine/src
+          ref: main
+          fetch-depth: 0
+      - name: Switch CLDR to CLDR_REF
+        run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      - name: Run invariant tests
+        run: |
+          cd unicodetools/mine/src
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

LICENSE

@@ -2,7 +2,7 @@ UNICODE LICENSE V3
 
 COPYRIGHT AND PERMISSION NOTICE
 
-Copyright © 2001-2023 Unicode, Inc.
+Copyright © 2001-2024 Unicode, Inc.
 
 NOTICE TO USER: Carefully read the following legal agreement. BY
 DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
@@ -37,3 +37,5 @@ Except as contained in this notice, the name of a copyright holder shall
 not be used in advertising or otherwise to promote the sale, use or other
 dealings in these Data Files or Software without prior written
 authorization of the copyright holder.
+
+SPDX-License-Identifier: Unicode-3.0

README.md

@@ -23,7 +23,7 @@ The tools maintainers use GH issues for issues with the code in this repo.
 
 ### Copyright & Licenses
 
-Copyright © 2001-2023 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries.
+Copyright © 2001-2024 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries.
 
 The project is released under [LICENSE](./LICENSE).
 

UnicodeJsps/Dockerfile

@@ -9,6 +9,7 @@ RUN cd source && gcc -I ../include/ -static -Os -o3 -o bidiref1 bidiref1.c bruti
 RUN ls -lh /build/source/bidiref1 && (/build/source/bidiref1 || true)
 # copy and unpack to /tmp/data
 ADD ./target/cldr-unicodetools.tgz /build/data/
+ADD ./target/generated.tgz /build/data/
 # move this into place (including unicodetools/unicodetools)
 RUN rm -rf /build/data/cldr/.git  # unneeded
 FROM jetty:9-jre11-alpine-eclipse-temurin AS run
@@ -24,9 +25,10 @@ ENV BIDIREFHOME /usr/local/share
 # copy the bidiref1 bin
 ENV BIDIREF1 /usr/local/bin/bidiref1
 COPY --from=cbuild /build/source/bidiref1 /usr/local/bin/
-RUN mkdir -p /var/lib/jetty/data/unicodetools
+RUN mkdir -p /var/lib/jetty/data/unicodetools/Generated
 COPY --from=cbuild /build/data/cldr /var/lib/jetty/data/cldr
 COPY --from=cbuild /build/data/unicodetools/ /var/lib/jetty/data/unicodetools/unicodetools/
-ENV JAVA_OPTIONS -DCLDR_DIR=/var/lib/jetty/data/cldr -DUNICODETOOLS_REPO_DIR=/var/lib/jetty/data/unicodetools
+COPY --from=cbuild /build/data/Generated /var/lib/jetty/data/unicodetools/Generated
+ENV JAVA_OPTIONS -DCLDR_DIR=/var/lib/jetty/data/cldr -DUNICODETOOLS_REPO_DIR=/var/lib/jetty/data/unicodetools -DUNICODETOOLS_GEN_DIR=/var/lib/jetty/data/unicodetools/Generated -Xmx4g
 # This is the default PORT. Override by setting PORT.
 EXPOSE 8080

UnicodeJsps/pom.xml

@@ -29,7 +29,7 @@
 		<!-- icu -->
 		<dependency>
 			<groupId>com.ibm.icu</groupId>
-			<artifactId>icu4j-for-cldr</artifactId>
+			<artifactId>icu4j</artifactId>
 		</dependency>
 
 		<dependency>

UnicodeJsps/src/main/java/org/unicode/cldr/tool/TablePrinter.java

@@ -451,6 +451,7 @@ private int findIdentical(Comparable[][] sortedFlat, int rowIndex, int colIndex)
         }
         return sortedFlat.length - rowIndex;
     }
+
     // to-do: prevent overlap when it would cause information to be lost.
     private BitSet breaksSpans = new BitSet();
 

UnicodeJsps/src/main/java/org/unicode/jsp/Confusables.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.CollectionUtilities;
 import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.text.Normalizer;
@@ -41,7 +40,7 @@ public static UnicodeMap<String> getMap() {
         UnicodeMap<String> result = new UnicodeMap<String>();
         for (String s : equivalents) {
             Set<String> others = new TreeSet<String>(equivalents.getEquivalences(s));
-            String list = "\u2051" + CollectionUtilities.join(others, "\u2051") + "\u2051";
+            String list = "\u2051" + String.join("\u2051", others) + "\u2051";
             for (String other : others) {
                 result.put(other, list);
             }

UnicodeJsps/src/main/java/org/unicode/jsp/Globe.java

@@ -495,6 +495,7 @@ private static void loadSourceMap(String sname) {
         }
         changeImage(frame);
     }
+
     /**
      * @param frame
      */
@@ -704,12 +705,14 @@ public Shape getClip() {
             }
             return clip;
         }
+
         /**
          * @return
          */
         public boolean usesOrigin() {
             return false;
         }
+
         // must set before use
         Transform set(double srcW, double srcH, double dstW, double dstH) {
             this.srcW = srcW;
@@ -724,6 +727,7 @@ Transform set(double srcW, double srcH, double dstW, double dstH) {
             clip = null;
             return this;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.
         // cos(-angle) =  cos(angle)
@@ -750,6 +754,7 @@ public final boolean transform(double x, double y, DPoint retcoord) {
             retcoord.y = lat_dstH.map(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= dstW && retcoord.y >= 0 && retcoord.y <= dstH;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.  Since inverting
         // the transform is also the same as negating the angle, itransform
@@ -774,18 +779,21 @@ public final boolean itransform(double x, double y, DPoint retcoord) {
             retcoord.y = srcH_lat.back(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= srcW && retcoord.y >= 0 && retcoord.y <= srcH;
         }
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _transform(DPoint retcoord);
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _itransform(DPoint retcoord);
 
         protected abstract Shape _getClip();
+
         /**
          * @param style
          * @return
@@ -866,6 +874,7 @@ public void _transform(DPoint retcoord) {
         public void _itransform(DPoint retcoord) {
             // nothing
         }
+
         /* (non-Javadoc)
          * @see Globe.Transform#_getClip()
          */
@@ -979,6 +988,7 @@ public static class TransformEqualAreaEllipse extends Transform {
                 debugTemp = false;
             }
         }
+
         // Area of a spherical cap is 2 pi r^2 (1-sin(lat))
         // Area of a circular segment is r^2 ( acos(p) - p sqrt(1-p^2)), where p = dist to chord/r
         // Thus we get the itransform easily:
@@ -1016,6 +1026,7 @@ public void _itransform(DPoint retcoord) {
         protected Shape _getClip() {
             return new Ellipse2D.Double(0, 0, dstW, dstH);
         }
+
         /**
          * @param in -1..1
          * @return value in 0..PI
@@ -1573,6 +1584,7 @@ public void imageComplete(int status) {
             consumer.imageComplete(status);
         }
     }
+
     /*
      public static double convertDegreesToDecimal(double degrees, double minutes, double seconds, boolean NorthOrEast) {
          double result = (degrees + minutes / 60 + seconds / 3600);
@@ -1932,6 +1944,7 @@ void fixContents() {
                 lineContents[c.yStart].add(c);
             }
         }
+
         /**
          * @param c
          * @return
@@ -2066,6 +2079,7 @@ private static void drawDegrees(
     }
 
     private static DPoint drawLineP1 = new DPoint();
+
     /*
 
      private static void drawLine(Graphics2D graphics2D, Transform trans, double x1, double y1, double x2, double y2) {
@@ -2111,6 +2125,7 @@ private static void drawLine2(Graphics2D graphics2D, Transform trans, double x,
 
     abstract static class TTransform {
         double x, y;
+
         // t is 0..1
         abstract void transform(double t);
     }
@@ -2174,11 +2189,13 @@ static class LineDrawer {
         transient double startX, startY, startT;
         // transient double endX, endY, endT;
         TTransform ttransform;
+
         // int segments = 0;
         LineDrawer(Graphics2D graphics2D, TTransform ttransform) {
             this.graphics2D = graphics2D;
             this.ttransform = ttransform;
         }
+
         // t is 0..1
         void draw(double startT, double endT) {
             this.startT = startT;

UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.CollectionUtilities;
 import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
@@ -46,6 +45,7 @@ public enum ScriptSpecials {
     public static final int EXTRA_COUNT = 16; // should be enough, hard working as UTC is!
 
     public static final Map<String, Integer> extraScripts = new ConcurrentHashMap<>(EXTRA_COUNT);
+
     /** Extended scripts; note that they do not have stable numbers, and should not be persisted. */
     public static final int
             // HANT = UScript.CODE_LIMIT,
@@ -391,7 +391,7 @@ public static String getNames(
             for (int i = value.nextSetBit(0); i >= 0; i = value.nextSetBit(i + 1)) {
                 names.add(ScriptTester.getScriptName(i, choice));
             }
-            return CollectionUtilities.join(names, separator).toString();
+            return String.join(separator, names);
         }
     }
 
@@ -552,6 +552,7 @@ public ScriptTester get() {
             }
             return new ScriptTester(character_scripts);
         }
+
         /**
          * Add list of scripts that are acceptable in combination together.
          *

UnicodeJsps/src/main/java/org/unicode/jsp/Typology.java

@@ -34,6 +34,7 @@ public class Typology {
     public static Relation<String, String> labelToPaths =
             new Relation(new TreeMap(), TreeSet.class);
     public static Map<String, Map<String, UnicodeSet>> label_parent_uset = new TreeMap();
+
     // public static Relation<String, String> pathToList = new Relation(new TreeMap(),
     // TreeSet.class);