Merge remote-tracking branch 'la-vache/main' into 172-C3

unicode-org · Oct 2, 2023 · 61c8f93 · 61c8f93
2 parents e089c69 + 340fd97
commit 61c8f93
Show file tree

Hide file tree

Showing 577 changed files with 3,900,014 additions and 603,489 deletions.
diff --git a/.github/workflows/build-jsp.yml b/.github/workflows/build-jsp.yml
@@ -1,8 +1,8 @@
 name: Build JSP
 
 env:
-  CURRENT_UVERSION: 15.1.0
-  PREVIOUS_UVERSION: 15.0.0  # not used at present
+  CURRENT_UVERSION: 16.0.0
+  PREVIOUS_UVERSION: 15.1.0  # not used at present
 
 on:
   push:

diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml
@@ -9,8 +9,8 @@ on:
     - '*'
 
 env:
-  CURRENT_UVERSION: 15.1.0
-  PREVIOUS_UVERSION: 15.0.0
+  CURRENT_UVERSION: 16.0.0
+  PREVIOUS_UVERSION: 15.1.0
 
 jobs:
 
@@ -207,34 +207,17 @@ jobs:
           # run GenerateEnums
           mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.props.GenerateEnums"  -Dexec.args="" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
           # apply formatting because generated file will not pass Java formatter
-          mvn spotless:apply
-          # TODO(#100) Get rid of need for fake version 13.1 of Unicode enum val
-          # Fail if we haven't committed changes from Generate Enums, but make exception for fake Unicode version hack (#100)
+          mvn spotless:apply '-DspotlessFiles=.*[\\/]org[\\/]unicode[\\/]props[\\/]UcdProperty(Values)?.java'
+          # Fail if we haven't committed changes from Generate Enums.
           echo "Check if GenerateEnums output has been committed to repo"
           echo "git diff --compact-summary"
           if git diff --compact-summary --exit-code; then
             echo "Success: Any output from GenerateEnums has already been committed."
             exit 0
           else
-            echo "git diff --name-status | awk '{print $2;}'"
-            names=`git diff --name-status | awk '{print $2;}'`
-            if [[ "$names" == "unicodetools/src/main/java/org/unicode/props/UcdPropertyValues.java" ]]; then
-              gitdiffstat=`git diff --stat | tail -n 1`
-              if [[ "$gitdiffstat" = " 1 file changed, 1 insertion(+), 2 deletions(-)" ]]; then
-                git diff | grep 'V13_1("13.1")'
-                if [ $? -eq 0 ]; then
-                  echo "Inferred: Only non-committed change after GenerateEnums is temporary V13_1 hack (issue #100)"
-                  echo "Check passed."
-                  exit 0
-                fi
-              fi
-            fi
             echo "Failure: GenerateEnums has changes that need to be committed."
             exit 1
           fi
-          # TODO(#100): Note: when running locally on a clean checkout, this command
-          # leaves a change in UcdPropertyValues.java by not preserving the comment
-          # about the fake Unicode version 13.1.
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

diff --git a/.github/workflows/push-jsp-on-tag.yml b/.github/workflows/push-jsp-on-tag.yml
@@ -58,7 +58,7 @@ jobs:
     - name: Get the version
       id: get_tag_name
       run: echo ::set-output name=GIT_TAG_NAME::${GITHUB_REF/refs\/tags\//}
-    - uses: RafikFarhad/push-to-gcr-github-action@v3
+    - uses: RafikFarhad/push-to-gcr-github-action@241707854fb71f655ec4e2a98bb16505f218bcc2
       with:
         gcloud_service_key: ${{ secrets.GCLOUD_SERVICE_KEY }}
         registry: us.gcr.io

diff --git a/.gitignore b/.gitignore
@@ -85,3 +85,4 @@ rules.mk
 
 .DS_Store
 /output
+/cldr
diff --git a/.project b/.project
@@ -16,12 +16,12 @@
 	</natures>
 	<filteredResources>
 		<filter>
-			<id>1625608051965</id>
+			<id>1683027586342</id>
 			<name></name>
 			<type>30</type>
 			<matcher>
 				<id>org.eclipse.core.resources.regexFilterMatcher</id>
-				<arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
+				<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
 			</matcher>
 		</filter>
 	</filteredResources>

diff --git a/LICENSE b/LICENSE
@@ -1,46 +1,39 @@
-UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+UNICODE LICENSE V3
 
-See Terms of Use for definitions of Unicode Inc.'s
-Data Files and Software.
+COPYRIGHT AND PERMISSION NOTICE
 
-NOTICE TO USER: Carefully read the following legal agreement.
-BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
-DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
-YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
-TERMS AND CONDITIONS OF THIS AGREEMENT.
-IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
-THE DATA FILES OR SOFTWARE.
+Copyright © 2001-2023 Unicode, Inc.
 
-COPYRIGHT AND PERMISSION NOTICE
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
 
-Copyright © 1991-2020 Unicode, Inc. All rights reserved.
-Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of the Unicode data files and any associated documentation
-(the "Data Files") or Unicode software and any associated documentation
-(the "Software") to deal in the Data Files or Software
-without restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, and/or sell copies of
-the Data Files or Software, and to permit persons to whom the Data Files
-or Software are furnished to do so, provided that either
-(a) this copyright and permission notice appear with all copies
-of the Data Files or Software, or
-(b) this copyright and permission notice appear in associated
-Documentation.
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
 
-THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT OF THIRD PARTY RIGHTS.
-IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
-NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
-DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
-DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
 
-Except as contained in this notice, the name of a copyright holder
-shall not be used in advertising or otherwise to promote the sale,
-use or other dealings in these Data Files or Software without prior
-written authorization of the copyright holder.
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@ maintainers for the production and vetting of data files for the UCD and
 other Unicode specs such as UCA, emoji, idna, and security.
 
 Do not use the Unicode data files in this repo for production.
-Do use the data files posted publicly on unicode.org.
+Do use the data files [posted publicly on unicode.org](https://www.unicode.org/releases/)
 
 There is some documentation for these tools in this repo, in the [docs folder](./docs/).
 
@@ -21,12 +21,10 @@ use the Unicode Contact Form: https://www.unicode.org/reporting.html
 Do not use the GitHub Issues feature in this repo for those.
 The tools maintainers use GH issues for issues with the code in this repo.
 
-### Licenses
+### Copyright & Licenses
 
-- Data and software is governed by the [Unicode Terms of Use](https://www.unicode.org/copyright.html)
-a copy of which is included as [LICENSE](./LICENSE).
+Copyright © 2001-2023 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries.
 
-### Copyright
+The project is released under [LICENSE](./LICENSE).
 
-© 1991 and later: Unicode, Inc. and others.
-License & terms of use: <https://www.unicode.org/copyright.html>
+A CLA is required to contribute to this project - please refer to the [CONTRIBUTING.md](https://github.com/unicode-org/.github/blob/main/.github/CONTRIBUTING.md) file (or start a Pull Request) for more information.
diff --git a/UnicodeJsps/Dockerfile b/UnicodeJsps/Dockerfile
@@ -11,7 +11,7 @@ RUN ls -lh /build/source/bidiref1 && (/build/source/bidiref1 || true)
 ADD ./target/cldr-unicodetools.tgz /build/data/
 # move this into place (including unicodetools/unicodetools)
 RUN rm -rf /build/data/cldr/.git  # unneeded
-FROM jetty:9-jre11-slim AS run
+FROM jetty:9-jre11-alpine-eclipse-temurin AS run
 ADD port-entrypoint.sh /port-entrypoint.sh
 ADD ./jetty.d/ROOT /var/lib/jetty/webapps/ROOT/
 ENTRYPOINT [ "/port-entrypoint.sh" ]

diff --git a/UnicodeJsps/pom.xml b/UnicodeJsps/pom.xml
@@ -80,7 +80,19 @@
 			<artifactId>unicodetools-testutils</artifactId>
 			<scope>test</scope>
 		</dependency>
-	</dependencies>
+
+		<dependency>
+			<groupId>org.owasp.encoder</groupId>
+			<artifactId>encoder</artifactId>
+			<version>1.2.3</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.owasp.encoder</groupId>
+			<artifactId>encoder-jsp</artifactId>
+			<version>1.2.3</version>
+		</dependency>
+			</dependencies>
 	<build>
 		<finalName>${project.artifactId}</finalName>
 		<plugins>

diff --git a/UnicodeJsps/src/main/java/org/unicode/jsp/CachedProps.java b/UnicodeJsps/src/main/java/org/unicode/jsp/CachedProps.java
@@ -5,6 +5,7 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
 import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.util.ICUUncheckedIOException;
 import com.ibm.icu.util.VersionInfo;
 import java.io.DataInputStream;
@@ -28,7 +29,7 @@
 import org.unicode.props.UnicodeProperty;
 
 public class CachedProps {
-    public static final boolean IS_BETA = false;
+    public static final boolean IS_BETA = true;
 
     public static final Splitter HASH_SPLITTER = Splitter.on('#').trimResults();
     public static final Splitter SEMI_SPLITTER = Splitter.on(';').trimResults();
@@ -43,7 +44,8 @@ public class CachedProps {
     final BiMultimap<String, String> nameToAliases = new BiMultimap<String, String>(null, null);
     final Map<String, BiMultimap<String, String>> nameToValueToAliases = new LinkedHashMap();
 
-    static CachedProps CACHED_PROPS = getInstance(VersionInfo.getInstance(14));
+    static VersionInfo jspVersionInfo = UCharacter.getUnicodeVersion();
+    static CachedProps CACHED_PROPS = getInstance(jspVersionInfo);
 
     static UnicodeProperty NAMES = CachedProps.CACHED_PROPS.getProperty("Name");
 

diff --git a/UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeJsp.java b/UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeJsp.java
@@ -25,6 +25,7 @@
 import org.unicode.idna.Idna2008;
 import org.unicode.idna.Uts46;
 import org.unicode.jsp.UnicodeUtilities.CodePointShower;
+import org.unicode.text.utility.Settings;
 
 public class UnicodeJsp {
 
@@ -420,7 +421,7 @@ public static String getVersions() {
                     : "<p style='border: 1pt solid red;'>Properties use ICU for Unicode V"
                             + UCharacter.getUnicodeVersion().getVersionString(2, 2)
                             + "; the beta properties support Unicode V"
-                            + CachedProps.CACHED_PROPS.version.getVersionString(2, 2)
+                            + VersionInfo.getInstance(Settings.latestVersion).getVersionString(2, 2)
                             + "&beta;. "
                             + "For more information, see <a target='help' href='https://unicode-org.github.io/unicodetools/help/changes'>Unicode Utilities Beta</a>.</p>";
 

diff --git a/UnicodeJsps/src/main/resources/org/unicode/jsp/IdentifierStatus.txt b/UnicodeJsps/src/main/resources/org/unicode/jsp/IdentifierStatus.txt
@@ -1,11 +1,11 @@
 # IdentifierStatus.txt
-# Date: 2022-08-26, 16:49:09 GMT
-# © 2022 Unicode®, Inc.
+# Date: 2023-05-16, 22:25:15 GMT
+# © 2023 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Security Mechanisms for UTS #39
-# Version: 15.0.0
+# Version: 15.1.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr39
 #
@@ -582,7 +582,8 @@ FA27..FA29    ; Allowed    # 1.1    [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK CO
 2B740..2B81D  ; Allowed    # 6.0  [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
 2B820..2CEA1  ; Allowed    # 8.0 [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
 2CEB0..2EBE0  ; Allowed    # 10.0 [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
+2EBF0..2EE4A  ; Allowed    # 15.1 [603] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE4A
 30000..3134A  ; Allowed    # 13.0 [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; Allowed    # 15.0 [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 112156
+# Total code points: 112759
diff --git a/UnicodeJsps/src/main/resources/org/unicode/jsp/IdentifierType.txt b/UnicodeJsps/src/main/resources/org/unicode/jsp/IdentifierType.txt
@@ -1,11 +1,11 @@
 # IdentifierType.txt
-# Date: 2022-08-26, 16:49:09 GMT
-# © 2022 Unicode®, Inc.
+# Date: 2023-05-16, 22:25:14 GMT
+# © 2023 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Security Mechanisms for UTS #39
-# Version: 15.0.0
+# Version: 15.1.0
 #
 # For documentation and usage, see https://www.unicode.org/reports/tr39
 #
@@ -576,10 +576,11 @@ FA27..FA29    ; Recommended                    # 1.1    [3] CJK COMPATIBILITY ID
 2B740..2B81D  ; Recommended                    # 6.0  [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
 2B820..2CEA1  ; Recommended                    # 8.0 [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
 2CEB0..2EBE0  ; Recommended                    # 10.0 [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
+2EBF0..2EE4A  ; Recommended                    # 15.1 [603] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE4A
 30000..3134A  ; Recommended                    # 13.0 [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
 31350..323AF  ; Recommended                    # 15.0 [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
 
-# Total code points: 112139
+# Total code points: 112742
 
 #	Identifier_Type:	Inclusion
 
@@ -1892,6 +1893,7 @@ A8F8..A8FA    ; Obsolete Not_XID               # 5.2    [3] DEVANAGARI SIGN PUSH
 2E9B..2E9E    ; Not_XID                        # 3.0    [4] CJK RADICAL CHOKE..CJK RADICAL DEATH
 2EA0..2EF2    ; Not_XID                        # 3.0   [83] CJK RADICAL CIVILIAN..CJK RADICAL J-SIMPLIFIED TURTLE
 2FF0..2FFB    ; Not_XID                        # 3.0   [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID
+2FFC..2FFF    ; Not_XID                        # 15.1   [4] IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION
 3001..3004    ; Not_XID                        # 1.1    [4] IDEOGRAPHIC COMMA..JAPANESE INDUSTRIAL STANDARD SYMBOL
 3008..301D    ; Not_XID                        # 1.1   [22] LEFT ANGLE BRACKET..REVERSED DOUBLE PRIME QUOTATION MARK
 301F..3020    ; Not_XID                        # 1.1    [2] LOW DOUBLE PRIME QUOTATION MARK..POSTAL MARK FACE
@@ -1903,6 +1905,7 @@ A8F8..A8FA    ; Obsolete Not_XID               # 5.2    [3] DEVANAGARI SIGN PUSH
 3190..3191    ; Not_XID                        # 1.1    [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK
 31C0..31CF    ; Not_XID                        # 4.1   [16] CJK STROKE T..CJK STROKE N
 31D0..31E3    ; Not_XID                        # 5.1   [20] CJK STROKE H..CJK STROKE Q
+31EF          ; Not_XID                        # 15.1       IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
 3248..324F    ; Not_XID                        # 5.2    [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE
 A67E          ; Not_XID                        # 5.1        CYRILLIC KAVYKA
 A720..A721    ; Not_XID                        # 5.0    [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE
@@ -2136,7 +2139,7 @@ FFFD          ; Not_XID                        # 1.1        REPLACEMENT CHARACTE
 1FB00..1FB92  ; Not_XID                        # 13.0 [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK
 1FB94..1FBCA  ; Not_XID                        # 13.0  [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON
 
-# Total code points: 5699
+# Total code points: 5704
 
 #	Identifier_Type:	Not_NFKC