Merge branch 'main' into add-script-sunuwar

unicode-org · Oct 11, 2023 · 9e852f0 · 9e852f0
2 parents 616f2e8 + db60be5
commit 9e852f0
Show file tree

Hide file tree

Showing 650 changed files with 3,901,858 additions and 641,099 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -15,6 +15,7 @@
 *.cc text diff=cpp
 *.classpath text
 *.cmd text eol=crlf
+*.config text eol=lf
 *.cpp text diff=cpp
 *.css text diff=css
 *.dsp text

diff --git a/.github/workflows/build-jsp.yml b/.github/workflows/build-jsp.yml
@@ -1,8 +1,8 @@
 name: Build JSP
 
 env:
-  CURRENT_UVERSION: 15.1.0
-  PREVIOUS_UVERSION: 15.0.0  # not used at present
+  CURRENT_UVERSION: 16.0.0
+  PREVIOUS_UVERSION: 15.1.0  # not used at present
 
 on:
   push:

diff --git a/.github/workflows/cache_retain.yml b/.github/workflows/cache_retain.yml
@@ -0,0 +1,58 @@
+# Copyright (C) 2023 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+#
+# This workflow is designed to keep specific caches on the main
+# branch from getting evicted according to the Github Actions policy
+# (currently in 2023: 7 days) in cases where the cost to construct
+# the cache is high, especially in cases where there is flakiness
+# (ex: network loss / throttling when downloading artifacts) involved in
+# constructing the cache.
+#
+# Preventing a cache from eviction using this workflow requires that:
+#  - the cache is not too big that it starves other caches 
+#    from using the shared cache quota for the repository
+#  - the cache key is specific enough to avoid cache collisions, according
+#    to good cache key design
+#  - the cache key is not overly specific to cause unnecessary cache misses
+#    (resulting in duplicate caches values, thereby wasting space), according
+#    to good cache key design
+#  - For more details, see: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
+
+name: Retain Specific Caches
+
+on:
+  schedule:
+    # Because the Github Actions cache eviction policy is every 7 days,
+    # this cron schedule is set to run every 6 days to ensure retention
+    - cron: '0 12 */6 * *'
+
+jobs:
+  retain-maven-cache:
+    name: Run all tests with Maven
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout and setup
+        uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      # Omitting running any build instructions.
+      #
+      # The point is to touch the cache before the TTL causes it to get evicted.
+      # For Github Actions, the cache is preserved only when the cache action is
+      # invoked and the job completes successfully.
+      #
+      # It seems sufficient for the CI job here to finish successfully by
+      # trivially running nothing.
diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml
@@ -9,8 +9,8 @@ on:
     - '*'
 
 env:
-  CURRENT_UVERSION: 15.1.0
-  PREVIOUS_UVERSION: 15.0.0
+  CURRENT_UVERSION: 16.0.0
+  PREVIOUS_UVERSION: 15.1.0
 
 jobs:
 
@@ -81,8 +81,14 @@ jobs:
         run: |
           mkdir -p Generated/BIN
 
-      - name: Run command - Build and Test
-        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+      # Since these are just examples to smoke-test the in-source build process,
+      # let’s not run the whole build and test suite, which is quite slow (6 min
+      # 26 s as of this writing).  Just run the invariant tests and smoke-test
+      # MakeUnicodeFiles.  We don’t even check that MakeUnicodeFiles doesn’t
+      # change anything, which makes little sense; but that is the job of the
+      # other job.
+      - name: Run invariant tests
+        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -91,14 +97,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  out-of-source-build:
-    name: Out-of-source Instructions
+
+  # Out-of-source build.
+  ucd-and-smoke-tests:
+    name: Check UCD consistency, invariants, smoke-test generators
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Unicode Tools
         uses: actions/checkout@v3
         with:
-          repository: unicode-org/unicodetools
           path: unicodetools/mine/src
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
@@ -136,6 +143,30 @@ jobs:
         run: |
           mkdir -p unicodetools/mine/Generated/BIN
 
+      - name: Run command - Make Unicode Files
+        run: |
+          cd unicodetools/mine/src
+          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check that UCD files are consistent
+        run: |
+          cd unicodetools/mine/src
+          ./py/copygenerateducd.py --out-of-source -y
+          git diff --compact-summary --exit-code || {
+            git diff --compact-summary |
+              awk '{
+                if (previous) {
+                  print "::error file="previous",title=File must be regenerated::Run org.unicode.text.UCD.Main build MakeUnicodeFiles and copy any changed files to unicodetools/data/ucd/dev."
+                }
+                previous=$1
+              }'
+            exit 1
+          }
+
+      # Only test once we know the UCD is internally consistent.
+      # MakeUnicodeFiles is much faster than this anyway.
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
@@ -151,13 +182,6 @@ jobs:
           path: |
             unicodetools/mine/Generated/UnicodeTestResults.*
 
-      - name: Run command - Make Unicode Files
-        run: |
-          cd unicodetools/mine/src
-          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
       # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava
       - name: Run command - AAC Order
         run: |
@@ -166,18 +190,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests
-      # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names
-      - name: Run command - UCA - collation validity log
-        run: |
-          cd unicodetools/mine/src
-          # invoke main() in class ...UCA.Main
-          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
-          # check for output file
-          compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
       # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md
       - name: Run command - IDNA
         run: |
@@ -207,34 +219,17 @@ jobs:
           # run GenerateEnums
           mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.props.GenerateEnums"  -Dexec.args="" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
           # apply formatting because generated file will not pass Java formatter
-          mvn spotless:apply
-          # TODO(#100) Get rid of need for fake version 13.1 of Unicode enum val
-          # Fail if we haven't committed changes from Generate Enums, but make exception for fake Unicode version hack (#100)
+          mvn spotless:apply '-DspotlessFiles=.*[\\/]org[\\/]unicode[\\/]props[\\/]UcdProperty(Values)?.java'
+          # Fail if we haven't committed changes from Generate Enums.
           echo "Check if GenerateEnums output has been committed to repo"
           echo "git diff --compact-summary"
           if git diff --compact-summary --exit-code; then
             echo "Success: Any output from GenerateEnums has already been committed."
             exit 0
           else
-            echo "git diff --name-status | awk '{print $2;}'"
-            names=`git diff --name-status | awk '{print $2;}'`
-            if [[ "$names" == "unicodetools/src/main/java/org/unicode/props/UcdPropertyValues.java" ]]; then
-              gitdiffstat=`git diff --stat | tail -n 1`
-              if [[ "$gitdiffstat" = " 1 file changed, 1 insertion(+), 2 deletions(-)" ]]; then
-                git diff | grep 'V13_1("13.1")'
-                if [ $? -eq 0 ]; then
-                  echo "Inferred: Only non-committed change after GenerateEnums is temporary V13_1 hack (issue #100)"
-                  echo "Check passed."
-                  exit 0
-                fi
-              fi
-            fi
             echo "Failure: GenerateEnums has changes that need to be committed."
             exit 1
           fi
-          # TODO(#100): Note: when running locally on a clean checkout, this command
-          # leaves a change in UcdPropertyValues.java by not preserving the comment
-          # about the fake Unicode version 13.1.
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -269,3 +264,61 @@ jobs:
           mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile  -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Out-of-source build.
+  uca:
+    name: Check UCA data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Unicode Tools
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/unicodetools
+          path: unicodetools/mine/src
+      - name: Get the CLDR_REF from pom.xml
+        id: cldr_ref
+        run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
+      - name: Verify CLDR checkout ref
+        run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
+      - name: Cache CLDR repository
+        uses: actions/cache@v3
+        with:
+          path: cldr/mine/src
+          key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
+          restore-keys: |
+            cldr
+      - name: Check out CLDR
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/cldr
+          path: cldr/mine/src
+          ref: main
+          fetch-depth: 0
+      - name: Switch CLDR to CLDR_REF
+        run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests
+      # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names
+      - name: Run command - UCA - collation validity log
+        run: |
+          cd unicodetools/mine/src
+          # invoke main() in class ...UCA.Main
+          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+          # check for output file
+          compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: "22 5 * * 5"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ javascript, python ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+        if: ${{ matrix.language == 'javascript' || matrix.language == 'python' }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/push-jsp-on-tag.yml b/.github/workflows/push-jsp-on-tag.yml
@@ -25,12 +25,15 @@ jobs:
         restore-keys: |
           cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
     - name: Check out CLDR
-      uses: actions/checkout@v1.2  # yes, v1 due to https://github.com/actions/checkout/issues/265
+      uses: actions/checkout@v3
       with:
         repository: unicode-org/cldr
         path: cldr
-        ref: ${{ steps.cldr_ref.outputs.CLDR_REF }}
-    - name: Backup Unicodetools and CLDR for jsps
+        ref: main
+        fetch-depth: 0
+    - name: Switch CLDR to CLDR_REF
+      run: cd cldr && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+    - name: Backup Unicodetools and CLDR for jsps  # this is needed only for the Docker build
       run:
         mkdir -p UnicodeJsps/target && tar -cpz --exclude=.git -f UnicodeJsps/target/cldr-unicodetools.tgz ./cldr ./unicodetools
     - name: Cache local Maven repository
@@ -55,12 +58,12 @@ jobs:
     - name: Get the version
       id: get_tag_name
       run: echo ::set-output name=GIT_TAG_NAME::${GITHUB_REF/refs\/tags\//}
-    - uses: RafikFarhad/push-to-gcr-github-action@v3
+    - uses: RafikFarhad/push-to-gcr-github-action@241707854fb71f655ec4e2a98bb16505f218bcc2
       with:
         gcloud_service_key: ${{ secrets.GCLOUD_SERVICE_KEY }}
         registry: us.gcr.io
         project_id: dev-infra-273822
         image_name: unicode-jsps
-        image_tag: ${{ steps.get_tag_name.outputs.GIT_TAG_NAME}}
+        image_tag: ${{ steps.get_tag_name.outputs.GIT_TAG_NAME }}
         dockerfile: ./UnicodeJsps/Dockerfile
         context: ./UnicodeJsps/
diff --git a/.gitignore b/.gitignore
@@ -85,3 +85,4 @@ rules.mk
 
 .DS_Store
 /output
+/cldr
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
@@ -0,0 +1,5 @@
+  --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+  --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+  --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+  --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
diff --git a/.project b/.project
@@ -16,12 +16,12 @@
 	</natures>
 	<filteredResources>
 		<filter>
-			<id>1625608051965</id>
+			<id>1683027586342</id>
 			<name></name>
 			<type>30</type>
 			<matcher>
 				<id>org.eclipse.core.resources.regexFilterMatcher</id>
-				<arguments>node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
+				<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
 			</matcher>
 		</filter>
 	</filteredResources>