Make the canonical decomposition invariant test easier to maintain (#… #871
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build.md | |
| on: | |
| push: | |
| branches: | |
| - '*' | |
| pull_request: | |
| branches: | |
| - '*' | |
| env: | |
| CURRENT_UVERSION: 16.0.0 | |
| PREVIOUS_UVERSION: 15.1.0 | |
| jobs: | |
| # Using the Java style formatter google-java-style provided by the Spotless | |
| # plugin configured in the root pom.xml using 4-space indents (AOSP style). | |
| # Spotless is configured to run only on files in this branch (PR) that differ | |
| # from origin/main | |
| formatter: | |
| name: Formatter + Style checker | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Unicode Tools | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # fetch all branches so that Spotless can resolve `origin/main` | |
| - name: Check Java style | |
| run: mvn spotless:check || (echo "Style checker failed. Formatting changes can be applied by 'mvn spotless:apply'" && exit 1) | |
| # Only run 1 or 2 commands for in-source build instructions, just to demonstrate | |
| # what translating between an out-of-source build and an in-source build would | |
| # look like. Rely on out-of-source build steps as the canonical way to | |
| # execute tools in CI and write corresponding documentation. | |
| in-source-build: | |
| name: In-source Instructions (Examples) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Unicode Tools | |
| uses: actions/checkout@v3 | |
| - name: Get the CLDR_REF from pom.xml | |
| id: cldr_ref | |
| run: echo "CLDR_REF="$(mvn help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} | |
| - name: Verify CLDR checkout ref | |
| run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty | |
| - name: Cache CLDR repository | |
| uses: actions/cache@v3 | |
| with: | |
| path: cldr | |
| key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} | |
| restore-keys: | | |
| cldr | |
| - name: Check out CLDR | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: unicode-org/cldr | |
| path: cldr | |
| ref: main | |
| fetch-depth: 0 | |
| - name: Switch CLDR to CLDR_REF | |
| run: cd cldr && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} | |
| - name: Backup Unicodetools and CLDR for jsps # this is needed only for the Docker build | |
| run: | |
| mkdir -p UnicodeJsps/target && tar -cpz --exclude=.git -f UnicodeJsps/target/cldr-unicodetools.tgz ./cldr ./unicodetools | |
| - name: Symlink CLDR working copy to be sibling of Unicode Tools | |
| run: | | |
| ln -s $(pwd)/cldr .. | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: 11 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Set up in-source output dir | |
| run: | | |
| mkdir -p Generated/BIN | |
| - name: Run command - Build and Test | |
| run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run command - Make Unicode Files | |
| run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| out-of-source-build: | |
| name: Out-of-source Instructions | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Unicode Tools | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: unicode-org/unicodetools | |
| path: unicodetools/mine/src | |
| - name: Get the CLDR_REF from pom.xml | |
| id: cldr_ref | |
| run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} | |
| - name: Verify CLDR checkout ref | |
| run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty | |
| - name: Cache CLDR repository | |
| uses: actions/cache@v3 | |
| with: | |
| path: cldr/mine/src | |
| key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} | |
| restore-keys: | | |
| cldr | |
| - name: Check out CLDR | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: unicode-org/cldr | |
| path: cldr/mine/src | |
| ref: main | |
| fetch-depth: 0 | |
| - name: Switch CLDR to CLDR_REF | |
| run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: 11 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Set up out-of-source output dir | |
| run: | | |
| mkdir -p unicodetools/mine/Generated/BIN | |
| - name: Run command - Build and Test | |
| run: | | |
| cd unicodetools/mine/src | |
| MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload UnicodeTestResults artifact | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: unicode-test-results | |
| path: | | |
| unicodetools/mine/Generated/UnicodeTestResults.* | |
| - name: Run command - Make Unicode Files | |
| run: | | |
| cd unicodetools/mine/src | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava | |
| - name: Run command - AAC Order | |
| run: | | |
| cd unicodetools/mine/src | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.tools.AacOrder" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests | |
| # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names | |
| - name: Run command - UCA - collation validity log | |
| run: | | |
| cd unicodetools/mine/src | |
| # invoke main() in class ...UCA.Main | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| # check for output file | |
| compgen -G "../Generated/UCA/*/CheckCollationValidity.html" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md | |
| - name: Run command - IDNA | |
| run: | | |
| cd unicodetools/mine/src | |
| # Confirm that the directory for the bin files exists | |
| compgen -G "../Generated/BIN" | |
| # "Delete all the bin files" as per instructions | |
| rm -rf ../Generated/BIN/* | |
| # run GenerateIdna | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.idna.GenerateIdna" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md | |
| - name: Run command - Generate IDNA Test | |
| run: | | |
| cd unicodetools/mine/src | |
| # run GenerateIdnaTest | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.idna.GenerateIdnaTest" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/newunicodeproperties.md#run-generateenumsjava | |
| - name: Run command - Generate Enums Test | |
| run: | | |
| cd unicodetools/mine/src | |
| # run GenerateEnums | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.props.GenerateEnums" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| # apply formatting because generated file will not pass Java formatter | |
| mvn spotless:apply '-DspotlessFiles=.*[\\/]org[\\/]unicode[\\/]props[\\/]UcdProperty(Values)?.java' | |
| # Fail if we haven't committed changes from Generate Enums. | |
| echo "Check if GenerateEnums output has been committed to repo" | |
| echo "git diff --compact-summary" | |
| if git diff --compact-summary --exit-code; then | |
| echo "Success: Any output from GenerateEnums has already been committed." | |
| exit 0 | |
| else | |
| echo "Failure: GenerateEnums has changes that need to be committed." | |
| exit 1 | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/security.md#generating | |
| - name: Run command - Generate Confusables | |
| run: | | |
| cd unicodetools/mine/src | |
| # run GenerateConfusables | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.GenerateConfusables" -Dexec.args="-c -b" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| # run Build & Test command again to rerun TestSecurity test to verify output is okay | |
| MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestSecurity -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/unihan.md#run-generateunihancollators | |
| - name: Run command - Generate Unihan Collators | |
| run: | | |
| cd unicodetools/mine/src | |
| # run GenerateUnihanCollators | |
| mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.draft.GenerateUnihanCollators" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # https://github.com/unicode-org/unicodetools/blob/main/docs/newunicodeproperties.md#checking-other-properties | |
| - name: Run command - Checking Other Properties | |
| run: | | |
| cd unicodetools/mine/src | |
| # run CheckProperties | |
| # Note: the test-compile phase/target needs to precede `compile exec:java` because | |
| # the source code is in src/test/java, not src/main/java, and we need the | |
| # code to compile before it is executed. | |
| mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |