Split the UCA checks into their own job, check UCD consistency (#562)

unicode-org · Oct 9, 2023 · c0ae922 · c0ae922
1 parent 9e3e387
commit c0ae922
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 26 deletions.
diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml
@@ -81,8 +81,14 @@ jobs:
         run: |
           mkdir -p Generated/BIN
 
-      - name: Run command - Build and Test
-        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+      # Since these are just examples to smoke-test the in-source build process,
+      # let’s not run the whole build and test suite, which is quite slow (6 min
+      # 26 s as of this writing).  Just run the invariant tests and smoke-test
+      # MakeUnicodeFiles.  We don’t even check that MakeUnicodeFiles doesn’t
+      # change anything, which makes little sense; but that is the job of the
+      # other job.
+      - name: Run invariant tests
+        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -91,14 +97,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  out-of-source-build:
-    name: Out-of-source Instructions
+
+  # Out-of-source build.
+  ucd-and-smoke-tests:
+    name: Check UCD consistency, invariants, smoke-test generators
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Unicode Tools
         uses: actions/checkout@v3
         with:
-          repository: unicode-org/unicodetools
           path: unicodetools/mine/src
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
@@ -136,6 +143,30 @@ jobs:
         run: |
           mkdir -p unicodetools/mine/Generated/BIN
 
+      - name: Run command - Make Unicode Files
+        run: |
+          cd unicodetools/mine/src
+          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check that UCD files are consistent
+        run: |
+          cd unicodetools/mine/src
+          ./py/copygenerateducd.py --out-of-source -y
+          git diff --compact-summary --exit-code || {
+            git diff --compact-summary |
+              awk '{
+                if (previous) {
+                  print "::error file="previous",title=File must be regenerated::Run org.unicode.text.UCD.Main build MakeUnicodeFiles and copy any changed files to unicodetools/data/ucd/dev."
+                }
+                previous=$1
+              }'
+            exit 1
+          }
+
+      # Only test once we know the UCD is internally consistent.
+      # MakeUnicodeFiles is much faster than this anyway.
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
@@ -151,13 +182,6 @@ jobs:
           path: |
             unicodetools/mine/Generated/UnicodeTestResults.*
 
-      - name: Run command - Make Unicode Files
-        run: |
-          cd unicodetools/mine/src
-          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
       # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava
       - name: Run command - AAC Order
         run: |
@@ -166,18 +190,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests
-      # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names
-      - name: Run command - UCA - collation validity log
-        run: |
-          cd unicodetools/mine/src
-          # invoke main() in class ...UCA.Main
-          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
-          # check for output file
-          compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
       # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md
       - name: Run command - IDNA
         run: |
@@ -252,3 +264,61 @@ jobs:
           mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile  -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Out-of-source build.
+  uca:
+    name: Check UCA data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Unicode Tools
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/unicodetools
+          path: unicodetools/mine/src
+      - name: Get the CLDR_REF from pom.xml
+        id: cldr_ref
+        run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
+      - name: Verify CLDR checkout ref
+        run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
+      - name: Cache CLDR repository
+        uses: actions/cache@v3
+        with:
+          path: cldr/mine/src
+          key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
+          restore-keys: |
+            cldr
+      - name: Check out CLDR
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/cldr
+          path: cldr/mine/src
+          ref: main
+          fetch-depth: 0
+      - name: Switch CLDR to CLDR_REF
+        run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests
+      # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names
+      - name: Run command - UCA - collation validity log
+        run: |
+          cd unicodetools/mine/src
+          # invoke main() in class ...UCA.Main
+          mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+          # check for output file
+          compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py
@@ -17,9 +17,10 @@
 
 
 def main():
+    out_of_source = '--out-of-source' in sys.argv[1:]
     cwd = Path().cwd()
     uversion = os.getenv("CURRENT_UVERSION")
-    genucddir = cwd / "Generated" / "UCD" / uversion
+    genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion
     if not genucddir.exists():
         raise Exception(f"Generated directory not found at {genucddir.absolute()}")
 
@@ -34,7 +35,7 @@ def main():
         print("THE FOLLOWING FILES WILL BE MOVED:\n")
         print("\n".join([f"{str(p.name)} --> {devucddir / p.relative_to(genucddir)}" for p in to_move]))  # noqa: E501
 
-        confirm = bool(sys.argv[-1] == "-y")  # enable running this in automation
+        confirm = bool("-y" in sys.argv[1:])  # enable running this in automation
         if not confirm:
             confirm = input("\nProceed [y/N]?").lower() == "y"