diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 3fd9e8e56..24d4dce1c 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -81,8 +81,14 @@ jobs: run: | mkdir -p Generated/BIN - - name: Run command - Build and Test - run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + # Since these are just examples to smoke-test the in-source build process, + # let’s not run the whole build and test suite, which is quite slow (6 min + # 26 s as of this writing). Just run the invariant tests and smoke-test + # MakeUnicodeFiles. We don’t even check that MakeUnicodeFiles doesn’t + # change anything, which makes little sense; but that is the job of the + # other job. + - name: Run invariant tests + run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -91,14 +97,15 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - out-of-source-build: - name: Out-of-source Instructions + + # Out-of-source build. + ucd-and-smoke-tests: + name: Check UCD consistency, invariants, smoke-test generators runs-on: ubuntu-latest steps: - name: Checkout Unicode Tools uses: actions/checkout@v3 with: - repository: unicode-org/unicodetools path: unicodetools/mine/src - name: Get the CLDR_REF from pom.xml id: cldr_ref @@ -136,6 +143,30 @@ jobs: run: | mkdir -p unicodetools/mine/Generated/BIN + - name: Run command - Make Unicode Files + run: | + cd unicodetools/mine/src + mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check that UCD files are consistent + run: | + cd unicodetools/mine/src + ./py/copygenerateducd.py --out-of-source -y + git diff --compact-summary --exit-code || { + git diff --compact-summary | + awk '{ + if (previous) { + print "::error file="previous",title=File must be regenerated::Run org.unicode.text.UCD.Main build MakeUnicodeFiles and copy any changed files to unicodetools/data/ucd/dev." + } + previous=$1 + }' + exit 1 + } + + # Only test once we know the UCD is internally consistent. + # MakeUnicodeFiles is much faster than this anyway. - name: Run command - Build and Test run: | cd unicodetools/mine/src @@ -151,13 +182,6 @@ jobs: path: | unicodetools/mine/Generated/UnicodeTestResults.* - - name: Run command - Make Unicode Files - run: | - cd unicodetools/mine/src - mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava - name: Run command - AAC Order run: | @@ -166,18 +190,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests - # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names - - name: Run command - UCA - collation validity log - run: | - cd unicodetools/mine/src - # invoke main() in class ...UCA.Main - mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION - # check for output file - compgen -G "../Generated/UCA/*/CheckCollationValidity.html" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md - name: Run command - IDNA run: | @@ -252,3 +264,61 @@ jobs: mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Out-of-source build. + uca: + name: Check UCA data + runs-on: ubuntu-latest + steps: + - name: Checkout Unicode Tools + uses: actions/checkout@v3 + with: + repository: unicode-org/unicodetools + path: unicodetools/mine/src + - name: Get the CLDR_REF from pom.xml + id: cldr_ref + run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} + - name: Verify CLDR checkout ref + run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty + - name: Cache CLDR repository + uses: actions/cache@v3 + with: + path: cldr/mine/src + key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} + restore-keys: | + cldr + - name: Check out CLDR + uses: actions/checkout@v3 + with: + repository: unicode-org/cldr + path: cldr/mine/src + ref: main + fetch-depth: 0 + - name: Switch CLDR to CLDR_REF + run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up out-of-source output dir + run: | + mkdir -p unicodetools/mine/Generated/BIN + + # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests + # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names + - name: Run command - UCA - collation validity log + run: | + cd unicodetools/mine/src + # invoke main() in class ...UCA.Main + mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + # check for output file + compgen -G "../Generated/UCA/*/CheckCollationValidity.html" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py old mode 100644 new mode 100755 index a1a8f2f73..1b64f116b --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -17,9 +17,10 @@ def main(): + out_of_source = '--out-of-source' in sys.argv[1:] cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") - genucddir = cwd / "Generated" / "UCD" / uversion + genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion if not genucddir.exists(): raise Exception(f"Generated directory not found at {genucddir.absolute()}") @@ -34,7 +35,7 @@ def main(): print("THE FOLLOWING FILES WILL BE MOVED:\n") print("\n".join([f"{str(p.name)} --> {devucddir / p.relative_to(genucddir)}" for p in to_move])) # noqa: E501 - confirm = bool(sys.argv[-1] == "-y") # enable running this in automation + confirm = bool("-y" in sys.argv[1:]) # enable running this in automation if not confirm: confirm = input("\nProceed [y/N]?").lower() == "y"