From 7bec328ad4eee4329acb08ecafca5d96d1cab774 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 18:37:27 +0200 Subject: [PATCH 01/17] Split out UCA --- .github/workflows/cli-build-instructions.yml | 76 ++++++++++++++++---- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 3fd9e8e56..aad6f70ab 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -91,8 +91,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - out-of-source-build: - name: Out-of-source Instructions + # Out-of-source build. + ucd-security-idna-emoji: + name: Check UCD, security, IDNA, & emoji data runs-on: ubuntu-latest steps: - name: Checkout Unicode Tools @@ -151,6 +152,7 @@ jobs: path: | unicodetools/mine/Generated/UnicodeTestResults.* + # TODO(egg): This should check that the Unicode files have been made, not just that they can be made. - name: Run command - Make Unicode Files run: | cd unicodetools/mine/src @@ -166,18 +168,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests - # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names - - name: Run command - UCA - collation validity log - run: | - cd unicodetools/mine/src - # invoke main() in class ...UCA.Main - mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION - # check for output file - compgen -G "../Generated/UCA/*/CheckCollationValidity.html" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md - name: Run command - IDNA run: | @@ -252,3 +242,61 @@ jobs: mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Out-of-source build. + uca: + name: Check UCA data + runs-on: ubuntu-latest + steps: + - name: Checkout Unicode Tools + uses: actions/checkout@v3 + with: + repository: unicode-org/unicodetools + path: unicodetools/mine/src + - name: Get the CLDR_REF from pom.xml + id: cldr_ref + run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} + - name: Verify CLDR checkout ref + run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty + - name: Cache CLDR repository + uses: actions/cache@v3 + with: + path: cldr/mine/src + key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} + restore-keys: | + cldr + - name: Check out CLDR + uses: actions/checkout@v3 + with: + repository: unicode-org/cldr + path: cldr/mine/src + ref: main + fetch-depth: 0 + - name: Switch CLDR to CLDR_REF + run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up out-of-source output dir + run: | + mkdir -p unicodetools/mine/Generated/BIN + + # https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests + # Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names + - name: Run command - UCA - collation validity log + run: | + cd unicodetools/mine/src + # invoke main() in class ...UCA.Main + mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + # check for output file + compgen -G "../Generated/UCA/*/CheckCollationValidity.html" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e832ca73e12d40e72de4900274f4f89286d02bb6 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 19:21:57 +0200 Subject: [PATCH 02/17] =?UTF-8?q?Use=20Josh=E2=80=99s=20fancy=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cli-build-instructions.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index aad6f70ab..a94e47f98 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -152,7 +152,6 @@ jobs: path: | unicodetools/mine/Generated/UnicodeTestResults.* - # TODO(egg): This should check that the Unicode files have been made, not just that they can be made. - name: Run command - Make Unicode Files run: | cd unicodetools/mine/src @@ -160,6 +159,20 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Check that UCD files are consistent + run: | + ./py/copygenerateducd.py + git diff --compact-summary --exit-code || { + git diff --compact-summary | + awk '{ + if (previous) { + print "::error file="previous",title=File must be regenerated::Run org.unicode.text.UCD.Main build MakeUnicodeFiles and copy any changed files to unicodetools/data/ucd/dev." + } + previous=$1 + }' + exit 1 + } + # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava - name: Run command - AAC Order run: | From 4a2d585b2ac0db94c51d3d2462018f5b3e1b2d57 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 19:47:47 +0200 Subject: [PATCH 03/17] cd --- .github/workflows/cli-build-instructions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index a94e47f98..6eadf3e1f 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -161,6 +161,7 @@ jobs: - name: Check that UCD files are consistent run: | + cd unicodetools/mine/src ./py/copygenerateducd.py git diff --compact-summary --exit-code || { git diff --compact-summary | From 20b6070f56cb6c9dda9a53cd8247e3832b1790bd Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 20:01:49 +0200 Subject: [PATCH 04/17] =?UTF-8?q?=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cli-build-instructions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 6eadf3e1f..b615870fd 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -162,7 +162,7 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - ./py/copygenerateducd.py + python3 -B ./py/copygenerateducd.py git diff --compact-summary --exit-code || { git diff --compact-summary | awk '{ From 980f3a0e15ccad86a6a7bd55bd0907ee917ad639 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 21:11:12 +0200 Subject: [PATCH 05/17] out-of-source --- .github/workflows/cli-build-instructions.yml | 2 +- py/copygenerateducd.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index b615870fd..0386fcdad 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -162,7 +162,7 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - python3 -B ./py/copygenerateducd.py + python3 -B ./py/copygenerateducd.py -- --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | awk '{ diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index a1a8f2f73..fb26fa3c0 100644 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -17,9 +17,10 @@ def main(): + out_of_source = '--out-of-source' in sys.argv()[1:] cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") - genucddir = cwd / "Generated" / "UCD" / uversion + genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion if not genucddir.exists(): raise Exception(f"Generated directory not found at {genucddir.absolute()}") @@ -34,7 +35,7 @@ def main(): print("THE FOLLOWING FILES WILL BE MOVED:\n") print("\n".join([f"{str(p.name)} --> {devucddir / p.relative_to(genucddir)}" for p in to_move])) # noqa: E501 - confirm = bool(sys.argv[-1] == "-y") # enable running this in automation + confirm = bool("-y" in sys.argv[1:]) # enable running this in automation if not confirm: confirm = input("\nProceed [y/N]?").lower() == "y" From b08b877ca63c2ddbeac8c0c8bd19bdd17cf562b0 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 01:40:23 +0200 Subject: [PATCH 06/17] chmod --- py/copygenerateducd.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 py/copygenerateducd.py diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py old mode 100644 new mode 100755 From 5cc59a35881aa5dfb5aa4dbff13711ae928a9812 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 01:41:25 +0200 Subject: [PATCH 07/17] =?UTF-8?q?#=EF=B8=8F=E2=83=A3=E2=9D=97=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cli-build-instructions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 0386fcdad..f4cb71d98 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -162,7 +162,7 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - python3 -B ./py/copygenerateducd.py -- --out-of-source -y + ./py/copygenerateducd.py --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | awk '{ From ee448b867d07f7809477f42a876fad923c2fb9d2 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Fri, 6 Oct 2023 19:22:51 +0200 Subject: [PATCH 08/17] meow --- unicodetools/data/ucd/dev/UnicodeData.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unicodetools/data/ucd/dev/UnicodeData.txt b/unicodetools/data/ucd/dev/UnicodeData.txt index 8f88960db..4a51459ab 100644 --- a/unicodetools/data/ucd/dev/UnicodeData.txt +++ b/unicodetools/data/ucd/dev/UnicodeData.txt @@ -19401,7 +19401,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; -10EC2;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC2;ARABIC LETTER MEOW WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC3;ARABIC LETTER TAH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC4;ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; From 0907430d3248c8a07a7e62da7a6c6c17110f31c2 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 02:17:12 +0200 Subject: [PATCH 09/17] stat --- .github/workflows/cli-build-instructions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index f4cb71d98..86405beaf 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -162,6 +162,7 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src + stat py/copygenerateducd.py ./py/copygenerateducd.py --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | From 4e5fc53c6e0ca71937ea9a2dc65ba77e1221f4c8 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 02:42:52 +0200 Subject: [PATCH 10/17] just chmod it --- .github/workflows/cli-build-instructions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 86405beaf..d70525f58 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -162,7 +162,7 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - stat py/copygenerateducd.py + chmod +x py/copygenerateducd.py # TODO(egg): Why do we need that despite my git update-index --chmod=+x? ./py/copygenerateducd.py --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | From 809052cae0d035ab1f8047e3ce2bfd74f4c5b327 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:00:28 +0200 Subject: [PATCH 11/17] no () --- .github/workflows/cli-build-instructions.yml | 53 ++++++++++++++++++-- py/copygenerateducd.py | 2 +- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index d70525f58..2b1ed4541 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -92,8 +92,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Out-of-source build. - ucd-security-idna-emoji: - name: Check UCD, security, IDNA, & emoji data + tests: + name: Build and test, including invariants runs-on: ubuntu-latest steps: - name: Checkout Unicode Tools @@ -152,6 +152,52 @@ jobs: path: | unicodetools/mine/Generated/UnicodeTestResults.* + # Out-of-source build. + ucd-consistency-and-smoke-tests: + name: Check UCD consistency, smoke-test generators + runs-on: ubuntu-latest + steps: + - name: Checkout Unicode Tools + uses: actions/checkout@v3 + with: + repository: unicode-org/unicodetools + path: unicodetools/mine/src + - name: Get the CLDR_REF from pom.xml + id: cldr_ref + run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} + - name: Verify CLDR checkout ref + run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty + - name: Cache CLDR repository + uses: actions/cache@v3 + with: + path: cldr/mine/src + key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} + restore-keys: | + cldr + - name: Check out CLDR + uses: actions/checkout@v3 + with: + repository: unicode-org/cldr + path: cldr/mine/src + ref: main + fetch-depth: 0 + - name: Switch CLDR to CLDR_REF + run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up out-of-source output dir + run: | + mkdir -p unicodetools/mine/Generated/BIN + - name: Run command - Make Unicode Files run: | cd unicodetools/mine/src @@ -162,7 +208,8 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - chmod +x py/copygenerateducd.py # TODO(egg): Why do we need that despite my git update-index --chmod=+x? + # TODO(egg): Why do we need to chmod despite my git update-index --chmod=+x? + chmod +x py/copygenerateducd.py ./py/copygenerateducd.py --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index fb26fa3c0..1b64f116b 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -17,7 +17,7 @@ def main(): - out_of_source = '--out-of-source' in sys.argv()[1:] + out_of_source = '--out-of-source' in sys.argv[1:] cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From 51b94562d5cb111a8b3a0d689fe8d22ee2e63230 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:18:27 +0200 Subject: [PATCH 12/17] trace --- .github/workflows/cli-build-instructions.yml | 79 +++++--------------- py/copygenerateducd.py | 2 + 2 files changed, 19 insertions(+), 62 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 2b1ed4541..8c3fccf06 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -91,70 +91,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Out-of-source build. - tests: - name: Build and test, including invariants - runs-on: ubuntu-latest - steps: - - name: Checkout Unicode Tools - uses: actions/checkout@v3 - with: - repository: unicode-org/unicodetools - path: unicodetools/mine/src - - name: Get the CLDR_REF from pom.xml - id: cldr_ref - run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT} - - name: Verify CLDR checkout ref - run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty - - name: Cache CLDR repository - uses: actions/cache@v3 - with: - path: cldr/mine/src - key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }} - restore-keys: | - cldr - - name: Check out CLDR - uses: actions/checkout@v3 - with: - repository: unicode-org/cldr - path: cldr/mine/src - ref: main - fetch-depth: 0 - - name: Switch CLDR to CLDR_REF - run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }} - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up out-of-source output dir - run: | - mkdir -p unicodetools/mine/Generated/BIN - - - name: Run command - Build and Test - run: | - cd unicodetools/mine/src - MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload UnicodeTestResults artifact - if: always() - uses: actions/upload-artifact@v3 - with: - name: unicode-test-results - path: | - unicodetools/mine/Generated/UnicodeTestResults.* # Out-of-source build. - ucd-consistency-and-smoke-tests: - name: Check UCD consistency, smoke-test generators + ucd-and-smoke-tests: + name: UCD consistency, invariants, smoke-test generators runs-on: ubuntu-latest steps: - name: Checkout Unicode Tools @@ -222,6 +162,21 @@ jobs: exit 1 } + - name: Run command - Build and Test + run: | + cd unicodetools/mine/src + MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload UnicodeTestResults artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: unicode-test-results + path: | + unicodetools/mine/Generated/UnicodeTestResults.* + # https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava - name: Run command - AAC Order run: | diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index 1b64f116b..36d7888ff 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -18,6 +18,8 @@ def main(): out_of_source = '--out-of-source' in sys.argv[1:] + print(sys.argv) + print(out_of_source) cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From e6755b922daaa408c24e8d121b7962754be28617 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:29:54 +0200 Subject: [PATCH 13/17] Comments, faster in-source examples, try tracing harder --- .github/workflows/cli-build-instructions.yml | 12 ++++++++++-- py/copygenerateducd.py | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 8c3fccf06..2b354c5fd 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -81,8 +81,14 @@ jobs: run: | mkdir -p Generated/BIN - - name: Run command - Build and Test - run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION + # Since these are just examples to smoke-test the in-source build process, + # let’s not run the whole build and test suite, which is quite slow (6 min + # 26 s as of this writing). Just run the invariant tests and smoke-test + # MakeUnicodeFiles. We don’t even check that MakeUnicodeFiles doesn’t + # change anything, which makes little sense; but that is the job of the + # other job. + - name: Run invariant tests + run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -162,6 +168,8 @@ jobs: exit 1 } + # Only test once we know the UCD is internally consistent. + # MakeUnicodeFiles is much faster than this anyway. - name: Run command - Build and Test run: | cd unicodetools/mine/src diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index 36d7888ff..612383c81 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -18,8 +18,8 @@ def main(): out_of_source = '--out-of-source' in sys.argv[1:] - print(sys.argv) - print(out_of_source) + print(sys.argv,file=sys.stderr) + print(out_of_source,file=sys.stderr) cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From 0ac85b17f641883131260b96716f9fc1426d7d11 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:34:44 +0200 Subject: [PATCH 14/17] gaaaah --- .github/workflows/cli-build-instructions.yml | 2 +- py/copygenerateducd.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index 2b354c5fd..fb5df22cd 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -100,7 +100,7 @@ jobs: # Out-of-source build. ucd-and-smoke-tests: - name: UCD consistency, invariants, smoke-test generators + name: Check UCD consistency, invariants, smoke-test generators runs-on: ubuntu-latest steps: - name: Checkout Unicode Tools diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index 612383c81..c856f1c82 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -18,8 +18,7 @@ def main(): out_of_source = '--out-of-source' in sys.argv[1:] - print(sys.argv,file=sys.stderr) - print(out_of_source,file=sys.stderr) + raise Exception(f"argv={sys.argv}, out_of_source={out_of_source}") cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From 3e7d2934e70e4a3818c76e0e3c18c9eb6cb0feff Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:42:10 +0200 Subject: [PATCH 15/17] That might explain things --- .github/workflows/cli-build-instructions.yml | 3 --- py/copygenerateducd.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-build-instructions.yml b/.github/workflows/cli-build-instructions.yml index fb5df22cd..24d4dce1c 100644 --- a/.github/workflows/cli-build-instructions.yml +++ b/.github/workflows/cli-build-instructions.yml @@ -106,7 +106,6 @@ jobs: - name: Checkout Unicode Tools uses: actions/checkout@v3 with: - repository: unicode-org/unicodetools path: unicodetools/mine/src - name: Get the CLDR_REF from pom.xml id: cldr_ref @@ -154,8 +153,6 @@ jobs: - name: Check that UCD files are consistent run: | cd unicodetools/mine/src - # TODO(egg): Why do we need to chmod despite my git update-index --chmod=+x? - chmod +x py/copygenerateducd.py ./py/copygenerateducd.py --out-of-source -y git diff --compact-summary --exit-code || { git diff --compact-summary | diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index c856f1c82..612383c81 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -18,7 +18,8 @@ def main(): out_of_source = '--out-of-source' in sys.argv[1:] - raise Exception(f"argv={sys.argv}, out_of_source={out_of_source}") + print(sys.argv,file=sys.stderr) + print(out_of_source,file=sys.stderr) cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From 81e0e3708a2e26fe89b46b95b487e44eebc63e67 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:49:05 +0200 Subject: [PATCH 16/17] Drop the traces --- py/copygenerateducd.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/py/copygenerateducd.py b/py/copygenerateducd.py index 612383c81..1b64f116b 100755 --- a/py/copygenerateducd.py +++ b/py/copygenerateducd.py @@ -18,8 +18,6 @@ def main(): out_of_source = '--out-of-source' in sys.argv[1:] - print(sys.argv,file=sys.stderr) - print(out_of_source,file=sys.stderr) cwd = Path().cwd() uversion = os.getenv("CURRENT_UVERSION") genucddir = (cwd / ".." if out_of_source else cwd) / "Generated" / "UCD" / uversion From bbeab8a15c4bdba4c94258b1683453179a624ec5 Mon Sep 17 00:00:00 2001 From: Robin Leroy Date: Sat, 7 Oct 2023 03:53:37 +0200 Subject: [PATCH 17/17] unmeow This reverts commit ee448b867d07f7809477f42a876fad923c2fb9d2. --- unicodetools/data/ucd/dev/UnicodeData.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unicodetools/data/ucd/dev/UnicodeData.txt b/unicodetools/data/ucd/dev/UnicodeData.txt index 4a51459ab..8f88960db 100644 --- a/unicodetools/data/ucd/dev/UnicodeData.txt +++ b/unicodetools/data/ucd/dev/UnicodeData.txt @@ -19401,7 +19401,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; -10EC2;ARABIC LETTER MEOW WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC2;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC3;ARABIC LETTER TAH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EC4;ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;;