Skip to content

Commit

Permalink
Merge remote-tracking branch 'la-vache/main' into unpoof-shriis
Browse files Browse the repository at this point in the history
  • Loading branch information
eggrobin committed Jun 7, 2024
2 parents c5b0807 + 8215285 commit 10cedb3
Show file tree
Hide file tree
Showing 274 changed files with 129,167 additions and 103,670 deletions.
103 changes: 101 additions & 2 deletions .github/workflows/cli-build-instructions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ jobs:
# change anything, which makes little sense; but that is the job of the
# other job.
- name: Run invariant tests
run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Expand Down Expand Up @@ -170,7 +170,7 @@ jobs:
- name: Run command - Build and Test
run: |
cd unicodetools/mine/src
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DEMIT_GITHUB_ERRORS
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -Dtest=!TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DEMIT_GITHUB_ERRORS
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Expand Down Expand Up @@ -275,6 +275,22 @@ jobs:
with:
repository: unicode-org/unicodetools
path: unicodetools/mine/src
- name: Checkout base UnicodeData.txt
if: ${{ github.event_name == 'pull_request'}}
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
path: base
sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
- name: Compare repertoire
if: ${{ github.event_name == 'pull_request'}}
run: |
# Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
set +e
diff base-repertoire.txt merged-repertoire.txt
echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
- name: Get the CLDR_REF from pom.xml
id: cldr_ref
run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
Expand Down Expand Up @@ -316,9 +332,92 @@ jobs:
- name: Run command - UCA - collation validity log
run: |
cd unicodetools/mine/src
echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
then set +e
fi
# invoke main() in class ...UCA.Main
mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
# check for output file
compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# Out-of-source build.
security:
name: Check security data invariants
runs-on: ubuntu-latest
steps:
- name: Checkout Unicode Tools
uses: actions/checkout@v3
with:
repository: unicode-org/unicodetools
path: unicodetools/mine/src
- name: Checkout base UnicodeData.txt
if: ${{ github.event_name == 'pull_request'}}
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
path: base
sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
- name: Compare repertoire
if: ${{ github.event_name == 'pull_request'}}
run: |
# Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
set +e
diff base-repertoire.txt merged-repertoire.txt
echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
- name: Get the CLDR_REF from pom.xml
id: cldr_ref
run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
- name: Verify CLDR checkout ref
run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
- name: Cache CLDR repository
uses: actions/cache@v3
with:
path: cldr/mine/src
key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
restore-keys: |
cldr
- name: Check out CLDR
uses: actions/checkout@v3
with:
repository: unicode-org/cldr
path: cldr/mine/src
ref: main
fetch-depth: 0
- name: Switch CLDR to CLDR_REF
run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
java-version: 11
- name: Cache local Maven repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up out-of-source output dir
run: |
mkdir -p unicodetools/mine/Generated/BIN
- name: Run invariant tests
run: |
cd unicodetools/mine/src
echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
then ERROR="::notice"
else ERROR="::error"
fi
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS 2>&1 | sed "s/^::error/$ERROR/"
STATUS=${PIPESTATUS[0]}
if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
then exit 0
else exit $STATUS
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion UnicodeJsps/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<!-- icu -->
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j-for-cldr</artifactId>
<artifactId>icu4j</artifactId>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ private int findIdentical(Comparable[][] sortedFlat, int rowIndex, int colIndex)
}
return sortedFlat.length - rowIndex;
}

// to-do: prevent overlap when it would cause information to be lost.
private BitSet breaksSpans = new BitSet();

Expand Down
3 changes: 1 addition & 2 deletions UnicodeJsps/src/main/java/org/unicode/jsp/Confusables.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.unicode.jsp;

import com.ibm.icu.dev.util.CollectionUtilities;
import com.ibm.icu.dev.util.UnicodeMap;
import com.ibm.icu.impl.Utility;
import com.ibm.icu.text.Normalizer;
Expand Down Expand Up @@ -41,7 +40,7 @@ public static UnicodeMap<String> getMap() {
UnicodeMap<String> result = new UnicodeMap<String>();
for (String s : equivalents) {
Set<String> others = new TreeSet<String>(equivalents.getEquivalences(s));
String list = "\u2051" + CollectionUtilities.join(others, "\u2051") + "\u2051";
String list = "\u2051" + String.join("\u2051", others) + "\u2051";
for (String other : others) {
result.put(other, list);
}
Expand Down
17 changes: 17 additions & 0 deletions UnicodeJsps/src/main/java/org/unicode/jsp/Globe.java
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,7 @@ private static void loadSourceMap(String sname) {
}
changeImage(frame);
}

/**
* @param frame
*/
Expand Down Expand Up @@ -704,12 +705,14 @@ public Shape getClip() {
}
return clip;
}

/**
* @return
*/
public boolean usesOrigin() {
return false;
}

// must set before use
Transform set(double srcW, double srcH, double dstW, double dstH) {
this.srcW = srcW;
Expand All @@ -724,6 +727,7 @@ Transform set(double srcW, double srcH, double dstW, double dstH) {
clip = null;
return this;
}

// Remember that the coordinate system is upside down so apply
// the transform as if the angle were negated.
// cos(-angle) = cos(angle)
Expand All @@ -750,6 +754,7 @@ public final boolean transform(double x, double y, DPoint retcoord) {
retcoord.y = lat_dstH.map(retcoord.y);
return retcoord.x >= 0.0 && retcoord.x <= dstW && retcoord.y >= 0 && retcoord.y <= dstH;
}

// Remember that the coordinate system is upside down so apply
// the transform as if the angle were negated. Since inverting
// the transform is also the same as negating the angle, itransform
Expand All @@ -774,18 +779,21 @@ public final boolean itransform(double x, double y, DPoint retcoord) {
retcoord.y = srcH_lat.back(retcoord.y);
return retcoord.x >= 0.0 && retcoord.x <= srcW && retcoord.y >= 0 && retcoord.y <= srcH;
}

/**
* @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
* (radians from -pi to pi)
*/
protected abstract void _transform(DPoint retcoord);

/**
* @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
* (radians from -pi to pi)
*/
protected abstract void _itransform(DPoint retcoord);

protected abstract Shape _getClip();

/**
* @param style
* @return
Expand Down Expand Up @@ -866,6 +874,7 @@ public void _transform(DPoint retcoord) {
public void _itransform(DPoint retcoord) {
// nothing
}

/* (non-Javadoc)
* @see Globe.Transform#_getClip()
*/
Expand Down Expand Up @@ -979,6 +988,7 @@ public static class TransformEqualAreaEllipse extends Transform {
debugTemp = false;
}
}

// Area of a spherical cap is 2 pi r^2 (1-sin(lat))
// Area of a circular segment is r^2 ( acos(p) - p sqrt(1-p^2)), where p = dist to chord/r
// Thus we get the itransform easily:
Expand Down Expand Up @@ -1016,6 +1026,7 @@ public void _itransform(DPoint retcoord) {
protected Shape _getClip() {
return new Ellipse2D.Double(0, 0, dstW, dstH);
}

/**
* @param in -1..1
* @return value in 0..PI
Expand Down Expand Up @@ -1573,6 +1584,7 @@ public void imageComplete(int status) {
consumer.imageComplete(status);
}
}

/*
public static double convertDegreesToDecimal(double degrees, double minutes, double seconds, boolean NorthOrEast) {
double result = (degrees + minutes / 60 + seconds / 3600);
Expand Down Expand Up @@ -1932,6 +1944,7 @@ void fixContents() {
lineContents[c.yStart].add(c);
}
}

/**
* @param c
* @return
Expand Down Expand Up @@ -2066,6 +2079,7 @@ private static void drawDegrees(
}

private static DPoint drawLineP1 = new DPoint();

/*
private static void drawLine(Graphics2D graphics2D, Transform trans, double x1, double y1, double x2, double y2) {
Expand Down Expand Up @@ -2111,6 +2125,7 @@ private static void drawLine2(Graphics2D graphics2D, Transform trans, double x,

abstract static class TTransform {
double x, y;

// t is 0..1
abstract void transform(double t);
}
Expand Down Expand Up @@ -2174,11 +2189,13 @@ static class LineDrawer {
transient double startX, startY, startT;
// transient double endX, endY, endT;
TTransform ttransform;

// int segments = 0;
LineDrawer(Graphics2D graphics2D, TTransform ttransform) {
this.graphics2D = graphics2D;
this.ttransform = ttransform;
}

// t is 0..1
void draw(double startT, double endT) {
this.startT = startT;
Expand Down
5 changes: 3 additions & 2 deletions UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.unicode.jsp;

import com.ibm.icu.dev.util.CollectionUtilities;
import com.ibm.icu.dev.util.UnicodeMap;
import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.lang.UProperty;
Expand Down Expand Up @@ -46,6 +45,7 @@ public enum ScriptSpecials {
public static final int EXTRA_COUNT = 16; // should be enough, hard working as UTC is!

public static final Map<String, Integer> extraScripts = new ConcurrentHashMap<>(EXTRA_COUNT);

/** Extended scripts; note that they do not have stable numbers, and should not be persisted. */
public static final int
// HANT = UScript.CODE_LIMIT,
Expand Down Expand Up @@ -391,7 +391,7 @@ public static String getNames(
for (int i = value.nextSetBit(0); i >= 0; i = value.nextSetBit(i + 1)) {
names.add(ScriptTester.getScriptName(i, choice));
}
return CollectionUtilities.join(names, separator).toString();
return String.join(separator, names);
}
}

Expand Down Expand Up @@ -552,6 +552,7 @@ public ScriptTester get() {
}
return new ScriptTester(character_scripts);
}

/**
* Add list of scripts that are acceptable in combination together.
*
Expand Down
1 change: 1 addition & 0 deletions UnicodeJsps/src/main/java/org/unicode/jsp/Typology.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class Typology {
public static Relation<String, String> labelToPaths =
new Relation(new TreeMap(), TreeSet.class);
public static Map<String, Map<String, UnicodeSet>> label_parent_uset = new TreeMap();

// public static Relation<String, String> pathToList = new Relation(new TreeMap(),
// TreeSet.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1348,6 +1348,7 @@ public static void getDifferences(
};

static Comparator<String> col = UnicodeSetUtilities.MAIN_COLLATOR;

// Collator.getInstance(ULocale.ROOT);
// static {
// ((RuleBasedCollator) col).setNumericCollation(true);
Expand Down Expand Up @@ -1958,6 +1959,7 @@ static Subheader getSubheader() {
static String removals =
new UnicodeSet("[\u1806[:di:]-[:cn:]]").complement().complement().toPattern(false);
static Matcher rem = Pattern.compile(removals).matcher("");

// TODO use UnicodeRegex

// static IdnaLabelTester getIdna2008Tester() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public void TestCollection() {
Set<Foo> w = Builder.with(EnumSet.noneOf(Foo.class)).add(Foo.foo1).freeze();
assertTrue("w", w.size() == 1 && w.contains(Foo.foo1));
}

/**
*
*
Expand Down
1 change: 1 addition & 0 deletions UnicodeJsps/src/test/java/org/unicode/jsptest/TestJsp.java
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ public void assertNoMatch(String message, String pattern, Object actual) {
false,
actual);
}

// return handleAssert(expected == actual, message, stringFor(expected),
// stringFor(actual), "==", false);

Expand Down
Loading

0 comments on commit 10cedb3

Please sign in to comment.