Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ICU-22908 DRAFT!!! MF2 ICU4J: Update spec tests and update implementation for recent spec changes #3206

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

ICU-22908 DRAFT!!! MF2 ICU4J: Update spec tests and update implementation for recent spec changes #3206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c532790
ICU-22908 MF2 ICU4J: Update spec tests and update implementation for …
mihnita Sep 22, 2024
01b1a3e
WIP
mihnita Jan 29, 2025
d9f9641
WIP: parser done, need to fix the errors
mihnita Jan 29, 2025
8593dde
Merge branch 'main' into mihai_22908
mihnita Jan 29, 2025
5de5b89
WIP: Fix shared test data errors
mihnita Jan 29, 2025
6ddf8a2
WIP: update test files, reduced number of failures (a bit)
mihnita Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ public void testAllKindOfNumbers() {
public void testSpecialPluralWithDecimals() {
String message;
message = ".local $amount = {$count :number}\n"
+ ".match {$amount :number}\n"
+ ".match $amount\n"
+ " 1 {{I have {$amount} dollar.}}\n"
+ " * {{I have {$amount} dollars.}}";
TestUtils.runTestCase(new TestCase.Builder()
Expand All @@ -338,7 +338,7 @@ public void testSpecialPluralWithDecimals() {
.expected("I have 1 dollar.")
.build());
message = ".local $amount = {$count :number minimumFractionDigits=2}\n"
+ ".match {$amount :number minimumFractionDigits=2}\n"
+ ".match $amount\n"
+ " one {{I have {$amount} dollar.}}\n"
+ " * {{I have {$amount} dollars.}}";
TestUtils.runTestCase(new TestCase.Builder()
Expand Down Expand Up @@ -367,7 +367,8 @@ public void testDefaultFunctionAndOptions() {

@Test
public void testSimpleSelection() {
String message = ".match {$count :number}\n"
String message = ".input {$count :number}\n"
+ ".match $count\n"
+ " 1 {{You have one notification.}}\n"
+ " * {{You have {$count} notifications.}}";

Expand All @@ -386,7 +387,9 @@ public void testSimpleSelection() {
@Test
public void testComplexSelection() {
String message = ""
+ ".match {$photoCount :number} {$userGender :string}\n"
+ ".input {$photoCount :number}\n"
+ ".input {$userGender :string}\n"
+ ".match $photoCount $userGender\n"
+ " 1 masculine {{{$userName} added a new photo to his album.}}\n"
+ " 1 feminine {{{$userName} added a new photo to her album.}}\n"
+ " 1 * {{{$userName} added a new photo to their album.}}\n"
Expand Down Expand Up @@ -437,8 +440,9 @@ public void testSimpleLocaleVariable() {
@Test
public void testLocaleVariableWithSelect() {
String message = ""
+ ".input {$count :number}\n"
+ ".local $exp = {$expDate :date year=numeric month=short day=numeric weekday=short}\n"
+ ".match {$count :number}\n"
+ ".match $count\n"
+ " 1 {{Your ticket expires on {$exp}.}}\n"
+ " * {{Your {$count} tickets expire on {$exp}.}}";

Expand Down
55 changes: 23 additions & 32 deletions icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ public static class StringPart implements PatternPart {
@Deprecated
public interface Expression extends PatternPart {
// Provides a common type for all kind of expressions:
// LiteralExpression, VariableExpression, FunctionExpression, UnsupportedExpression, Markup
// LiteralExpression, VariableExpression, FunctionExpression, Markup
}

/**
Expand All @@ -223,17 +223,17 @@ public interface Expression extends PatternPart {
@Deprecated
public static class LiteralExpression implements Expression {
public final Literal arg;
public final Annotation annotation;
public final Function function;
public final List<Attribute> attributes;

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
*/
@Deprecated
public LiteralExpression(Literal arg, Annotation annotation, List<Attribute> attributes) {
public LiteralExpression(Literal arg, Function function, List<Attribute> attributes) {
this.arg = arg;
this.annotation = annotation;
this.function = function;
this.attributes = attributes;
}
}
Expand All @@ -245,7 +245,7 @@ public LiteralExpression(Literal arg, Annotation annotation, List<Attribute> att
@Deprecated
public static class VariableExpression implements Expression {
public final VariableRef arg;
public final Annotation annotation;
public final Function function;
public final List<Attribute> attributes;

/**
Expand All @@ -254,9 +254,9 @@ public static class VariableExpression implements Expression {
*/
@Deprecated
public VariableExpression(
VariableRef arg, Annotation annotation, List<Attribute> attributes) {
VariableRef arg, Function function, List<Attribute> attributes) {
this.arg = arg;
this.annotation = annotation;
this.function = function;
this.attributes = attributes;
}
}
Expand All @@ -266,8 +266,19 @@ public VariableExpression(
* @deprecated This API is for technology preview only.
*/
@Deprecated
public interface Annotation {
// Provides a common type for FunctionAnnotation, UnsupportedAnnotation
public static class Function {
public final String name;
public final Map<String, Option> options;

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
*/
@Deprecated
public Function(String name, Map<String, Option> options) {
this.name = name;
this.options = options;
}
}

/**
Expand All @@ -276,16 +287,16 @@ public interface Annotation {
*/
@Deprecated
public static class FunctionExpression implements Expression {
public final FunctionAnnotation annotation;
public final Function function;
public final List<Attribute> attributes;

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
*/
@Deprecated
public FunctionExpression(FunctionAnnotation annotation, List<Attribute> attributes) {
this.annotation = annotation;
public FunctionExpression(Function function, List<Attribute> attributes) {
this.function = function;
this.attributes = attributes;
}
}
Expand Down Expand Up @@ -359,26 +370,6 @@ public VariableRef(String name) {
}
}

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
*/
@Deprecated
public static class FunctionAnnotation implements Annotation {
public final String name;
public final Map<String, Option> options;

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
*/
@Deprecated
public FunctionAnnotation(String name, Map<String, Option> options) {
this.name = name;
this.options = options;
}
}

/**
* @internal ICU 72 technology preview
* @deprecated This API is for technology preview only.
Expand Down
25 changes: 12 additions & 13 deletions icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@
import java.util.Locale;
import java.util.Map;

import com.ibm.icu.message2.MFDataModel.Annotation;
import com.ibm.icu.message2.MFDataModel.CatchallKey;
import com.ibm.icu.message2.MFDataModel.Declaration;
import com.ibm.icu.message2.MFDataModel.Expression;
import com.ibm.icu.message2.MFDataModel.FunctionAnnotation;
import com.ibm.icu.message2.MFDataModel.Function;
import com.ibm.icu.message2.MFDataModel.FunctionExpression;
import com.ibm.icu.message2.MFDataModel.InputDeclaration;
import com.ibm.icu.message2.MFDataModel.Literal;
Expand Down Expand Up @@ -163,14 +162,14 @@ private Pattern findBestMatchingPattern(
} else if (fph.getInput() instanceof MFDataModel.VariableExpression) {
MFDataModel.VariableExpression ve = (MFDataModel.VariableExpression) fph.getInput();
argument = resolveLiteralOrVariable(ve.arg, variables, arguments);
if (ve.annotation instanceof FunctionAnnotation) {
functionName = ((FunctionAnnotation) ve.annotation).name;
if (ve.function instanceof Function) {
functionName = ((Function) ve.function).name;
}
} else if (fph.getInput() instanceof LiteralExpression) {
LiteralExpression le = (LiteralExpression) fph.getInput();
argument = le.arg;
if (le.annotation instanceof FunctionAnnotation) {
functionName = ((FunctionAnnotation) le.annotation).name;
if (le.function instanceof Function) {
functionName = ((Function) le.function).name;
}
}
SelectorFactory funcFactory = standardFunctions.getSelector(functionName);
Expand Down Expand Up @@ -477,7 +476,7 @@ private static Map<String, Object> convertOptions(
private FormattedPlaceholder formatExpression(
Expression expression, Map<String, Object> variables, Map<String, Object> arguments) {

Annotation annotation = null; // function name
Function function = null; // function name
String functionName = null;
Object toFormat = null;
Map<String, Object> options = new HashMap<>();
Expand All @@ -486,7 +485,7 @@ private FormattedPlaceholder formatExpression(
if (expression instanceof MFDataModel.VariableExpression) {
MFDataModel.VariableExpression varPart = (MFDataModel.VariableExpression) expression;
fallbackString = "{$" + varPart.arg.name + "}";
annotation = varPart.annotation; // function name & options
function = varPart.function; // function name & options
Object resolved = resolveLiteralOrVariable(varPart.arg, variables, arguments);
if (resolved instanceof FormattedPlaceholder) {
Object input = ((FormattedPlaceholder) resolved).getInput();
Expand All @@ -504,11 +503,11 @@ private FormattedPlaceholder formatExpression(
} else if (expression
instanceof MFDataModel.FunctionExpression) { // Function without arguments
MFDataModel.FunctionExpression fe = (FunctionExpression) expression;
fallbackString = "{:" + fe.annotation.name + "}";
annotation = fe.annotation;
fallbackString = "{:" + fe.function.name + "}";
function = fe.function;
} else if (expression instanceof MFDataModel.LiteralExpression) {
MFDataModel.LiteralExpression le = (LiteralExpression) expression;
annotation = le.annotation;
function = le.function;
fallbackString = "{|" + le.arg.value + "|}";
toFormat = resolveLiteralOrVariable(le.arg, variables, arguments);
} else if (expression instanceof MFDataModel.Markup) {
Expand All @@ -523,8 +522,8 @@ private FormattedPlaceholder formatExpression(
}
}

if (annotation instanceof FunctionAnnotation) {
FunctionAnnotation fa = (FunctionAnnotation) annotation;
if (function instanceof Function) {
Function fa = (Function) function;
if (functionName != null && !functionName.equals(fa.name)) {
fatalFormattingError(
"invalid function overrides, '" + functionName + "' <> '" + fa.name + "'");
Expand Down
22 changes: 13 additions & 9 deletions icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,10 @@
import java.util.Set;
import java.util.StringJoiner;

import com.ibm.icu.message2.MFDataModel.Annotation;
import com.ibm.icu.message2.MFDataModel.CatchallKey;
import com.ibm.icu.message2.MFDataModel.Declaration;
import com.ibm.icu.message2.MFDataModel.Expression;
import com.ibm.icu.message2.MFDataModel.FunctionAnnotation;
import com.ibm.icu.message2.MFDataModel.Function;
import com.ibm.icu.message2.MFDataModel.FunctionExpression;
import com.ibm.icu.message2.MFDataModel.InputDeclaration;
import com.ibm.icu.message2.MFDataModel.Literal;
Expand Down Expand Up @@ -140,29 +139,31 @@ private boolean validateDeclarations(List<Declaration> declarations) throws MFPa
private void validateExpression(Expression expression, boolean fromInput)
throws MFParseException {
String argName = null;
Annotation annotation = null;
boolean wasLiteral = false;
Function function = null;
if (expression instanceof Literal) {
// ...{foo}... or ...{|foo|}... or ...{123}...
// does not declare anything
} else if (expression instanceof LiteralExpression) {
LiteralExpression le = (LiteralExpression) expression;
argName = le.arg.value;
annotation = le.annotation;
function = le.function;
wasLiteral = true;
} else if (expression instanceof VariableExpression) {
VariableExpression ve = (VariableExpression) expression;
// ...{$foo :bar opt1=|str| opt2=$x opt3=$y}...
// .input {$foo :number} => declares `foo`, if already declared is an error
// .local $a={$foo} => declares `a`, but only used `foo`, does not declare it
argName = ve.arg.name;
annotation = ve.annotation;
function = ve.function;
} else if (expression instanceof FunctionExpression) {
// ...{$foo :bar opt1=|str| opt2=$x opt3=$y}...
FunctionExpression fe = (FunctionExpression) expression;
annotation = fe.annotation;
function = fe.function;
}

if (annotation instanceof FunctionAnnotation) {
FunctionAnnotation fa = (FunctionAnnotation) annotation;
if (function instanceof Function) {
Function fa = (Function) function;
if (fa.options != null) {
for (Option opt : fa.options.values()) {
LiteralOrVariableRef val = opt.value;
Expand All @@ -184,7 +185,10 @@ private void validateExpression(Expression expression, boolean fromInput)
addVariableDeclaration(argName);
} else {
// Remember that we've seen it, to complain if there is a declaration later
declaredVars.add(argName);
if (!wasLiteral) {
// We don't consider {|bar| :func} to be a declaration of a "bar" variable
declaredVars.add(argName);
}
}
}
}
Expand Down
Loading
Loading