CLDR-17424 v45: mfwg2 spec pour (#3626)

common/dtd/messageFormat/message.dtd

@@ -0,0 +1,58 @@
+<!--
+Copyright © 1991-2024 Unicode, Inc.
+For terms of use, see http://www.unicode.org/copyright.html
+SPDX-License-Identifier: Unicode-3.0
+CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
+-->
+
+<!ELEMENT message (
+  (declaration | unsupportedStatement)*,
+  (pattern | (selectors,variant+))
+)>
+
+<!-- In a <declaration type="input">, the <expression> MUST contain a <variable> -->
+<!ELEMENT declaration (expression)>
+<!ATTLIST declaration
+  type (input | local) #REQUIRED
+  name NMTOKEN #REQUIRED
+>
+
+<!ELEMENT unsupportedStatement (expression)+>
+<!ATTLIST unsupportedStatement
+  keyword CDATA #REQUIRED
+  body CDATA #IMPLIED
+>
+
+<!ELEMENT selectors (expression)+>
+<!ELEMENT variant (key+,pattern)>
+<!ELEMENT key (#PCDATA)>
+<!ATTLIST key catchall (true | false) "false">
+
+<!ELEMENT pattern (#PCDATA | expression | markup)*>
+
+<!ELEMENT expression (
+  ((literal | variable), (functionAnnotation | unsupportedAnnotation)?, attribute*) |
+  ((functionAnnotation | unsupportedAnnotation), attribute*)
+)>
+
+<!ELEMENT literal (#PCDATA)>
+
+<!ELEMENT variable (EMPTY)>
+<!ATTLIST variable name NMTOKEN #REQUIRED>
+
+<!ELEMENT functionAnnotation (option)*>
+<!ATTLIST functionAnnotation name NMTOKEN #REQUIRED>
+
+<!ELEMENT option (literal | variable)>
+<!ATTLIST option name NMTOKEN #REQUIRED>
+
+<!ELEMENT unsupportedAnnotation (#PCDATA)>
+
+<!ELEMENT attribute (literal | variable)?>
+<!ATTLIST attribute name NMTOKEN #REQUIRED>
+
+<!ELEMENT markup (option*, attribute*)>
+<!ATTLIST markup
+  kind (open | standalone | close) #REQUIRED
+  name NMTOKEN #REQUIRED
+>

common/dtd/messageFormat/message.json

@@ -0,0 +1,242 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "https://github.com/unicode-org/cldr/blob/maint/maint-45/common/dtd/messageFormat/message.json",
+
+  "oneOf": [{ "$ref": "#/$defs/message" }, { "$ref": "#/$defs/select" }],
+
+  "$defs": {
+    "literal": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "literal" },
+        "value": { "type": "string" }
+      },
+      "required": ["type", "value"]
+    },
+    "variable": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "variable" },
+        "name": { "type": "string" }
+      },
+      "required": ["type", "name"]
+    },
+    "options": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "value": {
+            "oneOf": [
+              { "$ref": "#/$defs/literal" },
+              { "$ref": "#/$defs/variable" }
+            ]
+          }
+        },
+        "required": ["name", "value"]
+      }
+    },
+    "attributes": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "value": {
+            "oneOf": [
+              { "$ref": "#/$defs/literal" },
+              { "$ref": "#/$defs/variable" }
+            ]
+          }
+        },
+        "required": ["name"]
+      }
+    },
+
+    "function-annotation": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "function" },
+        "name": { "type": "string" },
+        "options": { "$ref": "#/$defs/options" }
+      },
+      "required": ["type", "name"]
+    },
+    "unsupported-annotation": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "unsupported-annotation" },
+        "source": { "type": "string" }
+      },
+      "required": ["type", "source"]
+    },
+    "annotation": {
+      "oneOf": [
+        { "$ref": "#/$defs/function-annotation" },
+        { "$ref": "#/$defs/unsupported-annotation" }
+      ]
+    },
+
+    "literal-expression": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "expression" },
+        "arg": { "$ref": "#/$defs/literal" },
+        "annotation": { "$ref": "#/$defs/annotation" },
+        "attributes": { "$ref": "#/$defs/attributes" }
+      },
+      "required": ["type", "arg"]
+    },
+    "variable-expression": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "expression" },
+        "arg": { "$ref": "#/$defs/variable" },
+        "annotation": { "$ref": "#/$defs/annotation" },
+        "attributes": { "$ref": "#/$defs/attributes" }
+      },
+      "required": ["type", "arg"]
+    },
+    "function-expression": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "expression" },
+        "annotation": { "$ref": "#/$defs/function-annotation" },
+        "attributes": { "$ref": "#/$defs/attributes" }
+      },
+      "required": ["type", "annotation"]
+    },
+    "unsupported-expression": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "expression" },
+        "annotation": { "$ref": "#/$defs/unsupported-annotation" },
+        "attributes": { "$ref": "#/$defs/attributes" }
+      },
+      "required": ["type", "annotation"]
+    },
+    "expression": {
+      "oneOf": [
+        { "$ref": "#/$defs/literal-expression" },
+        { "$ref": "#/$defs/variable-expression" },
+        { "$ref": "#/$defs/function-expression" },
+        { "$ref": "#/$defs/unsupported-expression" }
+      ]
+    },
+
+    "markup": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "markup" },
+        "kind": { "oneOf": [ "open", "standalone", "close" ] },
+        "name": { "type": "string" },
+        "options": { "$ref": "#/$defs/options" },
+        "attributes": { "$ref": "#/$defs/attributes" }
+      },
+      "required": ["type", "kind", "name"]
+    },
+
+    "pattern": {
+      "type": "array",
+      "items": {
+        "oneOf": [
+          { "type": "string" },
+          { "$ref": "#/$defs/expression" },
+          { "$ref": "#/$defs/markup" }
+        ]
+      }
+    },
+
+    "input-declaration": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "input" },
+        "name": { "type": "string" },
+        "value": { "$ref": "#/$defs/variable-expression" }
+      },
+      "required": ["type", "name", "value"]
+    },
+    "local-declaration": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "local" },
+        "name": { "type": "string" },
+        "value": { "$ref": "#/$defs/expression" }
+      },
+      "required": ["type", "name", "value"]
+    },
+    "unsupported-statement": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "unsupported-statement" },
+        "keyword": { "type": "string" },
+        "body": { "type": "string" },
+        "expressions": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/expression" }
+        }
+      },
+      "required": ["type", "keyword", "expressions"]
+    },
+    "declarations": {
+      "type": "array",
+      "items": {
+        "oneOf": [
+          { "$ref": "#/$defs/input-declaration" },
+          { "$ref": "#/$defs/local-declaration" },
+          { "$ref": "#/$defs/unsupported-statement" }
+        ]
+      }
+    },
+
+    "variant-key": {
+      "oneOf": [
+        { "$ref": "#/$defs/literal" },
+        {
+          "type": "object",
+          "properties": {
+            "type": { "const": "*" },
+            "value": { "type": "string" }
+          },
+          "required": ["type"]
+        }
+      ]
+    },
+    "message": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "message" },
+        "declarations": { "$ref": "#/$defs/declarations" },
+        "pattern": { "$ref": "#/$defs/pattern" }
+      },
+      "required": ["type", "declarations", "pattern"]
+    },
+    "select": {
+      "type": "object",
+      "properties": {
+        "type": { "const": "select" },
+        "declarations": { "$ref": "#/$defs/declarations" },
+        "selectors": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/expression" }
+        },
+        "variants": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "keys": {
+                "type": "array",
+                "items": { "$ref": "#/$defs/variant-key" }
+              },
+              "value": { "$ref": "#/$defs/pattern" }
+            },
+            "required": ["keys", "value"]
+          }
+        }
+      },
+      "required": ["type", "declarations", "selectors", "variants"]
+    }
+  }
+}

common/testData/messageFormat/README.md

@@ -0,0 +1,41 @@
+# Test Data for CLDR MessageFormat 2.0 Tech Preview
+
+For information about MessageFormat 2.0, see [Unicode Locale Data Markup Language (LDML): Part 9: Message Format](../../../docs/ldml/tr35-messageFormat.md)
+
+The files in this directory were originally copied from the [messageformat project](https://github.com/messageformat/messageformat/tree/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/__fixtures)
+and are here relicensed by their original author (Eemeli Aro) under the Unicode License.
+
+These test files are intended to be useful for testing multiple different message processors in different ways:
+
+- `syntax-errors.json` — An array of strings that should produce a Syntax Error when parsed.
+
+- `data-model-errors.json` - An object with string keys and arrays of strings as values,
+     where each key is the name of an error and its value is an array of strings that
+     should produce `error` when processed.
+     Error names are defined in ["MessageFormat 2.0 Errors"](../../../docs/ldml/tr35-messageFormat.md#errors) in the spec.
+
+- `test-core.json` — An array of test cases that do not depend on any registry definitions.
+  Each test may include some of the following fields:
+  - `src: string` (required) — The MF2 syntax source.
+  - `exp: string` (required) — The expected result of formatting the message to a string.
+  - `locale: string` — The locale to use for formatting. Defaults to 'en-US'.
+  - `params: Record<string, string | number | null | undefined>` — Parameters to pass in to the formatter for resolving external variables.
+  - `parts: object[]` — The expected result of formatting the message to parts.
+  - `cleanSrc: string` — A normalixed form of `src`, for testing stringifiers.
+  - `errors: { type: string }[]` — The runtime errors expected to be emitted when formatting the message.
+     If `errors` is either absent or empty, the message must be formatted without errors.
+  - `only: boolean` — Normally not set. A flag to use during development to only run one or more specific tests.
+
+- `test-function.json` — An object with string keys and arrays of test cases as values,
+  using the same definition as for `test-core.json`.
+  The keys each correspond to a function that is used in the tests.
+  Since the behavior of built-in formatters is implementation-specific,
+  the `exp` field should generally be omitted,
+  except for error cases.
+
+TypeScript `.d.ts` files are included for `test-core.json` and `test-function.json` with the above definition.
+
+Some examples of test harnesses using these tests, from the source repository:
+- [CST parse/stringify tests](https://github.com/messageformat/messageformat/blob/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/cst/cst.test.ts)
+- [Data model stringify tests](https://github.com/messageformat/messageformat/blob/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/data-model/stringify.test.ts)
+- [Formatting tests](https://github.com/messageformat/messageformat/blob/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/messageformat.test.ts)

common/testData/messageFormat/data-model-errors.json

@@ -0,0 +1,32 @@
+{
+    "Variant Key Mismatch": [
+        ".match {$foo :x} * * {{foo}}",
+        ".match {$foo :x} {$bar :x} * {{foo}}"
+    ],
+    "Missing Fallback Variant": [
+        ".match {:foo} 1 {{_}}",
+        ".match {:foo} other {{_}}",
+        ".match {:foo} {:bar} * 1 {{_}} 1 * {{_}}"
+    ],
+    "Missing Selector Annotation": [
+        ".match {$foo} one {{one}} * {{other}}",
+        ".input {$foo} .match {$foo} one {{one}} * {{other}}",
+        ".local $foo = {$bar} .match {$foo} one {{one}} * {{other}}"
+    ],
+    "Duplicate Declaration": [
+        ".input {$foo} .input {$foo} {{_}}",
+        ".input {$foo} .local $foo = {42} {{_}}",
+        ".local $foo = {42} .input {$foo} {{_}}",
+        ".local $foo = {:unknown} .local $foo = {42} {{_}}",
+        ".local $foo = {$bar} .local $bar = {42} {{_}}",
+        ".local $foo = {$foo} {{_}}",
+        ".local $foo = {$bar} .local $bar = {$baz} {{_}}",
+        ".local $foo = {$bar :func} .local $bar = {$baz} {{_}}",
+        ".local $foo = {42 :func opt=$foo} {{_}}",
+        ".local $foo = {42 :func opt=$bar} .local $bar = {42} {{_}}"
+    ],
+    "Duplicate Option Name": [
+            "bad {:placeholder option=x option=x}",
+            "bad {:placeholder ns:option=x ns:option=y}"
+    ]
+}